diff --git a/.agents/docs/2026-08-12-bench-suite-architecture-and-plan.md b/.agents/docs/2026-08-12-bench-suite-architecture-and-plan.md new file mode 100644 index 00000000..4b73dc48 --- /dev/null +++ b/.agents/docs/2026-08-12-bench-suite-architecture-and-plan.md @@ -0,0 +1,212 @@ +# `bench/` 构建引擎基准套件 —— 架构与实施计划 + +> 2026-08-12 +> 前置分析:[2026-08-12-modular-build-performance-deep-analysis.md](./2026-08-12-modular-build-performance-deep-analysis.md) +> 目标:把一次性的对比脚本,变成一套**可复用、跨平台、可扩展**的构建引擎基准。 + +--- + +## 0. 为什么要重做一遍 + +上一轮分析用的一次性脚本(bash + hyperfine)能回答"mcpp 和 xmake 谁快",但它有四个结构性缺陷,直接决定了它不能长期用下去: + +| 缺陷 | 后果 | +|---|---| +| 只支持 2 个引擎,加第 3 个要改 `run.sh` 主体 | 每加一个对比对象都动核心逻辑 | +| bash + hyperfine | **Windows 上跑不了**;而 mcpp 是三平台产品 | +| 被测对象只有 mcpp 自己 | 无法回答"模块化 vs 头文件"这个真正的问题 | +| 结果是 TSV,字段随手加 | 跨机器/跨时间的数据无法可靠合并 | + +新套件按四个角度设计:**优雅(加引擎=加一个文件)、架构稳定(协议与实现解耦)、兼容(旧数据可读)、跨平台(不依赖 shell)**。 + +--- + +## 1. 顶层结构 + +``` +bench/ ← 顶层目录,与 src/ tests/ docs/ 平级 + README.md 基准规范(可复用的那份文档) + mcpp.toml 基准工具本身就是一个 mcpp 工程 + src/ + main.cpp + protocol.cppm ★ 协议:结果 schema / 版本 / 序列化 + spec.cppm 矩阵与场景定义(数据,不是代码) + runner.cppm 计时循环:预热、重复、中位数 + registry.cppm 引擎注册表 + engines/ + engine.cppm 适配器契约 + mcpp.cppm cmake.cppm xmake.cppm meson.cppm bazel.cppm + fixture/ + generate.cppm 同一工程 → 头文件版 / 模块版 + emit_buildfiles.cppm 为每个引擎生成构建描述 + analysis/ + ninjalog.cppm graph.cppm report.cppm 构建剖析(--analyze) + platform.cppm 门面(主模块,export import 各分区) + platform/ + posix.cppm 分区:整文件宏控,非 POSIX 上不导出任何符号 + windows.cppm 分区:同上 + results/ 结果 + NOTES.md +``` + +**为什么基准工具本身用 mcpp 写**:它要在 Linux/macOS/Windows 上跑同一套逻辑。bash 在 Windows 上不可用,hyperfine 需要额外安装,而 mcpp 是本仓库必然存在的东西。**用 mcpp 构建 mcpp 的基准工具,顺带也是一次 dogfooding。** + +--- + +## 2. 协议模块(`bench.protocol`)—— 架构稳定性的锚点 + +这是整套设计里唯一"必须先定、之后不能随便改"的东西。 + +```cpp +export module bench.protocol; + +// 结果 schema 的版本。字段增删必须动它,读取侧据此决定兼容策略。 +export inline constexpr int kProtocolVersion = 1; + +export struct HostInfo { // 结果只有配上宿主才有意义 + std::string os, arch, cpu_model; + int logical_cores{}, physical_cores{}; + bool heterogeneous{}; // 13900K 的 8P+16E 不能当 24 个同构核读 + std::uint64_t ram_bytes{}; +}; + +export struct CellKey { // 一个测量单元的完整坐标 + std::string engine, compiler, profile, scenario, fixture, variant; +}; + +export struct Sample { double wall_s{}; int exit_code{}; }; + +export struct CellResult { + CellKey key; + std::vector samples; + double median_s{}, min_s{}, max_s{}; + std::string status; // ok | failed | skipped | unavailable + std::string note; // 失败或跳过的原因,必填 +}; +``` + +**三条不变量**,写死在协议里: + +1. **失败不得伪装成数据。** `status` 与 `median_s` 是两个字段;上一轮 `run.sh` 把失败写成 `0.000s`,就是因为没有这一层。 +2. **跳过必须带原因。** "bazel 不在这台机器上"和"bazel 跑失败了"是完全不同的结论。 +3. **宿主信息与结果同生共死。** 单独一个数字没有意义。 + +序列化为 JSON,字段名即上面的名字,顶层带 `protocol_version`。 + +--- + +## 3. 引擎适配器契约 + +```cpp +export struct Engine { + virtual ~Engine() = default; + virtual std::string_view name() const = 0; + // 这台机器上有没有?没有就 unavailable,不是 failed。 + virtual Availability probe() const = 0; + // 是否支持这个 fixture 变体(headers / modules) + virtual bool supports(Variant) const = 0; + virtual Result configure(const Job&) const = 0; + virtual Result build(const Job&) const = 0; + virtual Result clean(const Job&) const = 0; +}; +``` + +**加一个引擎 = 新增一个 `engines/.cppm` + 在 `registry.cppm` 注册一行。** 不动 runner、不动协议、不动 CI。 + +`supports(Variant, compiler)` 是必要的,而且**编译器是这个问题的一部分** —— 实测:bazel 9.2 + rules_cc 0.2.22 +配 clang 能构建 C++20 模块,配 gcc 则死在它自己的扫描器里(`aggregate-ddi: Invalid JSON string`, +它解析不了 GCC 的 P1689 输出);meson 1.10.2 两个编译器都不行(`module 'fx.a' not found`)。 +所以"bazel 支不支持模块"没有脱离具体运行的答案。不支持时报 `unavailable` **并附上得出该结论的那次测量**, +而不是硬跑出一个误导性的数字。 + +--- + +## 4. Fixture:同一工程的两种形态 + +**生成而非手写。** 手写两份"等价"的代码,几乎必然在某处不等价,而那正是被测量的东西。 + +生成器参数:单元数 `N`、依赖深度 `D`、每单元代码量 `L`。产出: + +``` +fixtures/synth-x/ + headers/ include/unit_k.hpp + src/unit_k.cpp (传统头文件 + 分离实现) + modules/ src/unit_k.cppm (模块接口单元) + modules-impl/ src/unit_k.cppm + src/unit_k_impl.cpp (接口 + 实现单元 ★) +``` + +第三种变体直接对应上一轮分析的 **F4 / §6.3**:把实现移出接口单元。有了它,"改一行函数体"的代价差异就是**测出来的**,不是推断的。 + +同时提供 **`--project ` 模式**:直接就地测量一个已存在的工程(mcpp 自身即基础用例),因为真实工程的依赖形状不是合成器能编出来的。该模式下 variant 轴坍缩为 `native`,并且 `edit-body` 会在测量前后**逐字节保存并恢复**被改的源文件 —— 包括构建失败的路径,那正是遗留改动最容易被忽略的时候。 + +--- + +## 5. 场景矩阵 + +| 维度 | 取值 | +|---|---| +| engine | `mcpp=`(可给多个,自动按版本标注)、cmake、xmake、meson、bazel | +| variant | headers, modules, modules-impl | +| profile | release, debug | +| scenario | cold, noop, touch-hub, edit-body, touch-leaf | +| compiler | gcc, clang, msvc(平台可用者) | + +**"优化前后"用两个真实二进制表达,不用模拟。** `--engines mcpp=<旧>,mcpp=<新>` 会注册两个引擎,各自向自己的二进制询问版本并据此标注(`mcpp@2026.8.11.3` / `mcpp@2026.8.12.1`)。 + +早期设计里有一个 `mcpp-opt` 引擎,靠在构建前后设 `SOURCE_DATE_EPOCH` 来**模拟**优化。已删除:**在 harness 里模拟一个改动,测的是 harness 对该改动的理解**,而且一旦真实实现与之分叉,它会静默地不再跟踪。优化属于 mcpp,基准测的是二进制。 + +矩阵是笛卡尔积但**不是全跑**:`spec.cppm` 用显式的 include/exclude 规则裁剪,CI 默认跑一个小集合,`workflow_dispatch` 可放开。 + +--- + +## 6. 平台拆分 + +采用 **xlings `src/platform/*.cppm` 的既定约定**:模块分区 + 整文件宏控。 + +| 关注点 | 位置 | +|---|---| +| 进程启动 + 墙钟计时 + 退出码 | `platform/posix.cppm`、`platform/windows.cppm` | +| CPU 型号 / 核数 / 异构判定 | 同上 | +| 环境变量读写 | 同上(`setenv` vs `SetEnvironmentVariableA`) | +| 组装与可移植部分(std::filesystem) | 主模块 `platform.cppm` | + +每个分区把**整个 body** 包在一个宏里,非目标平台**不导出任何符号**;两侧导出同名函数,于是任一构建中每个名字只有一份定义,**编译期自动选中**——不需要 stub,也不需要 `if constexpr` 派发。主模块 `export import :posix; :windows;` 后用 `export using` 提升。 + +结果:`#if defined(_WIN32)` 只出现在这两个分区里,runner / engines / protocol / fixture 全部零平台条件。 + +--- + +## 7. CI + +新增 `.github/workflows/bench.yml`: + +- `on: workflow_dispatch`(**只手动触发** —— 基准是重活,不该挂在每个 PR 上) +- 输入:`engines`、`scenarios`、`variants`、`fixture_size`、`runs` +- 矩阵:`ubuntu-24.04` × `macos-14` × `windows-2022`,各自的默认工具链 +- 产出:上传 `results/*.json` 为 artifact +- **不设阈值断言**:基准用于观察趋势,不用于 gate。把噪声变成红叉只会让人忽略它。 + +--- + +## 8. 实施阶段 + +| 阶段 | 内容 | 完成判据 | +|---|---|---| +| **A** | `bench/` 骨架:protocol + platform + runner + registry + mcpp 引擎 | 三平台能跑 `bench --engine mcpp --scenario cold --fixture self` 并产出合法 JSON | +| **B** | fixture 生成器(headers / modules / modules-impl) | 三个变体编译产物行为一致(同一断言集通过) | +| **C** | cmake / xmake / meson / bazel 适配器 | 缺失工具报 `unavailable` 且带原因,不是崩溃 | +| **D** | 构建剖析并入 `--analyze` + 结果合并 | 关键路径与 Python 实现交叉验证一致 | +| **E** | `bench.yml` CI | 手动触发在三平台跑通并上传 artifact | +| **F** | 文档 / 测试 / 版本 / PR / 验证 / 合入 / 发布 | 见目标清单 | + +**顺序是有依赖的**:A 定协议,之后所有阶段都写向它;B 之前 C 无处可跑;D 依赖 A 的结果格式。 + +--- + +## 9. 明确不做 + +- **不把基准挂进 PR CI**。噪声会淹没信号。 +- **不设性能回归阈值**。宿主差异(异构 CPU、云厂商邻居噪声)远大于多数真实回归。 +- **不重新实现计时统计学**。中位数 + min/max 足够;不做置信区间,因为样本量本来就小。 +- **不追求引擎功能对等**。引擎跑不了某个变体就报 unavailable —— 强行凑一个数字比没有数字更糟。 + 但"跑不了"必须是**测出来的**,不是假设的:最初这里写死了 `bazel supports(modules) = false`, + 而实际上加上 `module_interfaces` + `--experimental_cpp_modules --features=cpp_modules` 之后, + bazel 配 clang 是能构建并运行模块程序的。写死的能力判断会把一整列真实数据变成空白。 diff --git a/.agents/docs/2026-08-12-cold-build-optimization-plan.md b/.agents/docs/2026-08-12-cold-build-optimization-plan.md new file mode 100644 index 00000000..db6dd8ca --- /dev/null +++ b/.agents/docs/2026-08-12-cold-build-optimization-plan.md @@ -0,0 +1,334 @@ +# mcpp 冷构建深度优化方案 + +> 2026-08-12 +> 前置:[模块化构建性能深度分析](./2026-08-12-modular-build-performance-deep-analysis.md) · [bench 套件](./2026-08-12-bench-suite-architecture-and-plan.md) +> 范围:**冷构建**(`mcpp clean && mcpp build`)。增量侧的级联抑制已在 2026.8.12.1 落地。 + +--- + +## 0. 现状 + +`bench --project . --engines mcpp=<旧>,mcpp=<新> --scenarios cold`,mcpp 构建 mcpp 自身: + +| 版本 | 冷构建中位数 | +|---|---| +| 2026.8.11.3 | 78.70s | +| 2026.8.12.1(含 `bmi-equal`) | 78.57s | + +**持平,而且必然持平。** `bmi-equal` 修的是「重编后 BMI 未变则不级联」;冷构建里没有「上一份 BMI」,这条路径压根不适用。冷构建要快,必须解决另一组约束。 + +--- + +## 1. 三个互相独立的约束 + +### C1 —— 关键路径 = 100% 墙钟 + +``` +edges : 423 +makespan : 76.54 s +work (sum dur) : 303.36 s +avg parallelism: 3.96 x (of 32 hw threads) +critical path : 76.48 s = 100% of makespan +``` + +后 55% 的时间里,32 个硬件线程上只有 **1 个**编译进程在跑。**这不是调度器不够聪明,是图本身就是一条链。** + +### C2 —— 关键路径上 77% 的时间在生产无人等待的 `.o` + +BMI 在编译进度 **22.8%** 处就被**原子 `rename`** 就位(`strace` 证实:此后 982 个系统调用没有一个再碰它),而 ninja 的依赖模型只认「边结束」。于是每个导入者都要多等一段纯 codegen。 + +### C3 —— mcpp 从不传 `-j`,ninja 用默认的 `nproc + 2` + +这在**本机**是 34,在 62 GB 内存上没问题。但实测单个模块编译的峰值常驻内存: + +| 模块 | 峰值 RSS | +|---|---| +| `build/prepare.cppm`(最重) | **1,057 MB** | +| `build/plan.cppm`(中位偏上) | **561 MB** | + +⇒ 一台 64 核 / 32 GB 的机器会跑 66 路并发 × ~0.5–1 GB = **换页甚至 OOM**。`nproc + 2` 在核多内存少的机器上是**主动有害**的默认值。 + +> C3 与 C1/C2 正交:在图仍是一条链时,调低 `-j` 不会更慢(反正用不满),调高也不会更快。所以 C3 首先是**安全属性**,只有在 C1/C2 解决之后才变成性能属性。 + +--- + +## 2. 优化 A —— BMI 落盘即释放下游(最大项) + +### 2.1 收益已实测,不是模拟 + +机械改写 `build.ninja`、拆边、同一编译器进程,`bench/proto-bmi-release/`: + +| 方案 | 墙钟 | 产物 | +|---|---|---| +| baseline(边完成即释放) | **77.42s** | 19,347,008 B | +| **split(BMI 落盘即释放)** | **36.56s** | 19,347,008 B,可运行 | + +**2.12×,零额外 CPU**(每个模块仍然只有一个 `g++ -c`)。 + +### 2.2 图的形状 + +```ninja +rule cxx_module_bmi # 编译器起跑,BMI 落盘即返回 + command = $mcpp compile-module --phase=spawn --slot $slot --bmi $out --obj $obj_out -- $cxx ... + restat = 1 +rule cxx_module_obj # 等同一个编译器收尾,传播退出码 + command = $mcpp compile-module --phase=wait --slot $slot --obj $out + restat = 1 + +build $bmi : cxx_module_bmi $src | $ddi_dd + dyndep = $ddi_dd +build $obj : cxx_module_obj $bmi +``` + +下游**不需要改动**:dyndep 本来就让导入者依赖 `gcm.cache/X.gcm`,它们只是提前约 4 倍就绪。link 依赖 `obj/*.m.o`,仍然等全部对象。 + +**唯一不显然的改动**:dyndep 把依赖挂在扫描阶段记录的 `-fdeps-target` 上,也就是 `obj/X.m.o`。若不动它,**导入依赖会去门禁那条只负责等待的边,而真正做编译的边在没有任何 BMI 的情况下起跑**。所以扫描边的 `-fdeps-target` 要改指向 BMI。 + +⚠️ 改这个要小心:`cxx_scan` 规则把 `$compile_target` **用了两次** —— 一次 `-fdeps-target=`,一次 `-o`。改共享变量会把预处理输出对准 BMI 路径、有截断风险。必须拆成两个变量,只改 `-fdeps-target`。(原型里就是这么做的。) + +### 2.3 两个会让方案「看起来没用」的实现陷阱 + +**陷阱 1:分离出去的编译器继承了构建系统的 stdout/stderr 管道。** +ninja 判定一条边结束的依据是**管道 EOF,而不是直接子进程退出**。第一阶段即使提前 `exit 0`,只要后台编译器还持有那个 fd,ninja 就认为边还在跑。原型第一次运行就栽在这里:BMI 边中位数 2018 ms(= 完整编译时长),看起来像「这个想法没用」,实际是**测量被伪装成了 baseline**。 +⇒ 后台进程的 stdout/stderr 必须重定向到文件,由第二阶段回放(否则编译器警告与错误静默消失)。 + +**陷阱 2(已被实测缩小):`-j` 与编译器上限的关系。** + +我最初把原型第一次的 78.99s 归因于两件事:管道继承 **和** 「`-j` 必须远大于上限」。后来把 `-j` 单独扫了一遍,结论是**第二条基本不成立**: + +| ninja `-j` | 编译器上限 | 墙钟 | +|---|---|---| +| 32 | 32 | **37.84s** ← 最快 | +| 64 | 32 | 38.23s | +| 128 | 32 | 38.39s | +| 192 | 32 | 39.06s | + +**`-j` 越大反而略慢**(多出来的休眠边只是调度开销)。也就是说那次 78.99s **几乎全部是陷阱 1**,我把一个原因写成了两个。 + +正确的规则很简单:**`-j` 取编译器上限即可**;它不需要更大,也不应该更大。 + +### 2.4 进程生命周期:分离,但**不脱离进程组** + +这是设计里最容易做错的一处。 + +需求只有一条:**别占住 ninja 的管道**。它**不**要求 `setsid()`。 + +保持在同一进程组的直接好处:ninja 收到 Ctrl-C 时会把 SIGINT 发给整个进程组,**分离出去的编译器照样收到**。若为了「干净」而 `setsid()`,反而要自己实现中断清理,并且会留下孤儿编译器继续吃满 CPU。 + +- **POSIX**:`fork` → 子进程把 stdio 重定向到日志 → `exec` 编译器;**不调用 `setsid`**。父进程(spawn 阶段)轮询 BMI 后退出,编译器被 init 收养但仍在原进程组。 +- **Windows**:`CreateProcess` **不带** `DETACHED_PROCESS`,句柄重定向到文件,继承控制台 ⇒ Ctrl-C 正常。后续可加 Job Object 做强保证。 + +### 2.5 失败语义 + +编译器可能在 BMI 落盘**之后**才失败(codegen 阶段的 ICE —— xlings 迁移前的 xmake.lua 就因 GCC 15 在 `-O1/-O2` + modules 上 `tree-ssa-dce` ICE 而强制 `-Og`)。此时: + +- 导入者已经拿着一份**合法的** BMI 开始编译 —— 前端成功过,BMI 有效 +- `--phase=wait` 拿到非零退出码,该边失败,构建整体失败 + +⇒ **失败仍然被报告,只是更晚**,且下游做的是无害的额外工作。诊断顺序可能颠倒(下游错误先于上游失败出现),这一点要写进发布说明。 + +### 2.6 并发上限用什么实现 + +跨平台、无外部依赖:**原子 `mkdir` 令牌目录**。`mkdir` 在 POSIX 与 Windows 上都是原子的「要么成功要么 EEXIST」。令牌在编译器**退出时**释放(由 spawn 阶段的后台段释放),不是在 `wait` 边被调度时 —— 否则上限会被 ninja 的调度延迟放大。 + +不会死锁:持有令牌的进程从不等待另一个令牌。 + +--- + +## 3. 优化 B —— 硬件感知的并发选择(可选项) + +### 3.1 为什么 `nproc + 2` 是错的 + +两个原因,都不是「保守一点更好」这种口味问题: + +1. **内存**:实测每编译 0.5–1 GB(§1 C3)。`nproc+2` 在 64 核/32 GB 上必然换页。 +2. **异构**:i9-13900K 是 8 P-core + 16 E-core。`nproc` 报 32,但 E-core 编译吞吐约为 P-core 的 40%,SMT 兄弟核约 25%。**把 32 当成 32 个同构核,会把有效并行度估高一倍以上。** + +### 3.2 `auto` 的取值 + +``` +jobs_auto = clamp( min( cpu_budget, mem_budget ), 1, 64 ) + +cpu_budget = 异构 ? physical_cores // E-core 不按整核计 + : logical_cores +mem_budget = max( 1, (available_ram - reserve) / per_job_estimate ) + reserve = 2 GiB + per_job_estimate = 768 MiB // 实测中位 561 MB / 峰值 1057 MB +``` + +- 用 **available**(不是 total)内存:构建通常不是机器上唯一的东西 +- `per_job_estimate` 是**可配置常量**,不是猜测:它来自本仓库的实测,并且在文档里注明了来源,便于其他工程按自己的规模调整 +- 上限 64:再高时 ninja 自身的调度开销与文件系统争用开始显现 + +### 3.3 配置面 + +```toml +[build] +jobs = "auto" # 或一个整数;缺省 = 当前行为(不传 -j,由 ninja 决定) +``` + +``` +mcpp build --jobs N|auto +``` + +**默认不变。** 这是刻意的:改变默认并发会改变所有人的构建时长与内存占用,属于行为变更,应当先作为可选项验证一段时间。文档里把 `auto` 标为推荐值。 + +### 3.4 与优化 A 的关系 + +A 落地后仍然只需要**一个**数。§2.3 的实测表明 `-j` 取编译器上限就是最优,更大反而略慢,所以: + +``` +compiler_cap = jobs_auto +ninja_jobs = compiler_cap // 不放大 +``` + +(我一度以为这里需要 `cap * 6`,那是把陷阱 1 的症状误记到了陷阱 2 上;扫描数据见 §2.3。) + +--- + +## 4. 优化 C —— 关键路径感知的调度顺序 + +ninja 在多条就绪边之间的选择顺序是任意的。当图很窄时无所谓(现状后半段并发度 1.0),但 **A 落地后图会变宽**,此时先跑关键路径上的边就有价值。 + +mcpp 已经在扫描阶段拿到了完整模块图,算一次最长路径几乎免费。ninja 没有优先级 API,但可以通过**边的声明顺序**施加弱影响。 + +收益不确定,成本极低,排在 A 之后作为微调。 + +--- + +## 5. 优化 D —— 对象级缓存 + +codegen 占全部工作量的 **77%**。`bmi-equal` 让 BMI 稳定之后,`.o` 也可按内容哈希缓存。mcpp 已有 `~/.mcpp/build-cache/v1/` 用于依赖包,扩展到根包即可。 + +**只对重复的冷构建有效**(切分支来回、revert、CI 缓存恢复),对首次冷构建无效。 + +⚠️ 历史教训:本仓库出现过「命中也 100% 重编」的假缓存,骗了三个月。验收判据必须是**命中时确实跳过了编译**,不是日志里出现 `Cached`。 + +--- + +## 6. 明确不做 + +| 方向 | 为什么不做 | +|---|---| +| 降低优化档位 | 实测 `-O0` 相对 `-O2` 只快 **1.75×**,而产物运行时性能全丢 | +| 缩小 BMI / 降扇入 | 实测 `import std`(31.5 MB BMI)只多 **4.8 ms** —— GCC 导入本来就是惰性的 | +| 优化 scan / dyndep 阶段 | 合计 **0.8%** 的工作量 | +| 分布式编译(distcc/icecc) | 关键路径 100% ⇒ **在 A 之前是负收益**(只增加网络延迟) | +| `-fmodule-only` 两阶段 | 实测它**照样跑完整个 codegen 再丢弃**(15.93s vs 完整 15.95s) | + +--- + +## 7. 实施顺序与验收判据 + +| # | 动作 | 预期 | 验收判据 | +|---|---|---|---| +| **A1** | `mcpp compile-module --phase=spawn/wait` + 拆边 + 扫描 `-fdeps-target` 重定向 | 78.6s → **~37s** | 产物字节一致;**BMI 边中位数远小于 OBJ 边**(否则第一阶段没有提前退出,测的是 baseline);构建后无残留编译器进程;Ctrl-C 不留孤儿 | +| **A2** | Windows 侧(Job Object) | 同上 | Windows e2e 绿 | +| **B** ✅ | `--jobs N\|auto` + `[build] jobs`(**2026.8.12.1 已实施**) | 安全属性;A 之后转为性能属性 | 本机 `auto` → `-j24`(异构 ⇒ 物理核 24,内存预算更大);默认行为不变;8 个单测覆盖公式的每条分支 | +| **C** | 关键路径优先的边序 | 微调 | A 之后重测,无提升则回退 | +| **D** | 对象缓存 | 重复冷构建 | **命中时确实跳过编译**,不是日志说了算 | + +--- + +## 8. 与其他构建系统的对照(同一台机器、同一编译器) + +见 `bench/results/`。要点:xmake 在**同样的图形状**下也是延迟瓶颈(它同样走 GCC 单阶段),所以 A 不是「追平 xmake」,而是**两者都还没做的事**。 + +--- + +# 附录:2026-08-13 复测 —— 优化 A 的依据被重新确立,并纠正一处错误推理 + +上文写 A(BMI 提前释放)时,依据是一次原型测量。这次在 **mcpp 自身 80s 冷构建**上 +重新逐条量过,结论是 **A 成立,而且比原来写的更硬**;但中间我先得出过一个**相反且错误** +的结论,过程值得记下来。 + +## A.1 现状:100% 延迟受限 + +`bench --analyze` 于 mcpp 的 release 构建目录: + +``` +edges : 426 +makespan : 79.79 s +work (sum dur) : 314.08 s +avg parallelism: 3.94 x (of 32 hw threads) +critical path : 79.73 s = 100% of makespan +``` + +**关键路径就是墙钟本身。** 32 个硬件线程上平均并行度只有 3.94 —— 加核、加机器、 +分布式编译全部无效。关键链 26 跳,几乎全是 `cxx_module`,其中 +`mcpp.build.prepare` 单个模块 **16.1s**,占整个构建的 20%。 + +## A.2 ⚠️ 错误推理:用 `-fmodule-only` 判定「codegen 占多少」 + +第一反应是量「只产 BMI」要多久: + +``` +prepare BMI-only 16.18s full 16.27s → 99% +cli BMI-only 5.50s full 5.59s → 98% +plan BMI-only 5.36s full 5.43s → 99% +``` + +据此我一度判定 **A 不成立**:BMI 几乎就是全部成本,代码生成只有 1%,提前释放没有空间。 + +**这是错的。** GCC 的 `-fmodule-only` 并不跳过后端,它跑完整条流水线、只是不写目标文件。 +用它测「BMI 什么时候好」等于什么都没测。 + +## A.3 正确判据:BMI 文件何时**写完**,以及下游能否用 + +三步,缺一不可: + +1. **何时出现** —— 轮询 `.gcm`:`prepare` 的 BMI 在 **2.31s / 16.19s = 14%** 处出现。 + 但「出现」不等于「写完」(GCC 早创建、可能持续写)。 +2. **何时写完** —— 轮询到大小连续 150ms 不变。快照 785488 字节,与编译结束后的成品 + **逐字节相同**。 +3. **是否可用** —— 把早期快照放回 `gcm.cache/`,编译一个真实下游导入者 + (`mcpp.build.execute`):**exit 0**。 + +采样关键链上最重的 8 个模块: + +| 模块 | full | BMI 写完 | 占比 | +|---|---|---|---| +| mcpp.build.prepare | 16.20s | 2.50s | **15%** | +| mcpp.cli | 5.67s | 2.22s | 39% | +| mcpp.build.plan | 5.47s | 1.07s | 20% | +| mcpp.build.compile_commands | 4.74s | 1.03s | 22% | +| mcpp.build.execute | 4.52s | 1.19s | 26% | +| mcpp.libs.toml | 2.28s | 0.53s | 23% | +| mcpp.modgraph.scanner | 3.23s | 0.66s | 20% | +| mcpp.build.ninja | 3.65s | 1.00s | 27% | + +**中位约 22%。下游在等的 78% 是它根本不需要的代码生成。** + +## A.4 头寸 + +关键链 24 个模块节点合计 ~74.7s。若在 BMI 写完即解锁: +`74.7 × 0.22 ≈ 16.4s` + `obj/main.o` 4.78s + link 0.18s ≈ **21s**。 +此后构建转为吞吐受限,下限是 `work / 线程数 = 314 / 32 ≈ 9.8s`,按 60–70% 并行效率 +落在 15–20s。**综合预期 80s → 25–35s(2.3–3.2×)。** + +## A.5 实施形状(未实施) + +ninja 认为一条边完成 = 进程退出,所以必须让「BMI 好了」成为一个可观测事件: + +* **信号**:GCC 的 `-fmodule-mapper`(P1184)在 BMI 落盘时发 `MODULE-COMPILED`。 + 这是设计好的机制,不需要轮询文件大小(轮询只适合做上面这种一次性测量)。 +* **边的形状**:`cxx_module` 改为跑一个 mcpp 助手,它代管 mapper 协议,收到 + `MODULE-COMPILED` 后**把余下的 codegen 甩到后台并退出 0**。 +* **收口**:`cxx_link` 前置一条 `await-objects` 边,等所有后台 codegen 结束。 + 链接本来就在最后,目标文件是并行完成的,所以这条边通常不阻塞。 + +⚠️ 三个已知坑: + +1. **甩到后台的子进程会继承 ninja 的管道** —— 上一次原型就栽在这里:BMI 边的耗时 + 被记成整条编译的耗时,数字变成 78.99s,看起来像「这个想法不成立」。 + 子进程的 stdio 必须重定向到文件。 +2. **失败会迟到** —— 后台 codegen 失败时,`cxx_module` 边已经报成功了。 + `await-objects` 必须收集并复现每个失败,否则会变成链接期的一堆未定义符号。 +3. **作业槽会超订** —— ninja 以为边结束了,后台进程仍在吃 CPU。这在当前 + 3.94× 的并行度下是**想要**的,但在 `--jobs` 很大时需要重新标定。 + +## A.6 顺带:两条不需要改引擎的路 + +* **`mcpp.build.prepare` 一个文件 16.2s,占 20%。** 拆开它直接缩短关键链, + 且不引入任何调度复杂度。 +* **换编译器。** clang 在同类工程上整体快约 2.4×(此前测量),而关键链的形状不变。 diff --git a/.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md b/.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md new file mode 100644 index 00000000..2e8a9164 --- /dev/null +++ b/.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md @@ -0,0 +1,628 @@ +# 模块化 C++ 构建性能深度分析与优化方案 + +> 2026-08-12 — mcpp 2026.8.11.3 自举构建 / xmake v3.0.7 对照 +> 实测宿主:Intel i9-13900K(8 P-core + 16 E-core,32 线程)、62 GB RAM、Linux 6.8 +> 编译器:GCC 16.1.0(mcpp hermetic payload)、Clang 22.1.8 +> 被测工程:mcpp 自身 —— **137 个 `.cppm` 模块接口单元 + 1 个 `main.cpp`,56.6k 行** + +--- + +## 0. 一句话结论 + +mcpp 的自举构建**不是吞吐瓶颈,是延迟瓶颈**:关键路径 = 100% 墙钟时间,后 55% 的时间里 32 个硬件线程上只有 **1 个**编译进程在跑。而这条关键路径上 **77% 的时间在生成没有任何下游需要的 `.o`** —— 下游真正需要的 BMI 平均在编译进度 **22.8%** 处就已经原子落盘。 + +由此得到的最高价值优化不是"编得更快",而是**"更早释放下游"**。这条已经用真实原型验证过,不只是模拟:同一编译器、同样的编译器并发上限、产物一致,**冷构建 77.42s → 36.56s(2.12×),零额外 CPU 工作量**。 + +第二个发现更廉价也更刺眼:仓库里 2026-05-12 就设计并实现了"接口不变则不级联重编"的 BMI restat 机制,但它**从未生效过**——因为 GCC 把 wall-clock 时间戳写进了 BMI 文件内容本身。一个 `SOURCE_DATE_EPOCH` 让 touch 场景从 **73.0s 变成 0.22s**。 + +--- + +## 1. 分析方法与策略 + +### 1.1 策略:先证伪"编译器很慢",再定位"等待很久" + +面对"构建慢",默认假设通常是"编译器慢 / 代码太多"。这个假设**在本例中是错的**,而且错得很具体。分析按以下顺序推进,每一步都要求可证伪: + +| 步骤 | 问题 | 判据 | 结果 | +|---|---|---|---| +| 1 | 工作量 vs 墙钟 | `sum(edge duration)` / `makespan` | 309s / 79s = **3.91×**,32 线程只用上 12% | +| 2 | 是并行度不足还是关键路径长? | 计算真实关键路径 | 关键路径 = **79.01s = 100% 墙钟** → 纯延迟瓶颈 | +| 3 | 关键路径上的时间花在哪? | `-ftime-report` + `-fmodule-only` 对照 | 86% 在 `opt and generate` | +| 4 | 下游真的需要等 codegen 吗? | 轮询 BMI 落盘时刻 + `strace` | **不需要**,BMI 在 22.8% 处原子就位 | +| 5 | 能否让下游早走? | GCC 模块映射器协议实测 | 可以,`MODULE-COMPILED` 就是该信号 | +| 6 | 增量为何也这么慢? | 字节比对连续两次编译的 BMI | GCC 嵌了时间戳 → 级联抑制永久失效 | + +### 1.2 五个把结论带偏的测量陷阱(每一个都真的改变过结论) + +这些不是花絮,是复现本报告时必须避开的坑: + +1. **多输出边在 `.ninja_log` 里每个输出各写一行**,起止时间相同。按行求和会把编译耗时从 302s 读成 604s。必须按 `(start, end, command_hash)` 去重。 + +2. **模块的真实依赖边不在 `build.ninja` 里**,而在构建期生成的 dyndep 文件 `obj/*.ddi.dd` 中。只读 `build.ninja` 算出的关键路径是 **22s**,折入 dyndep 后是 **79s**——差 3.6 倍,足以得出完全相反的结论("并行调度有问题" vs "关键路径就是全部")。 + +3. **dyndep 把依赖挂在 `obj/X.m.o` 上,而导入者依赖的是同一条边的另一个输出 `gcm.cache/X.gcm`。** 不把同一条边的多个输出合并成单个图节点,最长路径走两跳就断了。 + +4. **ninja 是追加写 `.ninja_log` 的,且每次调用时钟从 0 重启。** 多次构建混在一起会算出"关键路径 > makespan"这种不可能的读数(xlings 那份日志第一次跑出 136%)。必须只取最后一次调用。 + +5. **最长路径必须按拓扑序松弛,不能用栈式 DFS。** DFS 里"跳过已在栈上的节点"这个防环写法,会把兄弟分支压入但尚未算完的依赖也当成 0,导致路径提前终止。我的 C++ 版最初就是这样,报出 **33.9s / 10 节点**,而真值是 **76.5s / 26 节点** —— 把"100% 延迟瓶颈"读成了"44%",结论直接反转成"加核有用"。 + +> 第 5 条是**靠交叉验证抓到的**:同一份日志,`bench --analyze`(C++)与独立的 Python 分析器在 makespan、工作量、逐规则耗时、并发曲线上**全部吻合**,唯独关键路径差 2.3 倍。旁证是并发曲线——末尾 40 秒的 1.0× 串行尾巴不可能与 34 秒的关键路径共存。 +> +> **凡是计算关键路径的东西,都要用第二个实现交叉验证。** 这五条已固化进 `bench/src/analysis/` 与 `bench/README.md`。 + +### 1.3 一个被推翻的假设(保留在此以免后人重走) + +**假设**:GCC 的 `-fmodule-only`("Only emit Compiled Module Interface")能跳过 codegen,从而低成本地拿到 BMI,做成两阶段编译。 + +**实测**:`-fmodule-only` 确实**不产出 `.o`**,但 `-ftime-report` 显示它**照样完整执行 `phase opt and generate`(13.65s / 86%)然后把结果丢弃**。总耗时 15.93s vs 完整编译 15.95s。 + +``` +完整编译 : 15.95s → .o + .gcm +-fmodule-only : 15.93s → 只有 .gcm(codegen 白做) +-fsyntax-only : 2.04s → 什么都不产出(证明前端只要 2s) +``` + +**结论**:GCC 在 16.1 上**没有**廉价产出 BMI 的开关。这是 QoI 缺陷,值得向上游报告。Clang 有(`--precompile`)。 + +### 1.4 工具链 + +| 工具 | 用途 | 关键用法 | +|---|---|---| +| `.ninja_log` + 自研分析器 | 每条边的起止毫秒 → 工作量/关键路径/并发曲线 | `bench --analyze ` | +| `hyperfine` 1.18 | 统计严谨的墙钟计时(中位数、prepare/cleanup 钩子) | 所有矩阵单元 | +| `strace -f -tt -e trace=openat,write,close,rename` | 单次编译内 BMI 文件的生命周期 | 证明 BMI 是**原子 rename** 就位 | +| `g++ -ftime-report` | cc1plus 内部分阶段耗时 | 定位 86% 在 codegen | +| `-fmodule-mapper=\|` | 实测 P1184 模块映射器协议 | 证明 `MODULE-COMPILED` 信号存在 | +| 逐字节 `cmp -l` | BMI 可复现性 | 定位到 4 字节时间戳 | +| 离散事件调度模拟器 | 用实测 t_bmi/t_total + 真实依赖图预测收益 | 贪心表调度,P 可扫 | +| **图改写原型** | 把模拟结论变成实测:机械拆边后跑真实构建 | `bench/proto-bmi-release/` | +| `perf` / `bpftrace` | 备用 —— 本轮**没有用上**:瓶颈在调度与 I/O 时序,不在 CPU 采样能看到的地方 | — | + +> 关于 `perf`:提前开了 `perf_event_paranoid=-1`,但整个分析没有用到采样剖析。定位靠的是**构建图的时间结构**(`.ninja_log`)和**单进程内的文件生命周期**(`strace -tt`)。这本身是一条方法论结论:构建性能问题通常不是"哪段代码热",而是"谁在等谁"。 + +--- + +## 2. 基线数据 + +### 2.1 mcpp 自举构建(GCC 16.1,`-O2`,`-j32`) + +`bench --analyze` 输出(可复现): + +``` +edges : 423 (137 cxx_module + 138 cxx_scan + 138 cxx_dyndep + 1 cxx_object + 1 link + 8 stage) +makespan : 76.54 s +work (sum dur) : 303.36 s +avg parallelism: 3.96 x (of 32 hw threads) +critical path : 76.48 s = 100% of makespan +verdict : LATENCY-bound. More cores will not help. +``` + +| rule | count | total_s | avg_ms | max_ms | %work | +|---|---|---|---|---|---| +| `cxx_module` | 137 | 296.29 | 2162.7 | 15892 | **97.7%** | +| `cxx_object` | 1 | 4.53 | 4527.0 | 4527 | 1.5% | +| `cxx_scan` | 138 | 1.75 | 12.7 | 105 | 0.6% | +| `cxx_dyndep` | 138 | 0.60 | 4.4 | 9 | 0.2% | +| `cxx_link` | 1 | 0.16 | 163.0 | 163 | 0.1% | +| `stage_file` | 8 | 0.02 | 2.8 | 12 | 0.0% | + +> 早前一次剖析读数为 makespan 79.07s / work 309.0s / CP 79.01s,同一结论;差异是运行间噪声。hyperfine 3 次中位数为 **77.55s**。 + +**扫描阶段只占 0.8%。** 每个 TU 三次进程调用(scan → dyndep → compile)的开销常被当成嫌疑犯,实测不是。这条要写进结论,以免有人去优化一个 1.8 秒的阶段。 + +### 2.2 并发度塌陷 + +``` + t= 0.0s 17.4x |################################# + t= 4.0s 9.7x |################## + t= 7.9s 5.2x |########## + t= 11.9s 6.2x |############ + t= 15.8s 4.9x |######### + t= 19.8s 8.7x |################ + t= 23.7s 7.3x |############## + t= 27.7s 2.4x |#### + t= 31.6s 2.0x |#### + t= 35.6s 1.0x |## ← 此后 44 秒(墙钟的 55%)全程单线程 + ... + t= 75.1s 1.0x |## +``` + +关键路径 24 层深: +`shell → linux → platform → manifest.types → manifest.toml → manifest → runtime_selection → runtime_binding → elf_runtime → loader_contract → plan → flags → compile_commands → ninja_backend → prepare(16.1s) → execute → configure → cmd_build → cli → main.o → link` + +> §2.1 的 79.07s 是被剖析的那一次完整重建;§4 表格里的 **77.55s** 是 hyperfine 3 次的中位数。两者一致,前者用于结构分析,后者用于对比。 + +### 2.3 BMI 落盘时刻 vs 编译总时长(全部 137 个模块,`-O2`) + +> 方法:逐个模块单独编译,轮询其 `.gcm` 出现的时刻。**这些是隔离测量**,没有 32 路并发下的内存带宽争用,因此合计 258.9s 低于真实构建的 309.0s(约 16%)。这个偏差对 §6.2 的模型 A 和 B **同向作用**,所以那里的**加速比(2.98×)比绝对秒数更可信**。 + +| | 秒 | +|---|---| +| BMI 产出耗时合计 | **59.1** | +| 完整编译耗时合计 | **258.9** | +| **下游白等的 codegen** | **199.8(77.2%)** | +| BMI 平均就绪进度 | **22.8%** | + +最热的几个: + +| 模块 | t_bmi | t_total | BMI 占比 | +|---|---|---|---| +| `build/prepare.cppm` | 2.29 | 15.99 | 14.3% | +| `cli.cppm` | 1.98 | 5.50 | 35.9% | +| `build/plan.cppm` | 0.92 | 5.44 | 17.0% | +| `platform/runtime_binding.cppm` | 0.98 | 5.40 | 18.1% | +| `doctor.cppm` | 1.05 | 5.07 | 20.8% | +| `manifest/toml.cppm` | 0.59 | 4.67 | 12.7% | + +### 2.4 BMI 是原子就位的(可安全提前消费) + +`strace` 抓到的 `build/plan.cppm` 编译过程: + +``` +10:25:01.058 编译开始 +10:25:01.855 openat("gcm.cache/mcpp.build.plan.gcm~", O_RDWR|O_CREAT|O_TRUNC) +10:25:01.950 close(fd) +10:25:01.950 rename("...gcm~", "...gcm") ← BMI 原子就位 +10:25:06.527 进程退出 ← 又跑了 4.58 秒纯 codegen +``` + +**写临时文件 + `rename()`** 意味着 BMI 要么不存在、要么完整,不存在撕裂读。这让"看到 BMI 就放行下游"在**构造上**是安全的,不需要额外加锁或校验。 + +--- + +## 3. 根因 + +### F1 — 构建是延迟瓶颈,加核完全无效 + +离散事件模拟(真实依赖图 + 实测每模块耗时): + +``` +P=8 72.0s → 并行度不是约束 +P=16 72.0s +P=24 72.0s +P=32 72.0s +P=64 72.0s ← 加到 64 线程,一秒都不会快 +``` + +### F2 — 关键路径上 77% 的时间在生产无人等待的 `.o` + +见 §2.3 / §2.4。GCC 单阶段模型下,BMI 与 `.o` 由同一个进程产出;ninja 的依赖模型只认"边结束",于是导入者被迫等到 codegen 收尾。 + +### F3 — GCC 把时间戳写进 BMI,级联抑制机制从未生效 + +`build.ninja` 的 `cxx_module` 规则实现了 2026-05-12 文档设计的 copy-if-different: + +```sh +cp -p $bmi_out $bmi_out.bak && && \ + if cmp -s "$bmi_out" "$bmi_out.bak"; then mv "$bmi_out.bak" "$bmi_out"; else rm -f "$bmi_out.bak"; fi +``` + +同一条命令连编两次,BMI 有 **4 字节**不同: + +``` +buildtime: 2026/08/12 02:25:01 UTC localtime: 2026/08/12 02:25:01 UTC +buildtime: 2026/08/12 02:25:33 UTC localtime: 2026/08/12 02:25:33 UTC + ^^ ^^ +``` + +2026-05-12 的文档写道:「GCC 每次都会重新生成 BMI 文件(即使内容相同**时间戳也变**),所以必须在构建系统层面做 copy_if_different」。**该判断只覆盖了文件 mtime,漏掉了时间戳被写进文件内容**,因此 `cmp -s` 同样必然失败,`restat` 永远认为 BMI 变了。 + +实测后果(touch 一个被 46 个模块导入、内容完全未变的 `platform.cppm`): + +| | 墙钟 | 重跑边数 | +|---|---|---| +| 现状 | **73.0s** | 180 | +| `SOURCE_DATE_EPOCH=<固定值>` | **0.22s** | 5 | + +**332×**,且正确性不变:真实接口变更仍然完整级联(180 边),回退亦然。 + +### F4 — 改函数体照样全量级联(架构问题,不是 bug) + +在 `src/ui.cppm`(27 个导入者)的**非 inline 函数体内加一行注释**,接口完全未动: + +| | 墙钟 | 重跑边数 | +|---|---|---| +| body-only 编辑 | **47.6s** | 66 | + +GCC 的 BMI 携带函数体(为了跨模块内联),所以任何编辑都会改变 BMI 字节。`SOURCE_DATE_EPOCH` 修不了这一类。 + +**真正的根因是工程结构**:mcpp 有 **137 个模块接口单元,却只有 1 个实现 TU(`main.cpp`)**——全部实现代码都写在接口单元里。因此每一次日常编辑的代价都是 O(导入者数),而不是 O(1)。 + +### F5 — 扫描/dyndep 阶段不是瓶颈 + +`cxx_scan` 1.83s + `cxx_dyndep` 0.48s = 全部工作量的 **0.8%**。每 TU 三次进程调用的设计**不需要优化**。 + +### F6 — `prepare.cppm` 单点占关键路径 20% + +16.1s,扇入 **61 个 BMI**(含 31MB 的 `std.gcm` 与 17MB 的 `mcpp.libs.json.gcm`)。 + +--- + +## 4. mcpp vs xmake 实测对比 + +同一份源码(137 `.cppm` + `main.cpp`)、**同一个 `g++` 二进制**(`xim-x-gcc/16.1.0`,由 `xmake.lua` 从 `mcpp.toml` 的 `[toolchain] default` 读取并钉死)、同样 `-std=c++23 -fmodules -O2`、同样 `-j32`。hyperfine 中位数,每格 3 次。 + +两侧各产出 **141 个 BMI** —— 模块集合一致,xmake 的 culling 没有偷偷少编东西。 + +| 场景 | mcpp | xmake | 判读 | +|---|---|---|---| +| **冷构建**(release `-O2`) | **77.55s** | 88.94s | mcpp 快 **1.15×** | +| **冷构建**(debug `-O0 -g`) | **44.23s** | 46.30s | mcpp 快 1.05× | +| **no-op** | 0.430s | **0.381s** | xmake 略快 | +| **touch hub 模块**(46 导入者,内容未变) | **73.79s** | 81.70s | **两者都退化到接近全量重建** | +| **改函数体**(27 导入者,接口未动) | **47.75s** | 52.19s | 两者都完整级联 | +| **touch `main.cpp`** | 5.40s | **5.28s** | 持平 | + +> **`-O0` 相对 `-O2` 只快 1.75×**(77.55 → 44.23)。对一个 codegen 占 77% 工作量的构建,这个比例偏低,再次印证前端与关键路径结构才是主导——**降优化档不是出路**(§6.5)。 +> +> 另可注意:release 档 mcpp 领先 14.7%,debug 档只领先 4.7%。优化档位越高,两个引擎的差距越明显。 + +### 4.1 冷构建差距的归因(两个已声明的不对称,都已量化) + +| 不对称 | 实测值 | 结论 | +|---|---|---| +| mcpp 从全局缓存 stage `std.gcm`,xmake 自己编译 | 编译 `std` 只要 **2.04s** | 只解释约 2s,**不是主因** | +| mcpp 有全局依赖构建缓存 | `mcpp build --cache=off` 冷构建 = **78.46s**(vs 77.55s) | 缓存只值 **0.9s**,**不是主因** | + +⇒ **11.4s 的差距扣除上述约 2s 后仍有约 9s(~10%),是真实的引擎差异**,不是缓存优势。 + +### 4.2 最重要的判读:两个引擎在增量场景下**一起失败** + +`touch-hub` 一栏是全表最关键的信息:一个内容**完全没变**的文件,mcpp 花 73.79s、xmake 花 81.70s,而冷构建分别是 77.55s / 88.94s —— **增量 ≈ 全量**。 + +两个独立实现的构建引擎表现几乎一致,说明这**不是某一家的实现质量问题**,而是 **C++ 命名模块 + GCC 的结构性问题**(§3 的 F3/F4)。这也意味着: + +> 在 GCC 上,任何构建系统都无法靠"更聪明的调度"解决增量问题——必须解决 BMI 的确定性(F3)与 BMI 携带函数体(F4)。 + +### 4.3 结论在第二个独立项目上复现:xlings + +只测一个项目得出的"结构性结论"不可信。**xlings** 是理想的对照:独立作者、独立代码库、同量级规模,且已从 xmake 迁移到 mcpp(用户给出的 `xmake.lua` 是迁移前的 ca25ab7)。 + +| | mcpp | xlings | +|---|---|---| +| 模块接口单元 / LOC | 137 / 56 555 | 110 / 46 253 | +| 冷构建 makespan | 79.07s | 51.74s | +| 总工作量 | 309.0s | 163.6s | +| **平均并行度**(32 线程) | **3.91×** | **3.16×** | +| **关键路径占墙钟** | **100%** | **100%** | +| 编译占总工作量 | 97.6% | 95.5% | +| 扫描 + dyndep 占比 | 0.8% | 1.7% | +| 关键链深度 | 24 | 24 | + +**两个项目的病理完全一致。** 这不是某个代码库的偶然结构,而是"C++23 命名模块 + GCC 单阶段 + 边完成即释放"这一组合的固有结果。 + +--- + +--- + +## 5. 被证伪的优化方向(先说不要做什么) + +投入之前先砍掉三条看起来合理、实测无效的路线。每条都有具体判据。 + +### ✗ 5.1 "缩小 BMI / 降低扇入"——BMI 体积几乎不要钱 + +直觉:`std.gcm` 31.5MB 被 134/138 个 TU 导入,`mcpp.libs.json.gcm` 17.1MB 被 17 个导入,反序列化必然很贵。 + +实测(空模块 vs 逐个加 import): + +| TU 内容 | 编译耗时 | +|---|---| +| 空模块(无 import) | 12.1 ms | +| `+ import std`(31.5 MB BMI) | 16.9 ms(**+4.8 ms**) | +| `+ import mcpp.libs.json`(17.1 MB) | 19.2 ms(**+2.3 ms**) | + +**GCC 的模块导入本来就是惰性的**(mmap + 按需具现)。BMI 体积基本不影响导入成本;真正花钱的是这个 TU **自己**的代码。 + +佐证:`corr(LOC, t_total) = 0.825`,平均 **4.6 ms/行**。编译时间由代码量驱动,不是由扇入驱动。 + +> 推论:`-fmodule-lazy` 大概率也没有收益(默认已惰性)。 + +### ✗ 5.2 "优化 scan / dyndep 阶段"——它只占 0.8% + +每个 TU 三次进程调用(scan → dyndep → compile)看着浪费,实测 `cxx_scan` 1.83s + `cxx_dyndep` 0.48s = 全部工作量的 **0.8%**。不要动。 + +### ✗ 5.3 "上分布式编译(distcc/icecc)"——关键路径 100%,分布式无处可分 + +关键路径 = 100% 墙钟意味着**任何时刻可并行的工作都已经并行完了**。模拟显示 P=64 与 P=16 完全同速。分布式编译在 F2 解决之前是纯粹的负收益(加了网络延迟)。 + +**顺序很重要:必须先做 §6.2,分布式才有意义。** + +--- + +## 6. 优化方案 + +按 **收益 / 成本** 排序。每条都给出判据与验证方式。 + +### 6.1 ✅【已实施 · 2026.8.12.1】让级联抑制真正生效(仅 GCC) + +**问题**:F3。**仅适用于 GCC** —— Clang 的 `.pcm` 实测字节稳定(§7.2),那边的级联抑制本来就在工作。 + +**已按方案 A 实施**(2026.8.12.1):把"BMI 是否相等"的判据从裸 `cmp` 换成**时间戳无关比较**。新增 `mcpp bmi-equal `(内部子命令),跳过 BMI 内的 `buildtime:` / `localtime:` 字段。`cxx_module` 规则里把 + +```sh +cmp -s "$bmi_out" "$bmi_out.bak" +``` +换成 +```sh +$mcpp bmi-equal "$bmi_out" "$bmi_out.bak" +``` + +**方案 B(附赠可复现构建)**——注入 `SOURCE_DATE_EPOCH`。取值**不能是当前时间**(那等于没改),候选:git commit 时间 / manifest version 派生的常量。加 `[build] reproducible = true` 开关。 + +**副作用**:方案 B 会改变 `__DATE__` / `__TIME__` 的值。用户代码可能依赖,因此不宜作为默认。**方案 A 无此问题,应作为默认;方案 B 作为可选项。** + +**实测收益(落地后,由 `bench --project` 测 mcpp 构建自身)**: + +| 场景 | 2026.8.11.3 | 2026.8.12.1 | +|---|---|---| +| `noop` | 0.27s | 0.19s | +| **`touch-hub`**(46 导入者,内容未变) | **73.99s** | **0.45s** | + +正确性由单测从**两侧**钉死(`tests/unit/test_bmi_equivalent.cpp`):真实接口变更仍完整级联。 + +> **⚠️ 目前仅 POSIX。** `ninja_backend` 的 Windows 分支写着 *"skip BMI restat optimization (requires POSIX shell)"* —— 整套 backup/compare/restore 是用 shell 的 `if` / `cp` / `cmp` 拼的,cmd.exe 上没有对应写法,所以 **Windows 一直没有任何级联抑制**。 +> +> 这个限制现在可以解除了:`bmi-equal` 已经是 mcpp 的子命令,把剩下的 backup/restore 也收进一个 `mcpp bmi-guard --bmi -- ` 里,整个序列就变成**一个进程、零 shell**,两个平台共用同一条规则。这是本条优化的直接后续,不需要新的设计。 + +**验证方式**:`bench/run.sh --scenario touch-hub`,并**必须同时验证**接口变更场景仍然级联——只测 touch 分不清"级联被正确抑制"和"级联坏了"。 + +### 6.2 【L1·最大收益】BMI 落盘即释放下游 + +**问题**:F1 + F2。这是全部方案里收益最高的一条。 + +**核心事实**(已实测): +- BMI 在编译进度 22.8% 处**原子 rename** 就位(§2.4),之后 77% 的时间下游在空等 +- GCC 的模块映射器协议**主动发送 `MODULE-COMPILED `**(实测报文见 §7),这正是"CMI 就绪"信号 +- 同一份 CPU 工作量,**一个编译进程都不多** + +**模拟收益**(真实依赖图 + 实测每模块 t_bmi/t_total): + +| 模型 | makespan | 加速 | +|---|---|---| +| A 边完成即释放(现状) | 72.0s | 1.00× | +| **B BMI 落盘即释放** | **24.1s** | **2.98×** | +| C codegen 完全离开关键路径(理论上界) | 15.4s | 4.66× | + +且核数重新变得有意义:现状 P=16 与 P=64 同为 72s;方案 B 下 P=8→42.7s、P=32→24.1s。 + +#### ✅ 已用真实原型实测验证(不是只有模拟) + +把 mcpp 生成的 `build.ninja` 机械改写成"每模块两条边、共用一个编译进程",在同一个构建目录、同一编译器、**同样的编译器并发上限(≤32)**下 A/B: + +| 方案 | 墙钟 | 产物 | +|---|---|---| +| baseline(边完成即释放) | **77.42s** | 19,347,008 B | +| **split(BMI 落盘即释放)** | **36.56s** | 19,347,008 B,`--version` 正常 | + +**实测 2.12×**,零额外 CPU 工作量(每个模块仍然只有一个 `g++ -c`)。自检:BMI 边中位数 883ms vs OBJ 边中位数 3041ms —— 确认第一阶段真的提前退出了。 + +实测 2.12× 低于模拟 2.98×,差距来自原型的 bash 轮询(5ms)与 `mkdir` 信号量开销,以及模拟使用的是隔离编译耗时(§2.3 注)。生产实现(用映射器协议或原生 job control)应更接近模拟值。 + +#### ⚠️ 原型暴露的两个实现陷阱(任何真实实现都会踩) + +**陷阱 1:分离出去的编译器继承了构建系统的 stdout/stderr 管道。** +ninja 判定一条边结束的依据是**管道 EOF,而不是直接子进程退出**。第一阶段即使提前 `exit 0`,只要后台编译器还持有那个 fd,ninja 就认为边还在跑。第一次原型运行就栽在这里:BMI 边的中位数是 2018ms(= 完整编译时长),看起来像"这个想法没用",实际是**测量被伪装成了 baseline**。 +修法:后台进程的 stdout/stderr 重定向到文件,由第二阶段回放(否则编译器警告与错误会静默消失)。 + +**陷阱 2:ninja 的 `-j` 必须远大于编译器并发上限。** +编译器一旦分离就不再占用 ninja 槽位,并发改由信号量约束。若 `-j` 与信号量上限相同,槽位会被"卡在信号量上"和"等 codegen"的休眠边占满,就绪前沿饿死,调度退化成 baseline。原型第一次正是 `-j32` + 上限 32 ⇒ 78.99s(比 baseline 还慢)。 +修法:`-j` 取编译器上限的数倍(原型用 6×),CPU 并行度仍由信号量精确控制。 + +**三条实现路径:** + +| | 机制 | 成本 | 风险 | +|---|---|---|---| +| **(A) 拆边 + 监督进程** | 每个模块拆成 `cxx_module_bmi`(BMI 落盘即退出)+ `cxx_module_obj`(等 codegen 收尾并传播退出码),共用一个编译进程 | 改 `ninja_backend` + 一个新 helper 子命令 | ninja 槽位记账;中断时需清理后台进程;Windows 无 fork(用 Job Object) | +| **(B) Clang 原生两阶段** ⭐ | `--precompile` → `.pcm`,再 `-c x.pcm` → `.o`。**Clang 本来就支持**,mcpp 目前只在 `std` 模块上用了它,项目模块走的是单阶段 `-fmodule-output=` | 只改构建图,**无需监督进程/信号量/映射器** | 已实测:总 CPU 只多 **7%**,关键路径份额降到 **25%**(§7.3),**风险接近零** | +| **(C) mcpp 作为模块映射器服务** | `-fmodule-mapper=`,mcpp 阻塞应答 `MODULE-IMPORT` 直到 BMI 就绪,生产者发 `MODULE-COMPILED` 时立即放行 | 最大改动 | **死锁**:并发槽被"等 import"的编译器占满而生产者排不进来 ⇒ 必须按拓扑序做准入控制 | + +**建议路线**:**(B) 先落地**(Clang 上零风险拿到收益,macOS/Windows 默认即 llvm)→ **(A) 覆盖 GCC** → (C) 作为下一代架构。 + +**给 GCC 上游的反馈**:`-fmodule-only` 文档写的是 "Only emit Compiled Module Interface",实际仍完整执行 codegen 再丢弃(§1.3 实测)。若上游修复,方案 A 可退化成两条普通 ninja 边,复杂度大降,并直接逼近模型 C 的 4.66×。 + +### 6.3 【L2·架构】把实现移出模块接口单元 + +**问题**:F4。body-only 编辑仍然 47.8s。 + +**根因是工程结构**:mcpp 有 **137 个模块接口单元、1 个实现 TU**。GCC 的 BMI 携带函数体(为跨模块内联),所以接口单元里的**任何**编辑都改变 BMI 字节 ⇒ 代价 O(导入者),而不是 O(1)。 + +**方案**:非 inline、非模板的函数体迁往模块实现单元(`module mcpp.ui;`,不带 `export`)。实现单元**不产生 BMI**,编辑它只重编 1 个 TU。 + +**优先靶点**(按 LOC × 扇入): + +| 模块 | LOC | 扇入 | 现状单次编辑代价 | +|---|---|---|---| +| `build/prepare.cppm` | **6476**(占全项目 11%) | — | 15.99s 自身 + 关键路径 20% | +| `ui.cppm` | 723 | 27 | 47.8s(实测) | +| `platform/platform.cppm` | — | 46 | 73.0s(实测,内容未变时) | +| `manifest/manifest.cppm` | — | 30 | — | + +**成本**:重构工作量大,可增量推进(先动上表 4 个)。需要先确认 mcpp 的 scanner / glob 对实现单元的支持体验。 + +**注意**:`prepare.cppm` 6476 行本身就是独立问题——它是关键路径末端最重的单点(占 20%),即使不迁实现,拆分它也直接缩短关键路径。 + +### 6.4 【L1】对象级缓存 + +codegen 占全部工作量的 **77%**。BMI 稳定后(§6.1),`.o` 也可按内容哈希缓存。mcpp 已有 `~/.mcpp/build-cache/v1/` 用于依赖包,扩展到根包即可。 + +**收益场景**:切分支来回、revert、CI 缓存恢复。**不改善**首次冷构建。 + +### 6.5 【不推荐】降低优化档位 + +**实测否定了这条**:`-O0` 相对 `-O2` 只快 **1.75×**(77.55s → 44.23s)。对一个 codegen 占 77% 工作量的构建,这个比例说明降档拿不到成比例的收益——前端与关键路径结构才是主导。而代价是产物运行时性能全丢。 + +⇒ **不值得**。相比之下 §6.2(2.12× 实测)与 §7.3(Clang 两阶段)既不牺牲产物质量,收益也更大。 + +**生态先例仅作参考**:xlings 迁移前的 `xmake.lua` 因 GCC 15 在 `-O1/-O2` + C++23 modules 上 ICE(`tree-ssa-dce`)而在 Linux 上强制 `-Og`。那是**规避编译器崩溃**,不是性能选择。 + +--- + +## 7. 跨平台与不同编译器 + +### 7.1 实测:同一份代码,Clang 比 GCC 快 2.42× + +`mcpp build`,同一工程、同一 `-O2`、同样 137+1 个编译单元、同样的边数,只换工具链: + +| | GCC 16.1.0 | Clang 22.1.8 | | +|---|---|---|---| +| **冷构建墙钟** | 77.55s | **32.08s** | **2.42×** | +| 总工作量(所有边耗时之和) | 309.0s | **123.0s** | 2.51× | +| 平均并行度 | 3.91× | **3.89×** | 一样低 | +| **关键路径占墙钟** | **100%** | **100%** | 一样是延迟瓶颈 | +| 最重单点 `prepare.m.o` | 16.1s | 7.1s | 2.27× | +| 产物大小 | 19.3 MB | 5.7 MB | (libc++ + 默认 strip 差异) | + +**这两组数字要一起读:** + +- Clang 让**每个单元**便宜 2.5 倍 —— 这是纯粹的编译器前端/后端效率差距,**零工程成本**; +- 但 Clang 的构建**结构性病理与 GCC 完全一样**:并行度 3.89×、关键路径 100%。换编译器**不解决** F1/F2。 + +⇒ **§6.2 的 BMI 提前释放与"换 Clang"是正交的、可以相乘的**,不是二选一。粗略叠加后 mcpp 自举冷构建有望进入 **15s 量级**(当前 77.55s)。 + +### 7.2 编译器能力矩阵(全部实测,不是查文档) + +| 能力 | GCC 16.1.0 | Clang 22.1.8 | +|---|---|---| +| 廉价"只产 BMI" | ✗ `-fmodule-only` **仍完整跑 codegen 再丢弃**(15.93s vs 15.95s) | ✓ `--precompile` **1.80s vs 单阶段 7.18s(3.99×)** | +| BMI 在编译进度多早落盘 | 22.8% | 25.3% —— **同样的问题** | +| BMI 字节可复现 ⇒ F3 | ✗ 嵌 `buildtime`/`localtime`,需 §6.1 | ✓ **字节稳定,无需任何处理** | +| 精简 BMI 能否免掉 body 编辑级联 ⇒ F4 | ✗ | ✗ **`-fmodules-reduced-bmi` 实测无效** | +| 模块映射器协议(P1184) | ✓ 实测可用,`MODULE-COMPILED` 即就绪信号 | ✗(用 `-fmodule-file=`) | +| 惰性导入 | ✓ 默认即惰性(§5.1) | ✓ | + +### 7.3 关键实测:Clang 的两阶段编译几乎是白送的 + +对同一个 `build/prepare.cppm`: + +| | 耗时 | 落在关键路径上的部分 | +|---|---|---| +| 单阶段 `-fmodule-output`(mcpp 现状) | 7.18s | **7.18s** | +| 两阶段 · phase 1 `--precompile` | **1.80s** | **1.80s** | +| 两阶段 · phase 2 `.pcm → .o` | 5.90s | 0(可完全并行) | +| 两阶段合计 | 7.71s | — | + +**总 CPU 只多 7%,关键路径上的份额降到 25%。** 而且实现上只需要**把一条 ninja 边拆成两条**——不需要监督进程、不需要信号量、不需要映射器服务,§6.2 那两个实现陷阱一个都不会遇到。 + +⇒ **这是整份报告里性价比最高的一条:Clang 上改构建图即可,风险接近零。** + +### 7.4 关于 F4 的更正 + +早期设计文档(2026-05-12 §3.4)寄望于 Clang 的 `-fmodules-reduced-bmi` 来消除"改实现也级联"。**实测不成立**: + +| 编辑方式 | GCC BMI 变化 | Clang BMI 变化 | Clang + reduced-bmi | +|---|---|---|---| +| 函数体内加一行注释(移动行号) | 变 | 变 21 974 B | 变 21 974 B | +| **不改变行数**的函数体内编辑 | 变 **14 B** | 变 14 354 B | 变 14 354 B | + +两个编译器都会变 ⇒ 都会级联。**F4 只能靠 §6.3 的工程结构调整解决**(把实现移出接口单元),没有编译器开关可用。 + +### 7.5 平台结论 + +- **F3(BMI 不确定性)是 GCC 独有的** —— Clang 上 mcpp 现有的级联抑制本来就在工作。这意味着 §6.1 是 GCC 专项修复。 +- **F1/F2(延迟瓶颈)两个编译器都有**,且 mcpp 目前在 Clang 上也走单阶段(`-fmodule-output=`),白白放弃了 `--precompile`。 +- **换编译器与改调度是正交的**:Clang 已经快 2.42×,叠加两阶段后关键路径还能再降约 4×。 +- **Windows** 无 `fork`,§6.2(A) 的监督进程需 Job Object;但 Windows 默认已是 `llvm@20.1.7`,走 §7.3 的两阶段即可绕开。 +- **macOS** 默认 `llvm@22.1.8`,同样直接受益。 +- **musl / 交叉目标** 不影响本分析任何结论(瓶颈在前端与调度,不在 libc)。 + +**跨平台注意**: +- **Windows** 无 `fork`,§6.2(A) 的监督进程需用 Job Object 保证中断时不残留;MSVC 两阶段天然 +- **macOS** 默认 llvm ⇒ §6.2(B) 收益立即可得 +- **musl / 交叉目标** 不影响本分析的任何结论(瓶颈在前端与调度,不在 libc) + +--- + +## 8. 建议落地顺序 + +按"收益 ÷ 风险"排,每条都给出**验证判据**——判据不满足就不算做完。 + +| # | 动作 | 适用 | 实测/预估收益 | 风险 | 验证判据 | +|---|---|---|---|---|---| +| **1** | §7.3 **Clang 走原生两阶段**(`--precompile` + `-c x.pcm`) | Clang(macOS/Windows 默认,Linux 可选) | 关键路径份额 7.18s→1.80s(**3.99×**),总 CPU 仅 +7% | **极低**:只改构建图 | 冷构建墙钟下降;`.pcm` 与单阶段产物等价;`mcpp test` 全绿 | +| **2** | §6.1 **BMI 比较忽略时间戳**(`mcpp bmi-equal`) | 仅 GCC | touch 场景 **73.0s → 0.22s** | 低 | **必须双侧钉**:内容未变→不级联;接口变更→**仍完整级联**(只测前者分不清"修好了"和"坏了") | +| **3** | §6.2(A) **GCC 侧 BMI 落盘即释放** | GCC | **实测 2.12×**(77.42→36.56s) | 中:进程生命周期管理 | 产物一致 + 构建后无残留编译器进程 + 中断可清理;⚠️ 必检 BMI 边耗时是否**真的**远小于 OBJ 边 | +| **4** | §6.3 **实现移出接口单元**(先动 `prepare.cppm` 6476 行) | 全平台 | body 编辑 47.8s → 预计个位数秒 | 中:重构量大,可增量 | 改一个实现单元后重编 TU 数 = 1 | +| **5** | §6.4 **对象级缓存** | 全平台 | 切分支/revert 场景 | 低 | 命中时**确实跳过编译**(⚠️ 历史上出现过"命中也 100% 重编"的假缓存) | +| **6** | §6.2(C) **mcpp 作为模块映射器服务** | GCC | 逼近模型 C(4.66×) | 高:需拓扑序准入控制防死锁 | 大规模工程下无死锁、无饥饿 | + +**明确不做**:§5.1 缩小 BMI/降扇入、§5.2 优化扫描阶段、§5.3 分布式编译(在 #3 之前无效)、§6.5 降优化档。 + +**关于默认工具链**:Linux 默认 `gcc@16.1.0` 比 `llvm@22.1.8` 慢 **2.42×**(§7.1)。这是个值得单独评估的决策——但它与上表正交,不是替代关系。 + +--- + +## 附录 A:复现方式 + +> 本报告成文时用的是一次性 bash + hyperfine 脚本;它已被 `bench/` 取代 —— 一套用 mcpp 写的跨平台基准套件,见 [bench 架构与实施计划](./2026-08-12-bench-suite-architecture-and-plan.md)。下列命令是当前的复现方式。 + +```bash +# 基准(同一个二进制,跨三平台) +cd bench && mcpp build +./target/*/*/bin/bench --list # 本机装了哪些引擎 +./target/*/*/bin/bench --engines mcpp,mcpp-opt,cmake,xmake \ + --variants headers,modules,modules-impl \ + --scenarios cold,noop,touch-hub,edit-body + +# 构建剖析器(同一个二进制的 --analyze 模式) +./target/*/*/bin/bench --analyze ../target/x86_64-linux-gnu/ + +# BMI 提前释放原型的 A/B +bench/proto-bmi-release/run_proto.sh +``` + +测量契约见 `bench/README.md`;结果按运行分目录,索引见 `bench/results/README.md`, +本文这批数据的出处在 `bench/results/hyperfine-20260812/NOTES.md`。 + +## 附录 B:关键原始数据 + +### B1 每模块 BMI 落盘时刻(-O2,137 个单元合计) + +``` +sum t_bmi = 59.1 s sum t_total = 258.9 s BMI 占比 22.8% 白等 199.8 s +``` + +### B2 模块图形态 + +``` +单元 138 依赖边 736 平均扇出 5.3 总 LOC 56 555 平均 4.6 ms/行 +corr(LOC, t_total) = 0.825 + +扇出 top3 prepare 60 · doctor 27 · ninja_backend 24 +扇入 top4 std 134 · mcpp.platform 46 · mcpp.manifest 30 · mcpp.ui 27 +纯聚合模块 platform.cppm(9 条 export import,0 行实码)· pm.cppm · manifest.cppm +``` + +### B3 模块导入的边际成本(证伪"BMI 太大"这条路线) + +``` +空模块 12.1 ms → + import std(31.5 MB)16.9 ms → + import json(17.1 MB)19.2 ms +``` + +### B4 GCC 的 BMI 非确定性 + +``` +连续两次相同编译,BMI 差 4 字节: + buildtime: 2026/08/12 02:25:01 UTC localtime: 2026/08/12 02:25:01 UTC + buildtime: 2026/08/12 02:25:33 UTC localtime: 2026/08/12 02:25:33 UTC +设 SOURCE_DATE_EPOCH 后:字节完全一致,且 localtime 字段消失 +``` + +### B5 BMI 的原子提交(strace,`build/plan.cppm`) + +``` +10:25:01.058 编译开始 +10:25:01.855 openat("gcm.cache/mcpp.build.plan.gcm~", O_RDWR|O_CREAT|O_TRUNC) +10:25:01.950 close → rename(...gcm~, ...gcm) BMI 原子就位 +10:25:06.527 进程退出 之后 982 个系统调用,无一再碰它 +``` + +### B6 GCC 模块映射器协议实测报文 + +``` +--> HELLO 1 GCC '' ; <-- HELLO 1 mapper-probe gcm.cache ; +--> MODULE-REPO <-- PATHNAME gcm.cache +--> MODULE-EXPORT probe.a <-- PATHNAME probe.a.gcm +--> MODULE-COMPILED probe.a <-- OK ← 这就是「CMI 已就绪」信号 +``` + +⚠️ 协议用**行尾 ` ;`** 表示批处理续行,应答必须镜像该标记,否则 GCC 会把第 N 个应答配到第 N+1 个请求上。 diff --git a/.agents/docs/2026-08-13-build-optimization-status.md b/.agents/docs/2026-08-13-build-optimization-status.md new file mode 100644 index 00000000..5daa1cec --- /dev/null +++ b/.agents/docs/2026-08-13-build-optimization-status.md @@ -0,0 +1,693 @@ +# 构建性能优化:综合报告(2026-08-13) + +本文报告 L1–L4 四条杠杆的**当前状态**、每条的**依据**、以及一次**被回退的实施**。 +架构与方案在 `2026-08-13-build-performance-architecture.md`;这里只讲做到了哪里。 + +--- + +## 0. 一句话结论 + +**目标达成。** 同一份 pinned 源码(@8219584)、同一台机器、`eef8a4c` 上复测: + +| | schedule=off | schedule=on | 比值 | +|---|---|---|---| +| **gcc 16.1.0** | 80.58s | **35.13s** | **2.29×** | +| **llvm 22.1.8** | 32.40s | **18.04s** | **1.80×** | + +**L1 与 L2 叠加**:gcc/off 80.58s → clang/on **18.04s = 4.47×**,远在 50s 线内。 +两者正交 —— L1 换编译器(压常数),L2 改图的形状,所以相乘而不是相加。 + +拆分调度下 noop **重建 0 个产物**(`.o`/`.gcm`/`.pcm` 一个都没动)。 + +### 而且是通用的,不是把 mcpp 这一个工程调快 + +| 工程 | 规模 | schedule=off | schedule=on | 比值 | +|---|---|---|---|---| +| **mcpp** | 138 模块 / 57k 行 | 79.9s | **34.80s** | **2.30×** | +| **xlings** | 110 模块 / 46k 行 | 112.92s | **33.41s** | **3.38×** | + +xlings 是**独立作者、独立代码库**的对照(openxlings/xlings @ b1563fe), +效果比开发它的那个工程**更大**。两个工程都 noop 无重建 +(mcpp 0.21s;xlings 的 10.77s 全部是依赖解析开销,`.ninja_log` 增量 **0 条边**)。 + +⚠️ xlings 那一栏有一处不对称:off 那次编译了 `mcpplibs.xpkg`(5 个单元), +on 那次命中了缓存。5 个单元相对 80s 的差值可以忽略,但记在这里而不是抹掉。 + +⚠️ **修正一次错误归因。** 本文先前写「schedule 基础层导致段错误,已整批回退」。 +重新施加后逐条复现:**基础层 rc=0**。那两次崩溃用的二进制**都包含当时未提交的图拆分 +发射** —— 崩的是那部分。基础层已恢复,`auto` 为 off、`on` 才启用拆分形状。 + +--- + +## 0b. L2 的覆盖面 —— gcc 与 clang 都已落地 + +两个编译器各支持**其中一种**机制,不可互换,`policy` 决策一次: + +| 编译器 | 形状 | 为什么只能是它 | +|---|---|---| +| gcc | `detach-codegen` | 无廉价的 BMI-only 模式(`-fmodule-only` 要花掉整编译的 99%),但 gcc 用 `rename()` 发布 BMI,所以"文件出现"是可靠信号 | +| clang | `two-phase` | 反过来:clang 用 `O_TRUNC` 就地写 BMI(读者会看到半个文件),但它有真正廉价的 BMI-only 调用 | + +### clang 这条的两个坑,都不是"接边"的问题 + +**坑 1:`--precompile` 发出来的不是同一种 BMI。** + + -fmodule-output= … -c 7.35s BMI 9,102,984 B (reduced) + --precompile 1.81s BMI 18,402,920 B (FULL) + --precompile -Xclang -emit-reduced-module-interface + 1.67s BMI 9,102,968 B (reduced) + (clang 22.1.8,src/build/prepare.cppm) + +`--precompile` 单独用又快又对**看起来**成立,实际上它发的是 *full* BMI —— +因为它的产物本来是要喂回去做 codegen 的。把 full BMI 发布给下游不是等价替换: +小模块上体积涨约 16 倍(`mcpp.platform` 19,424 → 313,452 B),而且在 mcpp 自己的 +模块图上直接让 clang 22.1.8 编错一个下游 TU: + + error: call to implicitly-deleted default constructor of + 'formatter, wchar_t>' + +—— 一个**窄**格式串,报错点在 `std` 里面,离真因三个文件远。同一个 TU 对着 reduced BMI +编译通过。所以 reduced 不是优化,是契约:`bmiOnlyFlags` 必须逐字节复现它。 + +⚠️ 这个坑的判据是**体积**,不是"编过了"。先做出来的版本能跑完 fixture、 +noop 干净、增量传播正确,**在 137 个模块的真实工程上才炸**。 + +**坑 2:object 边只能重编源码,不能读 BMI。** + +reduced BMI 不能拿去 codegen,于是 object 边是 `-c <源码>`(不带 `-fmodule-output`, +BMI 归 A 边所有,两条边不能写同一个文件)。代价是**前端跑两遍**;收益是下游只等 A 边。 + +两条边彼此**独立**(object 边不等 BMI 边),所以 codegen 可以整体落在图的后面。 + +### dyndep:一个源码两条边,两条都要记录 + +P1689 只知道 object(`primary-output` 取自被扫描命令的 `-o`),BMI 边没有记录时 +ninja 不是警告而是**整图拒绝**: + + ninja: build stopped: 'pcm.cache/mcpp.version_req.pcm' not mentioned in + its dyndep file 'obj/version_req.cppm.ddi.dd' + +—— 报的是边,不是缺失的记录,指向的是无辜的一侧。 + +解法是 `mcpp dyndep --split-module`:给 BMI **和** primaryOutput 各写一条记录。 +不是"把目标改成 BMI" —— 两条边都要解析同一批 import,都需要同一批隐式输入。 +**不要去改扫描的 `-o`**:GCC 那边共用 `-o` 与 `-fdeps-target` 已经造成过 +"扫描去写还不存在的 `gcm.cache/`" —— **在 mcpp 自己的仓库上不暴露** +(那个目录早被上一次构建建好),换个全新工程立刻失败。 + +### 实测(pinned 源码 @ 8219584,clang 22.1.8) + +| 并发 | schedule=off | schedule=on | 比值 | +|---|---|---|---| +| `-j4` | 56.34s | **37.60s** | 1.50× | +| `-j8` | 34.00s | **25.55s** | 1.33× | +| `-j32` | 32.03s | **17.95s** | **1.78×** | + +前端跑两遍要多花 CPU,但**在试过的每个并发档位上都是净赢**,所以没有加核数门槛。 + +**正确性判据是产物而不是退出码**:两条臂的 **130 个目标文件逐字节相同**。 +(BMI 有 101 个不同 —— 两臂在不同的指纹目录下,BMI 里烙了输出目录的绝对路径; +这正是"对照放两个目录会让路径冒充差异"那条,所以 BMI 差异在这里不构成证据。) + +--- + +## 1. 四条杠杆的状态 + +| | 杠杆 | 状态 | 依据 | +|---|---|---|---| +| **L1** | 按次选择工具链 `--toolchain` | **已实施** | 实测 81.8 → **32.6s**(2.51×) | +| **L2** | 下游在 BMI 可用时即开始 | **已实施**(`bmi_schedule = "on"`,gcc + clang) | gcc 79.9 → **34.8s**(2.30×);clang 32.0 → **17.95s**(1.78×) | +| **L3** | 定义移出接口单元 | **不做** —— 已量出它治的是 L2 同一个病 | 实测:对 mcpp **−6.2%**,对 cmake +92.3% | +| **L4** | 拆 `build.prepare` | **已实施**(架构收益;性能上为零) | 实测:**0**,原因见下 | + +### L1:做成了「按次选择」,没有换默认 + +`mcpp build --toolchain llvm@22.1.8` —— 实测 **81.83s → 32.61s(2.51×)**,已达标。 + +**换默认**才是那个不能做的动作:它让所有已发布包的指纹失效(全生态一次性重编), +三平台 llvm 版本还不统一(Windows 20.1.7 vs Linux/macOS 22.1.8), +且牵涉 `-static-libstdc++` 与 libc++/libstdc++ 的 ABI 选择 —— 需要协调。 +**按次选择不需要任何人配合,收益却是同一个 2.51×。** + +⚠️ 它**不改变形状**:clang 下 makespan 32.20s / 关键路径 32.15s = 仍然 **100%**, +并行度 3.90×,与 gcc 完全一致。clang 只是每模块便宜 2.5 倍。**L2 因此仍然必要。** + +### L3:明确不进这个 PR + +L3 指的是**改 mcpp 自己的 138 个模块**——把定义从接口单元移到实现单元。 +它不是构建引擎的能力,而是**被构建工程的写法**。 + +**决定:待合入的 PR 不动 mcpp 源码的实现风格。** 理由: +本轮的目标是优化 **mcpp 的构建性能**(引擎能力),而"改被测工程的结构来提速" +是另一件事——它对所有用 mcpp 的工程都适用,却要求每个工程改写自己的代码。 +把两者混进同一个 PR,会让一次引擎改动挟带一次跨全库的风格变更。 + +**可以做的**:拉一个临时分支/PR,只为**量出具体收益**(推算是链 74.6s → ~10.4s), +测完即弃,不合入。收益数字回填到本文。 + +### ⚠️ 更正:L3 是单项收益最大的杠杆,不是"绕行方案" + +下面那段用**未标定的旧 fixture**得出"L3 只值 +8%",**那个数字不可信** —— +那份 fixture 每个 TU 有 74% 是编译器启动、`weight` 旋钮推不动成本(见 bench/README §1a)。 +用标定后的 fixture(`--preset standard`)重测的 2×2: + +| | `modules`(定义在接口) | `modules-impl`(定义移到实现单元) | +|---|---|---| +| **schedule=off** | 17.76s | **5.30s** | +| **schedule=on** | 12.45s | **5.00s** | + +* **L3 单独:3.35×** · **L2 单独:1.43×** · **L2+L3:3.55×** + +**它们叠加,但只叠一点点**,因为**两者治的是同一份浪费、只是从两头下手**: +L3 把 codegen 从接口单元搬走,L2 是不等那份 codegen。做了任何一个, +另一个就没多少可买。而**单项收益 L3 远大于 L2**。 + +⚠️ 注意 L2 在这里只有 1.43×,而在 mcpp 真实源码上是 2.30× —— +fixture 单元的 codegen/parse 比例与真实模块不同,**不要跨工作负载搬运比值**。 + +**所以"极致性能"的答案是:引擎侧 L2 + 工程侧 L3,而 L3 是更大的那一半。** +L3 仍然不进这个 PR(它改的是被构建工程的写法),但它的定位从 +"给没有 L2 的引擎准备的绕行方案"更正为**最有效的单项优化**, +文档提示应当据此改写。 + +### 旧的分析(基于未标定 fixture,保留作为对照) + +不用拉分支:bench 的 `modules-impl` 变体测的**正是** L3(定义写在接口单元 vs 移到实现 +单元),数据已经在 `bench/results/five-way-20260812/` 里。同一 fixture、同一编译器: + +| 场景 | mcpp:modules → modules-impl | cmake:modules → modules-impl | +|---|---|---| +| cold (gcc) | 3.53 → 3.25s **(+7.8%)** | 13.05 → 12.80s (+2.0%) | +| cold (clang) | 2.50 → 2.19s **(+12.3%)** | 4.00 → 3.96s (+0.9%) | +| **edit-body (gcc)** | 0.29 → 0.31s **(−6.2%)** | **10.29 → 0.79s (+92.3%)** | +| edit-body (clang) | 0.46 → 0.31s (+32.5%) | 2.62 → 0.42s (+84.0%) | + +**看 `edit-body` 那两行。** 把定义移出接口单元,给 cmake 带来 **92%** 的提升, +给 mcpp 带来 **−6%**(即没有)。原因是同一个:改函数体时接口没变, +cmake 按 BMI 的 mtime 级联,mcpp 比 BMI 的内容。**L3 是给没有 L2 的引擎准备的绕行方案。** +引擎做了这件事之后,工程再去重构,在这条轴上什么都买不到。 + +真正留下来的是 **cold 上 +8%~12%** —— 接口单元变薄,关键链就变短。这是真的, +但它是"顺手的好设计",不是一条值得为性能去改 138 个模块的理由。 + +**所以 L3 的文档提示是:先要引擎的 L2,再谈重构。** 顺序反了会做很多白工。 + +### L4:已实施,而且实测收益为零 —— 这一点比数字本身重要 + +抽出 `mcpp.build.prepare_inputs`(341 行,cfg() 谓词 + 指纹规范化), +`prepare.cppm` 6521 → 6186 行,两个函数 re-export 所以调用方零改动。 + +**构建时间没有变化**(off 79.23s / on 34.54s)。原因在拆分前就分析出来了, +实测只是确认:**抽出来的东西成了 prepare 的依赖,链只会变长不会变短** —— +`… → prepare_inputs → prepare → …` 仍然串行,prepare 少掉的成本正好由新模块付掉。 + +要缩短关键路径,抽出的部分必须是 prepare 的**兄弟**(被 prepare 的**导入者**直接用)。 +已查清:configure 只用 `BuildContext`,execute 用 `BuildContext` + `prepare_build`, +其余只用 `prepare_build`;而链上是 prepare → execute → configure, +execute 离不开 `prepare_build`,所以抽类型也没人能离开这条链。 +真正有效的是拆 `prepare_build` 本身。 + +而且 **L2 落地后这件事的收益又小了一截**:一个接口现在只阻塞导入者约 22% 的编译, +不是全部。所以这次拆分按**架构**理由留下(6500 行的模块本就该拆),不按性能理由。 + +### 这条界线 + +⚠️ **这是本轮最重要的一条界线。** 「优化 mcpp 的构建性能」指的是**通用构建性能**, +不是把 mcpp 这一个工程调快。**通过改被测目标来变快,不能算数** —— +它对别人的工程一点用都没有,而且会让基准失去意义。 + +L2 是通用的:引擎侧的 2.30× / 3.38× 在**两个互不相关的工程**上都成立, +不要求任何人改写自己的代码。L3/L4 只对被改写的那个工程生效。 + +**所以 L3 和 L4 都不进这个 PR**,它们降级为**文档里的提示**: +想更快的工程可以这么做,收益在临时分支上量、量完即弃,数字回填到本文。 + +两者仍然不是同一类动作,区别记在下面: + +* **L4 是架构改动** —— `build.prepare` 6521 行、16.4s、占关键链 22%,是唯一的真离群点。 + 形状比"把大文件拆小"苛刻得多: + + ⚠️ **把一部分抽成 prepare 的依赖,是让链变长而不是变短。** + `… → 新模块 → prepare → …` 仍然串行,只是多了一跳;prepare 少掉的那点成本 + 被新模块自己的成本抵掉。抽出来的东西必须是 prepare 的**兄弟** —— + 被 prepare 的**导入者**直接使用,才能与 prepare 并行编译。 + + 实际调查(谁 import prepare、用了什么): + + configure.cppm 只用 BuildContext(一个类型) + execute.cppm 用 BuildContext + prepare_build + doctor / pack / cli.cmd_build 只用 prepare_build + + 所以把 `BuildContext` 抽成叶子模块能让 `configure` 不再依赖 prepare —— + **但链上是 prepare → execute → configure**,而 `execute` 需要 `prepare_build`, + configure 仍在 execute 之后。**净收益为零。** + + 真正能缩短关键路径的是**拆 `prepare_build` 本身**,让 `execute` 只依赖它的一部分。 + 那是对一个 6521 行模块的深度重构,不是一次抽取,需要单独立项 —— + 而且按上面那条界线,它属于**工程侧建议**,不属于本轮的引擎优化。 +* **L3 是实现风格改动** —— 把定义从接口单元移到实现单元,要动 138 个模块。 + 它对代码的组织方式提出要求,而收益只对改写了的那个工程生效。 + **不做**;要量收益就在临时分支上测,测完即弃。 + +⚠️ **两者都不得触碰 bench 钉住的那份基准 mcpp 源码**(`bench/projects/mcpp/` 指向的 +被测树的快照)。那份源码是**测量基准**:改了它,前后两次测量就不再是同一个工作负载, +所有比值失效。 + +**L2 与它们的区别**:引擎侧的 2.30× 对**每一个** mcpp 工程都生效, +不要求任何人改写自己的代码;L3/L4 只对被改写的那个工程生效。 + +### L2 做到哪里 + +**已实施并验证通过**的部分: + +* `src/build/schedule/policy.cppm` —— 纯函数决策表(每个编译器一种机制), + 7 条单测两侧钉死;`requested_switch` 是唯一读开关的地方。 +* `src/build/schedule/detach_codegen.cppm` —— gcc 的运行期。 + **实测:阶段一在 2.30s / 16.15s = 14% 返回,目标文件正确,两阶段 rc=0。** +* 可观测性:`# mcpp:graph=normal;schedule=detach-codegen` 写进图头, + `--verbose` 打印决策理由。 +* 失效靠**指纹**而不是守卫:换调度就换构建目录,旧形状的图结构上不可达。 + +**已全部接上。**(下段保留当时的记录,因为两个失效点值得留证) +历史记录 —— 曾经未接上的:图的拆分发射。它需要按 §2.3/§2.4 把 BMI 边的 `depfile` 接到 +P1689 扫描的产出上;第一版发射(未提交)会让 `mcpp build` **段错误(rc=139)**, +而**同一棵树上不含它的二进制 rc=0** —— 这一点是逐条复现出来的, +先前把整批基础层当成元凶是错误归因。 + +`auto` 现在是 **off**:调度改错是**静默**的(漏掉一条头文件依赖不会报错, +只会不再重编),不该凭一台机器的结果成为默认。 + +--- + +## 2. 已经钉死、下次不必重走的四件事 + +这些是本轮最有价值的产出,全部有实测支撑: + +1. **GCC 原子发布 BMI,clang 不是。** + strace:GCC 写 `.gcm~` 再 `rename()`;clang 以 `O_TRUNC` **直写最终路径**。 + ⇒ 「看文件出现」对 GCC 成立、对 clang **不成立**(会读到写了一半的 `.pcm`)。 + +2. **两种机制互补,不是二选一。** + clang 有便宜的 BMI-only 调用(实测 `src/build/prepare.cppm`:**1.67s vs 7.35s**); + GCC 没有(`-fmodule-only` 要 **99%** 的时间 —— 它不跳过后端,只是不写目标文件), + 但 GCC 原子发布 BMI 而 clang 不。**装反是静默的。** + + ⚠️ 早先这一条写的是「`--precompile` 0.78s / `-c` 自 pcm 0.70s,总 CPU 只多 9.6%」。 + **那条路走不通**:`--precompile` 发的是 *full* BMI(见 §0b 坑 1)。真实数字是 + 1.67s + 7.31s ≈ **多 22% CPU**,object 边重编源码而不是读 BMI。 + +3. **depfile 在 BMI 之后写出。** + 实测:depfile 16.39s,BMI 2.36s,整条编译 16.55s。 + ⇒ 拆分后的 BMI 边**不能**用编译器自己的 depfile(边结束时它还不存在); + 挂到对象边则头文件变更时 `bmi-await` 立刻返回、**什么都不重编**。 + 两种朴素挂法都会**静默丢掉头文件跟踪**。 + +4. **P1689 扫描的 depfile 是可用的替代来源。** + 实测对照:它相对编译的 depfile **只缺 `.gcm`**(那是 dyndep 在管的), + **头文件全覆盖**;且 `cxx_scan` **没有**声明 `depfile`,ninja 不会消费掉它。 + ⇒ 这条路是通的,只是还没接上。 + +--- + +## 3. 顺带修掉的:`mcpp test` 热跑 189.7s → 2.15s(88×) + +不属于构建引擎,但属于同一个问题的同一种病 ——「每次都重做一件上一次已经做完的事」。 + +用户报「每次运行单元测试都很慢,是不是没有并行测试功能」。**先量,分解就把前提否掉了**: + + finished in 189.74s (build 187.70s + run 1.78s) + +83 个测试的**运行阶段只有 1.78s**。「没有并行执行」属实,但它不是慢的原因。 + +给 `NinjaBackend::build` 加了分段计时(`-v` → `build/stage:`,留在代码里),一次热跑: + + loader-tags total=158701ms calls=85 <-- 98% + ninja total= 575ms calls=85 + compile-commands total= 443ms calls=85 + runtime-validate total= 160ms calls=85 + emit-ninja total= 88ms calls=85 + +没有这个分解只能猜 —— 我先后猜过 emit_ninja / compile_commands / hermetic,全错 +(三者合计 < 15ms)。 + +三处修复: + +| # | 改动 | 收益 | +|---|---|---| +| 1 | rule E(`check_and_record_loader_tags`)只解析 stat 变了的产物;没变的从 `resolution.json` 把判定**读回来** | 189.7s → 5.3s | +| 2 | `-k 0` 批量构建成功 ⇒ 跳过每个测试的复驱动(~39ms × 83) | 5.3s → 3.9s | +| 3 | 并行跑测试(>1 个时捕获输出、整块打印;=1 个时保持前台流式) | 3.9s → **2.15s** | + +⚠️ **1 的回归测试形状**:便宜的写法(跳过没变的就完事)会让记录缩水成「这次重链了 +什么」,于是「记录为空」和「全部合规」长得一模一样 —— 正是 rule E 存在的理由。 +**纯 noop 抓不到它**(noop 时什么都不重写,坏记录也完好);要**只碰一个目标的源码** +(一个重链、一个不重链)。e2e 214 按这个形状写,并按错误实现验过先红。 + +⚠️ **3 为什么不是无脑并行**:多个测试直接流式输出会逐行交错,失败的断言变得无法归属 —— +而归属正是那个循环存在的理由。所以 >1 时捕获、结束时整块打印;=1 时保持流式,因为 +那是调试场景,挂住的时候尤其需要实时输出。 + +--- + +## 4. 本 PR 当前包含什么 + +引擎侧: + +* **L1** `--toolchain SPEC` / `MCPP_TOOLCHAIN`:按次选工具链,不动 manifest、不动指纹。 +* **L2** `bmi_schedule = "on"` / `MCPP_BMI_SCHEDULE`:gcc `detach-codegen` + clang `two-phase`, + 决策集中在 `src/build/schedule/policy.cppm`,运行期在 `src/build/schedule/`。 + 默认 `auto` = off。 +* **L4** 抽出 `src/build/prepare_inputs.cppm`(架构收益,性能为零 —— 见 §1)。 +* `--jobs N|auto`。 +* **BMI 等价性判断改用 `mcpp bmi-equal`**,替代永远不可能成功的 `cmp -s` + (GCC 把时间戳写进 BMI 内容)。真实工程实测: + `touch-hub` **84.53s → 0.44s**(对 cmake **192×**,对上一版 mcpp **174×**)。 + ⚠️ `edit-body` 无提升(18.29s vs 18.30s)且**这是对的** —— + 改函数体确实改变 BMI,级联是必需的。这一行是区分 + 「避免不必要的工作」与「避免工作」的对照。 +* `mcpp test` 的三处提速(§3)。 + +其余是 bench 套件、规范与数据(见 `bench/README.md`、`bench/results/`)。 + +## 5. CI 与合入 + +* `curl: (52) Empty reply from server` 那一类红是引导下载失败,12 秒内即挂、与代码 + 无关。判据:失败 job 的日志里没有任何测试名,只有 curl 的退出码。已由 + `.github/tools/fetch_release.sh`(`--retry-all-errors` + 归档校验)根治。 +* ⚠️ **更正:此前这里写的是「反复出现的红**全部**是下载失败」,那是错的。** + bench 那条线上真正的问题是**没有红** —— 见 §7。把所有说不清的红都归给网络, + 正是让那件事多存活了几周的原因。 +* **未合入**,按要求。 + +## 7. bench 套件审计:一整条绿色的 CI 什么都没有测 + +用户报「bench 卡住而且没有进度」。查下去发现卡住只是最表层的症状。 + +**`bench (macos/clang/fixture)` 报成功,实际是 6 ok / 48 failed / 18 unavailable**; +唯一过的 6 个格子全是 cmake 的 `headers` 变体。三个 xlings job 报成功,**一个测量 +都没有**。这个状态持续了数周。 + +### 六个互相独立的真因 + +| # | 缺陷 | 为什么没被发现 | +|---|---|---| +| 1 | 每个引擎拿到的编译器不同 —— CI 用 `command -v g++` = runner 的 gcc 13.3.0,而 mcpp 用自己 registry 的 16.1.0 | cmake 配不出 modules、xmake 把 gcc 编崩,都被记成对引擎的「真实发现」 | +| 2 | 构建工具版本随 runner 漂移(镜像自带 cmake 3.31.6,没有 4.0 的 `import std` 键) | 同上 | +| 3 | 被测工程运行时从默认分支 clone —— `--hub src/xlings.cppm` 早就不存在 | 每个格子报 `skipped`,harness 退出 0 | +| 4 | `--hub`/`--body` 按 harness 的 cwd 解析,不是按工程目录 | 只有「测你正站着的树」时才对,即 mcpp 测自己 | +| 5 | harness **永远返回 0** | 「测到了东西」这件事从来没有被断言过 | +| 6 | xmake 的 `--buildir` 相对 `-P` 解析,`clean()` 删的是另一个目录 | `cold 0.60s` 状态 `ok`、带样本 —— **每一个 xmake 真实工程 cold 数字都是假的** | + +### 一条贯穿的形状 + +**每一个都是「失败看起来像成功」,而不是「失败没被处理」。** 套件的协议不变量 1 +写着「失败不得看起来像测量」,但它只覆盖了单个 cell 的 `status` 字段 —— 没覆盖 +退出码、没覆盖「量到的是不是真的那件事」。 + +现在补上的断言,按发现顺序: + +* `failed` 或「一个 ok 都没有」⇒ 非零退出;已知缺口写进 `allow_failed` 且必须带 + `KNOWN GAP` 说明(守卫检查)。 +* `cold` 必须大于同引擎同 variant 的 `2 × noop` —— 否则它没重建。**不是性能阈值**, + 是内部一致性。 +* `hub`/`body` 必须在钉住的树里真实存在(靠子模块才可检查)。 +* 工具版本必须是精确版本;`reference_mcpp` 必须等于 `.xlings.json` 的 bootstrap pin。 +* 扰动**形态**写进 note —— `edit-comment` 在有函数体的单元里插注释(行号全移、BMI + 真的变了、级联是对的)和在没有函数体的单元末尾追加(什么都没动)是两个不同的 + 问题,而套件用一个名字同时回答了它们。 + +### 可观测性(用户最初报的那件事) + +* 进度实时打到 stderr 并逐行 flush; +* 每条 configure/build 有超时(默认 1800s),超时 kill 并报 `TIMED OUT after Ns`; +* 失败时直接打出子进程日志尾部; +* 子进程日志改为**追加**、由 runner 每个 cell 清空一次 —— 此前计时构建那一行 + `build ok, spent 0.111s` 会把前面 configure 的输出整个擦掉,这正是 xmake 那条 + 0.60s 一开始无法诊断的原因。 + +### 这批数字里最该记住的一条 + +`edit-comment` 在 mcpp 自己的工程上是 **199×**,在 xlings 上是 **1.00×**。 +不是优化时灵时不灵 —— 是 mcpp 的 hub 恰好没有函数体。**mcpp 测自己永远看不到 +这件事**,这就是独立控制目标存在的全部理由。 + +## 6. 下一步(按顺序) + +1. **`auto` 是否翻成 on**:这是发布决策不是技术缺口 —— 指纹里带了 schedule, + 翻默认会让所有已发布包全量重建一次。等三平台 CI 见过 `on` 之后再单独立项。 +2. **msvc**:`/ifcOnly` 的代价与 `.ifc` 是否原子发布都还没测。猜错是静默的 + (半个 BMI 不是诊断,是编错),所以保持 `None`。 +3. **L3 作为书写约定**:优先施加于链上那 19 个模块(见 §1),不回改存量。 +4. **换默认工具链**单独立项(生态决策,见 §L1)。 + + +## 8. ⚠️ L2 有一个未修好的正确性缺陷 —— 现阶段不应推荐开启 + +`bmi_schedule = "on"` 在**增量重建**上会让导入者撞到不存在的 BMI: + + failed: gcm.cache/fx.unit_1.gcm + fx.unit_0: error: failed to read compiled module: No such file or directory + fx.unit_0: note: imports must be built before being imported + +**复现**(生成的 fixture,modules variant,四个场景稳定失败): + + bench --engines 'mcpp[schedule=on]=' --variants modules \ + --scenarios touch-hub,touch-leaf,edit-body,edit-comment \ + --preset standard --runs 2 --compiler payload:gcc + +### 已经确定的 + +* **不是编译器之间的竞态**:`-j1` 一样复现。 +* **窗口是设计带来的、而且很大**:phase 1 在 spawn 编译器**之前**就把旧 BMI + rename 进 `.bak`,直到编译器发布新的为止,这个模块在磁盘上**没有 BMI**。 + 实测一次增量重建中该文件消失约 **208ms**(2ms 采样 × 104 次命中)。 +* **失败模态是安全的那一种**:该路径下 BMI 是**缺失**而不是**陈旧**,所以永远 + 是响亮的失败,不会产出一个「成功但错误」的构建。这一点是量出来的,不是希望。 +* 真实工程(mcpp 自己、xlings 两种风格)上没有复现 —— 只有 fixture 的紧密 + unit_0→unit_1 链会撞上。**这就是合成 fixture 的价值**,我此前把它当成 + 「不如真实工程可信的那一半」,是错的。 + +### 已经修掉但**不是**本缺陷成因的 + +`compile_release_at_bmi` 的 `read_rc` 分支返回成功却从不 `settle_bmi`(见 +`8c9f239`)。它确实是个真缺陷 —— 上一份 BMI 一直停在 `.bak`,而且那个单元的 +**等价性检查从未运行**,也就是说级联抑制对最便宜的那些单元是静默关闭的。修完 +之后 `.bak` 残留归零,**但四个场景照样失败**。 + +### 还没搞清楚的 + +在 `-j1`、且 dyndep 明确写着 `unit_1.gcm: dyndep | unit_0.gcm` 的情况下,导入者 +为什么仍然会在那 208ms 的窗口里被 ninja 调度。下一步应当是 `ninja -d explain` +配合边级时间线,而不是继续静态推理 —— 这一条我已经猜错过一次。 + +### 因此 + +* **`auto` 绝不能翻成 on**,直到这条修好; +* 所有已发布的 `bmi_schedule` 数字都是在缺陷存在时取的,README 里已标注不可引用; +* 修法方向:要么让 BMI 在整个重建期间保持可读(先编译到临时路径、成功后再原子 + 替换,而不是先把旧的挪走),要么让导入者的边真正等到 BMI **重新发布**之后。 + 前者更像是对的 —— 「先移走再重建」本身就在制造一个不存在的中间态。 + + +## 8b. 第四次尝试之后:把 L2 从 CI 里撤出,并说明现状 + +**已修好的两件事(独立成立,与下面那条无关):** + +* `compile_release_at_bmi` 的 `read_rc` 分支返回成功却不 `settle_bmi` —— + 上一份 BMI 停在 `.bak`,而且那个单元的**等价性检查从未运行**,级联抑制对最便宜 + 的单元静默关闭。 +* 编译失败时不再把单元留在「完全没有 BMI」的状态。 + +**「导入者读不到 BMI」已经修好。** 原设计在 spawn 编译器**之前**就把旧 BMI +`rename` 走,于是模块在磁盘上有约 208ms 没有 BMI(实测)。改成**复制**一份到 +`.bak`、原件留在原地,并用「文件身份(size+mtime)发生变化」而不是「文件存在」 +来判断发布。`failed to read compiled module` 不再出现。 + +⚠️ 中间踩的两个坑,都值得记住: +* `std::filesystem::file_size(p, ec)` 失败时返回 `(uintmax_t)-1`。我在检查 `ec` + **之前**就把它写进结构体,于是「文件不存在」与默认构造的哨兵不相等 —— phase 1 + 第一次轮询就认为「变了」,在编译器产出任何东西之前返回,所有 object 边报 + `no compiler was started … phase 1 did not run`。 +* `copy_file` 给副本盖的是**拷贝时刻**的 mtime。而 `settle_bmi` 恢复这份副本正是 + 为了让 mtime **不前进**、让 ninja 的 restat 掐断级联。不显式把原 mtime 带过去, + 恢复反而把 mtime 推前 —— `touch-hub` 变成 12.61s(冷构建 12.37s), + **六个格子全报 `ok`**。状态列抓不到这个,只有数字能。 + +**~~仍然没修好的~~ 已于 2026-08-14 修好。** 上一版这里写的方向是对的,并且就是最终 +的修法:BMI 的 restat 抑制不能连带抑制**这个单元自己的 object 边**。 + +`ninja_backend.cppm` 里 object 边原本是 + + build : cxx_module_obj # 唯一输入是 BMI + +现在是 + + build : cxx_module_obj | + +源码作为输入给了它一个 restat 清不掉的「脏」的理由;BMI 降为隐式输入,顺序不变 +(`bmi-await` 不能跑在 phase 1 之前)。 + +**⚠️ 但真实症状比这里记的严重得多,而且我记错了它的形态。** 上一版把它记成 +「挂在链接上」,于是它读起来像 fixture 特有的边角情况,整整一周没人再看。实际的 +一般形态是**静默产出错误的二进制**: + + export int leaf_value() { return 1; } // 改成 42,重建 + + Finished dev in 0.02s <- 报成功,8 条边只跑了 3 条 + ./repro -> 1 <- 源码写的是 42 + +`undefined reference` 只是「符号原本就不存在」时的特例。**记录一个缺陷时,记最一般 +的形态,不要记你第一次撞见的那个** —— 后者会让所有人低估它。 + +**⚠️ 它还污染了已发布的数字,这一点当时完全没人察觉。** 被跳过的 object 边与链接是 +构建欠下的工作,所以那一列量的比一次构建少。`bmi_schedule=on` 的 +`touch-hub 0.22s` / `edit-comment 0.18s` 在修复后是 **0.44s / 0.44s**,比默认档还略 +慢 —— 因为那两行本来就没有级联可省。`cold`(35.43→36.36)与 +`edit-body`(30.17→30.48)不受影响,头条结论成立。 + +**⚠️ 我为这件事写的守卫失败了,记在这里比记成功更有用。** 思路是:轮询引擎写入的 +那棵树,引擎退出后还有文件落盘就判定「构建只是返回了、并没有结束」。三次尝试: +(1) 看错了目录 —— `job.build_dir` 是给 cmake/xmake 的 `-o`,mcpp 写的是 +`/target`;(2) 窗口取 300ms,而要检测的**尾巴本身就是一次编译**,实测 +1.24s 才落盘;(3) 改成轮询到稳定、上限 3s、按引擎声明的产物目录 —— 诊断确认它 +**确实在跑、确实拿到了基线**,但对仍带缺陷的二进制在本套件的 touch-hub 流程里 +不触发。于是撤掉:**一条无法被证明能抓到目标的守卫,和没有守卫无法区分。** + +顺带订正我自己的一句话:我先前把「已发布数字量的是没跑完的构建」当成结论写进了 +README。手工重建确实能观察到 `mcpp build` 0.56s 返回、`cc1plus` 仍在跑、object +1.24s 后落盘;但在 harness 自己的流程里复现不出来。把一次手工观察当成对已发布 +数字的解释,是过度归因 —— 能站住的只有「object 边和链接被 restat 清掉了」。 + +**因此 CI 里暂时不跑 `+schedule=on` 这条臂。** 修好之后可以放回去;门槛仍是 §8 的 +复现全绿。 + +**这条 bug 我连错五次**(误诊 settle_bmi、哨兵不匹配、mtime 没带过去、object 边、 +以及把症状记成了链接错误)。下一个人应该从「object 边与 BMI 边的 restat 语义不同」 +开始。 + + +## 9. bench 跑起来之后暴露的两个**与 bench 无关**的既有缺陷 + +这两个都不是这次改动引入的 —— 是矩阵此前根本没在测,所以从来没人看见。 + +### 9a. macOS 上 mcpp 编不了依赖的 `build.mcpp` 助手 + +`bench (macos/clang/xlings-2026.8.13.1)`: + + error: dependency 'xpkg': build.mcpp failed to compile (exit 1): + dyld[21445]: Symbol not found: __ZdaPv + clang++: error: unable to execute command: Abort trap: 6 + +`__ZdaPv` 是 `operator delete[](void*)`。助手链接过了,运行期找不到 libc++。 +与 [[build-mcpp-helper-self-containment]] 同一类问题(glibc 靠 rpath、musl 与 PE +才需 `-static`),但 macOS 这条此前没有覆盖。 + +**影响面比 bench 大**:任何在 macOS 上依赖带 `build.mcpp` 的包的工程都会踩到。 + +### 9a-2. 同一个机制,更大的面:macOS 上**每个**子进程都被污染 + +不只是 `build.mcpp` 助手。bench 的 macOS 格子里 cmake / bazel / 参照 mcpp +**三个引擎一起**挂在同一处: + + dyld: Symbol not found: __ZdaPv + Referenced from: …/XcodeDefault.xctoolchain/usr/bin/ld ← Apple 自己的链接器 + Expected in: …/registry/…/lib/libc++.1.0.dylib + +**Apple 的 `ld` 自己就链 libc++**,而 registry 的那份 libc++ 进了动态加载器的搜索 +路径,于是 `ld` 还没开始链接就 abort。判据:**被测的 mcpp 那条臂在同一次运行里 +全绿** —— 三个引擎同样地挂、一个不挂,说明问题在环境而不在任何一个引擎。 + +⚠️ **把 macOS 上的 payload libc++ flag 全部去掉之后,它照样发生。** 所以污染不是 +经由链接 flag 进来的,而是经由 `DYLD_*`(很可能是 mcpp/xlings 为了让自己的载荷 +二进制跑起来而设的),然后被所有子进程继承。 + +macOS 的 bench 格子因此进 `excluded`,原因写在 matrix.json 里。**没有继续猜** —— +本机是 Linux,复现不了,而这条已经让我烧掉好几轮 CI。 + +### 9b. mbedtls 的 registry 源码包缺 `framework/` 子模块 + + mbedtls-3.6.1/CMakeLists.txt:304 + framework/CMakeLists.txt not found. + Run `git submodule update --init` from the source tree. + +挡住的是 xlings cmake arm 的最后四分之一(ftxui / libarchive / lua 三个已经接好、 +能用)。注意 **mcpp 自己构建 mbedtls 不会踩到**,说明 mcpp 走的根本不是 mbedtls +的 CMake 路径。 + +解法有三条,都需要决策而不是我单方面选:vendor 那个不大的 `framework` 目录、 +改走 mbedtls 的 Makefile、或者让 registry 把完整树打进包里。 + +## 10. CI 的核心 job 全红 —— 根因是 clang 22 的模块误编译,不是环境 + +**判据(同一份分支代码,只换编译器):** + +| 编译器 | `mcpp test` | +|---|---| +| clang 22.1.8(仓库默认) | `unit/test_elf_runtime` **SIGSEGV**,82 passed / 1 failed | +| gcc 16.1.0 | `unit/test_elf_runtime ... ok`,**83 passed / 0 failed** | + +backtrace 落在 libc++ 的 `__assign_with_sentinel`,调用方是 +`mcpp::platform::elf::inspect_elf_runtime@mcpp.platform.elf_runtime` —— +**那个文件本分支一行没改**(`git log origin/main..HEAD -- src/platform/elf_runtime.cppm` +为空),而 main 用同一个 clang 是 80 passed / 0 failed(在独立 worktree 里实测, +不是看 CI)。也就是说:别处的新增改变了这个函数的代码生成。这与仓库里已记录的 +`clang-modules-unused-fn-miscompile` 是同一类现象。 + +崩的那个用例叫 `RejectsUnsupportedOrTruncatedElfWithoutGuessing` —— 它只对一个 +9 字节文本和一个 6 字节文件调 `inspect_elf_runtime`,而头部检查 +(`bytes.size() < 0x40`)本该直接挡住。 + +### 定位它花了多少弯路,以及为什么 + +**转折点是给 CI 加了一个 `continue-on-error` 的 verbose 探针**,它把崩溃钉在 +`stage("runtime-validate")` **打印之前** —— 即 `validate_changed_artifacts` 内部。 +顺着这条线才在本地找到那个稳定复现的单测。 + +在此之前逐条否掉的七个假设(每条都有实验,不是推理): + +1. `--jobs auto` 导致 OOM —— `mcpp.toml` 根本没设 `jobs`,默认不走该路径 +2. xmake 3.1.0 —— 切过去后 fixture 本地 1 ok / 0 failed +3. mcpp-index 变动 —— 最后一次提交 08-12,不在窗口内 +4. xim 索引载荷 —— 近期无 llvm/glibc 变动 +5. 容器镜像 —— 在同一个 `debian:stable-slim` 里精确复现 CI 那一步:`hello 195`,rc=0 +6. 全新空 registry(强制重下全部载荷)—— rc=0 +7. 「这是本分支的代码缺陷」—— 被同一批日志否掉:`toolchain: gcc` 里崩的是**已发布的 + 2026.8.11.3**,hermetic 里崩的是本分支构建的,两个版本都崩 + +⚠️ **本来可以早几个小时定位。** 我在很早就记录过「本地 `mcpp test` 有 1 个失败: +`unit/test_elf_runtime`」,当时把它当孤立小问题;它和 CI 的 `exit 139` 是**同一个 +信号、同一个子系统**。一个和 CI 症状同信号的本地失败,永远值得先连起来看。 + +⚠️ 还有一个我一度当成证据的错误推理:「文档提交(经核实只有两个 markdown、38 行) +让 9 个核心 job 全红 ⇒ 必是环境问题」,以及后来的「重跑能复现 ⇒ 确定性 ⇒ 不是环境」。 +后一步是错的:**确定性失败同样可以来自一个已经变了的外部状态**。真正的解释是第三种: +缺陷一直在,变的是**构建状态是否让那条代码路径被走到**(缓存命中 vs 冷构建)。 + +### 触发点:一行混进提交的工具链变更(我自己造成的) + +`git bisect`(判据=该单测是否 SIGSEGV)指向 `f51e6ab`,而它对 `mcpp.toml` 的改动是: + + [toolchain] + -default = "gcc@16.1.0" + +default = "llvm@22.1.8" + +**这一行与那个提交的主题(xmake 臂、libarchive 覆盖包)毫无关系,提交说明里一个字 +都没提** —— 是 `git add -A` 扫进去的。它把 mcpp 自身的默认工具链从 gcc 换成了 +clang,于是每一次 CI 构建都撞上 clang 22 的模块误编译。 + +这也解释了那个「纯文档提交让 9 个核心 job 全红」的怪事:**工具链早在它的上一个 +提交就被换掉了**,文档提交只是第一个跑完整套核心 job 的提交。我当时的推理 +「文档提交不可能弄坏构建 ⇒ 必是环境问题」前半句是对的,但我从没想到去查**前一个 +提交里混进了什么**。 + +改回 `gcc@16.1.0` 后:`83 passed; 0 failed`。 + +⚠️ **教训:`git add -A` 会把无关改动带进一个主题明确的提交。** 这次带进去的是 +一行工具链切换,代价是几小时的排查,而排查方向一直被"提交说明说它只改了 bench" +误导。提交前看 `--stat` 里有没有主题之外的文件,是最便宜的防线。 +CI 里那个诊断探针(`ci-linux-e2e.yml` 的 "diagnose: verbose trace")已在根因确认后 +删除 —— 它的使命就是把崩溃钉在 `stage("runtime-validate")` 打印之前,做到了。 + +**修复确认**:改回 gcc 后,先前全红的六个核心 job(build+unit tests / toolchain: gcc / +hermetic e2e / integration / toolchain: musl+llvm / cross-build aarch64)在 CI 上 +**全部转绿**。 diff --git a/.agents/docs/2026-08-13-build-performance-architecture.md b/.agents/docs/2026-08-13-build-performance-architecture.md new file mode 100644 index 00000000..a92ae23c --- /dev/null +++ b/.agents/docs/2026-08-13-build-performance-architecture.md @@ -0,0 +1,274 @@ +# mcpp 构建性能:架构层面的分析与方案(2026-08-13) + +目标:`mcpp clean && mcpp build --release` 从 **79.9s** 降到 **50s 以内**。 + +本文只用实测数字。每个结论后面都注明它是**测出来的**还是**推算的**,推算的给出推算方式。 + +--- + +## 0. 基线 + +| | | +|---|---| +| 工程 | mcpp 自身,138 个模块接口单元 + `src/main.cpp`,57k 行,全部 `import std;` | +| 主机 | 13th Gen Intel i9-13900K,32 逻辑 / 24 物理(异构),64 GiB | +| 工具链 | `gcc@16.1.0`(mcpp 自带载荷),release = `-std=c++23 -fmodules -O2` | +| 冷构建 | **79.92s** | + +对照(同机、同源码,构建描述在 `bench/projects/mcpp/`): + +| 引擎 | 冷构建 | +|---|---| +| mcpp | 80.0s | +| cmake 4.0.2 + ninja | 94.5s | +| xmake v3.0.7 | 94.6s | + +**四个引擎都在 15% 以内。** 这说明瓶颈不在调度实现,而在所有引擎共有的那个东西 —— 图的形状。 + +--- + +## 1. 结构性事实:100% 关键路径 + +`bench --analyze`: + +``` +edges : 426 +makespan : 79.79 s +work (sum dur) : 314.08 s +avg parallelism: 3.94 x (of 32 hw threads) +critical path : 79.73 s = 100% of makespan +``` + +**关键路径等于墙钟本身。** 32 个硬件线程上平均只跑到 3.94 路并行。 + +直接推论,每一条都实测印证过: + +* **加核无效。** `-j4/8/16/32` = 101.3 / 81.0 / 80.0 / 79.9s。从 8 到 32 提升 1.4%。 +* **分布式编译无效。** 关键路径是串行依赖,不是资源不足。 +* **换引擎无效。** cmake/xmake 走同一条链,差异 ≤ 18%。 + +**换了编译器,形状也不变。** clang 下重测: + +``` +makespan : 32.20 s ← gcc 79.79 s +work (sum dur) : 125.49 s ← gcc 314.08 s +avg parallelism: 3.90 x ← gcc 3.94 x +critical path : 32.15 s = 100% of makespan +``` + +clang 不是调度得更好,而是**每个模块便宜 2.5 倍**;100% 关键路径、3.9 路并行这两个结构特征一模一样。 +所以「换 clang」是把常数压小,不是把问题解决 —— 工程再长大一倍,它同样会顶到墙。 + +--- + +## 2. 成本解剖:一次模块编译的钱花在哪 + +`-ftime-report`,`src/build/prepare.cppm`(16.2s,链上最贵的一跳): + +| 阶段 | 秒 | 占比 | +|---|---|---| +| **opt and generate** | 14.08 | **86%** | +| ├ callgraph functions expansion | 11.00 | 67% | +| └ callgraph ipa passes | 2.77 | 17% | +| phase parsing | 1.32 | 8% | +| template instantiation | 0.95 | 6% | +| module import | 0.51 | 3% | + +**一次模块接口编译的 86% 是代码生成 —— 而它的导入者一个字节都用不到。** + +这一点有独立的三重验证(见 §5.2 的方法):BMI 在编译的 **15%–39%(中位 ~22%)** 处 +就已经原子落盘、与最终产物**逐字节相同**、且**真实下游导入者能用它编译成功**。 + +⚠️ 一个错误推理值得记下来:我先用 `-fmodule-only` 量「只产 BMI 要多久」,得到 99%, +据此判定「codegen 只占 1%,没有优化空间」。**这是错的** —— GCC 的 `-fmodule-only` +不跳过后端,只是不写目标文件。这个 flag 不能用来回答这个问题。 + +--- + +## 3. 图的形状:19 跳的导入链 + +按源码 `import` 关系建图,用实测编译耗时加权求最长路径: + +``` +最长导入链 = 19 个模块,链上编译耗时合计 74.6s(全图 306.0s) + + 1.02s mcpp.platform.env + 2.84s mcpp.platform.process + 0.02s mcpp.platform ← 纯 re-export 门面 + 1.98s mcpp.manifest.types + 5.21s mcpp.manifest.toml + 0.02s mcpp.manifest ← 纯 re-export 门面 + 1.82s mcpp.platform.xlings.runtime_selection + 5.65s mcpp.platform.runtime_binding + 4.03s mcpp.platform.elf_runtime + 2.06s mcpp.build.loader_contract + 6.08s mcpp.build.plan + 2.82s mcpp.build.flags + 4.76s mcpp.build.compile_commands + 3.69s mcpp.build.ninja + 16.41s mcpp.build.prepare ← 22% of the chain + 4.70s mcpp.build.execute + 2.21s mcpp.build.configure + 3.49s mcpp.cli.cmd_build + 5.77s mcpp.cli +``` +（尾部还有 `obj/main.o` 4.78s + link 0.18s,不在模块链内但在关键路径上。） + +两个观察: + +1. **这条链是真实的分层**:platform → manifest → build → cli。它不是偶然的耦合, + 压平它等于破坏架构。**所以「重构掉这条链」不是一个可行方案。** +2. **成本是摊开的,不是集中的。** 142 次模块编译:中位 1.99s, + top5 占 13%、top10 占 21%、top20 占 35%;44 个 <1s,66 个 1–3s,30 个 3–6s,2 个 >6s。 + 只有 `build.prepare`(16.4s)是真离群。 + **推论:「把最胖的模块拆了」不是通用解**,它只在那一跳上有效。 + +--- + +## 4. 四条杠杆 + +| | 杠杆 | 冷构建 | 依据 | 改动面 | +|---|---|---|---|---| +| **L1** | 默认工具链换 clang | 79.9 → **32.2s**(2.48×) | **实测** | 一行 manifest | +| **L2** | 引擎:下游在 BMI 可用时即开始 | gcc 80.5 → **39.2s**(2.05×) | **实测**(原型 A/B) | 引擎中等 | +| **L3** | 源码:定义移出接口单元 | 链 74.6 → ~10.4s | 推算(§2 的 86%) | 138 个模块 | +| **L4** | 源码:拆 `build.prepare` | 链 −8~11s | 推算 | 一个模块 | + +L1 与 L2 **可叠加**(一个压常数、一个改形状),叠加后推算 ~15s。 + +### L1 —— 换 clang(实测 2.48×) + +零引擎改动,单独就达标。但它**不改变形状**(§1),而且有三个必须先回答的问题: + +* mcpp 在三平台上的 llvm 载荷是否都可用、版本是否统一(目前 `windows = "llvm@20.1.7"`, + 与 Linux/macOS 的 22.1.8 不同)。 +* 换默认工具链会让**所有已发布包的指纹失效**,全生态一次性重编。 +* ABI:`-static-libstdc++` 与 libc++/libstdc++ 的选择;共享库不得内嵌 C++ 运行时 + (已知问题,见 `origin-precedence-and-shared-lib-cxx-runtime`)。 + +**这是一个生态决策,不是性能决策。** 它应当单独立项。 + +### L2 —— 让下游在 BMI 可用时就开始,而不是等编译器退出 + +**这是唯一一条既改变形状、又不需要改源码结构的路。** + +⚠️ **这一节的形状改过一次。** 最初写成「POSIX 用 fork 甩开 codegen,Windows 不支持」, +被指出「cmake / mcpp / xmake / bazel 不都是跨平台的吗」。追下去发现两件事, +它们把方案从「一个技巧勉强套两个编译器」改成了**按编译器族选机制**: + +| 编译器 | BMI 可用时刻 | 机制 | 可移植性 | +|---|---|---|---| +| GCC 16.1 | 编译的 **~22%** | 原子 rename + 甩开 codegen | 需要一个比本进程活得久的监督进程 | +| Clang 22.1 | **57%** | **原生两阶段**,两条普通 ninja 边 | 完全可移植,零进程把戏 | + +**Clang 实测**:`--precompile` 0.78s、`-c` 自 `.pcm` 0.70s,两阶段总 CPU 比单阶段(1.36s) +多 **9.6%**,而下游解锁点从 100% 提前到 **57%**。 + +**而且 clang 不能用 GCC 那套**:strace 证实它以 `O_TRUNC` **直接写最终路径**,没有 rename —— +「看文件出现」对它不成立,读者会读到写了一半的 `.pcm`。反过来,GCC 没有便宜的两阶段 +(`-fmodule-only` 要 99% 的时间,见 §2 的错误推理)。**两条路互为补集,不是二选一。** + +策略落在已有的 `BmiTraits`(`src/toolchain/model.cppm`)里,和 `moduleOutputPrefix`、 +`bmiSearchPrefix` 并列 —— 那正是「同一个决策只推导一次」的位置: + +``` + clang → TwoPhase cxx_precompile: x.cppm -> x.pcm + cxx_object_pcm: x.pcm -> x.o + gcc → DetachCodegen cxx_module_bmi: 编译器启动 → BMI 原子落盘 → 本边退出 0 + cxx_module_obj: 等该进程结束 → 回放输出 → 传播退出码 + msvc → None(待调研:/ifcOnly 是否便宜、.ifc 是否原子发布) +``` + +两种形状下,**下游 import 只依赖 BMI 边,link 依赖 object 边**。 + +「比本进程活得久的进程」用**派生一个 `mcpp` 监督子进程**实现,不用 `fork()` —— +派生进程在 Windows 上同样成立,所以 DetachCodegen 也不是 POSIX 专属; +它的**前提**(BMI 原子发布)才是编译器专属的。 + +原型 A/B(同构建目录、同编译器、同 flags、同编译器并发上限、同源码集, +**唯一差异是图的形状**): + +``` +baseline rc=0 wall=80.51s ninja -j32 compilers<=32 binary=19362456 +split rc=0 wall=39.23s ninja -j192 compilers<=32 binary=19362456 +BMI 边: n=142 中位 940ms OBJ 边: n=143 中位 3091ms +产物可运行;残留游离编译器进程 0 +``` + +⚠️ **四个已知坑,全部踩过:** + +1. **后台子进程会继承 ninja 的管道。** ninja 认为一条边结束是**管道 EOF**,不是直接子进程退出。 + 继承管道会让提前退出**完全不可见**,每条 BMI 边被记成整条编译的耗时 —— 第一次原型 + 就是这样得出「这个想法不成立」的。子进程的 stdio 必须重定向到文件,由 obj 边回放。 +2. **`ninja -j` 必须远大于编译器并发上限。** 编译器一旦脱离,就不再占 ninja 的槽; + 若 `-j` 等于上限,槽会被「正在睡觉的边」占满、就绪前沿饿死,调度退化成 baseline。 + 两条边的并发必须用**独立的信号量**(原子 `mkdir` 令牌)来限,而不是靠 `-j`。 +3. **失败会迟到。** BMI 落盘之后 codegen 才失败时,模块边已经报成功了。 + obj 边必须收集并**复现**每一个失败,否则会变成链接期一堆看不懂的未定义符号。 +4. **图必须自己声明形态。** `build.ninja` 是共享可变状态,快路径会重放它。 + 拆分与否必须写进 `# mcpp:graph=` 那一行,否则换了开关之后快路径会重放旧形状的图 + (这正是 #387/#407 的形状)。 + +**还有一个附带收益**:现在的 BMI 等价性判断是写在 ninja 命令里的一段 POSIX shell, +**Windows 上整段跳过**。把它移进 `mcpp` 子命令后,Windows 也能享受级联抑制。 + +### L3 —— 把定义移出接口单元(推算,链 74.6 → ~10.4s) + +§2 说一次接口编译 86% 是 codegen。这些 codegen 之所以发生在**接口单元**里, +是因为定义写在接口里。移到实现单元后: + +* 接口单元只剩 parse + 实例化 ≈ 现成本的 14%,**它们才是链上的节点**; +* codegen 搬到实现单元 —— 它们是 DAG 的**叶子**(没人 import),完全可并行; +* 总 work 不变,关键路径推算 74.6 × 0.14 ≈ 10.4s + `main.o` 4.78s ≈ **15s**, + 之后转为吞吐受限(314s / 24 线程 ≈ 13s)。 + +这与 bench 的 `modules-impl` 变体测的是同一件事。**但它要动 138 个模块**, +是一次跨越整个代码库的重构,不能一次做完,也不该为了性能而牺牲可读性 —— +它应当作为**新代码的书写约定**逐步生效,并优先用在**链上那 19 个模块**。 + +### L4 —— 拆 `build.prepare`(推算,链 −8~11s) + +16.41s,占链的 22%,是唯一的真离群点。拆成互不依赖的兄弟模块可直接缩短关键路径。 +**⚠️ 拆成链式的两个模块等于什么都没做** —— 必须是兄弟。 + +--- + +## 5. 建议顺序 + +1. **先做 L2。** 唯一改变形状、且不需要改源码结构的路;实测 2.05×,单独就把 80s 打到 39s, + 达成 <50s 目标。附带把级联抑制带到 Windows。 +2. **L4 紧随其后。** 一个模块的改动,收益可直接测量,且与 L2 叠加。 +3. **L1 单独立项。** 是生态决策(指纹失效、三平台载荷、ABI),不该混进性能 PR。 +4. **L3 作为约定长期生效**,优先施加于链上的 19 个模块。 + +## 6. 明确不做 + +* **加核 / 更大的 `-j` / 分布式编译。** 已实测:`-j8 → -j32` 只快 1.4%。 +* **压平模块分层。** §3:那条链是真实的架构分层,不是偶然耦合。 +* **为了性能降低优化档。** 改的是产物,不是构建。 +* **缓存自己的 BMI 跨构建复用。** 冷构建的定义就是没有缓存;这条对 §0 的目标无效。 + +--- + +## 7. 方法学附注 + +### 7.1 判据必须是「下游能不能用」,不是「文件在不在」 + +`.gcm` **出现**在编译的 14% 处,但「出现」不等于「写完」。三步缺一不可: + +1. 轮询到大小连续 150ms 不变 → 快照; +2. `cmp` 快照与编译结束后的成品 → **逐字节相同**; +3. 把快照放回 `gcm.cache/`,编译一个**真实下游导入者** → **exit 0**。 + +只做第 1 步会把「文件被创建」当成「BMI 可用」。 + +### 7.2 对照组 + +「改函数体后 BMI 差 2 字节」看起来像铁证。跑一次**同一份源码编译两次**的对照: +差的是**同样两个偏移**,上下文是 `buildtime:` / `localtime:` 的秒位。 +没有这个对照就会得出相反结论。 + +### 7.3 关键路径必须按拓扑序松弛 + +用栈式 DFS 求最长路径(带防环)会把 76.5s / 26 节点读成 33.9s / 10 节点, +把「100% 关键路径」读成「44%」,结论完全反过来。必须用 Kahn 拓扑序松弛。 diff --git a/.agents/docs/2026-08-14-bench-local-first-design.md b/.agents/docs/2026-08-14-bench-local-first-design.md new file mode 100644 index 00000000..40924ac4 --- /dev/null +++ b/.agents/docs/2026-08-14-bench-local-first-design.md @@ -0,0 +1,221 @@ +# bench 改为「本地真跑、CI 不跑」:方案与 Linux 实测计划(2026-08-14) + +**状态:待 review,尚未实施。** + +--- + +## 0. 一句话 + +把 bench 从 CI 里整个删掉,改成**一条本地可重复的命令**产出数据;当前只承诺 +**Linux**,其他平台在文档里明确标 `(未测试)`;标准数据集固定为 **3 轮**。 + +--- + +## 1. 为什么删 CI bench —— 不是因为它慢,是因为它测不到东西 + +删除的理由必须是事实,不是偏好。当前 `bench/matrix.json` 的账: + +| | | +|---|---| +| 格子数 | 10 | +| 外部引擎臂总数 | 32 | +| **被 `allow_failed` 豁免的** | **12(37%)** | +| 其中 xmake | 在测 4 / 豁免 6 —— **豁免多于在测** | +| 其中 cmake | 在测 5 / 豁免 5 | + +也就是说:一个自称「跨引擎对比」的矩阵,有三分之一的对比臂从未产出过数字,而 +job 是绿的。这不是"还没修完",这是**这个东西在 CI 上跑的形态本身就不成立**: + +1. **共享 runner 上测的是 runner,不是引擎。** 两核机器、邻居噪声、镜像随时更新。 + 同一份代码在 runner 上 cold 是 243s,在开发机上是 79s —— 三倍差距不来自 mcpp。 +2. **外部工具的缺口不是 mcpp 的缺陷,却由 mcpp 的 CI 承担。** xmake 编不了 + libc++ 的 `import std`、bazel 的 MSVC 依赖扫描器坏掉、rules_cc 不支持 + Windows PIC —— 每一个都要在 mcpp 的矩阵里挂一条豁免,而豁免越多,绿色越没意义。 +3. **一次矩阵约两小时,而它回答的问题("性能有没有退")在 PR 粒度上并不需要每次回答。** + 真正需要它的时刻是**发版前**和**做完一次优化后**,那两个时刻都有人在场。 + +**保留 CI 里的哪一部分**:`tests/e2e/230_bench_harness.sh`(套件自身能构建、能跑出一份 +合法报告)继续留在 e2e 里。删掉的是**跑真实工作负载的那个矩阵**,不是套件的自测。 + +--- + +## 2. 删掉什么、留下什么 + +**删除** +- `.github/workflows/bench.yml` 整个文件 + +**保留,但改变角色** +- `bench/matrix.json` —— 从「CI 跑哪些格子」变成「**标准数据集的定义**」。 + 这一点很关键:清单仍然只有一份,只是读它的人从 workflow 变成了本地脚本。 +- `bench/src/**` —— harness 本身不动。 +- `bench/results/**` —— 数据仍然入库。 + +**新增** +- `bench/run-standard.sh`(名字待定)—— 读 `matrix.json`,跑**当前平台**的格子, + 3 轮,输出到 `bench/results/--/`。一条命令,不带参数也能跑。 + +**必须同步改的守卫(否则删完 e2e 直接红)** +- `tests/e2e/233_bench_matrix.sh` 现在有 5 处读 `.github/workflows/bench.yml`: + 第 48/53 行(存在性)、427(workflow 读 matrix.json)、438(`--runs` 与 + dispatch 默认值)、475/480(runner 镜像不得内联)。这些检查的**对象**要从 + workflow 换成 `bench/run-standard.sh`,判据不变:**清单只有一份、脚本读它而不重复它**。 +- `tests/e2e/232_workflow_syntax.sh` 会少一个文件,无需改。 + +--- + +## 3. 标准数据集 —— 只承诺能证明的 + +### 3.1 平台 + +| 平台 | 承诺 | README 写法 | +|---|---|---| +| **Linux x86_64** | 本次真实跑出 | 完整数据 | +| macOS | 不跑 | `(未测试)` | +| Windows | 不跑 | `(未测试)` | + +`(未测试)` 不是 `(不支持)`。差别要在文档里写明:**没有数据 ≠ 不能用**,而 +"曾经有过一版数据、但那版数据是在有缺陷的 harness 上取的"更要写明 —— 见 §5。 + +### 3.2 格子(Linux) + +从现有 10 格里取 Linux 的 6 格,并**去掉所有需要豁免的外部臂**: + +| 工作负载 | 编译器 | 引擎 | 说明 | +|---|---|---|---| +| `fixture` | gcc | mcpp, cmake, xmake | 三引擎,唯一能控制 variant 轴的地方 | +| `fixture` | clang | mcpp, cmake, xmake, bazel | 四引擎 | +| `mcpp-2026.8.11.3` | gcc | mcpp, xmake | cmake 有未定案缺口 → 不列 | +| `mcpp-2026.8.11.3` | clang | mcpp, cmake | xmake 有已复现缺口 → 不列 | +| `xlings-2026.8.11.2` | gcc | mcpp | 两个外部臂都有缺口 | +| `xlings-2026.8.13.1` | gcc | mcpp | 同上,与上一行成对(两种代码风格) | + +**原则:标准集里不出现豁免。** 一条臂要么在测,要么不在这张表里 —— 它的缺口写进 +§5 的「已知缺口」清单,附证据。这样"标准数据"里的每一个数字都是真的跑出来的, +不需要读者去分辨哪些是豁免掉的。 + +### 3.3 轮次 + +**3 轮**(`--runs 3`),取中位数,同时记录 min/max。 + +理由:n=1 没有离散度,而现有 README 每张表都标 `n=1` 并附带"不要比较个位数字"的 +警告 —— 那个警告本身就说明 n=1 不够。3 轮 × 6 格 × 5 场景在开发机上约 40–60 分钟, +是一个人可以在一次会话里跑完的量。 + +### 3.4 场景 + +沿用现有六个:`cold` / `noop` / `touch-hub` / `touch-leaf` / `edit-body` / +`edit-comment`。真实工程略去 `touch-leaf`(现有 note 已说明:真实的树里没有 +"没人 import 且足够稳定可以点名"的单元)。 + +--- + +## 4. 本地实测计划(Linux) + +### 4.1 前置条件(脚本要检查并明确报错,而不是继续跑) + +1. 工具链载荷齐备:`xim-x-gcc/16.1.0`、`xim-x-llvm/22.1.8`、`xim-x-binutils`、 + `xim-x-glibc` +2. 外部工具按 `matrix.json` 的 `tools` 钉住:cmake 4.0.2 / xmake 3.1.0 / bazel 9.2.0 +3. 子模块已 checkout(三个 pinned 工作负载) +4. 依赖包已解包(`mcpplibs.cmdline` 等)—— 现在 cmake 臂会 FATAL_ERROR,好过静默少编 + +### 4.2 步骤 + +```bash +# 1. 构建被测的 mcpp 与 harness +mcpp build --release +cd bench && mcpp build --release && cd .. + +# 2. 拉取钉住的工作负载 +git submodule update --init + +# 3. 跑标准集(一条命令,内部按 matrix.json 展开 Linux 的格子) +bash bench/run-standard.sh + +# → bench/results/2026-08-14-linux-x86_64/*.json + report.md +``` + +### 4.3 记录什么 + +每份报告已经带 `host`(CPU 型号、核数、内存、是否异构)。再补两件: + +- **被测 mcpp 的版本与 commit** —— 现在只记版本号,同一版本可以有不同 commit +- **跑完的时间戳与总耗时** —— 用于判断两次数据是否可比 + +### 4.4 判据(跑完必须核对,不能只看退出码) + +1. 退出码 0 +2. `failed` 为 0、**`waived` 为 0**(标准集里不该有豁免) +3. 每个 `cold` 都通过既有的两条不变量(vs 自身 noop ≥2×、vs 同侪 ≤20×) +4. 三轮的 min/max 跨度不超过中位数的 ±20% —— 超了就是机器有噪声,数据不发布 + +--- + +## 5. README 要改成什么样 + +现在的 `bench/README.md` 是**一份实验记录**:900 行,大量"这里曾经错在哪"。 +那些内容有价值,但它不是一个人第一次想跑 bench 时该读到的东西。 + +**改成三层:** + +``` +bench/README.md +├─ 一、怎么跑(前 40 行,可以照抄的命令) +├─ 二、标准数据(Linux 表格 + 其他平台标 (未测试)) +└─ 三、方法与已知缺口(现有内容,往后放) +``` + +**「怎么跑」必须是可重复的**:给出完整命令序列、预期耗时、以及"跑完怎么判断这份 +数据能不能用"(§4.4 的四条)。现在的 README 里这些信息散在 §6 和 §10。 + +**已知缺口单独成节**,每条写:现象、归属(我们的 / 上游的)、复现方式、试过什么。 +当前应当列入: + +| 缺口 | 归属 | 证据 | +|---|---|---| +| xmake + libc++ 编不了真实工程的 `import std` | 上游 | 本机复现,`--sdk` 与两种工具链都试过 | +| cmake 在 CI runner 上 `__CMAKE::CXX23` 不可用 | 未定 | 外部可查项全部与能跑通的机器一致 | +| cmake 读 xlings 依赖包时 `manifest has no sources` | **我们的** | 本机 rc=0,runner 上某个 `compat-x-*` 缺 `sources`;**具体包名待补** | +| bazel MSVC 依赖扫描器 | 上游 rules_cc | Windows 专有 | + +**中文版同步**,并保留"以英文版为准"的声明。 + +--- + +## 6. 现有已发布数据怎么处理 + +`bench/results/` 里已有 30 个 JSON。**不删**,但要在 README 里分层: + +- **标准数据**:本次 Linux 实测那一份(3 轮),README 的表格只引用它 +- **历史数据**:其余的,标注日期与当时的 harness 状态 + +⚠️ 特别地:`bmi_schedule=on` 那一列的旧数据是在 §8b 缺陷修复**之前**取的, +`touch-hub`/`edit-comment` 两格量的工作比构建欠下的少(已在 README §8b 记录)。 +本次重测应当把这一列一并重取,让标准表里不再混有修复前的数字。 + +--- + +## 7. 风险与取舍 + +| 取舍 | 得 | 失 | +|---|---|---| +| CI 不再跑 bench | 省两小时/次;不再用绿色掩盖 12 条豁免 | **性能回归不再自动发现** | +| 只承诺 Linux | 数字都是真的 | 跨平台差异无数据 | +| 标准集不含豁免臂 | 表里每个数字都真跑过 | 表变小(32 臂 → 约 14 臂) | + +**最大的失**是第一条。缓解:把「发版前跑一次标准集」写进发布流程文档 +(`release-publish-pipeline` 那条);优化类 PR 在描述里附本地数据。 + +**不做的事**:不搞"CI 里跑一个缩水版 bench"。缩水版会重新引入"绿色代表什么" +的模糊地带 —— 这正是要删掉它的原因。 + +--- + +## 8. 待 review 的决策点 + +1. `bench/run-standard.sh` 这个名字与位置,是否放在 `bench/` 下 +2. 标准集的格子清单(§3.2)是否合适 —— 尤其是否要保留 `xlings` 的两格 + (它们只有 mcpp 一条臂,是"两种代码风格对比"而非"引擎对比") +3. 3 轮是否够;要不要对 `cold` 单独加轮次 +4. `bench/matrix.json` 是否继续承载"标准集定义",还是另起一个更小的文件 +5. §6 的历史数据:保留还是清理到只剩标准集 diff --git a/.agents/docs/2026-08-15-bench-resumable-design.md b/.agents/docs/2026-08-15-bench-resumable-design.md new file mode 100644 index 00000000..b67c06f0 --- /dev/null +++ b/.agents/docs/2026-08-15-bench-resumable-design.md @@ -0,0 +1,173 @@ +# bench 可断续:每个测量点独立落盘,进度可算可显示(2026-08-15) + +**状态:方案,待实施。** + +--- + +## 0. 现状(读代码确认,不是推测) + +| | | +|---|---| +| 落盘时机 | **只在一次 `bench` 调用结束时**,`std::ofstream out(opts->out, trunc)` 一把写完 | +| 断点续跑 | **完全没有** —— 源码里没有 resume / checkpoint / journal 任何一处 | +| 内层循环 | `engine × variant × scenario`,轮次在 `Runner::measure` 内部 | +| 进度显示 | 有实时行(`[123.4s] engine/tc/…/scenario run 2/3`),但**没有总量**,所以看不出"还剩多少" | +| `run-standard.sh` | 一格一次 `bench` 调用;格与格之间也没有续跑 | + +**后果**:任何中断 —— 我杀进程、机器重启、一个 cell 超时、我改错一个文件 —— +都让**整次调用**的工作归零。这两天我已经因此丢掉三整轮数据,每轮数小时: + +1. 裸名 `mcpp` 测成了旧版 → 全轮作废 +2. 字段错位导致 cmake 拿错目录 → 全轮作废 +3. 装 cmake 4.4.2 中途换掉 PATH → 全轮作废 + +**这不是运维失误的代价,是设计缺陷的代价。** 一个跑七小时、且没有任何中间态的 +批处理,注定会被它自己的运行环境打断。 + +--- + +## 1. 可保存的最小单元 + +正如你指出的,单元是: + +``` +os · toolchain · project · variant · scenario · engine · run_index +``` + +一个单元的产物只有一个数:那一轮的墙钟秒数(以及退出码)。这七个字段是它的 +**完整坐标**,而当前的报告已经带了前六个 —— 缺的只是 `run_index` 和"逐条落盘"。 + +**总量是可算的**,开跑前就能算出: + +``` +N = Σ(每个 cell) engines × variants × scenarios × runs +``` + +本机 Linux 标准集:约 **1000 个单元**(fixture 两格各 54/90 个测量点 × 3 轮, +五个真实工程格各 15 个 × 3 轮)。 + +--- + +## 2. ⚠️ 难点不在"存盘",在"这条记录属于哪一次运行" + +断点续跑最危险的失败模式不是丢数据,是**把两次运行的数据拼在一起而没人发现**。 + +这不是假设 —— **今天就发生了一次**:上一轮被 `kill` 的进程没有立刻死,在我 +`rm -rf` 之后才写出报告,于是 `clang-fixture.json`(90 格,20:21 那轮)和 +`gcc-fixture.json`(72 格,20:32 这轮)同名并列在一个目录里。唯一能分辨的线索是 +JSON 里的 `started_at`。如果我直接拿这个目录生成表格,**两轮数据会被无声地拼进 +一张表**。 + +加了 resume 之后,这个风险从"偶然"变成"系统性":resume 的本质就是"把旧记录 +当成本次的结果"。所以: + +> **没有身份校验的 resume,就是一台自动拼接不同运行的机器。** + +### 运行身份(run key) + +每条记录必须带一个 key,resume 只接受 key 完全相同的记录: + +| 成分 | 为什么 | +|---|---| +| 被测 mcpp 的 **版本 + commit** | 同一分支上每个 commit 版本号相同 —— 这正是任务 #16 | +| 参照 mcpp 的版本 | 老版本列换了就不是同一张表 | +| cmake / xmake / bazel 版本 | 今天 cmake 4.0.2→4.4.2 中途切换过 | +| 编译器载荷版本 | gcc 16.1.0 / llvm 22.1.8 | +| 每个工作负载的 **submodule commit** | 被测的树不能漂 | +| harness 自身的 commit | 计时逻辑改了,数就不可比 | +| host 指纹(CPU 型号 + 核数 + 内存) | 换机器就不是同一次测量 | + +key 不匹配时:**拒绝续跑,不静默重测**。打印哪一项变了 —— +"cmake 4.0.2 → 4.4.2,已有 412 条记录作废" 比"重新开始"有用得多。 + +--- + +## 3. 设计 + +### 3.1 journal(append-only) + +`/journal.jsonl`,每测完一个单元立刻 append 一行并 `flush`: + +```json +{"key":"","os":"linux","toolchain":"gcc","project":"mcpp-2026.8.11.3", + "variant":"native","scenario":"cold","engine":"mcpp@2026.8.13.1","run":2, + "wall_s":80.65,"exit":0,"at":"2026-08-15T04:31:02Z"} +``` + +* **append-only**,不改写:一次 kill 最多丢掉正在测的那一个单元 +* **JSONL**:半行(进程被杀在写一半)可以被解析器跳过,而半个 JSON 对象会让 + 整个文件无法读取 +* 最终报告仍然生成,由 journal 归约而来 —— 报告变成**派生物**,journal 是真源 + +### 3.2 resume + +启动时:读 journal → 丢弃 key 不匹配的行 → 得到"已完成单元集合" → 计划里减掉它们。 + +``` +resuming: 412/1004 units already recorded (41%), 592 remaining + skipped: gcc/fixture (54/54), clang/fixture (90/90) + partial: gcc/mcpp-2026.8.11.3 — 7 of 45 units +``` + +⚠️ **seed build 不是单元,必须重做。** 增量场景要求一棵"已经是最新"的树,而那 +是 seed build 建立的状态,不在 journal 里。所以恢复一个 cell 时:seed 重跑一次, +然后只补缺的轮次。这要写进文档,否则会有人以为 resume 是完全免费的。 + +### 3.3 进度 + +总量开跑前就能算,所以进度行带上分母: + +``` +[ 512.3s] (412/1004, 41%) mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3 run 2/3 +``` + +再加一个粗略的 ETA:用**已完成单元的中位耗时**外推,而不是平均 —— cold 与 noop +差三个数量级,平均值毫无意义。 + +### 3.4 CLI + +| 选项 | 行为 | +|---|---| +| (默认) | 有 journal 且 key 匹配 → 自动续跑;key 不匹配 → 拒绝并说明哪一项变了 | +| `--fresh` | 忽略 journal,从头测(旧 journal 改名保留) | +| `--dry-run` | 打印计划与总量,不测 | + +`run-standard.sh` 传同一个 `--out-dir`,于是**格与格之间**也自动续跑:被打断后 +重跑同一条命令即可。 + +--- + +## 4. 实施步骤 + +1. `protocol.cppm`:`RunKey` 结构 + journal 行的序列化/反序列化 +2. `main.cpp`:开跑前算总量;`Runner::measure` 每轮后 append + flush +3. `main.cpp`:启动时读 journal、校验 key、从计划中减去已完成单元 +4. 进度行加 `(n/N, x%)` 与 ETA +5. 报告从 journal 归约生成(现有 `Report` 结构不变,只换填充来源) +6. `run-standard.sh`:传 `--out-dir`,去掉每格一个 `--out` +7. **任务 #16 合并进来**:run key 需要被测 mcpp 的 commit,这本来就是要补的 +8. e2e 守卫: + - 杀掉一次运行再续跑,结果与不中断跑完**逐格相同** + - key 变化时**拒绝**续跑(反向必须验) + - 半行 journal 不会让读取崩溃 + +--- + +## 5. 不做什么 + +* **不做跨机器共享 journal。** host 是 key 的一部分,换机器就是另一次测量。 +* **不做单元级并行。** 并行会让每个单元的计时互相污染,而这是个基准。 +* **不缓存构建产物来跳过 seed。** seed 建立的是"树已是最新"的状态,把它缓存起来 + 就是在测缓存,不是测构建。 + +--- + +## 6. 这件事的判据 + +实施完成后,下面这句必须成立并且**被测试钉住**: + +> 在任意时刻杀掉 bench,重跑同一条命令,最终报告与从未中断跑完的报告**逐格相同**; +> 而当任何一项被钉住的东西变了,它**拒绝**续跑并说出是哪一项。 + +后半句和前半句一样重要。今天那次两轮混在一个目录里的事故说明:**能续跑但不校验 +身份,比不能续跑更危险** —— 前者只是浪费时间,后者会产出看起来正常的错误数据。 diff --git a/.agents/docs/2026-08-15-issues-412-422-analysis.md b/.agents/docs/2026-08-15-issues-412-422-analysis.md new file mode 100644 index 00000000..25f5dce0 --- /dev/null +++ b/.agents/docs/2026-08-15-issues-412-422-analysis.md @@ -0,0 +1,359 @@ +# #412 #415 #416 #417 #418 #421 #422 —— 逐条核实与修复方案(2026-08-15) + +**状态:分析 + 方案,待 review。全部对 HEAD 重新核实过,不是照抄 issue。** + +七条都是 2026-08-11/12 写的,分支此后动过。下面每一条先给**核实结论**(还在? +行号变了没?issue 的推断成立吗?),再给方案。**三处 issue 自身需要修正**,标了 ⚠️。 + +| # | 核实 | 性质 | 改动面 | 建议 | +|---|---|---|---|---| +| 416 | 仍在,但 ⚠️ **issue 的因果与症状都不成立**(实测) | 正确性(后果远小于所述) | 小 | 先复现 | +| 422 | 仍在,且真因比 issue 说的更好修 | 正确性(MSVC 不可用) | 小 | **先做** | +| 412 | 三条全在 ⚠️ 编号冲突 | 文档/覆盖 | 小 | **先做** | +| 418 | 两条全在,但字段有同名的活字段 ⚠️ | 死代码 | 小 | 做 | +| 415 | 仍在(`Origin` 无 `Artifact` 档) | 可观测性 | 中 | 做 | +| 417 | 仍在,但**真因未定位** | 首次体验 | 中 | 先探针 | +| 421 | 仍在;⚠️ 文档有一句是错的 | 能力边界 | 大 | 拆成三档 | + +--- + +## #416 — `obj/std.o` 无条件链进每个单元 + +### 核实 + +仍在。行号已从 issue 写的 1362-1381 变到 **`ninja_backend.cppm:1648` / `:1658`**: + +```cpp +case LinkUnit::Binary: +case LinkUnit::TestBinary: + if (has_std_artifacts) ins += " " + escape_ninja_path(std_o_dst); +case LinkUnit::SharedLibrary: + if (has_std_artifacts) ins += " " + escape_ninja_path(std_o_dst); +``` + +`has_std_artifacts` 的含义是「这条工具链有预建的 std 模块」,与**这个链接单元是否 +需要**无关 —— 这一点确实成立。但 issue 声称的**后果**不成立,见下面两处修正。 + +### 方案 + +把条件从「工具链有 std」收窄到「这个单元的闭包里真的有人 `import std`」。 + +判定数据**已经在手**,不需要新的扫描:模块图里每个 TU 的 `requires_` 是扫描器 +填的,`import std` 会出现在里面。所以: + +``` +needs_std(link_unit) = 该单元的任一 TU 的 requires_ 含 "std" / "std.compat" + ∪ 该单元链接的任一【本工程】静态库/对象的同一判定(传递闭包) +``` + +⚠️ **传递性必须递归,不能只看本单元的源码。** issue 自己点了这条,而它有前科: +依赖的 BMI 跨版本毒化那次(#405)就是「边存在但没有任何人依赖它」。一个单元自己 +不写 `import std`,但它 import 的模块接口写了,链接时同样需要 `std.o`。 + +### ⚠️ 两处对 issue 的修正(实测,不是推理) + +**修正 1:`std.o` 不可能是 `libstdc++.so.6` 的来源。** + +``` +$ nm --defined-only obj/std.o → 1 个符号:_ZGIW3std(模块 std 的全局初始化器) +$ nm --undefined-only obj/std.o → 0 个 +$ ls -l obj/std.o → 1280 字节 +``` + +**零个未定义符号**,所以它拖不动任何共享库。issue 把 +「纯 C 的 compat 包多一条 `NEEDED libstdc++.so.6`」归因给 `std.o`,这条因果不成立。 + +真正的嫌疑是**链接驱动**:`ninja_backend.cppm` 一律用 `$cxx`(g++)链接,而 g++ +总是加 `-lstdc++`;全仓 `--as-needed` **只**用在 `-latomic` 那一处 +(`flags.cppm:240`),所以 `-lstdc++` 会无条件成为 `NEEDED`,与 `std.o` 无关。 + +**修正 2:头号症状在本机产物上复现不出来。** + +``` +$ readelf -d ~/.mcpp/registry/subos/default/lib/libXau.so.6 | grep NEEDED + libc.so.6 ← 只有这一条,没有 libstdc++.so.6 +$ nm -D libXau.so.6 | grep -c _ZGIW → 0 +``` + +这份产物的来源版本不明(可能是 #414 之后重建的)。**实施前必须先用当前 mcpp +重新构建一个纯 C 的 compat 包并 `readelf -d`**,确认症状是否还在 —— +否则可能在修一个已经不存在的问题,而真正的 `-lstdc++` 通道没人动。 + +### 这两处修正如何改变方案 + +* **仍然该做**:`std.o` 无条件链进纯 C 单元本身就是错的(一个不该在那里的对象), + 即使它的后果比 issue 说的小得多。 +* **但判据要拆开**:「`readelf -d` 没有 `libstdc++.so.6`」这条**不是**收窄 `std.o` + 就能达成的,它属于 `-lstdc++` / `--as-needed` / 链接驱动那条线。两件事要分成 + 两个 issue,否则会出现「改完了,判据仍然不满足」。 +* **风险比想象中低**:`std.o` 唯一的符号是模块初始化器,而 `import std` 的 TU 会 + 引用它。所以谓词若**漏判**(该链没链),结果是**链接期 undefined `_ZGIW3std`** + —— 响亮的失败,不是静默错误。这让递归传递闭包不必一次写到完美。 + +--- + +## #422 — MSVC:`cxx_runtime` 到不了 std 模块 + +### 核实 + +仍在,而且**真因比 issue 推测的更具体**。`src/toolchain/msvc.cppm:529` 构建 std 的 +命令是: + +``` +cl /nologo /EHsc /O2 /W0 /c /ifcOutput /Fo: +``` + +**一个 `/MT` 或 `/MD` 都没有** —— 用的是 cl 的默认(`/MT`)。工程在 +`cxx_runtime = "host-coupled"` 下编 `/MD`,于是 `_MSVC_MT` / `_MSVC_MD` 对不上, +C5050 之后是真正的 C2375。 + +### ⚠️ issue 给了两条路,其实做第一条就白送第二条 + +issue 说「要么 std 构建遵守 `cxx_runtime`,要么 std 缓存按它分键」。核实发现 +**std 的缓存身份键里已经含 `std_build_commands`(实际命令行)** +(`src/toolchain/stdmod.cppm:136`,且 `metadata_matches` 的 14 个键里也有它)。 + +所以只要把 runtime flag 加进那条命令,缓存目录会**自动**随之分叉,两者不可能再 +静默背离。不需要单独设计缓存键。 + +### 方案 + +1. `msvc.cppm` 的 std 命令构造函数接收「有效 runtime 契约」,发 `/MT` 或 `/MD` + (以及 debug profile 下的 `/MTd` / `/MDd`)。 +2. 契约来源必须与工程 TU **同一处推导**,否则就是又一个「同一决策两处推导」。 + 工程侧的推导在 `src/build/flags.cppm`(`dist::parse_contract` / `contractByRole`), + std 侧要读同一个结果。 +3. 顺带:GCC / Clang 侧不需要动 —— 它们的 std 模块不带 runtime ABI 开关。 + 但**注释要写明为什么只有 MSVC 需要**,否则下一个人会以为漏了。 + +### 判据 + +* `cxx_runtime = "host-coupled"` 的工程在 MSVC 上**能构建并运行**(现在是失败) +* 同一台机器上,`host-coupled` 与 `self-contained` 两个工程各自拿到**不同的** + std 缓存目录 —— 用 `ls /std/` 直接看,两个 key +* ⚠️ 反向也要验:**只**改 `cxx_runtime` 不改别的,std 必须重建。否则说明 flag + 没进 `std_build_commands`,缓存键没分叉,问题只是被当前的冷缓存掩盖了 + +--- + +## #412 — 三处过期的 MSVC 陈述 + +### 核实 + +三条全在: + +| | 位置 | 现状 | +|---|---|---| +| 1. 劝退用户的假 note | `src/toolchain/lifecycle.cppm:664` | 仍在 | +| 2. 与自身断言矛盾的头注释 | `tests/e2e/95_msvc_system_toolchain.sh:9` | 仍在 | +| 3. `217` 在两个平台整条被跳过 | `tests/e2e/217_module_extensions.sh:2` `# requires: gcc` | 仍在 | + +### ⚠️ issue 的方案里有一个编号冲突 + +issue 建议「补一条 `219`」。**`219` 已被 `219_runtime_search_farm_is_last.sh` 占用。** +新测试要另取编号(当前最大是 234,故用 **235**)。 + +### 方案 + +1. 删 `lifecycle.cppm:664` 那句 note。它的实际效果是**劝退一个能用的功能**。 + 替换成陈述现状还是直接删掉,取决于是否有已知限制要说——若无,直接删, + 「没有消息」比「过期的消息」好。 +2. 删 `95` 头注释第 9 行。 +3. 新增 `tests/e2e/235_module_extensions_msvc_llvm.sh`:不依赖 `gcc` 能力, + 按平台选默认工具链(Windows→msvc,macOS→llvm),验证 `.ixx` 走完 + build→link→run。 + +⚠️ 第 3 条是唯一有实质价值的:前两条是删字,第三条是**补一条从未被走过的路**。 +`.ixx` + `module_extensions` + MSVC 这条组合推理上必然通,但推理不是测量 —— +这正是 #411 里指出的同一形状。 + +--- + +## #418 — 两处只写不读的死字段 + +### 核实 + +两条都仍在,但 ⚠️ **`cxxRuntimeTests` 这个名字下有两个字段,只有一个是死的**: + +| 字段 | 位置 | 状态 | +|---|---|---| +| `BuildConfig::cxxRuntimeTests` | `types.cppm:450` | **活的** —— toml.cppm:967 解析,flags.cppm:760/848 读 | +| `TargetEntry::cxxRuntimeTests` | `types.cppm:653` | **死的** —— 解析处(toml.cppm:1379)只读标量 `cxx_runtime`,应用处(prepare.cppm:1154)只应用 `cxxRuntime` | + +`contractByRole` 全仓两处命中(`flags.cppm:62` 声明、`:875` 写),**确认无读取方**。 + +实施时不要 `grep cxxRuntimeTests` 后一把删 —— 会删掉活的那个。 + +### 方案 + +* **`TargetEntry::cxxRuntimeTests`:删。** 理由:per-target 通道目前只支持标量, + 而标量已覆盖所有角色。补全表形式解析是**扩表面积**,而这个通道还没有用户要求它。 + 一个不生效的配置项比没有更糟 —— 但补全它等于新增一个需要长期维护的语义。 +* **`contractByRole`:接出去,不删。** 它记录「每个角色实际拿到的契约(降级之后)」, + 而 #414 之后共享库的契约按格式分档,「我这个 `.so` 到底拿到了哪一档」是用户会 + 真的问的问题,现在只能靠 `readelf` 自己看。写进 `resolution.json`,让 + `mcpp why` / `mcpp doctor` 能回答。 + +### 判据 + +* `TargetEntry::cxxRuntimeTests` 不存在,且 `[target.].cxx_runtime_tests` + 写在 manifest 里会**报未知键**(而不是被静默忽略) +* `contractByRole` 有真实消费方并被测试覆盖:一个共享库工程的 `resolution.json` + 里能读到它拿到的档位,且与 `readelf` 的实际观测一致 + +--- + +## #415 — `$ORIGIN` 不在 runtime 闭包里 + +### 核实 + +仍在。`src/platform/runtime_search.cppm` 的 `Origin` 枚举只有 +`Payload / Package / SubosFarm / HostDefault`,**没有 `Artifact` 档**(全文件 0 处 +命中 "Artifact")。`$ORIGIN` 由 `src/build/plan.cppm:475` 在 per-unit flags 通道 +单独发出。 + +模块自称「Search order = decreasing immutability. This is the one invariant in this +module」,而**最关键的那个目录不在这个排序里** —— #414 修的正是它排错了位置。 + +### 方案(采纳 issue 的轻档,不做重档) + +* `runtime_search.cppm`:`Origin` 加 `Artifact`,`rank()` 置于 `Package` 与 + `SubosFarm` 之间,补 `to_string()`; + ⚠️ `is_machine_local()` 返回 **false** —— `$ORIGIN` 随产物走,不是机器局部的。 + 这一条写错会让 `pack` 的判断反过来(虽然 `pack` 当前不 import 这个模块, + 但那是巧合,不是契约)。 +* `plan.cppm:679` `runtime_search_closure()`:把产物输出目录加进去。 +* 显示侧:`prepare.cppm` 写记录、`doctor.cppm` 打标签。 + +**重档(让闭包成为唯一的 rpath 生产者)不做**:`$ORIGIN` 本质是 per-unit 的 +(只有消费共享库的单元才需要),挪进全局闭包要给闭包引入 per-unit 概念, +改动量与风险明显更大,而收益只是消灭「两个生产者」这个洁癖。 + +### 判据 + +e2e 219 能把「记录的闭包」与「产物的 DT_RPATH」**逐项**比对并通过, +**不对 `$ORIGIN` 做任何过滤或例外** —— 现在它必须过滤,那个过滤就是缺口的证据。 + +--- + +## #417 — 全新 MCPP_HOME 首次构建,rule B 对每个产物报 inconclusive + +### 核实 + +仍在。两条消息在 `src/platform/elf_runtime.cppm:773` / `:791`,**按产物逐条发**, +所以一个图形工程刷 13 条。 + +### ⚠️ 真因未定位,不要照着 issue 的推理直接改 + +issue 自己写了「精确接缝还需要一次探针确认 —— **不要照着这个推理直接改**」。 +这条要照办。当前证据只支持这个描述: + +* 首次运行:`binding.loader` 与 `binding.library_dirs` 为空,而 `binding.search_dirs` + **有**值(已记下 `/lib`) +* 第二次运行:两者都有值,警告消失 +* 磁盘上二者首次运行时**都存在** + +「binding 求值早于 farm 落盘」是**自然读法**,不是已证事实。search_dirs 有值而 +另外两个没有,说明填充它们的**不是同一段代码或同一时刻** —— 这一点本身就值得先查。 + +### 方案:先探针,再修 + +1. **探针**:在 `runtime_binding.cppm:337-380` 的填充点打一条 verbose,记录 + 「此刻 `/lib64`、`/lib` 是否存在、里面有没有 `libc.so.6` 和 + `ld-linux-*`」。全新 MCPP_HOME 跑一次,把真实接缝钉死。 +2. 按探针结果二选一: + * 若确是**时序**:把 binding 的求值推迟到 farm 落盘之后(或在首次构建时重求值一次)。 + * 若是**查找逻辑**在首次运行时的目录形态下失效(例如那时还是符号链接的符号链接): + 修查找,而不是改时序。 +3. **无论哪一种,诊断都要改**:同一个原因对 13 个产物刷 26 条,是噪声不是信息。 + 改成 binding 层面**只说一次**「此刻无法求值,原因 X」。 + +### ⚠️ 必须写进代码注释的一条 + +**即使 rule B 完全正常,它也抓不到 #414 那个崩溃。** +`elf_runtime.cppm:761-801` 只比对 libc / PT_INTERP 的同一性,不管其它 SONAME +解析到谁。修这个 issue 不能替代 #414 的顺序修复,也不要当成它的兜底。 + +### 判据 + +全新 MCPP_HOME 的第一次构建:要么 rule B 给出真实判决(pass/mismatch), +要么 binding **只说一次**「此刻还无法求值」,而不是对每个产物各刷两条。 + +--- + +## #421 — 扫描器 M1:条件 import 与头单元(来自 XRGUI 的真实适配) + +### 核实 + +两条限制都仍在:`scanner.cppm:660`(条件 import)、`:666`(头单元)。扫描器是 +**纯词法**的,用 `if_depth > 0` 判定,**不看条件是否可判定**。 + +### ⚠️ 文档里有一句是错的,这是最该先修的 + +`docs/05-mcpp-toml.md:414`(中文版 §2.3 同): + +> `defines` … also reaches the P1689 module scan, **which is what makes a +> macro-guarded `import` resolvable** + +**这句话对 mcpp 不成立。** `defines` 确实会进**编译器的** P1689 扫描(`.ddi`), +但 mcpp **自己的**前置扫描器在看到条件块里有 `import` 时就直接报错,根本走不到 +宏求值。用户按文档写一个宏保护的 `import`,拿到的是 `forbidden in M1`。 + +**这一条不需要设计,只需要把文档改对** —— 而且它是七条里唯一一个「文档承诺了一个 +不存在的能力」,优先级应当最高。 + +### 三档方案(按代价递增,建议只做前两档) + +**档 A(必做,零风险):把文档改对。** 说明 `defines` 到达的是编译器的扫描, +而 mcpp 的前置扫描器目前拒绝一切条件块内的 `import`。 + +**档 B(建议做):诊断可操作化。** 现在只说 "forbidden in M1"。改成: +* 指出**是哪个宏**在保护它; +* 若该宏出现在 `[build].defines` / `[targets.*].defines` 里,直接说 + 「这个条件是可判定的,当前未实现求值(见 #421)」; +* 若不在(例如 `__cpp_lib_stacktrace` 这种工具链内建宏),说 + 「依赖工具链内建宏,不可判定」。 + +代价很小,而它把「适配一个真实工程」的排查成本从「逐个文件试」降到「读一条消息」。 + +**档 C(不建议现在做):真正求值可判定的条件。** +若条件只依赖 `defines` 里出现过的宏,扫描器信息是足够的。但这意味着扫描器要引入 +一个**最小预处理器**(`#if` 表达式求值、`defined()`、嵌套、`#elif`),而 +「实现一个不完整的预处理器」的失败形态是**静默扫错依赖图**,不是报错。 +M1 的取舍(纯词法 + 硬拒)正是为了避免这个。要做也应当先把边界写死: +只支持 `#ifdef X` / `#ifndef X` / `#if defined(X)` 三种字面形态,别的一律仍然拒。 + +**头单元**:维持现状(拒绝)。头单元本身是泥潭,而 issue 的报告者也说 +「不是说 M1 的取舍不对」。但档 B 的诊断应当同时指出**替代写法** +(把 `import ;` 改成 GMF 里的 `#include `),这是他实际采用的做法。 + +### 附:一处诊断透传(优先级最低) + +`module : private;` 在 GCC 16 的 P1689 扫描路径下报 `module already declared` +(编译路径报的是 `sorry, unimplemented: private module fragment`)。这是 GCC 的 +消息差异,不是 mcpp 的缺陷。扫描器既然已经在做词法解析,识别出这一句并附一句 +「当前工具链未实现私有模块片段」能省掉一轮误导性排查。**可做可不做。** + +--- + +## 建议的顺序 + +1. **#421 档 A** —— 改一句文档,消除「承诺了不存在的能力」。零风险,最高性价比。 +2. **#422** —— MSVC 上 `host-coupled` 现在**根本不能用**,而修法很小且缓存键白送。 +3. **#416** —— ⚠️ **先复现**:用当前 mcpp 重建一个纯 C compat 包,确认 + `readelf -d` 里还有没有 `libstdc++.so.6`。症状若已不在,这条降级为 + 「清理一个不该在那里的对象」,并把 `-lstdc++` / `--as-needed` 另开一条。 +4. **#412** —— 两条删字 + 一条真正的覆盖补齐(注意编号用 235)。 +5. **#418** —— 死字段;⚠️ 注意两个同名字段只有一个该删。 +6. **#415** —— 可观测性,改动中等,判据明确(219 不再需要过滤 `$ORIGIN`)。 +7. **#417** —— **先探针再修**,不要照 issue 的推理直接动时序。 +8. **#421 档 B** —— 诊断可操作化。 +9. #421 档 C / 私有片段透传 —— 暂不做。 + +## 一条贯穿的观察 + +七条里有**四条**(412、418、421 档 A、415)本质是同一件事: +**写下来的状态与实际行为脱节** —— 过期的 note、不生效的字段、承诺了不存在能力的 +文档、自称唯一排序却漏了最关键一项的模型。它们都不是「功能缺失」,而是 +**「代码与它自己的说明书不一致」**,而这类问题的共同代价是:读的人据此做决定, +然后浪费掉的时间不会归因到这里。 diff --git a/.agents/docs/2026-08-15-module-edit-granularity.md b/.agents/docs/2026-08-15-module-edit-granularity.md new file mode 100644 index 00000000..cb4f3eb0 --- /dev/null +++ b/.agents/docs/2026-08-15-module-edit-granularity.md @@ -0,0 +1,137 @@ +# 改一处实现,重编多少?—— 模块写法、编译器、与 BMI 的实际行为(2026-08-15) + +**状态:实测结论。每一条都有对照,推翻的中间结论也记在这里。** + +被测:`mcpp 2026.8.13.1`(本分支自举产物)· gcc 16.1.0 / llvm 22.1.8 · Linux x86_64 + +--- + +## 0. 一句话 + +> **让「改实现不级联」这件事跨编译器成立的,只有把定义放进 `.cpp` 实现单元。 +> 在单个 `.cppm` 里把声明和定义分开写,对增量构建零收益。** + +--- + +## 1. 三种写法的对照(8 个导入者,只改函数体) + +| 写法 | gcc:重编 object | clang:重编 object | +|---|---|---| +| ① 单 `.cppm`,声明+定义写在一起 | 1 / 11 | **9 / 12** | +| ② 单 `.cppm`,**声明与定义分开写** | 1 / 11 | **9 / 12** | +| ③ `.cppm` + `.cpp` 实现单元 | 2 / 12 | **2 / 13** | + +**clang 上 ② 与 ① 完全一样。** 边界在**翻译单元**,不在文件内的位置:模块接口 +单元里的一切都属于同一个 TU,clang 把其中的定义序列化进 BMI,与它写在第几行无关。 +实现单元(`module X;`,无 `export`)是**另一个 TU**,内容不进任何 BMI,因此没有 +任何导入方能依赖它。 + +gcc 上 ①② 便宜,是因为 gcc 不把非模板函数体写进 BMI —— 那是**实现选择,不是语言 +保证**,而 clang 不这么做。 + +--- + +## 2. ⚠️ gcc 的「便宜」比看起来窄得多:插一行就级联 + +§1 里 gcc 的 1/11 是用**同行替换**测出来的(`a+=i%3` → `a+=i%3+1`)。改成**插入 +一行**——也就是日常写代码的样子——结论翻转: + +| 编辑 | 后续声明是否移位 | 重编 object | BMI 重写 | +|---|---|---|---| +| 同行替换,不改行数 | 否 | 1 / 11 | 0 | +| 在**函数体内**插一行 | 是 | **10 / 11** | **9** | +| 在**声明之间**插一行 | 是 | **10 / 11** | **9** | +| 在**文件末尾**追加一行 | 否 | 1 / 11 | 0 | + +**真因:GCC 的 BMI 记录后续声明的源码行号。** 只要插入使它们移位,BMI 就变, +级联就发生 —— 与被改函数体的语义无关。追加到文件末尾之后没有任何声明,所以不变。 + +于是 §1 gcc 那一列要这样读:**只有当编辑不改变行数时才成立**,而增删一行代码是 +常态。**实际开发中,gcc 上单 `.cppm` 同样会级联。** + +`.cppm` + `.cpp` 不受影响:实现单元根本不产 BMI,行号移位无处可去。 + +--- + +## 3. 这解释了 mcpp 自己 `edit-body` 为什么是 80 秒 + +基准的 `edit-body` 扰动是**插入**一条语句(`insert_into_first_body`),不是同行替换。 +在 `mcpp-2026.8.11.3/src/version_req.cppm` 上,插入点落在 `Version::str()` 的 +`for` 循环里,其后还有大量声明 —— 于是 BMI 变了,整图级联,80.87s。 + +* BMI 确实变了:`mcpp bmi-equal` 判为**不等价**; +* 对照成立:**同一份源码编两次,BMI 逐字节相同**(没有时间戳噪声)。 + +**mcpp 没有缺陷。** BMI 真的不同,restat 正确地拒绝抑制。级联的原因是**行号移位**, +不是「函数体的语义变了」。 + +--- + +## 4. ⚠️ 生成的 fixture 没有复现这个行为 + +标准集里 gcc/fixture 的 `modules` × `edit-body` 是 **0.94s**(不级联),而 mcpp 真实 +工程是 **80.87s**。同一个场景名,差两个数量级。 + +原因:fixture 的 `unit_0.cppm` 里,被扰动的函数是**文件中最后一个声明**,插入点之后 +没有任何声明可以移位,所以 BMI 不变。 + +**fixture 在这一点上不保真**,而这正是 `--project` 模式存在的理由。任何从 fixture +的 `edit-body` 推出的关于真实工程的结论都不成立。 + +--- + +## 5. 同一个场景名在量两件相反的事 + +`edit-body` 按 variant 改的文件不同(`bench/src/fixture/generate.cpp`): + +| variant | 改的文件 | 实际问的问题 | +|---|---|---| +| `headers` | `unit_0.cpp` | 普通 TU,必不级联 | +| `modules` | `unit_0.cppm` | 接口单元 → **可能级联,取决于是否移位 / 编译器** | +| `modules-impl` | `unit_0_impl.cpp` | 实现单元 → **必不级联** | + +`modules` 与 `modules-impl` 的期望结果是**相反**的,却共用一行。文档必须写明, +否则读者会把两个不同的问题当成同一个指标的两次测量。 + +--- + +## 6. 被推翻的中间结论(留档,免得重走) + +1. **「GCC 16.1 的 BMI 不含函数体 ⇒ 改函数体不必级联」** —— 只对**不改行数**的编辑 + 成立。之前那条记录是用同源两编 + 同行替换得到的,不能推广到日常编辑。 +2. **「`Version::str()` 是导出类的成员函数,所以 GCC 把它序列化进 BMI」** —— 错。 + §1 的 ② 直接反证:成员函数体同行替换不级联。真因是行号移位(§2)。 +3. **「插入一行只要不动 `export` 声明就没事」** —— 错。移位的是**声明的位置**, + 不是声明的内容。 + +--- + +## 7. 方法学:这一轮差点得出三个错误结论 + +| 差点错在哪 | 怎么发现的 | +|---|---| +| `sed '2a\'` 空追加在这台机器上是**空操作**,于是「插空行不级联」 | 打印**改动前后的行数**,发现 6→6 | +| 插入的注释把同一行剩下的 `} return a; }` **注释掉了**,代码不成立,`object=1` 是构建失败的假象 | 每次测量都**断言构建成功**,而不只看重编了几个 | +| 用同行替换代表「改函数体」,得出 gcc 不级联 | 换成与基准**同形**的插入扰动,结论翻转 | + +三条都是同一个形状:**扰动本身没有被验证**。测「改一处会重编多少」时,必须先证明 +那一处真的按预期被改了,且改完仍然能编。 + +--- + +## 8. 结论与待办 + +**给写模块的人:** + +* 要「改实现不级联」,把定义放进 **`.cpp` 实现单元**。这是唯一跨编译器成立的办法。 +* 在单个 `.cppm` 里分开写声明和定义是**代码组织**上的整洁,增量构建上**零收益**。 +* gcc 上单 `.cppm` 看起来便宜,只在编辑不改行数时成立。 + +**给这个套件:** + +- [ ] SPEC 写明 `edit-body` 按 variant 改的是哪个文件,以及两者期望相反(§5) +- [ ] README 修正:gcc 上 `edit-body` 的级联来自**行号移位**,不是「接口变了」(§3) +- [ ] fixture 的 `modules` 变体应在被扰动函数之后放置声明,否则不保真(§4); + 改了会让已发布的 `edit-body` 数据不可比,需要重测并声明 +- [ ] 考虑增加一个**不改行数**的扰动(如 `edit-body-inplace`),把「语义变了」与 + 「行号移了」分开量 —— 现在它们混在一行里 diff --git a/.agents/docs/2026-08-15-xmake-clang-import-std.md b/.agents/docs/2026-08-15-xmake-clang-import-std.md new file mode 100644 index 00000000..0e25a6ac --- /dev/null +++ b/.agents/docs/2026-08-15-xmake-clang-import-std.md @@ -0,0 +1,95 @@ +# xmake + clang 的 `import std`:错误消息把人指向了死路(2026-08-15) + +**结论先说**:xmake 那句 `maybe try to add --sdk=` 在这个场景下是 +**死路**。真正的开关是 **runtime**,而 `--sdk` 只在 runtime 已经把库判定成 libc++ +之后才会被读到。为此我试了三轮 `--sdk`,全部落空。 + +--- + +## 1. 现象 + +``` +warning: std and std.compat modules not found! maybe try to add --sdk= or install libc++ +error: missing std dependency for module mcpp.build.flags +``` + +在 Linux + 载荷 clang(`xim-x-llvm/22.1.8`)+ 真实工程上稳定复现;同一个 clang 在 +生成的 fixture 上没有问题,同一个工程用 gcc 也没有问题。 + +## 2. 真因(读 xmake 源码得到,不是推测) + +`rules/c++/modules/support.lua`: + +```lua +function get_cpplibrary_name(target) + if target:has_runtime("c++_shared", "c++_static") then return "c++" -- libc++ + elseif target:has_runtime("stdc++_shared", ...) then return "stdc++" -- libstdc++ + ... + -- 没有指定 runtime 时,按平台回落 + elseif target:is_plat("linux", ...) then return "stdc++" +end +``` + +`rules/c++/modules/clang/support.lua` 再按这个值分支: + +```lua +if cpplib == "c++" then + -- 找 libc++.modules.json;找不到才提示 --sdk +elseif cpplib == "stdc++" then + -- 委托给 gcc 的实现 +end +wprint("std and std.compat modules not found! maybe try to add --sdk=...") +``` + +我们的工具链**从未声明 runtime**,所以 Linux 上回落成 `stdc++` —— xmake 跑去 +**LLVM 载荷里**找 GCC 的 modules.json,当然没有,于是走到函数末尾那句通用警告。 + +**`--sdk` 只在 `c++` 分支里被读到。** 我们从来没进过那条分支,所以传它没有任何 +效果 —— 而消息本身就是这么建议的。 + +⚠️ **这是"错误消息指向了一个它自己没走到的分支"**:那句话是函数**末尾**的兜底 +warning,不属于任何一个分支,却引用了只有其中一个分支才用得到的选项。 + +## 3. 有效的组合(实测) + +| 尝试 | 结果 | +|---|---| +| `--sdk=` | 无变化 | +| 内建 `llvm` 工具链 + `--sdk` | 无变化 | +| target 上 `set_runtimes("c++_static")` | 无变化(自定义 standalone 工具链拿不到这套接线) | +| 配置层 `--runtimes=c++_static` + 自定义工具链 | 无变化 | +| **内建 `llvm` 工具链 + `--sdk` + `--runtimes=c++_static`** | **警告消失,开始编译模块 BMI** | + +载荷里 `lib//libc++.modules.json` **一直存在** —— 正是 `c++` 分支要找的 +那个文件。 + +已落到 `bench/src/engines/xmake.cppm`:载荷 clang 走内建 `llvm` 工具链并同时传 +`--sdk` 与 `--runtimes`。 + +## 4. 推进之后撞到的下一个问题(真上游缺陷) + +``` +.../include/c++/v1/__format/format_functions.h:99:30: + error: call to implicitly-deleted default constructor of + 'formatter, wchar_t>' + ... + note: in instantiation of 'std::basic_format_string' +``` + +**这与 mcpp 自己修过的是同一个缺陷**,见 +`.agents/docs/.../clang-precompile-emits-full-bmi`:clang 的 `--precompile` 发的是 +**full BMI**,把它发布给下游会让 clang 22 编错一个下游 TU —— 窄格式串报 +`formatter<..., wchar_t>`,报错点在 std 头文件里,离真因很远。mcpp 的解法是 +`-Xclang -emit-reduced-module-interface`。 + +xmake 的 clang 模块实现目前发布的是 full BMI,所以真实工程上会撞到同一个坑。 +**这一条是上游的**,但现在被精确定性了,而不是"xmake 不行"。 + +## 5. 给下一个人的判据 + +* **不要按错误消息的建议行动,先确认那条建议属于哪个分支。** 这次的建议在兜底 + warning 里,而兜底 warning 按定义是"所有分支都没走成"。 +* **xmake 选 std 模块看的是 C++ 库,而库是从 runtime 推的。** 用 clang + libc++ + 时,`--runtimes=c++_static`(或 `c++_shared`)是必需的,不是可选优化。 +* **自定义 `standalone` 工具链拿不到 runtime 的接线。** 需要 runtime 语义时用内建 + 工具链(`llvm`),把载荷通过 `--sdk` 指进去。 diff --git a/.github/actions/bootstrap-mcpp/action.yml b/.github/actions/bootstrap-mcpp/action.yml index cf0628eb..acf20308 100644 --- a/.github/actions/bootstrap-mcpp/action.yml +++ b/.github/actions/bootstrap-mcpp/action.yml @@ -77,8 +77,12 @@ runs: *) tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" ;; esac WORK=$(mktemp -d) - curl -fsSL -o "${WORK}/${tarball}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + # Retried and verified — see .github/tools/fetch_release.sh. A bare curl + # here was the single largest source of unexplained CI red on this repo + # (`curl: (52) Empty reply from server`). + bash "$REPO_DIR/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "${WORK}/${tarball}" tar -xzf "${WORK}/${tarball}" -C "${WORK}" "${WORK}/${tarball%.tar.gz}/subos/default/bin/xlings" self install export PATH="$HOME/.xlings/subos/default/bin:$PATH" @@ -113,8 +117,11 @@ runs: REPO_DIR="$(pwd)" WORK=$(mktemp -d) zipfile="xlings-${XLINGS_VERSION}-windows-x86_64.zip" - curl -fsSL -o "${WORK}/${zipfile}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" + # Same helper as the unix leg. This is the leg that kept failing, and a + # fix applied to only one of them is a fix half the CI does not get. + bash "$REPO_DIR/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" \ + "${WORK}/${zipfile}" cd "${WORK}" unzip -q "${zipfile}" "$WORK/xlings-${XLINGS_VERSION}-windows-x86_64/subos/default/bin/xlings.exe" self install diff --git a/.github/actions/setup-macos-llvm/action.yml b/.github/actions/setup-macos-llvm/action.yml index 8e2ba0a2..66314a0a 100644 --- a/.github/actions/setup-macos-llvm/action.yml +++ b/.github/actions/setup-macos-llvm/action.yml @@ -36,8 +36,11 @@ runs: run: | WORK=$(mktemp -d) tarball="xlings-${XLINGS_VERSION}-macosx-arm64.tar.gz" - curl -fsSL -o "${WORK}/${tarball}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + # Retried and verified — .github/tools/fetch_release.sh. A bare curl + # here is the `curl: (52) Empty reply from server` flake. + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "${WORK}/${tarball}" tar -xzf "${WORK}/${tarball}" -C "${WORK}" XLINGS_DIR="${WORK}/xlings-${XLINGS_VERSION}-macosx-arm64" "$XLINGS_DIR/subos/default/bin/xlings" self install diff --git a/.github/tools/fetch_release.sh b/.github/tools/fetch_release.sh new file mode 100755 index 00000000..86490c95 --- /dev/null +++ b/.github/tools/fetch_release.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# fetch_release.sh — download a release archive, and mean it. +# +# ONE implementation for every bootstrap point in the repo, same reason +# install_pinned_mcpp.sh is: the two legs of bootstrap-mcpp had a bare +# `curl -fsSL` each, and a fix applied to one of them is a fix half the CI does +# not get. +# +# WHAT KEPT BREAKING. The Windows legs fail regularly with +# +# curl: (52) Empty reply from server +# Error: Process completed with exit code 52 +# +# — the GitHub release CDN accepting the connection and then closing it with no +# response. It is transient and it is not rare: it accounted for essentially +# every unexplained red on this branch, always inside 12 seconds, always with no +# test name in the log. +# +# ⚠️ `curl --retry` ALONE DOES NOT COVER IT. `--retry` handles timeouts and a +# specific list of 5xx responses; an empty reply is a *transport* error and is +# not on that list. `--retry-all-errors` (curl 7.71+) is the flag that does, and +# it is the one that was missing. The outer loop below is not redundant with it: +# it also re-runs when the bytes arrive but do not form a readable archive, which +# curl considers a complete success. +# +# WHY THE ARCHIVE IS OPENED HERE. A truncated download is not detected by curl — +# `-f` only checks the HTTP status. Without this check the failure surfaces later +# as `tar: unexpected EOF` or `unzip: cannot find zipfile directory`, several +# steps away from the download that actually failed, and reads like a corrupt +# release rather than a flaky fetch. +set -euo pipefail + +url="${1:?usage: fetch_release.sh }" +dest="${2:?usage: fetch_release.sh }" + +attempts="${FETCH_ATTEMPTS:-5}" + +verify() { + case "$dest" in + *.tar.gz|*.tgz) tar -tzf "$dest" >/dev/null 2>&1 ;; + *.zip) unzip -tqq "$dest" >/dev/null 2>&1 ;; + # Nothing to open: fall back to "it is not empty", which still catches + # the zero-byte result an interrupted transfer leaves behind. + *) [ -s "$dest" ] ;; + esac +} + +for i in $(seq 1 "$attempts"); do + rm -f "$dest" + # --retry-all-errors is what covers exit 52; the rest bound how long a single + # attempt may hang. --max-time is generous because these archives are tens of + # megabytes on a shared runner. + if curl -fsSL \ + --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 20 --max-time 600 \ + -o "$dest" "$url"; then + if verify; then + [ "$i" -eq 1 ] || echo "fetch_release: succeeded on attempt $i" >&2 + exit 0 + fi + echo "fetch_release: attempt $i downloaded $(wc -c < "$dest" 2>/dev/null || echo 0)" \ + "bytes but the archive does not open" >&2 + else + echo "fetch_release: attempt $i failed to download" >&2 + fi + # Back off before retrying: an immediate retry against a CDN that just + # dropped the connection tends to be dropped again. + [ "$i" -lt "$attempts" ] && sleep $(( i * 5 )) +done + +echo "fetch_release: giving up after $attempts attempts" >&2 +echo " url : $url" >&2 +echo " dest: $dest" >&2 +exit 1 diff --git a/.github/tools/newest_artifact.sh b/.github/tools/newest_artifact.sh new file mode 100755 index 00000000..855a5786 --- /dev/null +++ b/.github/tools/newest_artifact.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# newest_artifact.sh — print the most recently built +# copy of a binary under a target/ tree. +# +# WHY THIS IS NOT `find ... | head -1`. mcpp lays artifacts out under +# target///bin/, and the fingerprint changes +# whenever the toolchain, the standard or the flags do. A tree that has been +# built more than once therefore holds SEVERAL binaries with the same name, and +# `find | head -1` picks whichever the filesystem happens to list first. +# +# That is not hypothetical: it picked a two-and-a-half-hour-old bench binary on +# the first machine it ran on, and the run that followed silently exercised code +# that had already been replaced. In CI the same line would benchmark a stale +# mcpp and report the numbers as the new one's — a wrong answer with no symptom, +# which is the only kind this suite really has to defend against. +# +# `-printf` is GNU-only and macOS ships BSD find, so the mtime comes from a +# per-file `stat` call whose flag differs by platform. Both spellings are here +# because the alternative is a script that works on Linux and silently returns +# the wrong file everywhere else. +set -euo pipefail + +dir="${1:?usage: newest_artifact.sh }" +name="${2:?usage: newest_artifact.sh }" + +[ -d "$dir" ] || { echo "newest_artifact: no such directory: $dir" >&2; exit 1; } + +mtime() { + # GNU coreutils first, then BSD/macOS. Windows runners use git-bash, which + # ships GNU stat. + stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0 +} + +best="" +best_t=-1 +# `bin/` and `bin/.exe` — anchored on the bin/ directory so a +# same-named object or intermediate elsewhere in target/ cannot win. +while IFS= read -r f; do + [ -f "$f" ] || continue + t=$(mtime "$f") + if [ "$t" -gt "$best_t" ]; then best_t=$t; best=$f; fi +done </dev/null) +EOF + +if [ -z "$best" ]; then + echo "newest_artifact: no '$name' under $dir/*/*/bin/" >&2 + find "$dir" -maxdepth 4 -type d -name bin >&2 2>/dev/null || true + exit 1 +fi + +printf '%s\n' "$best" diff --git a/.github/workflows/bootstrap-macos.yml b/.github/workflows/bootstrap-macos.yml index 2ed36f70..d4d2d3d6 100644 --- a/.github/workflows/bootstrap-macos.yml +++ b/.github/workflows/bootstrap-macos.yml @@ -31,8 +31,9 @@ jobs: run: | WORK=$(mktemp -d) tarball="xlings-${XLINGS_VERSION}-macosx-arm64.tar.gz" - curl -fsSL -o "${WORK}/${tarball}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "${WORK}/${tarball}" tar -xzf "${WORK}/${tarball}" -C "${WORK}" "${WORK}/xlings-${XLINGS_VERSION}-macosx-arm64/subos/default/bin/xlings" self install echo "$HOME/.xlings/subos/default/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/cross-build-test.yml b/.github/workflows/cross-build-test.yml index 6ecaf210..68a072ff 100644 --- a/.github/workflows/cross-build-test.yml +++ b/.github/workflows/cross-build-test.yml @@ -121,8 +121,9 @@ jobs: XLINGS_VERSION: '2026.8.11.2' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - curl -fsSL -o "/tmp/${tarball}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" tar -xzf "/tmp/${tarball}" -C /tmp "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install export PATH="$HOME/.xlings/subos/default/bin:$PATH" @@ -258,8 +259,9 @@ jobs: XLINGS_VERSION: '2026.8.11.2' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - curl -fsSL -o "/tmp/${tarball}" \ - "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/d2learn/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" tar -xzf "/tmp/${tarball}" -C /tmp "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install export PATH="$HOME/.xlings/subos/default/bin:$PATH" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de8e79d1..088fe558 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -100,8 +100,9 @@ jobs: run: | if [ ! -x "$HOME/.xlings/subos/default/bin/xlings" ]; then tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - curl -fsSL -o "/tmp/${tarball}" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" tar -xzf "/tmp/${tarball}" -C /tmp "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install fi @@ -291,8 +292,9 @@ jobs: XLINGS_VERSION: '2026.8.11.2' run: | tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - curl -fsSL -o "/tmp/${tarball}" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" tar -xzf "/tmp/${tarball}" -C /tmp "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" @@ -359,7 +361,14 @@ jobs: # NOT interpolated from it, so check_version_pins.sh scans for them # explicitly (they were absent from the old lock-step comment). XLA="xlings-2026.8.11.2-linux-aarch64.tar.gz" - if curl -fsSL -o "/tmp/$XLA" \ + # NOT fetch_release.sh: this asset is OPTIONAL and the `if` is the + # point — an arch with no prebuilt xlings must fall through quietly, + # while the helper retries a 404 five times before giving up. The one + # flag that matters here is --retry-all-errors: `curl: (52) Empty + # reply from server` is a transport error, so plain --retry does not + # cover it. + if curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 20 --max-time 600 -o "/tmp/$XLA" \ "https://github.com/openxlings/xlings/releases/download/v2026.8.11.2/$XLA"; then tar -xzf "/tmp/$XLA" -C /tmp XLBIN=$(find /tmp/xlings-2026.8.11.2-linux-aarch64 -path '*/bin/xlings' -type f | head -1) @@ -445,8 +454,9 @@ jobs: if [ ! -x "$HOME/.xlings/subos/default/bin/xlings" ]; then WORK=$(mktemp -d) tarball="xlings-${XLINGS_VERSION}-macosx-arm64.tar.gz" - curl -fsSL -o "${WORK}/${tarball}" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "${WORK}/${tarball}" tar -xzf "${WORK}/${tarball}" -C "${WORK}" "${WORK}/xlings-${XLINGS_VERSION}-macosx-arm64/subos/default/bin/xlings" self install fi @@ -630,8 +640,9 @@ jobs: REPO_DIR="$(pwd)" WORK=$(mktemp -d) zipfile="xlings-${XLINGS_VERSION}-windows-x86_64.zip" - curl -fsSL -o "${WORK}/${zipfile}" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" \ + "${WORK}/${zipfile}" cd "${WORK}" unzip -q "${zipfile}" "$WORK/xlings-${XLINGS_VERSION}-windows-x86_64/subos/default/bin/xlings.exe" self install diff --git a/.gitignore b/.gitignore index a0e61a87..5f31875a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ # mcpp's own build output (mcpp build / mcpp pack) target/ +# mbench's run cache: one fingerprinted directory per configuration +.mbench/ +# Python bytecode from bench/tools/*.py +__pycache__/ +*.py[cod] # mcpp's per-workspace xlings sandbox + lockfile + diagnostic logs /.xlings/ @@ -23,3 +28,18 @@ doctor.log *.ddi compile_commands.json .cache/ + +# xmake control-arm build (bench/: xmake.lua builds mcpp for the +# build-engine benchmark; these are its artifact + resolved-config dirs) +/build/ +/.xmake/ + +# benchmark scratch. bench-work/ holds generated fixtures (regenerated on every +# run, and large); reports are per-host and belong in an artifact, not in git. +/bench-work/ +/bench/bench-work/ +bench-report.json +bench/bench-report.json +# --project mode writes the measured build's stdout/stderr next to the project +bench-child.log +.mcpp.toml.bench-backup diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..301b5448 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,59 @@ +# The benchmark's measured WORKLOADS, all pinned. +# +# A benchmark has two halves: the engine under test, and the workload it is +# given. The engine is the binary and is SUPPOSED to move between runs. The +# workload is not — and both of these were moving. +# +# * xlings was `git clone --depth 1` of its default branch at run time, so the +# target changed with every upstream push. `--hub src/xlings.cppm` had been +# naming a file that no longer existed for months: every xlings cell reported +# `skipped`, every xlings job reported success, and nobody had a reason to +# look. +# * mcpp's own sources were `--project $GITHUB_WORKSPACE`, i.e. the checkout, +# so every commit on a branch silently changed the thing being measured. The +# same defect, just harder to see because the drift was our own. +# +# A submodule is a PIN, not a vendored snapshot: the commit is in the diff, it is +# reviewed like any other change, `git submodule update --init` gives everyone +# the tree CI measured, and `tests/e2e/233_bench_matrix.sh` can check that each +# `hub`/`body` still exists in it. Bumping one is a deliberate act that +# invalidates the previous ratios on purpose. +# +# WHY TWO COPIES OF xlings. They are the two code styles being compared: +# +# xlings-2026.8.11.2 (b1563fe) 110 .cppm + 2 .cpp — implementation lives +# inside each interface unit +# xlings-2026.8.13.1 (f072075) 110 .cppm + 92 .cpp — implementation split out +# +# Same authors, same 46k lines, same module graph; the question is what the split +# costs or saves on an incremental build. That is the `modules` vs `modules-impl` +# axis the generated fixture has, on a real tree. +# +# ONE description serves every pin of a project +# (bench/projects//{CMakeLists.txt,xmake.lua}): they glob +# `src/**/*.{cppm,cpp}`, which is the same rule mcpp itself infers from, so no +# style needs its own file, an environment switch, or a branch. +# +# `ignore = dirty` on every one of them: a measurement RUNS these trees, and the +# engines write into them — `mcpp build` writes `mcpp.lock`, cmake and xmake put +# objects under `build/`. Those are the engines doing their job, and they made +# `git status` in this repository report two modified submodules after every +# bench run. The only state that matters here is WHICH COMMIT each workload is +# pinned to, and that is still reported: `ignore = dirty` hides working-tree +# changes, never a moved gitlink. +# +# The source files themselves are restored by the harness (`SourceGuard`), so a +# leftover perturbation is already a bug rather than something to ignore — and +# `git submodule foreach git status` still shows one when it happens. +[submodule "bench/projects/xlings/xlings-2026.8.11.2"] + path = bench/projects/xlings/xlings-2026.8.11.2 + url = https://github.com/openxlings/xlings + ignore = dirty +[submodule "bench/projects/xlings/xlings-2026.8.13.1"] + path = bench/projects/xlings/xlings-2026.8.13.1 + url = https://github.com/openxlings/xlings + ignore = dirty +[submodule "bench/projects/mcpp/mcpp-2026.8.11.3"] + path = bench/projects/mcpp/mcpp-2026.8.11.3 + url = https://github.com/mcpp-community/mcpp + ignore = dirty diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e6c172..b8c22053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,85 @@ > 本文件追踪 `mcpp-community/mcpp` 公开仓的版本演进。 > 格式参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/)。 +## [2026.8.13.1] — 2026-08-13 + +### 性能 + +- **⚠️ 「接口没变就不级联重编」的机制从设计之日起从未生效过 —— 现已修复。** + + `cxx_module` 规则会保留上一份 BMI、重编、然后在内容相同时把旧文件换回去,让 + ninja 的 `restat` 判定输出未变、从而**不重建导入者**。这套机制 2026-05-12 就设计 + 并实现了,判据是 `cmp -s`。 + + 但 GCC 把 wall-clock **写进了 BMI 的内容**: + + ``` + buildtime: 2026/08/12 02:25:01 UTC + localtime: 2026/08/12 02:25:01 UTC + ``` + + 同一份源码相隔一秒的两次编译,BMI 差**恰好 4 个字节** —— `cmp` 于是永远报「变了」, + 这条快路径**一次都没有走通过**。当年的设计说明只预见到 GCC 会重写文件(mtime 抖动) + 并据此开出内容比较的药方,没有预见到时间戳本身就是内容,所以药方按原样写出来就不 + 可能生效。 + + 新增 `mcpp bmi-equal`(内部子命令,由 ninja 规则调用),比较时掩掉这两个字段。 + 刻意**不用 `SOURCE_DATE_EPOCH`**:那会把整个编译的 epoch 钉死,从而改变**用户代码**里 + `__DATE__` / `__TIME__` 的展开结果;掩码只改变 mcpp 认为「什么算相等」,别的什么都不动。 + + 构造上保守:找不到预期字段、或两份文件对字段位置的判断不一致时,回落为严格比较 —— + 它可以把等价的 BMI 判成不同,但**绝不会**把不同的 BMI 判成相同。 + + 实测(用 `bench/` 套件测 mcpp 构建 mcpp 自身,touch 一个被 46 个模块导入、**内容未变** + 的文件): + + | 场景 | 2026.8.11.3 | 2026.8.12.1 | + |---|---|---| + | `noop` | 0.27s | 0.19s | + | **`touch-hub`** | **73.99s** | **0.45s** | + + ~164×。正确性由单测从**两侧**钉死:只测「等价的判相等」会放过一个恒返回 true 的实现, + 而那比原缺陷更糟 —— 它会静默吞掉所有真实的级联。 + +### 新增 + +- **`bench/` —— 构建引擎基准套件(顶层目录,用 mcpp 自己写)。** + + 起因是一次实测:mcpp 的自举构建**不是吞吐瓶颈,是延迟瓶颈** —— 关键路径 = 100% + 墙钟,后 55% 的时间里 32 个硬件线程上只有 1 个编译进程在跑;而这条关键路径上 + **77% 的时间在生产没有任何下游需要的 `.o`**(BMI 在编译进度 22.8% 处就原子落盘 + 了)。同样的病理在 xlings(110 模块、独立作者)上完全复现,说明这是 + 「C++23 命名模块 + GCC 单阶段 + 边完成即释放」的结构性结果,不是某一家的实现问题。 + + 详见 `.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md`。 + + 上一轮用的是一次性 bash + hyperfine 脚本,有四个致命缺陷:只支持两个引擎、 + **Windows 上根本跑不了**、被测对象只有 mcpp 自己、结果格式随手加字段。新套件: + + - **跨平台**:C++23 写成、由 mcpp 构建,三平台同一套逻辑。平台差异按 xlings + `src/platform/*.cppm` 的约定拆成**模块分区 + 整文件宏控** —— 非目标平台不导出 + 任何符号,同名定义全局只有一份,编译期自动选中。`#if defined(_WIN32)` 只出现在 + 两个分区里,runner / engines / protocol / fixture 全部零平台条件。 + - **协议先行**:`bench.protocol` 带 `protocol_version`,并把三条不变量写进类型 —— + 失败不得伪装成数据(非 ok 的格**没有** timing 字段,而不是 0)、跳过必须带原因 + (`unavailable` ≠ `failed`)、结果与宿主同生共死(含**异构 CPU 标记**:13900K 的 + 32 线程不能当 32 个同构核读)。 + - **加一个引擎 = 加一个文件**:`bench.engines.Engine` + `registry.cppm` 一行。 + 已接入 mcpp / mcpp-opt / cmake / xmake / meson / bazel。 + - **同一工程三种形态**:`headers` / `modules` / `modules-impl`,由生成器产出而非 + 手写 —— 手写两份「等价」代码几乎必然在某处不等价,而那正是被测量的东西。 + - **`--analyze`**:同一个二进制还能剖析任意 ninja 构建目录(工作量 / makespan / + 关键路径 / 并发曲线),固化了五个会**反转结论**的解析陷阱。 + + CI:`.github/workflows/bench.yml`,**仅手动触发**、覆盖 linux/macOS/windows、 + **不设性能阈值**(宿主方差远大于多数真实回归,把噪声变成红叉只会让人忽略它)。 + +### 说明 + +本版本不改变 mcpp 的构建行为;`bench/` 是独立工程,`mcpp build` 不受影响。 +分析报告给出的优化方案(BMI 落盘即释放、BMI 时间戳归一、实现移出接口单元) +按收益/风险排序记录在文档中,尚未实施。 + ## [2026.8.11.3] — 2026-08-11 ### 修复 diff --git a/README.md b/README.md index 8a2339c7..80426bd7 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,54 @@ import mcpplibs.cmdline; +## Benchmark + +Building **mcpp itself** — 137 module interface units, 57k lines, every one of +them `import std;` — with four engines handed the **same compiler binary**. +Each cell is the median of **3 samples** and how many times faster it is than +cmake. Every column comes from **one run**. + + +| scenario | `mcpp` | `mcpp +opt` | `mcpp (old)` | `cmake` | `xmake` | +|---|---|---|---|---|---| +| `cold` | 86.69s · 1.1x | **35.73s · 2.6x** | 86.75s · 1.1x | 91.74s · 1.0x | 90.54s · 1.0x | +| `noop` | **0.16s · 2.0x** | 0.18s · 1.8x | 0.24s · 1.3x | 0.32s · 1.0x | 0.38s · 0.8x | +| `touch-hub` | **0.42s · 197.7x** | 0.42s · 197.2x | 81.72s · 1.0x | 83.21s · 1.0x | 82.48s · 1.0x | +| `edit-body` | 80.87s · 1.1x | **29.83s · 2.9x** | 81.19s · 1.1x | 85.30s · 1.0x | 84.33s · 1.0x | +| `edit-comment` | **0.40s · 207.0x** | **0.40s · 207.0x** | 79.11s · 1.1x | 83.21s · 1.0x | 82.15s · 1.0x | + +`cold` nothing built yet · `noop` nothing at all · `touch-hub` mtime only, content unchanged · `edit-body` a real edit inside a function body · `edit-comment` a comment added to a hub interface.
+
+`mcpp` = mcpp@2026.8.13.1, the build under test · `mcpp +opt` = the SAME binary as `mcpp`, with the opt-in key `[build] bmi_schedule = "on"` (off by default) · `mcpp (old)` = mcpp@2026.8.11.3, the previously published release.
+Linux x86_64 · i9-13900K · gcc 16.1.0 · n=3 · pinned workload `a749e9f` · +cmake 4.4.2 / xmake 3.1.0 · `-` would mean not measured, and there is none here · +min/max sit within 4% of every median above 1s · +data: [`standard-20260814-linux-x86_64`](bench/results/standard-20260814-linux-x86_64/). + +* **Cascade suppression accounts for the `touch-hub` and `edit-comment` rows.** + cmake and xmake decide by timestamp and rebuild every downstream unit. mcpp + compares the BMI the compiler has just produced against the previous one and + skips the cascade when the interface is unchanged. This is default behaviour + and requires no configuration. The `mcpp (old)` column measures the previous + release at 81.72s, level with cmake, so the effect is new in this revision. +* **`edit-body` is the control case**, and it is the row where the cascade is + genuinely owed: mcpp is 1.1x rather than 200x, and an engine faster here would + be omitting work. `+opt` does not omit it either — it performs the same work + 2.9x faster. Worth stating precisely: the perturbation **inserts a line**, and + under GCC that shifts the recorded source location of every declaration after + it, which changes the BMI. The cascade follows from the changed BMI, not from + the edited body — measured in + [`.agents/docs/2026-08-15-module-edit-granularity.md`](.agents/docs/2026-08-15-module-edit-granularity.md). +* **`bmi_schedule` is opt-in and disabled by default** (`auto` resolves to off). + It moves code generation off the critical path, so it helps only where a + cascade is required: `cold` 86.69s → 35.73s, `edit-body` 80.87s → 29.83s. On + the two rows where mcpp already skips the cascade it yields no improvement. + An incorrect scheduling change fails silently rather than loudly, so the + default is not changed on the evidence of a single machine. + +📊 **[Methodology, pinned versions, and the full data → +`bench/README.md`](bench/README.md)** · [简体中文](bench/README.zh-CN.md) + ## Platform Support mcpp's identity model has two orthogonal axes: a **toolchain** is diff --git a/README.zh-CN.md b/README.zh-CN.md index 30192ac6..1ab445d5 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -300,6 +300,46 @@ import mcpplibs.cmdline; +## 性能对比 + +用**四个构建引擎**编译 **mcpp 自己** —— 137 个模块接口单元、57k 行、每一个都 +`import std;` —— 并且**给它们同一个编译器二进制**。每格是 **3 轮的中位数**,以及 +相对 cmake 的倍率。所有列出自**同一次跑**。 + + +| 场景 | `mcpp` | `mcpp +优化` | `mcpp (旧版)` | `cmake` | `xmake` | +|---|---|---|---|---|---| +| `cold` | 86.69s · 1.1x | **35.73s · 2.6x** | 86.75s · 1.1x | 91.74s · 1.0x | 90.54s · 1.0x | +| `noop` | **0.16s · 2.0x** | 0.18s · 1.8x | 0.24s · 1.3x | 0.32s · 1.0x | 0.38s · 0.8x | +| `touch-hub` | **0.42s · 197.7x** | 0.42s · 197.2x | 81.72s · 1.0x | 83.21s · 1.0x | 82.48s · 1.0x | +| `edit-body` | 80.87s · 1.1x | **29.83s · 2.9x** | 81.19s · 1.1x | 85.30s · 1.0x | 84.33s · 1.0x | +| `edit-comment` | **0.40s · 207.0x** | **0.40s · 207.0x** | 79.11s · 1.1x | 83.21s · 1.0x | 82.15s · 1.0x | + +`cold` 还没编过 · `noop` 什么都没改 · `touch-hub` 只碰 mtime,内容不变 · `edit-body` 真的改了一个函数体 · `edit-comment` 在 hub 接口里加一行注释。
+
+`mcpp` = mcpp@2026.8.13.1,被测的这一版 · `mcpp +优化` = **和 `mcpp` 同一个二进制**,开了 opt-in 的 `[build] bmi_schedule = "on"`(默认关闭) · `mcpp (旧版)` = mcpp@2026.8.11.3,上一个已发布版。
+Linux x86_64 · i9-13900K · gcc 16.1.0 · n=3 · 锁定的工作负载 `a749e9f` · +cmake 4.4.2 / xmake 3.1.0 · `-` 表示未测,本表没有 · +所有大于 1s 的中位数 min/max 都在 ±4% 以内 · +数据:[`standard-20260814-linux-x86_64`](bench/results/standard-20260814-linux-x86_64/)。 + +* **`touch-hub` 与 `edit-comment` 两行由级联抑制决定。** + cmake 与 xmake 按时间戳判断,重编全部下游单元;mcpp 将编译器刚产出的 BMI 与上 + 一份比较,接口未变则不触发级联。这是默认行为,无需任何配置。`mcpp (旧版)` 一列 + 测得上一个发布版为 81.72s,与 cmake 同量级,因此该效果在本版本中才生效。 +* **`edit-body` 为对照组**,也是级联确实欠着的那一行:mcpp 为 1.1x 而非 200x, + 在这一行更快的引擎意味着省略了应做的工作。`+优化` 同样不省略,只是把同一份工作 + 加快 2.9 倍。有一点需要说准:该扰动是**插入一行**,而 GCC 会因此移动其后所有声明 + 的源码位置记录,BMI 随之改变。级联来自变化后的 BMI,而非被编辑的函数体 —— 实测见 + [`.agents/docs/2026-08-15-module-edit-granularity.md`](.agents/docs/2026-08-15-module-edit-granularity.md)。 +* **`bmi_schedule` 为 opt-in,默认关闭**(`auto` 解析为 off)。它将代码生成移出关键 + 路径,因此仅在级联必需时有效:`cold` 86.69s → 35.73s、`edit-body` 80.87s → + 29.83s;而在 mcpp 本已跳过级联的两行上没有收益。调度错误的表现是静默失效而非 + 报错,因此不以单台机器的证据变更默认值。 + +📊 **[方法、锁定的版本、完整数据 → `bench/README.zh-CN.md`](bench/README.zh-CN.md)** · +[English](bench/README.md) + ## 平台支持 mcpp 的身份模型是两条正交轴:**工具链** = `family@version`(family ∈ gcc | llvm | msvc), diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..16f79b53 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,1119 @@ +# `bench/` — build-engine benchmark suite + +**English** · [简体中文](README.zh-CN.md) + +A cross-platform harness for measuring **build engines** against each other on +the **same C++ sources**, and for measuring what C++20 named modules actually +cost compared to headers. + +Written in C++23 and built by mcpp, so it runs identically on Linux, macOS and +Windows — a shell-based harness cannot, and this suite replaced one that could +only run on Linux. + +--- + +## Running it + +### The standard data set — one command + +```bash +mcpp build --release # the mcpp under test +(cd bench && mcpp build --release) # the harness +git submodule update --init # the pinned workloads + +bash bench/run-standard.sh # → bench/results/standard---/ +``` + +That is the whole procedure. The script reads [`matrix.json`](matrix.json), +selects the cells for the machine it is on, runs **3 samples** of each, and +refuses to finish quietly: a failing cell prints `NOT PUBLISHABLE` and the +script exits non-zero. + +| option | | +|---|---| +| `--dry-run` | print the plan and stop | +| `--resume` | continue an interrupted run — see below | +| `--runs 1` | faster, and **not publishable** — one sample has no dispersion | +| `BENCH_PRINT_ARGV=1` | print the argv it would pass, instead of running | +| `BENCH_ROOT=` | run from a copy of the script against that repository | + +**Expect one to several hours.** Seven cells, every engine each cell lists, +three samples, on workloads whose cold build is 30–120 seconds. + +### If it gets interrupted + +```bash +bash bench/run-standard.sh --resume +``` + +The unit of measurement is one sample — + +``` +project · variant · scenario · engine · run index +``` + +— and the harness appends each one to `.mbench//journal.jsonl` the +moment it is measured, flushing as it goes. **An interruption costs the sample +in flight and nothing else.** Three whole runs were lost to this before it +existed: a bare `mcpp` measuring the released binary, a field-order bug feeding +cmake the wrong directory, and a `cmake` upgrade that changed which binary was +on PATH halfway through. + +The `` is one hash over the **whole configuration** — engines, +variants, scenarios, sample count, compiler, project, fixture shape, and `--id`. +Same shape as a build directory: the same configuration resumes, a different one +lands somewhere else instead of overwriting. `mbench` prints which it got: + +``` +run id : 4e58a816 (resuming, 73 unit(s) recorded) +run id : a0840b10 (fresh) +``` + +⚠️ **Resume is where a benchmark quietly splices two runs together**, so two +things are deliberate: + +* The fingerprint **excludes the mcpp binary**. Including it would restart from + zero on every rebuild — exactly when resume is worth having. Each record + carries the version it was measured with instead, and adopting one measured + with a different version is **reported**, not silent. +* **Seed builds are not samples and are redone.** An incremental scenario needs + a tree that is already up to date, and no journal can hold that state. A + resumed cell pays its seed again before it can skip anything. + +`--resume` is a flag rather than the default because the two behaviours guard +against opposite mistakes: without it a second invocation must never write +beside the first one's reports, and a stopped run that kept writing after a +`rm -rf` did once leave a 90-cell file next to a 72-cell one, distinguishable +only by reading `started_at` out of the JSON. + +### Before you publish anything from it + +The script checks the first of these and says so; it cannot check the rest. + +1. **it exited 0** — no cell failed, and nothing is pre-excluded, so a failure + is a real failure on *this* machine +2. **min/max within about ±20% of the median** — wider means the machine was + busy, and the numbers describe that instead +3. **a failure reproduced by hand before it is called a gap** — see §5 + +### Driving the harness directly + +```bash +BENCH=$(ls -t bench/target/*/*/bin/mbench | head -1) + +# generated fixtures, across engines and source forms +"$BENCH" --engines mcpp=,cmake,xmake,bazel \ + --variants headers,modules,modules-impl \ + --scenarios cold,noop,touch-hub,edit-body \ + --compiler payload:gcc --runs 3 + +# a real project +"$BENCH" --project bench/projects/mcpp/mcpp-2026.8.11.3 \ + --buildfiles bench/projects/mcpp \ + --engines mcpp=,mcpp= --compiler payload:gcc \ + --scenarios noop,touch-hub --hub src/platform/platform.cppm --runs 3 +``` + +⚠️ **Always pass mcpp as a PATH, never as the bare name `mcpp`.** The bare name +resolves through PATH to the xlings shim, which re-picks its version from the +working directory — and for `--project` that directory is the measured tree, +which carries its own pin. A whole run once reported `mcpp@2026.8.11.3` in every +cell: the released binary, not the branch, with nothing failing to say so. + +Each `mcpp=` labels itself from the version that binary reports, so two +releases never collapse into one row. That is how "did this release get faster?" +is answered — by running both, not by emulating one in the harness. + +### Where CI went + +There is no CI job for this. There was, and it measured far less than it looked +like: 10 cells, 32 foreign-engine arms, **12 of them waived** — xmake had more +arms waived than measured — while the job went green. On top of that a shared +runner measures the runner (the same tree took 243s there and 79s here), and +most of what was being waived is other people's tools rather than mcpp. + +`tests/e2e/230_bench_harness.sh` still runs on every PR: it builds the suite and +checks it produces a valid report. What is not automated is the measuring. + +**Run the standard set before a release, and after any change that claims a +performance effect.** + +--- + +## The standard data + +### What this run actually covers — and what it does not + +`bench/results/standard-20260814-linux-x86_64/` · **696 measured samples** · +3 samples per cell · Linux x86_64 · i9-13900K. + +`-` below means **not measured**. It never means "not applicable" and never +means zero: a gap that is not marked is a gap that gets read as a result. + +| toolchain | project | cells | outcome | +|---|---|---|---| +| gcc | `fixture` | 72 | 72 ok | +| clang | `fixture` | 90 | 90 ok | +| gcc | `mcpp-2026.8.11.3` | 25 | 25 ok | +| clang | `mcpp-2026.8.11.3` | 25 | 20 ok, **5 failed** (xmake — see below) | +| gcc | `xlings-2026.8.11.2` | 25 | 25 ok | +| gcc | `xlings-2026.8.13.1` | - | **not measured** — the run was stopped here | +| clang | `xlings-2026.8.11.2` | - | **not measured** — the run was stopped here | +| macOS (all cells) | - | - | **not measured** | +| Windows (all cells) | - | - | **not measured** | + +The run was stopped after five of seven cells: at ~55s per sample the remaining +two were another ~2 hours and neither changes a published conclusion — the +two-code-style comparison is complete on gcc, and clang is covered on the mcpp +workload. **Re-running `bash bench/run-standard.sh --resume` fills them in +without repeating anything**, because every sample above is journaled. + +#### The one failure, and why it is a finding rather than a gap + +`clang / mcpp-2026.8.11.3 / xmake` failed all five scenarios with +`seed build exited 255`. Reproduced by hand, the cause is exact: + +``` +__format/format_functions.h:99:30: error: call to implicitly-deleted default + constructor of 'formatter, wchar_t>' +``` + +on a **narrow** format string. xmake's default shape on clang is +`--precompile` → `.pcm` → `-c .pcm`, which requires clang's **full** BMI, and +publishing a full BMI to importers makes clang 22.1.8 miscompile a downstream +TU. mcpp is immune because it publishes reduced BMIs and recompiles the source +for the object edge. This is upstream, filed as +[#424](https://github.com/mcpp-community/mcpp/issues/424) — not something xmake +got wrong, and not something this suite should paper over with `allow_failed`. + +#### One declared outlier + +`gcc / xlings-2026.8.11.2 / mcpp+schedule=on / touch-hub` was measured as +`[1.77, 20.82, 1.79]` — a 1066% spread around a 1.79s median. Re-measured +immediately afterwards at 8 samples it was +`[1.79, 1.79, 1.79, 1.79, 1.78, 1.79, 1.81, 1.79]`, a 1% spread with no +outlier, so the 20.82s was machine noise rather than an intermittent cascade +failure. That probe is kept beside the run as +[`probe-touch-hub-outlier-8-samples.json`](results/standard-20260814-linux-x86_64/probe-touch-hub-outlier-8-samples.json) +— a claim about a published number has to be checkable too. **The published cell is left exactly as measured**: splicing a second +run's samples into a first run's report is the one thing this suite must never +do. The probe is evidence about the number, not a replacement for it. + +### What is pinned, and why every one of these is pinned + +A benchmark number is only worth the list of things that were held still while +it was taken. Every row below was loose at some point in this suite's short +life, and every one of them produced a table that was measuring something other +than what it said. + +| what | pinned to | declared in | +|---|---|---| +| cmake | **4.4.2** | `matrix.json` → `tools` | +| xmake | **3.1.0** | `matrix.json` → `tools` | +| bazel | **9.2.0** | `matrix.json` → `tools` | +| gcc | **16.1.0** | `bench/src/toolchain.cppm` | +| clang / libc++ | **22.1.8** (Windows: 20.1.7) | `bench/src/toolchain.cppm` | +| reference mcpp | **2026.8.11.3** | `matrix.json` → `reference_mcpp` | +| mcpp (the workload) | **2026.8.11.3** — `a749e9f` | submodule `projects/mcpp/mcpp-2026.8.11.3` | +| xlings (combined style) | **2026.8.11.2** — `b1563fe` | submodule `projects/xlings/xlings-2026.8.11.2` | +| xlings (split style) | **2026.8.13.1** — `f072075` | submodule `projects/xlings/xlings-2026.8.13.1` | +| mcpp under test | the checkout | built locally; `run-standard.sh` passes its PATH | + +**Everything is installed by xlings**, at those exact versions, on every runner. +`xlings install cmake@4.4.2 xmake@3.1.0 bazel@9.2.0 mcpp@2026.8.11.3` installs +the set; `run-standard.sh` refuses to start if an installed tool does not report +the pinned version, because a tool that varies is a variable the report does not +record and the reader cannot see. + +Four things this bought, each of which had already gone wrong: + +* **cmake 3.31.6** is what the GitHub runner images ship. It does not have the + CMake 4.0 experimental key for `import std`, so *every module cell failed to + configure*. With 4.0.2 they pass. +* **`command -v g++`** on those images is gcc 13.3.0. cmake cannot configure + C++23 modules with it and xmake crashes it with an internal compiler error — + while mcpp quietly used its own registry's gcc 16.1 regardless. The table read + `48 failed / 6 ok` and was still called a comparison of build engines. The + suite now hands **every** engine the driver out of mcpp's own payload + (`--compiler payload:gcc`), which is its fairness rule finally enforced rather + than merely written down. +* **The measured workloads moved.** xlings was `git clone --depth 1` of its + default branch at run time, so the target changed with every upstream push — + `--hub src/xlings.cppm` had been naming a file that no longer existed for + months, every xlings cell reported `skipped`, and every xlings job reported + success. mcpp's own sources had the same defect in a form that is harder to + see: `--project $GITHUB_WORKSPACE` made the checkout the workload, so every + commit on a branch silently changed the thing being measured. **The engine + under test is the binary and is supposed to move; the workload is not.** + All three are git submodules now, and the guard checks that each `hub` and + `body` exists in the pinned tree. +* **Only one mcpp was measured.** A report that says how fast this branch is, + without saying whether it got faster, is not what a benchmark on a pull + request is for. + +> **Not held still, and deliberately so:** the runner hardware. See §4a. + +> **Two ways the measured tree does get written to.** Neither affects a timing, +> but both leave a dirty submodule: +> +> 1. **The engine's own bookkeeping.** `mcpp build` writes `mcpp.lock`, cmake and +> xmake write into `build/`. That is the engine doing its job — a real user's +> build does it too — so it is not something the harness should prevent. +> 2. **A hard-killed run.** +> The editing scenarios save a file's exact bytes and restore them however +> the function exits — including on a failed build — but that is a +> destructor, and a destructor does not run under `SIGKILL`. Interrupt a +> `--project` run hard enough and the perturbation is still there. They are +> named `bench_nonce_*`, so they are easy to recognise in a diff. +> +> `git submodule foreach 'git checkout -- .'` undoes both. + +> **Not exercised by these numbers:** mcpp's split build schedule +> (`[build] bmi_schedule = "on"`) is opt-in until it has been verified on every +> platform, so both mcpp binaries run with it off. Its effect is measured +> separately in `.agents/docs/2026-08-13-build-optimization-status.md`. + +### The headline numbers, and where they come from + +> ⚠️ **`bmi_schedule` has a known correctness bug — do not quote these numbers.** +> On the generated fixture's `modules` variant, four scenarios fail with +> `failed to read compiled module: No such file or directory` in an importer. +> It reproduces at `-j1`, so it is not a race between compilers: phase 1 parks +> the previous BMI in `.bak` *before* spawning the compiler, and the file is +> measurably absent for ~208 ms of every rebuild. Every `bmi_schedule` figure +> below was taken with that defect present. See +> `.agents/docs/2026-08-13-build-optimization-status.md` §8. + +**Read the real-project table first.** A synthetic fixture is for isolating one +variable; it is not evidence about anyone's build. Where the two disagree, the +real project is right and the fixture is telling you about its own shape. + +#### mcpp itself — the pinned workload, 137 modules, 57k lines, gcc 16.1.0 + +`bench/projects/mcpp/mcpp-2026.8.11.3` (`a749e9f`), measured in place with +`--buildfiles projects/mcpp/`, i9-13900K, **n=1** (see the caveat below). +Ratios against cmake. + +| scenario | `mcpp@2026.8.11.3` | `mcpp@2026.8.13.1` | `+bmi_schedule=on` | `cmake` | `xmake` | +|---|---|---|---|---|---| +| `cold` | 79.46s · 0.86x | 79.54s · 0.86x | **36.36s · 0.39x** | **92.33s** · 1.00x | 90.30s · 0.98x | +| `noop` | 0.34s · 1.21x | 0.16s · 0.57x | 0.16s · 0.57x | **0.28s** · 1.00x | 0.38s · 1.36x | +| `touch-hub` | 76.53s · 0.92x | **0.40s · 0.005x** | 0.44s · 0.005x | **83.39s** · 1.00x | 82.08s · 0.98x | +| `edit-body` | 77.33s · 0.90x | 76.24s · 0.89x | **30.48s · 0.36x** | **85.64s** · 1.00x | 84.61s · 0.99x | +| `edit-comment` | 75.69s · 0.91x | **0.38s · 0.005x** | 0.44s · 0.005x | **82.96s** · 1.00x | 82.73s · 1.00x | + +Four things this says, and the fixture can say none of them: + +1. **On a cold build nobody wins, and that is the correct answer.** Every engine + is within 15% of the others, because mcpp's cold build is **100% critical + path** — 79.7s of a 79.8s makespan, average parallelism 3.94 of 32 hardware + threads. All of them walk the same 26-deep chain of module interfaces, and + scheduling cannot shorten a chain. The generated fixture puts mcpp at `0.26x` + here; that is an artefact of a workload whose units cost 0.09s each, and + quoting it as a cold-build advantage would be dishonest. +2. **The cold-build lever is the opt-in schedule, not the release.** 79.46s → + 79.54s between the two releases is no change at all; `bmi_schedule = "on"` + takes it to 36.36s. Everything else in this table is release-over-release; + that column is a *setting*. + + **And the schedule column is not a free upgrade.** On `touch-hub` and + `edit-comment` it is 0.44s against the default's 0.40s and 0.38s — slightly + WORSE, because those are exactly the rows where mcpp already skips the + cascade, so the split graph adds edges and buys nothing. It pays where a + cascade is genuinely owed (`cold`, `edit-body`) and nowhere else. +3. **The daily loop is where the engines differ**, by ~190x on this project: + touching a hub interface costs cmake and xmake a full 83-second rebuild + because they decide by timestamp, and 0.40s for an engine that compares the + BMI it just produced against the previous one. +4. **`edit-body` is the control.** mcpp is deliberately *not* fast there (0.89x): + the interface genuinely changed, so the cascade is owed. An engine that were + fast on that row would have skipped work it owed. + +> **The xmake column is from a SEPARATE run.** Its numbers in the original +> five-arm run were invalid — xmake normalises `--buildir` to a path relative to +> `-P` and then resolves it against the process cwd, so `clean()` had been +> removing a directory it never wrote to and `cold` came back at **0.60s** with +> status `ok`. Fixed (the engine now runs from `-P`) and re-measured on the same +> machine; `cold` went 0.58s → 90.95s in the isolated check and 90.30s here. +> Recorded rather than quietly re-run, because the two halves of this table were +> not taken in the same minute. + +> **n=1, so read the ratios and not the digits.** §4a R2 asks for dispersion and +> a single sample has none. Two rows also sit near their own engine's resolution +> floor: mcpp's `touch-hub` and `edit-comment` are 2.5x and 2.4x its own `noop`, +> just above R1's 2x line, so *"about two orders of magnitude"* is supported and +> *"0.40 versus 0.38"* is not. + +> **`edit-comment` here is the `end-of-file` form.** mcpp's hub has no function +> body, so the comment is appended rather than inserted, and no line numbers +> move. On a hub that does have bodies the same scenario legitimately cascades — +> see the xlings table below and SPEC.md §4. The cell's `note` records which +> form ran. + +#### xlings — the same question on someone else's codebase, in two code styles + +110 modules, 46k lines, different authors, never tuned for this. The two pins +are the same project either side of one refactor. Ratios against the released +mcpp. + +#### The three engines, on the combined tree + +First measurement in which all three arms produce a running binary — the cmake +and xmake columns below were `failed` cells until the arms were finished, and the +table that stood here was mcpp-against-mcpp for that reason. + +| scenario | **mcpp** `bmi_schedule=on` | mcpp default | cmake | xmake | +|---|---|---|---|---| +| `cold` | **37.56s** · 3.18x | 92.49s · 1.29x | 119.46s · 1.00x | 105.02s · 1.14x | +| `noop` | **0.72s** · 0.50x | 0.74s · 0.49x | 0.36s · 1.00x | 0.40s · 0.90x | +| `touch-hub` | **1.04s** · 93.93x | 1.79s · 54.96x | 98.16s · 1.00x | 98.16s · 1.00x | +| `edit-body` | **29.65s** · 3.32x | 88.38s · 1.11x | 98.43s · 1.00x | 98.00s · 1.00x | +| `edit-comment` | **30.44s** · 3.22x | 93.81s · 1.04x | 97.98s · 1.00x | 97.97s · 1.00x | + +**Both mcpp columns are here because one of them was misleading on its own.** +The default column is what a user gets today; `bmi_schedule=on` is the opt-in +split schedule, and leaving it out understated mcpp badly — `edit-body` reads +1.11x in the default column and 3.32x with the schedule on. + +* **`edit-body` and `edit-comment` are not "no advantage".** The cascade really + is owed in both (the perturbed function body lives in an interface unit, so + the BMI genuinely changes). The default column shows mcpp doing that owed work + at cmake's pace; the schedule column shows it doing the SAME work 3.3x faster, + by publishing each BMI as soon as it exists instead of after code generation. +* **`noop` is the one mcpp loses outright**, in both columns: 0.72–0.74s against + cmake's 0.36s (0.49x). That is per-invocation overhead — the number a user + feels on every edit-build cycle, and mcpp is the slowest of the three at doing + nothing at all. +* **`edit-comment` is 1.04x in the default column, not the 200x the mcpp + workload shows.** The comment lands INSIDE an inline function body that xlings + keeps in its interface unit, so the BMI genuinely changes and the cascade is + owed. The cell's note records which form ran; see §3, and do not read this as + the optimisation failing. +* **`touch-hub` is the real cascade-suppression result: 54.96x.** Content + unchanged, so mcpp compares the BMI it just produced against the previous one + and skips 45 importers. cmake and xmake decide by timestamp and rebuild all of + them — to within 0.00s of each other, which is what two timestamp-driven + engines should look like. +* **⚠️ These `bmi_schedule=on` cells were taken BEFORE the §8b fix, and are + therefore suspect in the same way the mcpp table's were.** All ten reported + `ok` — status cannot see a build that stopped early. The two mcpp-workload + cells that were affected there (`touch-hub`, `edit-comment`) doubled once the + object edge stopped being cleaned by the cascade's own restat; `cold` and + `edit-body` did not move. This table's `cold` and `edit-body` are the two + quoted above, so the headline holds, but it has not been re-run. When it is, + the file to compare against is `bench/results/xlings-3way-20260814/`. + +xlings `2026.8.11.2`, gcc 16.1.0 payload, Linux x86_64 · i9-13900K · n=1 · +`--baseline cmake`. Raw report: `bench/results/xlings-3way-20260814/`. +**This table splices two runs, which is worth stating rather than leaving to be +discovered.** `mcpp default`, `cmake` and `xmake` come from +`xlings-combined-3way.json`; the `bmi_schedule=on` column comes from +`xlings-schedule.json`, because the schedule arm was never measured in the same +run as the other two engines. That second file carries its OWN default arm, and +it does not match this one exactly — `cold` 91.61s there against 92.49s here, +about 1%, which is ordinary run-to-run spread on an untuned desktop. Read the +schedule speed-up from the within-run pair (91.61 → 37.56, **2.44x**) rather +than across the columns (92.49 → 37.56, 2.46x); the cross-engine ratios in the +table are the ones taken within `xlings-combined-3way.json`. + +#### The same three engines on the SPLIT tree + +Same project, implementations moved out of the interface units. This is the axis +the two pins exist for, and it changes the answer more than the engine does. + +| scenario | **mcpp** | cmake | xmake | +|---|---|---|---| +| `cold` | **27.59s** · 1.82x | 50.13s · 1.00x | 41.90s · 1.20x | +| `noop` | **0.79s** · 0.44x | 0.34s · 1.00x | 0.50s · 0.68x | +| `touch-hub` | **1.32s** · 20.08x | 26.60s · 1.00x | 31.68s · 0.84x | +| `edit-body` | **1.79s** · 0.75x | 1.35s · 1.00x | 1.49s · 0.91x | +| `edit-comment` | **24.17s** · 1.09x | 26.35s · 1.00x | 31.28s · 0.84x | + +xlings `2026.8.13.1`, `modules-impl`, same host and payload as above. Raw +report: `bench/results/xlings-3way-20260814/xlings-split-3way.json`. + +* **The refactor beats every engine choice on this workload.** `cold` falls from + 92.49s to 27.59s for mcpp — 3.35x — and cmake's own cold falls 119.46s → 50.13s + (2.38x). Moving implementations out of interface units buys more than switching + build tool does. +* **`edit-body` is 0.75x — mcpp is SLOWER than cmake here**, 1.79s against 1.35s. + With the body in a `.cpp`, one object recompiles and nothing cascades, so the + scenario measures per-invocation overhead rather than graph reasoning — the + same fixed cost `noop` shows. On this axis mcpp has no advantage to offer and + the number says so. +* **`touch-hub` still pays: 20.08x.** Smaller than the combined tree's 54.96x + because there is simply less downstream work left to skip. + +#### The two code styles, mcpp against mcpp + +Older run, kept because it is the only side-by-side of the two pinned +styles. Both columns are mcpp, so the foreign arms being unfinished at the +time does not affect it. + +| scenario | combined `2026.8.11.2` old → new | split `2026.8.13.1` old → new | what the split buys | +|---|---|---|---| +| `cold` | 97.01s → 92.48s | 30.33s → 29.78s ⁽ⁿ⁼³⁾ | **3.11x** | +| `noop` | 1.55s → 0.72s | 1.62s → 0.76s | — | +| `touch-hub` | 89.39s → **1.76s** (50.6x) | 24.87s → **1.30s** (19.1x) | 1.35x | +| `edit-body` | 89.46s → 88.33s | 2.73s → **1.77s** | **49.96x** | +| `edit-comment` | 95.40s → 95.02s | 25.09s → 25.29s | 3.76x | + +Those `new` columns are the DEFAULT build. With the opt-in BMI schedule on the +same binary: + +| tree | scenario | default | `+bmi_schedule=on` | | +|---|---|---|---|---| +| combined | `cold` | 92.95s | **43.26s** | **2.15x** | +| combined | `edit-body` | 91.66s | **30.19s** | **3.04x** | +| split | `cold` | 27.62s | 29.72s | 0.93x | +| split | `edit-body` | 1.79s | 1.79s | 1.00x | + +**The two levers overlap, and the code style is the bigger one.** The schedule +buys time by letting importers start as soon as a BMI exists — so it only helps +when there is a cascade to overlap. Splitting the implementations out removes +the cascade instead: 92.95s → 27.62s cold and 91.66s → 1.79s on an edit, after +which the schedule has nothing left to win (and costs a little on `cold`). + +If you are choosing one, choose the code style. The schedule is what helps a +codebase that has not made that change. + +* **Splitting implementations out of the interface units is worth 2.6x on a cold + build and ~50x on `edit-body`.** That is the largest single effect in this + whole suite, and it is a *code style*, not an engine feature. +* **`touch-hub` reproduces the engine result on a codebase nobody tuned for it** + — 50.6x, against 190x on mcpp's own tree. Different magnitude, same mechanism. +* ⚠️ **The `cold` row was nearly published as a 23% REGRESSION.** At n=1 the + split tree read `29.13s → 35.88s`, i.e. the new mcpp slower. Re-measured at + n=3 it is `30.33s → 29.78s` — marginally *faster*. The single pair had simply + caught the new arm near the old arm's maximum: the old arm's spread is + **19.1%** (29.92–35.72), a hair under the 20% that §4a R2 calls noisy, while + the new arm's is 4.7%. This is R2 doing exactly what it is for, and it is why + every other row here says n=1 rather than pretending otherwise. +* **`edit-comment` does not improve at all here (1.00x), and that is correct.** + xlings' hub has 56 function bodies, so inserting a comment moves every + subsequent line; GCC records inline-body source locations in the BMI, the BMI + genuinely changes, and the cascade is owed. mcpp's own hub has none, which is + the entire reason that row reads 199x there and 1.00x here. **A project + measuring itself cannot discover this.** + +#### The generated fixture — 40 units, fan-in 3 + + +Full data: [`results/five-way-20260812/`](results/five-way-20260812/). Useful +because it is the only place `headers` / `modules` / `modules-impl` can be +compared as a controlled variable, and because it covers clang and bazel too. + +`modules`, **gcc 16.1.0**: + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | +|---|---|---|---|---| +| `cold` | 3.61s · 0.28x | 3.53s · 0.27x | **13.05s** · 1.00x | 11.46s · 0.88x | +| `noop` | 0.15s · 0.46x | 0.14s · 0.42x | **0.34s** · 1.00x | 0.32s · 0.94x | +| `touch-leaf` | 0.39s · 0.39x | 0.30s · 0.31x | **0.99s** · 1.00x | 1.16s · 1.17x | +| `touch-hub` | 3.61s · 0.35x | **0.29s · 0.03x** | **10.32s** · 1.00x | 11.13s · 1.08x | +| `edit-comment`| 3.67s · 0.36x | **0.30s · 0.03x** | **10.31s** · 1.00x | 10.55s · 1.02x | +| `edit-body` | 3.65s · 0.35x | **0.29s · 0.03x** | **10.29s** · 1.00x | 11.15s · 1.08x | + +`modules`, **clang 22.1.8**: + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | bazel | +|---|---|---|---|---|---| +| `cold` | 2.65s · 0.66x | 2.50s · 0.62x | **4.00s** · 1.00x | 13.19s · 3.30x | 3.19s · 0.80x | +| `noop` | 0.18s · 0.57x | 0.18s · 0.54x | **0.32s** · 1.00x | 0.32s · 0.99x | 0.20s · 0.63x | +| `touch-hub` | 0.35s · 0.13x | 0.28s · 0.10x | **2.67s** · 1.00x | 12.76s · 4.79x | 0.23s · 0.08x | +| `edit-body` | 0.52s · 0.20x | 0.46s · 0.17x | **2.62s** · 1.00x | 12.68s · 4.84x | 2.84s · 1.08x | + +**What the old-vs-new column is actually showing.** Under gcc, 3.65s → 0.29s on +`edit-body` is not a scheduling change. Both releases have the same mechanism — +compare the BMI the compiler just produced against the previous one, and when +they are equivalent put the old file back so ninja's `restat` sees no change — +but 2026.8.11.3 compared **bytes**, and GCC writes `buildtime:`/`localtime:` +stamps into every BMI. No two BMIs were ever byte-equal, so the suppression had +never once fired since it was written in May. + +Under clang the same rows barely move, because clang's cold build is already +3.3× cheaper than gcc's and there is far less cascade to avoid. That is the +whole argument for the toolchain being an axis: **the answer is not the same +multiple on both**, so a suite that pinned one compiler would publish one of +these two numbers as if it were the answer. + +> ⚠️ Those numbers were taken with **cmake 4.0.2 / xmake 3.0.7**, before the +> pins in the table above. They are quoted here because they are a real, +> reproducible, in-repo result file; CI now runs the pinned versions and the +> tables are refreshed from its artifacts. Do not mix rows from the two. + +--- + +## 1. What is measured + +**The build engine**, i.e. the graph it constructs and the order it schedules — +not the compiler, not package resolution, not download speed. + +And, orthogonally, **the source form**: the same project emitted three ways. + +| variant | shape | question it answers | +|---|---|---| +| `headers` | `unit_k.hpp` declares, `unit_k.cpp` defines | the status quo baseline | +| `modules` | `unit_k.cppm` declares **and** defines | what most module code looks like | +| `modules-impl` | `unit_k.cppm` declares, `unit_k_impl.cpp` defines | does splitting implementation out of the interface stop edit cascades? | + +`modules-impl` gives the "move bodies out of interface units" advice a number. +What that number is turns out to depend on the compiler, and an earlier version +of this paragraph asserted the **opposite** of the measurement: + +* GCC 16.1 does **not** put the body of an exported non-template function into + the BMI. Editing such a body changes the object file and leaves the BMI + byte-identical apart from its embedded timestamps. +* So the cascade other engines pay for that edit is avoidable, and the engines + split by their *decision rule*: compare the BMI's **content** (mcpp, 0.3 s) or + trust its **mtime** (cmake and xmake, ~10 s). +* Templates and inline functions in an interface unit **do** change the BMI. The + advice survives; its justification is narrower than it was written to be. + +Establishing this needs a control — compile the *same* source twice and diff the +BMIs. The differing bytes land at the same offsets either way, inside +`buildtime:`/`localtime:`. Without that control the timestamp reads as a content +change and the conclusion inverts. + +### 1a. The workload must actually be the workload + +A size knob that does not move the cost is worse than no knob: it makes a +benchmark look tunable while it measures something else. The first version of +this fixture failed exactly there. + +| | cost per unit (gcc 16.1, x86_64) | +|---|---| +| empty module | 0.17 s | +| **old fixture unit, `weight 6`** | **0.23 s** — 74% of it compiler startup | +| old fixture unit, `weight 40` | 0.28 s — a 6.7x knob bought 20% | +| one unit with a realistic global module fragment | 0.97 s | +| **mcpp's own units** (57k lines / 139 units — the checkout when this table was taken, not the 137-unit pinned workload) | **0.57 s** | + +The old `weight` emitted O(weight²) instantiations of one trivial `constexpr` +recursion — a few hundred at weight 40, which a compiler does in microseconds. +Unit *count* scaled cost linearly at 0.088 s each; `weight` did not scale it at +all. **The suite was largely measuring `g++` starting up.** + +The workload is now built from what actually costs time in real C++: standard +library headers, plus instantiation over **distinct types** so blocks cannot +share instantiations. Cost is `0.38 s + 0.066 s × weight`, and the knob is +verified to move: at 20 units, `weight` 0 / 4 / 12 gives 4.7 s / 18.0 s / 31.4 s +cold. + +**Rule.** Any future knob must come with a measured sweep showing it changes +cost, in this file. A knob without one is assumed inert. + +### 1b. Named sizes + +A benchmark whose size is a free-form triple of numbers cannot be compared +between two people. `--preset` names it, and the default shape **is** `standard` +so that "no flags" and `--preset standard` cannot mean different things. + +| preset | units | fan-in | weight | mcpp cold (gcc, modules) | +|---|---|---|---|---| +| `smoke` | 4 | 2 | 1 | ~2 s — CI and the e2e test, not for publication | +| `standard` | 20 | 3 | 4 | ~18 s — what published results use | +| `large` | 60 | 3 | 6 | minutes | + +--- + +## 2. Fairness invariants + +| # | Invariant | How it is enforced | +|---|---|---| +| I1 | Identical compiler **binary** across engines | `--compiler ` is threaded into cmake (`-DCMAKE_CXX_COMPILER`), xmake (`CXX`), bazel (`CC` + `--action_env`). mcpp uses its hermetic payload — a **declared asymmetry**, see §5. | +| I0 | Optimisations are measured, never emulated | Engines are parameterised by BINARY (`mcpp=`). The harness contains no "what if we also set X" mode: emulating a change measures the harness's idea of it and silently stops tracking the implementation. | +| I2 | Identical source set | All variants come from one generator; no engine globs its own inputs. | +| I3 | Identical language level | C++23 everywhere; `import std;` is **absent from every fixture** (see §5). | +| I4 | Same parallelism | `--jobs N` is passed to every engine that accepts one. | +| I5 | A failure can never look like a measurement | `status` and timings are separate protocol fields; a non-ok cell carries **no** median. | +| I6 | A skip carries its reason | `unavailable` (not installed / cannot build this variant) is distinct from `failed`, and both require a note. | + +--- + +## 2b. Two modes + +| mode | fixture | when | +|---|---|---| +| **generated** (default) | `--units/--fanin/--weight` synthesise the same project in three source forms | comparing **source forms**, and engines against each other on identical input | +| **project** (`--project DIR`) | an existing tree, measured **in place** | comparing **engine binaries** on a real codebase — mcpp building itself is the base case | + +### Measure a PINNED SNAPSHOT, never your working tree + +```bash +BASE=$(git merge-base origin/main HEAD) +mkdir -p ~/.local/share/mcpp-bench-src/mcpp +git archive "$BASE" | tar -x -C ~/.local/share/mcpp-bench-src/mcpp + +bench --project ~/.local/share/mcpp-bench-src/mcpp \ + --engines mcpp=/path/to/old,mcpp=/path/to/new \ + --scenarios cold --hub src/platform/platform.cppm +``` + +Benchmarking the tree you are editing does not merely add noise — it produces +**wrong results that look real**. Measured here: a job-count sweep reported +`rc=1` at three different job counts in a row, which read as "the design fails +above 16 concurrent compiles". The actual cause was that a new module had been +added to the working tree between generating `build.ninja` and running the +sweep, so every arm was building a source set its graph did not know about. A +snapshot pinned to a commit cannot drift underneath a measurement. + +A plain `git archive` (not a clone or worktree) is deliberate: no `.git`, no +shared state, nothing that a branch switch in the real repo can reach. + +In project mode the variant axis collapses to `native`: the project is whatever +it already is, and generating over it would destroy the thing being measured. +Scenarios that perturb a file need to be told which one (`--hub`, `--leaf`, +`--body`); without it they report `skipped` **with the reason** rather than +picking a file and producing a number that looks valid. + +`edit-body` rewrites a source file. In project mode that file belongs to the +user, so its exact bytes are captured before and restored afterwards — including +when the build fails, which is precisely when a leftover edit would be missed. + +--- + +## 3. Scenarios + +| Scenario | Perturbation | What it exercises | +|---|---|---| +| `cold` | `clean()`, then time **configure + build** | full graph construction + every compile | +| `noop` | nothing | the up-to-date check / fast path | +| `touch-hub` | mtime bump on the most-depended-on unit, **content unchanged** | can the engine prove the interface did not change and stop the cascade? | +| `edit-comment` | insert a **comment** into the most-depended-on unit — bytes change, interface does not | mtime is no longer enough; only comparing the produced BMI avoids the cascade | +| `edit-body` | insert a **numbered `volatile` statement** into a function body | the everyday developer loop: real codegen change, interface untouched | +| `touch-leaf` | mtime bump on a unit nobody depends on | recompile 1 + link | + +Two details that are easy to get wrong and change the answer: + +* **`cold` includes configure.** cmake keeps its configure output inside the + build directory that `clean` removes, so building without re-configuring simply + fails. Timing configure separately would also be wrong: the user waits for both, + and engines that fold configure into the build (mcpp, bazel) would get a + discount for it. +* **`edit-body` uses a counter**, and the counter is in the *identifier*. An + idempotent edit is a real edit on run 1 and a bare `touch` on runs 2..N — a + different, much cheaper scenario, silently dragging the median toward it. The + inserted statement is `volatile`, so no optimiser can delete it and hand back + the previous object file, and its name carries the nonce, because + perturbations ACCUMULATE within a cell and a fixed name redeclares itself on + run 2. +* **`edit-body` and `edit-comment` are separate on purpose.** They were one + scenario, named `edit-body`, that inserted a comment — so every "N times + faster on edits" number it produced was really a statement about comments. + Splitting them costs one extra column and makes each number mean its name. + + On GCC 16.1 both happen to be cheap for the same underlying reason, and it is + worth stating because it is easy to misread as a bug: **GCC does not encode + the body of an exported non-template function into the BMI.** Editing such a + body changes the object file and leaves the BMI byte-identical apart from its + embedded `buildtime:`/`localtime:` stamps, so skipping the importers is + correct, not a missed rebuild. Establishing that requires a control — compile + the *same* source twice and diff: the differing bytes land at the same offsets, + inside the timestamps. + +--- + +## 4. Statistical method + +* Medians, with min/max. No confidence intervals: sample counts are small by + necessity and a computed interval would imply more rigour than exists. +* **The harness default is 3 runs per cell; CI's automatic runs take 1.** They + answer different questions. A push or a pull request asks "did this change + break or move anything", and one sample answers it — three would spend most of + a two-hour matrix on dispersion nobody reads. Numbers destined for a table are + taken by hand: `workflow_dispatch` with `runs: 3` (or more), or `--runs N` + locally. Every published table here says `n=1` because it was taken that way, + which is exactly what §4a R2 asks a reader to account for. + (It used to be 3 for `cold` and 5 for incremental scenarios. Flattened to 3: + the split was an accident of when each scenario was added, and having two + answers made "how many samples is this" a question rather than a fact.) +* One **untimed seed build** per cell: an incremental scenario is only incremental + against an up-to-date tree, and it warms the page cache so run 1 is not + systematically slower. +* Page cache is deliberately left **warm**. A cold-page-cache build is not a + situation developers live in, and dropping caches adds variance unrelated to + the engine. +* The harness never lets build output reach its own stdout; child streams go to + `/logs/-.log`. A mixed stream cannot be parsed — and + the log lives under the WORK root, never inside the measured tree, so a + `--project` run cannot drop scratch into someone's repository. + +### 4a. Validity rules — when a cell must NOT be compared + +Every result file carries `median_s`, `min_s`, `max_s` and every raw `sample`. +Two rules decide whether a number means anything, and both are computable from +those fields alone — no trust in the harness required. + +**R1 — resolution.** Each engine's `noop` row for the same variant is its floor: +what it costs to ask "is anything out of date?" before any work happens. A cell +within **2x of its own engine's `noop`** is measuring process startup and +bookkeeping, not building, and must not be read as a build comparison. + +> This is why the `headers` rows read the way they do at small sizes. With the +> old fixture, `cmake` `noop` was 0.33 s and `cmake` `edit-body` was 0.79 s — +> 2.4x, right at the edge. The three fastest engines sat inside a 0.15 s band +> that is *entirely* startup. Those cells were never a ranking. + +**R2 — dispersion.** If `(max_s − min_s) / median_s > 0.20`, the cell is noisy +and only order-of-magnitude claims survive it. Report it, do not silently +re-run: a cell that needs re-running to look stable is a cell whose number +depends on the machine's mood. + +Neither rule is applied automatically. Automatic suppression hides data; the +rules are stated so a reader applies them, and so a table that violates them is +visibly wrong rather than quietly wrong. + +### 4b. What this suite deliberately does not do + +* **No CPU pinning, no governor forcing, no `nice`.** Developers do not build + that way. The cost is variance, which R2 exposes rather than hides. +* **No cache dropping.** A cold page cache is not a situation anyone builds in, + and it adds variance unrelated to the engine. +* **No engine-specific tuning.** Each engine gets the same standard, the same + sources, the same compiler binary, the same optimisation level, and whatever + its own documentation says is the normal way to build. Tuning one engine and + not the others is how build-system benchmarks usually go wrong. +* **No confidence intervals.** Run counts are small by necessity; a computed + interval would imply rigour that is not there. Medians with min/max and the + raw samples are what the data supports. + +### 4c. Practices this follows, and what it is not + +Adopted, with the source of the practice: + +| practice | from | here | +|---|---|---| +| full disclosure — host, tool versions, exact command, all flags | SPEC's run rules | §11 + every engine's version recorded by the engine itself | +| no benchmark-specific tuning | SPEC's run rules | §4b | +| warm-up run excluded from the timing | hyperfine, Google Benchmark | one untimed seed build per cell | +| report dispersion, not just a central value | hyperfine | `min_s`/`max_s`/`samples` + R2 | +| distinguish "cannot run" from "ran and failed" | — | `unavailable` vs `failed`, both requiring a reason | +| a versioned, machine-readable result format | — | `protocol_version` | + +**What this is not.** It is not an audited or certified benchmark, there is no +reviewing body, and the numbers are single-host. Reproducing a published table +requires the same preset, the same engine versions and a comparable machine — +all of which the result file states, which is the point. Treat cross-machine +comparison of absolute seconds as invalid; compare **ratios within one table**. + +--- + +## 5. Declared asymmetries + +These cannot be removed, so they are stated rather than hidden. + +* ~~**mcpp uses its own hermetic toolchain.**~~ **CLOSED, and it was not an + asymmetry — it was a hole.** mcpp resolves gcc/llvm from its registry and + ignores the `--compiler` every other engine is handed, which for the generated + fixture is harmless (the harness writes that manifest) and for a real project + is not: the pinned workloads say `gcc@16.1.0`, so on a clang cell cmake and + xmake ran clang while mcpp quietly ran gcc — a compiler comparison wearing an + engine-comparison label. `--compiler payload:gcc|clang` now resolves the driver + out of mcpp's own registry for *every* engine, and the mcpp engine translates + the same request into `MCPP_TOOLCHAIN` from the same version constants. Stated + here because it stood as a "declared asymmetry" for a while, and a thing you + can fix should not stay on this list. +* **The `+schedule=on` arm is the same binary, not a different engine.** mcpp's + BMI schedule is a key in the MEASURED PROJECT's manifest and the workloads are + pinned (one belongs to someone else), so the harness reaches it through + `MCPP_BMI_SCHEDULE` and labels the arm `mcpp@+schedule=on`. It is an + option under test, and it is on the same row set as the default so the two are + read together rather than across runs. +* **No fixture says `import std;`.** Engines differ wildly in how — and whether — + they can build the std module (CMake needs a per-version experimental UUID, + meson has no story). That difference would dominate every measurement. The + fixtures reach the standard library through the global module fragment, which + every engine handles identically. **This suite measures module machinery, not + std-module support.** +* **The three arms do not obtain their dependencies the same way, and their + `cold` columns are therefore not the same quantity.** xlings links ftxui, + libarchive, lua and mbedtls, which mcpp's registry ships as SOURCE. The cmake + and bazel arms compile those sources themselves — from each package's own + `.xpkg.lua`, so the file list matches mcpp's exactly — which means their + `cold` includes ~470 dependency translation units. The xmake arm declares them + through xrepo instead, the way xlings' own `xmake.lua` does, so it links + libraries xrepo built earlier and its `cold` does not include them. + `xmake clean` does not evict the xrepo package cache, so this is stable across + runs rather than a first-run artefact — but it is a real difference in + workload, not a difference in engine speed. Compare `cold` across engines on + the FIXTURE, which has no third-party dependencies at all; on xlings, compare + the incremental scenarios, where no arm rebuilds a dependency. +* **bazel's cold is not a cold machine.** It keeps a warm server and an action + cache outside the workspace. `clean` here is deliberately *not* `--expunge`, + which would also discard the toolchain and turn the measurement into + provisioning. Every bazel cell says so in its note. +* **Module support is a property of the engine *and* the compiler.** `supports()` + therefore takes both, and a `false` becomes `unavailable` **with the + measurement that produced it** — never a slow number. As measured here: + + | engine | modules with clang | modules with gcc | + |---|---|---| + | mcpp | yes | yes | + | cmake ≥ 3.28 | yes | yes | + | xmake 3.x | yes | yes | + | bazel 9.2 + rules_cc 0.2.22 | **yes** | no — `aggregate-ddi failed … Invalid JSON string`, i.e. its ddi aggregator cannot parse GCC's P1689 output | + | meson 1.10.2 | no — `fatal error: module 'fx.a' not found`; no attribute declares an interface unit | no | + + So a gcc run and a clang run legitimately have **different sets of populated + cells**, and a table must say which compiler it used before its `unavailable` + rows mean anything. +* **bazel module builds are forced to one object flavour.** `cc_binary` registers + the ddi-aggregation action for both the PIC and the non-PIC object sets but + names the output `.CXXModules.json` for both, so analysis aborts before + any compilation: + + ``` + Attempted action contains artifacts not in previous action: _objs/fx/unit_0.pic.ddi + Previous action contains artifacts not in attempted action: _objs/fx/unit_0.ddi + Outputs: are equal + ``` + + The adapter passes `--force_pic` — to **every** variant, so bazel's own + headers-vs-modules rows stay comparable, and PIC rather than + `--features=-supports_pic` because it yields a PIE executable, which is what + the other engines produce by default. + +--- + +## 6. Result protocol + +Results are JSON, versioned by `protocol_version` (currently **1**). Any field +addition, removal or semantic change bumps it. + +```json +{ + "protocol_version": 1, + "started_at": "2026-08-12T12:04:42Z", + "host": { "os": "linux", "arch": "x86_64", "cpu_model": "...", + "logical_cores": 32, "physical_cores": 24, + "heterogeneous": true, "ram_bytes": 67147722752, "toolchain": "..." }, + "cells": [ { "engine": "mcpp", "compiler": "gcc", "profile": "release", + "scenario": "cold", "fixture": "synth-40x3", "variant": "modules", + "status": "ok", "note": "...", "runs": 3, + "median_s": 12.345, "min_s": 12.100, "max_s": 12.600, + "samples": [12.1, 12.345, 12.6] } ] +} +``` + +`heterogeneous` is not decoration: on a 13900K, "32 cores" is 8 P-cores + 16 +E-cores, and every average-parallelism figure has to be read against that. + +A non-ok cell has **no timing keys at all** rather than zeros — a reader that +forgets to check `status` gets a missing key (loud) instead of a `0.0` (silent). + +--- + +## 7. Extending + +**Adding an engine** is one new module implementing `bench::engines::Engine` plus +one line in `registry.cppm`. The runner, protocol, scenarios and CI do not change. + +**Adding a scenario** is one enum value in `spec.cppm` plus one case in +`Runner::perturb`. + +**Platform work** goes in `src/platform/{posix,windows}.cppm`. Each guards its +whole body with a single macro and exports the same names, so exactly one +definition exists per build and the compiler selects it — no stubs, no dispatch. +`#if defined(_WIN32)` appears in those two files and nowhere else in the suite. +(Same convention as xlings' `src/platform/*.cppm`.) + +--- + +## 8. Analysis mode + +The same binary profiles an existing ninja build directory: + +``` +bench --analyze target/x86_64-linux-gnu/ +``` + +reporting work, makespan, **critical path** and the concurrency profile. The +number to read first is the critical path as a percentage of makespan: at ~100% +the build is latency-bound and more cores buy nothing. + +Five parsing traps it exists to get right — each one changed a conclusion during +the original analysis: + +1. A multi-output edge (`build a.o | a.gcm : cxx_module`) writes **one + `.ninja_log` line per output**, sharing start/end. Summing lines double-counts + compile time (302 s reads as 604 s). +2. For a modules build the real edges live in the **dyndep files** + (`obj/*.ddi.dd`), not in `build.ninja`. Ignoring them made mcpp's critical path + measure 22 s instead of 79 s. +3. dyndep attaches deps to `obj/X.m.o`, but importers depend on the *other* output + of that edge, `gcm.cache/X.gcm`. Unless every output of an edge is one graph + node, the longest-path walk terminates after two hops. +4. ninja **appends** to `.ninja_log` and restarts its clock each invocation. A log + touched by several builds mixes overlapping ranges; the tell is a critical path + **above 100% of makespan** (xlings' log first read as 136%). +5. Longest path must be relaxed in **topological order**. A stack DFS's + "skip what is on the stack" cycle guard also skips a dependency a sibling + pushed but has not finished, scoring it 0 — reported **33.9 s over 10 nodes** + where the truth is **76.5 s over 26**, turning a 100%-critical-path build into + a 44% one and inverting the diagnosis. + +> **Cross-check anything that computes a critical path.** Every other metric +> agreed between two independent implementations while this one was wrong by 2.3x. + +--- + +## 8b. The `bmi_schedule` defect, and the numbers it produced + +Fixed on **2026-08-14**. Kept here because the way it hid is more instructive +than the fix, and because three tables point at this section. + +**The defect.** Under `bmi_schedule = "on"` with gcc, each module interface unit +gets two ninja edges: a BMI edge that returns as soon as the compiler publishes +the BMI, and an object edge that waits for that same compiler to finish. The +object edge's only input was the BMI. + +That is the same file the cascade suppression deliberately leaves untouched. +When the new BMI turns out equivalent to the previous one, mcpp puts the +previous file back so its mtime does not advance — which is exactly what stops +39 importers from rebuilding, and is correct. But ninja's `restat` then cleans +every edge whose only reason to be dirty was that output, and the object edge +was one. It was skipped, and the link with it. + +**Editing a function body does not change a GCC BMI** — bodies are not in it — +so this is not a corner case, it is the common one. Minimal reproduction: + +```cpp +export module repro.leaf; +export int leaf_value() { return 1; } // change to 42, rebuild +``` + + Finished dev in 0.02s <- reported success, 3 of 8 edges run + ./repro8b -> 1 <- the source says 42 + +No link error and no diagnostic. The detached compiler wrote the correct object +0.2s later, after ninja had already decided not to link it. On the generated +fixture the same skip surfaces as `undefined reference to +unit_19_value@fx.unit_19()`, which is this defect in the case where the symbol +did not exist beforehand — that is the form this section used to describe, and +describing only that form is why it read as fixture-specific for a week. + +**The fix** is one line of graph shape: the object edge takes the SOURCE as an +input, with the BMI as an implicit one so ninja still orders it after phase 1. +The unit's own object and the cascade to its importers are different questions +and now have different edges. + +**What it did to the published numbers.** The skipped object edge and link were +work the build owed, so the column was timing less than a build. The two +"instant" cells of `bmi_schedule=on` doubled once that work came back: + +| scenario | as published | re-measured with the fix | | +|---|---|---|---| +| `cold` | 35.43s | 36.36s | unchanged | +| `noop` | 0.16s | 0.16s | unchanged | +| `touch-hub` | 0.22s | **0.44s** | was not finished | +| `edit-body` | 30.17s | 30.48s | unchanged | +| `edit-comment` | 0.18s | **0.44s** | was not finished | + +The headline claims survive — `cold` and `edit-body` were doing real work and +are still 2.5x and 2.8x. What did not survive is the idea that the schedule +helps everywhere: on the two rows where the cascade is already skipped it is +now visibly *slower* than the default, which is the honest shape. + +Raw data: `bench/results/schedule-refix-20260814/`. The invariants in +`bench/src/main.cpp` did not catch this — both are `cold`-only, and 0.22s +against a 0.16s `noop` is not anomalous anyway. **THIS GAP IS STILL OPEN.** An +invariant was written for it — poll the tree the engine writes to and fail if +anything lands after the engine exits — and it does not fire on the binary that +still has the defect, in this suite's own touch-hub flow. It was removed rather +than kept: a guard that cannot be shown to catch the case it was written for is +indistinguishable from no guard, and the whole point of this section is that +things which look like coverage are the expensive kind of wrong. Watching a +hand-run rebuild DOES show `mcpp build` returning in 0.56s with `cc1plus` still +running and the object landing 1.24s later, so the phenomenon is real; what is +missing is a check that sees it from inside the harness. + +--- + +## 9. Real projects — `bench/projects/` + +| target | what it is for | +|---|---| +| [`mcpp/`](projects/mcpp/) | mcpp building itself, with cmake/xmake/bazel descriptions beside it | +| [`common/`](projects/common/) | the per-engine payload logic both projects share — one branch per compiler family | +| [`xlings/`](projects/xlings/) | an **independent** codebase (110 modules / 46k lines, different authors) — the control that separates "a faster build engine" from "a faster benchmark target" | + +⚠️ **An engine change that only helps the project it was developed on is not an +engine change.** The split module schedule was developed against mcpp (2.30x) +and reproduces on xlings at **3.38x**; that second number is the one that makes +it a general result. Conversely, restructuring a target's modules speeds up that +target and nobody else's — see `.agents/docs/2026-08-13-build-optimization-status.md` +§L3/L4 for why those stay out of the engine's own PR. + +### 9a. Building mcpp itself — `bench/projects/mcpp/` + +Separate from the generated fixtures, `bench/projects/mcpp/` carries one build +description per foreign engine for **mcpp itself** — the control arm for "same +real project, different engine". Synthetic fixtures cannot reproduce the +dependency shape of a real 138-module codebase, and the difference is not small: +on the fixture mcpp's module cold build is 0.26x cmake, on mcpp's own source it +is **0.85x**. Both arms exist because either alone misleads. + +```bash +bench --project . --buildfiles bench/projects/mcpp \ + --engines mcpp=,mcpp=,cmake,xmake \ + --compiler --baseline cmake \ + --hub src/platform/platform.cppm \ + --leaf src/version.cppm \ + --body src/build/stage.cppm +``` + +`--buildfiles` is what keeps these files **out of the repository root**. mcpp is +built by mcpp; a CMakeLists.txt and an xmake.lua at the root are files every +contributor has to learn to ignore, and one of them actively broke something: +`scripts/bootstrap-macos.sh` generates its own root `xmake.lua` when none is +present, and a bench-owned file at that path silently pre-empted it. cmake is +pointed at the directory with `-S`, xmake with `-P`; mcpp reads the project's +own manifest and ignores the flag. Copying the descriptions into the tree for +the duration of a run was the alternative, and it writes into the user's +repository, which this harness refuses to do. + +| engine | builds mcpp? | +|---|---| +| mcpp | yes — it is mcpp's own manifest | +| cmake 4.4.2 | yes — needs `CMAKE_CXX_MODULE_STD 1` and the experimental UUID **for that exact cmake**: the key is a different value in every release and is compiled into the binary, so a 4.0 UUID silently leaves `import std` off in 4.4 | +| xmake 3.1.0 | yes | +| meson 1.10.2 | no — no way to declare an interface unit, and no `import std;` | +| bazel 9.2.0 | not in the gcc table. `import std;` **is** buildable (libc++ ships the std module as ordinary source — see `bench/projects/mcpp/MODULE.bazel` for the working recipe), but bazel's modules need clang, so a bazel column belongs in a clang-baselined table or it breaks invariant I1 | + +### The cmake description has two traps worth knowing + +* **`FILE_SET CXX_MODULES` requires every file under a base directory.** The + `mcpplibs.cmdline` dependency lives in the registry, outside the tree, so it + needs its own file set with an explicit `BASE_DIRS`. +* **`add_compile_options()` does not reach the `std` module.** CMake generates + that target itself, so directory-scope options miss it: the std module then + compiles against the compiler's default libc headers while every mcpp unit + compiles against `--sysroot`, and the build dies on a type that exists in both + (`conflicting type for imported declaration 'char _IO_FILE::_unused2 [20]'`). + The error names neither the flag nor the target that is wrong. Use + `CMAKE_CXX_FLAGS`. + +The xmake description pins the compiler by reading `[toolchain] default` out of `mcpp.toml`, because +the registry holds several GCCs and "newest directory wins" only *happens* to +agree with the pin. Verify before quoting anything from it: + +```bash +xmake f -P bench/projects/mcpp -y -m release --toolchain=mcpp-gcc +xmake show -P bench/projects/mcpp -t mcpp | grep 'compiler (cxx)' # must be mcpp's binary +``` + +> An earlier revision called `set_toolchains()` unconditionally, which silently +> overrode `xmake f --toolchain=llvm`: the "clang" cell was in fact compiled by +> g++, and the only tell was that its number landed within noise of the gcc cell. +> That check above is not ceremony. + +## 10. (moved) + +Running the suite is described at the top of this file. + +## 11. Host record + +A result is only meaningful next to its host, and the report carries it +automatically. When quoting numbers by hand, quote the CPU model, whether it is +**heterogeneous**, thread count, RAM, compiler version and engine versions too. diff --git a/bench/README.zh-CN.md b/bench/README.zh-CN.md new file mode 100644 index 00000000..8ceede65 --- /dev/null +++ b/bench/README.zh-CN.md @@ -0,0 +1,516 @@ +# `bench/` — 构建引擎基准套件 + +[English](README.md) · **简体中文** + +一个跨平台测量工具：在**同一份 C++ 源码**上比较不同的**构建引擎**，并测量 +C++20 具名模块相对于头文件到底付出/节省了什么。 + +用 C++23 写、由 mcpp 构建，所以 Linux / macOS / Windows 上跑法完全一致 —— 它 +替换掉的那套 shell 脚本只能在 Linux 上跑。 + +--- + +## 怎么跑 + +### 标准数据集 —— 一条命令 + +```bash +mcpp build --release # 被测的 mcpp +(cd bench && mcpp build --release) # harness +git submodule update --init # 锁定的工作负载 + +bash bench/run-standard.sh # → bench/results/standard-<日期>--/ +``` + +全部流程就是这四行。脚本读 [`matrix.json`](matrix.json),挑出**当前机器**对应的 +格子,每格跑 **3 轮**,并且**不会安静地结束**:有格子失败就打印 `NOT PUBLISHABLE` +并以非 0 退出。 + +| 选项 | | +|---|---| +| `--dry-run` | 只打印计划 | +| `--resume` | 接着上次没跑完的继续 —— 见下 | +| `--runs 1` | 更快,但**不可发布** —— 单样本没有离散度 | +| `BENCH_PRINT_ARGV=1` | 打印它将要传的 argv,不实际运行 | +| `BENCH_ROOT=` | 从脚本副本运行,指向该仓库 | + +### 运行中断后的恢复 + +```bash +bash bench/run-standard.sh --resume +``` + +测量的最小单元是一个样本 —— + +``` +工程 · variant · 场景 · 引擎 · 轮次 +``` + +—— 每测完一个就 append 一行到 `.mbench/<指纹>/journal.jsonl` 并 flush。 +**一次中断至多损失正在测量的那一个样本。** 在此机制存在之前,曾有三轮完整数据 +因中断而作废:裸名 `mcpp` 解析到已发布二进制、字段错位使 cmake 收到错误的目录、 +以及运行途中安装 cmake 4.4.2 替换了 PATH 上的 cmake。 + +`<指纹>` 是**整份配置**的一个哈希 —— 引擎、变体、场景、轮数、编译器、工程、 +fixture 形状、`--id`。和构建目录一个形状:同配置命中续跑,改配置落到另一个目录 +而不是覆盖。`mbench` 会把结果打出来: + +``` +run id : 4e58a816 (resuming, 73 unit(s) recorded) +run id : a0840b10 (fresh) +``` + +⚠️ **断点续跑正是「把两次跑无声拼在一起」最容易发生的地方**,所以有两条是刻意的: + +* 指纹**不含 mcpp 二进制**。含了的话每次重编都从零开始 —— 而那恰恰是续跑最有用 + 的时候。每条记录改为携带当时的版本,认领到版本不同的记录时**会报出来**,不是 + 默默用掉。 +* **seed build 不是样本,必须重做。** 增量场景要求一棵「已经是最新」的树,而那个 + 状态没有任何 journal 装得下。所以恢复一个格子要先重跑 seed,然后才能开始跳过。 + +`--resume` 是**开关而不是默认**,因为两种行为防的是相反的错误:没有它时,第二次 +调用绝不能写在第一次的报告旁边 —— 曾经有一次被停掉的跑在 `rm -rf` 之后才写完 +报告,于是 90 格的文件和 72 格的文件并排躺着,唯一能分辨的线索是 JSON 里的 +`started_at`。 + +**预计一到数小时。** 七个格子、每格所有引擎、三轮,而工作负载的冷构建是 +30–120 秒。 + +### 发布这些数字之前 + +第 1 条脚本会替你查并明确报告,后两条它查不了。 + +1. **退出码为 0** —— 没有格子失败;标准集不预先排除任何东西,所以一次失败就是 + **这台机器上**的真实失败 +2. **min/max 在中位数 ±20% 以内** —— 更宽说明机器当时有噪声,数字描述的是噪声 +3. **把失败先手工复现,再称之为缺口** —— 见 §5 + +### 直接驱动 harness + +```bash +BENCH=$(ls -t bench/target/*/*/bin/mbench | head -1) + +"$BENCH" --project bench/projects/mcpp/mcpp-2026.8.11.3 \ + --buildfiles bench/projects/mcpp \ + --engines mcpp=<被测>,mcpp=<已发布> --compiler payload:gcc \ + --scenarios noop,touch-hub --hub src/platform/platform.cppm --runs 3 +``` + +⚠️ **mcpp 必须以路径指定,不得使用裸名 `mcpp`。** 裸名经 PATH 解析到 xlings shim, +而 **shim 会按当前工作目录重新解析版本** —— `--project` 模式下该目录即被测的树, +而每个工作负载各自锁定了不同版本。曾因此有一整轮将七个格子**全部**测成 +`mcpp@2026.8.11.3`(已发布的旧版本,而非被测分支),且**没有任何失败提示这一点**。 + +每个 `mcpp=` 以**该二进制自身报告的版本**作为标签,因此两个版本不会合并为 +同一行。「这个版本是否变快」即以此回答:两个版本各自实际运行一遍。 + +### 关于 CI + +**本套件没有 CI job。** 曾经有,但其实际测量范围远小于表面所示:10 个格子、32 条外部 +引擎臂,**其中 12 条被豁免**(xmake 被豁免的比在测的还多),而 job 仍为绿色。此外, +共享 runner 测量的是 runner 本身(同一棵树在其上为 243s,在开发机上为 79s), +且被豁免的多为第三方工具而非 mcpp。 + +`tests/e2e/230_bench_harness.sh` 仍然每个 PR 都跑:它构建套件并检查能产出一份 +合法报告。**不再自动化的是测量本身。** + +**发版前运行一次标准集;任何声称影响性能的改动之后同样运行一次。** + +--- + +> 本文是英文版 [`README.md`](README.md) 的对照翻译。两份内容一致；如有出入， +> 以英文版为准（CI 与守卫测试读的是英文版里引用的文件名）。 + +### 三份文档，分别回答什么 + +| 文件 | 回答 | +|---|---| +| **本文 / README.md** | 一次计时**怎么取**，以及什么是刻意不控制的 | +| [`SPEC.md`](SPEC.md) | **测什么**：六个轴、为什么 cmake 是基线、一个格子「无定义」意味着什么 | +| [`matrix.json`](matrix.json) | **标准集有哪些格子** —— 唯一真源,由 `bench/run-standard.sh` 读取 | + +格子清单只出现在其中**一处**。写两遍的矩阵就是会自相矛盾的矩阵，而且矛盾是无声 +的：两份副本看起来都对。`tests/e2e/233_bench_matrix.sh` 就是用来保证这一点的。 + +--- + +## 标准数据 + +### 这一次跑到底覆盖了什么 —— 以及没覆盖什么 + +`bench/results/standard-20260814-linux-x86_64/` · **696 个测量点** · +每格 3 轮 · Linux x86_64 · i9-13900K。 + +下表里的 `-` 表示**没测**。它不表示「不适用」,更不表示 0 —— 没有被标出来的缺口, +读者会当成结果。 + +| 工具链 | 工程 | 格数 | 结果 | +|---|---|---|---| +| gcc | `fixture` | 72 | 72 ok | +| clang | `fixture` | 90 | 90 ok | +| gcc | `mcpp-2026.8.11.3` | 25 | 25 ok | +| clang | `mcpp-2026.8.11.3` | 25 | 20 ok,**5 failed**(xmake,见下) | +| gcc | `xlings-2026.8.11.2` | 25 | 25 ok | +| gcc | `xlings-2026.8.13.1` | - | **未测** —— 跑到这里停了 | +| clang | `xlings-2026.8.11.2` | - | **未测** —— 跑到这里停了 | +| macOS(全部格) | - | - | **未测** | +| Windows(全部格) | - | - | **未测** | + +七格跑了五格就停:按每个点约 55 秒算,剩下两格还要约 2 小时,而它们不改变任何 +已发布的结论 —— 两种代码风格的对比在 gcc 上已经完整,clang 在 mcpp 工作负载上 +也已覆盖。**重跑 `bash bench/run-standard.sh --resume` 就能补上,且不会重复任何 +已测的点**,因为上面每一个点都在 journal 里。 + +#### 唯一的失败,以及它为什么是「发现」而不是「缺口」 + +`clang / mcpp-2026.8.11.3 / xmake` 五个场景全部 `seed build exited 255`。 +手工复现后根因是精确的: + +``` +__format/format_functions.h:99:30: error: call to implicitly-deleted default + constructor of 'formatter, wchar_t>' +``` + +而那是一个**窄**格式串。xmake 在 clang 上的默认形状是 +`--precompile` → `.pcm` → `-c .pcm`,这条链**必须**要 clang 的 **full** BMI, +而把 full BMI 发布给 importer 会让 clang 22.1.8 编错一个下游 TU。mcpp 不受影响, +因为它发布 reduced BMI、object 边重编源码。这是**上游**问题,已记为 +[#424](https://github.com/mcpp-community/mcpp/issues/424) —— 不是 xmake 写错了, +也不应以 `allow_failed` 豁免。 + +#### 一个已声明的离群点 + +`gcc / xlings-2026.8.11.2 / mcpp+schedule=on / touch-hub` 测到的是 +`[1.77, 20.82, 1.79]` —— 中位数 1.79s,离散度 1066%。紧接着用 8 轮重测得到 +`[1.79, 1.79, 1.79, 1.79, 1.78, 1.79, 1.81, 1.79]`,离散度 1%、没有离群点, +所以那 20.82s 是机器噪声,不是级联抑制偶发失效。那次重测的报告就放在同一个目录: +[`probe-touch-hub-outlier-8-samples.json`](results/standard-20260814-linux-x86_64/probe-touch-hub-outlier-8-samples.json) +—— 关于一个已发布数字的论断,本身也必须可核。**已发布的那一格原样保留**: +把第二次跑的样本拼进第一次跑的报告,正是这套套件绝不能做的事。那次重测是 +关于这个数字的**证据**,不是它的替代品。 + +--- + +## 0. 锁定了哪些内容，以及每一项为何必须锁定 + +一个基准数字的价值，等于取它时被摁住不动的那张清单。下面每一行都曾经是松的， +而每一行松着的时候，产出的表格测的都不是它自己声称的东西。 + +| 项目 | 锁定为 | 声明位置 | +|---|---|---| +| cmake | **4.4.2** | `matrix.json` → `tools` | +| xmake | **3.1.0** | `matrix.json` → `tools` | +| bazel | **9.2.0** | `matrix.json` → `tools` | +| gcc | **16.1.0** | `bench/src/toolchain.cppm` | +| clang / libc++ | **22.1.8**（Windows：20.1.7） | `bench/src/toolchain.cppm` | +| 参照 mcpp | **2026.8.11.3** | `matrix.json` → `reference_mcpp` | +| mcpp（被测工作负载） | **2026.8.11.3** — `a749e9f` | 子模块 `projects/mcpp/mcpp-2026.8.11.3` | +| xlings（合并风格） | **2026.8.11.2** — `b1563fe` | 子模块 `projects/xlings/xlings-2026.8.11.2` | +| xlings（分离风格） | **2026.8.13.1** — `f072075` | 子模块 `projects/xlings/xlings-2026.8.13.1` | +| 被测 mcpp | 当前 checkout | 本地构建,由 `run-standard.sh` 以路径传入 | + +**全部由 xlings 安装**,版本精确。`xlings install cmake@4.4.2 xmake@3.1.0 bazel@9.2.0 mcpp@2026.8.11.3` 装齐这一套;`run-standard.sh` 在开跑前核对, +版本对不上就拒绝启动 —— 一个会变的工具是报告没有记录、读者也看不见的变量。 + +这解决了四件已经真实发生过的事： + +* **cmake 3.31.6** 是 GitHub runner 镜像自带的版本。它没有 CMake 4.0 的 + `import std` 实验开关键，所以*每一个 module 格子都 configure 失败*。换成 + 4.0.2 之后全过。 +* **`command -v g++`** 在那些镜像上是 gcc 13.3.0。cmake 用它配不出 C++23 + modules，xmake 直接把它编崩（internal compiler error）—— 而 mcpp 一直悄悄用 + 自己 registry 里的 gcc 16.1。表格是 `48 failed / 6 ok`，却仍然被当作「构建引擎 + 对比」。现在**每个**引擎都拿到 mcpp 自己载荷里的那个驱动 + （`--compiler payload:gcc`）：套件的公平性规则终于被执行，而不只是写在注释里。 +* **被测工作负载会漂移。** xlings 是运行时 `git clone --depth 1` 默认分支的， + 所以目标随上游每次 push 而变 —— `--hub src/xlings.cppm` 指的文件已经消失 + 好几个月，每个 xlings 格子都报 `skipped`，每个 job 都报成功。mcpp 自己的 + 源码是同一个缺陷、但更难看见的形式：`--project $GITHUB_WORKSPACE` 让 + checkout 成了工作负载，于是分支上每一次提交都在悄悄改变被测对象。 + **被测的引擎是那个二进制、它本来就该变；工作负载不该变。** 现在三个都是 + git 子模块，守卫会检查每个 `hub`/`body` 在锁定的树中确实存在。 +* **只测了一个 mcpp。** 一份只说「这个分支有多快」、却不说「有没有变快」的报告， + 不是 pull request 上的基准该给的东西。 + +> **刻意不摁住的**:跑它的那台机器。数据里记了 CPU 型号、核数、内存,见英文版 §4a。 + +> **这些数字没有覆盖的**：mcpp 的分离式调度（`[build] bmi_schedule = "on"`）在所有 +> 平台验证通过前是 opt-in 的，所以两个 mcpp 二进制都是关着它跑的。它的效果单独 +> 测量，见 `.agents/docs/2026-08-13-build-optimization-status.md`。 + +--- + +## 1. 公平性不变量 + +一次对比只有在这五条同时成立时才有意义，任何一条破了，测的就是别的东西： + +1. **同一个编译器二进制** —— 不是「同一个 family」，是同一个文件。 +2. **同样的语言开关** —— `-std=c++23`、release 下同样的优化级别。 +3. **同一份源码集合** —— 用 glob 而不是手写清单，手写清单会悄悄漂移。 +4. **同样的产物形态** —— 一个可执行文件，同样的静态/动态标准库选择。 +5. **同一个标准库形态** —— `import std;`，不是头文件垫片。 + +`bench/projects/common/` 里放的就是「让这个引擎驱动出 mcpp 同样的进程树」这件事 +的共享实现（cmake 与 xmake 各一份），**按编译器 family 分支**，因为编译器是一个 +真实的轴而不是一个标签。 + +--- + +## 2. 两种模式 + +* **fixture 模式** —— 生成一棵合成树，参数化（`--preset` / `--units` / + `--fanin` / `--weight`）。唯一能同时给出 `headers` / `modules` / + `modules-impl` 三种形态的地方，所以也是 *variant* 轴真正成为受控变量的地方。 +* **`--project` 模式** —— 原地测量一棵真实的树。**永远不写入被测仓库**： + 编辑类场景会先存下文件字节、无论函数怎么退出都还原（`SourceGuard`），子进程 + 日志一律落在 `--work` 目录里。 + +真实工程测的是「锁定的快照」，而非你的工作区 —— 见 §「Measure a PINNED +SNAPSHOT」。 + +--- + +## 3. 场景 + +| 场景 | 扰动 | 问的问题 | +|---|---|---| +| `cold` | 没有构建目录 | 完整建图 + 全量编译 | +| `noop` | 什么都不动 | 「已经是最新」有多便宜 | +| `touch-hub` | 给被大量 import 的单元改 mtime，**内容不变** | 引擎能不能证明接口没变？ | +| `edit-comment` | 往同一个单元里插一条注释 | 字节**确实**变了但接口没变 —— 只有比较产出 BMI 的引擎能止住级联 | +| `edit-body` | 函数体内部一处真实语义修改 | 日常循环。接口单元里的内联函数体，BMI 合理地变了，级联是**对的** | +| `touch-leaf` | 给没人 import 的单元改 mtime | 重编 1 个 + 链接 | + +`edit-comment` 与 `edit-body` 是**刻意分开**的：不分开的话，一个能跳过纯注释重建 +的引擎就可以宣传成「改代码快 12 倍」，而那实际上是一句关于注释的话。 +`edit-body` 是反方向的对照 —— 那里没有引擎应该快，快了就是漏了该做的活。 + +--- + +## 4. 一个格子可以是「无定义」的，并且必须说明原因 + +| 状态 | 含义 | +|---|---| +| `ok` | 测到了，有 `samples` | +| `failed` | 引擎跑了但没产出产物 —— 这是**发现**，不是缺口 | +| `unavailable` | 这台机器上没装 | +| `skipped` | 这个引擎表达不了这个格子 —— `note` 说明缺什么 | + +**退出码**：只有「测到了东西」且「没有 `failed`」才返回 0。这不是时间阈值 —— +共享 runner 上设时间阈值只会把正常波动变成没人看的红叉。缺了这条的代价很具体： +一个「通过」的矩阵 job 实际是 6 ok / 48 failed / 18 unavailable，而每个 xlings +job 一个测量都没有，这个状态持续了好几周。 + +`failed` 确实属于已知缺口时，写进那个格子的 `allow_failed`，并且**必须**在 +`note` 里带 `KNOWN GAP` —— 守卫会检查。一个不说明原因的豁免就是一个被藏起来的 +失败。 + +--- + +## 4b. 三个引擎在 combined 树上的实测 + +**第一次三条臂都能产出可运行二进制的测量** —— 在此之前 cmake 与 xmake 两列一直是 +`failed`,所以这里原本只有 mcpp 对 mcpp。 + +| 场景 | **mcpp** `bmi_schedule=on` | mcpp 默认 | cmake | xmake | +|---|---|---|---|---| +| `cold` | **37.56s** · 3.18x | 92.49s · 1.29x | 119.46s · 1.00x | 105.02s · 1.14x | +| `noop` | **0.72s** · 0.50x | 0.74s · 0.49x | 0.36s · 1.00x | 0.40s · 0.90x | +| `touch-hub` | **1.04s** · 93.93x | 1.79s · 54.96x | 98.16s · 1.00x | 98.16s · 1.00x | +| `edit-body` | **29.65s** · 3.32x | 88.38s · 1.11x | 98.43s · 1.00x | 98.00s · 1.00x | +| `edit-comment` | **30.44s** · 3.22x | 93.81s · 1.04x | 97.98s · 1.00x | 97.97s · 1.00x | + +**两列 mcpp 都在这里,是因为只放一列会误导。** 默认列是用户今天拿到的东西; +`bmi_schedule=on` 是可选的分离调度,不放它会严重低估 mcpp —— `edit-body` 在默认 +列是 1.11x,开了调度是 3.32x。 + +* **`edit-body` 与 `edit-comment` 不是「没有优势」。** 两者的级联都是**欠的** + (被扰动的函数体在接口单元里,BMI 真的变了)。默认列显示 mcpp 以 cmake 的速度 + 做完这份欠下的工作;调度列显示它把**同一份工作**做快了 3.3 倍 —— 靠的是 BMI + 一产出就发布,而不是等代码生成结束。 +* **`noop` 是 mcpp 真正输的一项**,两列都输:0.72–0.74s 对 cmake 的 0.36s + (0.49x)。这是每次调用的固定开销,也是用户在每一次「改一行、构建一次」里都 + 感受得到的数字 —— 什么都不做时,mcpp 是三者中最慢的。 +* **默认列的 `edit-comment` 是 1.04x,不是 mcpp 自身工作负载上的 200x。** 注释插 + 进了 xlings 保留在接口单元里的内联函数体,BMI 真的变了,级联是**欠的**。格子的 + note 会记录当次是哪种形态(见 §3),不要把它读成优化失效。 +* **`touch-hub` 才是级联抑制的真结果:54.96x。** 内容没变,mcpp 拿编译器刚产出的 + BMI 和上一份比,跳过 45 个导入者;cmake 与 xmake 按时间戳判断,把它们全部重建 + —— 两者相差 0.00s,这正是两个时间戳驱动的引擎该有的样子。 +* **⚠️ 这里的 `bmi_schedule=on` 格子取自 §8b 修复之前,因此和 mcpp 那张表一样可疑。** + 十个格子全报 `ok` —— 状态列看不见「构建提前收工」。在 mcpp 工作负载上受影响的两 + 个格子(`touch-hub`、`edit-comment`)在 object 边不再被级联自己的 restat 清掉之后 + 翻了一倍;`cold` 与 `edit-body` 没动。上面引用的正是 `cold` 与 `edit-body`,所以 + 结论成立,但这张表**尚未重跑**。重跑时的对照文件是 + `bench/results/xlings-3way-20260814/`。 + +xlings `2026.8.11.2`,gcc 16.1.0 载荷,Linux x86_64 · i9-13900K · n=1 · +`--baseline cmake`。原始报告:`bench/results/xlings-3way-20260814/`。 +**这张表拼接了两轮测量,与其等人自己发现,不如写在这里。** `mcpp 默认`、`cmake`、 +`xmake` 三列来自 `xlings-combined-3way.json`;`bmi_schedule=on` 一列来自 +`xlings-schedule.json` —— 调度臂从未和另外两个引擎在同一轮里测过。后一个文件**自己 +也有一条默认臂**,且与这里并不完全相同:`cold` 在那边是 91.61s,这里是 92.49s, +差约 1%,属于未调优桌面机上正常的轮间波动。调度带来的加速请按**同轮那一对**读 +(91.61 → 37.56,**2.44x**),而不是跨列读(92.49 → 37.56,2.46x);表里的跨引擎 +比值取自 `xlings-combined-3way.json` 内部。 + + +## 4c. 同样三个引擎在 split 树上 + +同一个工程,实现从接口单元里搬了出去。这正是两个 pin 存在的意义 —— 而它对结果的 +影响**比换构建工具更大**。 + +| 场景 | **mcpp** | cmake | xmake | +|---|---|---|---| +| `cold` | **27.59s** · 1.82x | 50.13s · 1.00x | 41.90s · 1.20x | +| `noop` | **0.79s** · 0.44x | 0.34s · 1.00x | 0.50s · 0.68x | +| `touch-hub` | **1.32s** · 20.08x | 26.60s · 1.00x | 31.68s · 0.84x | +| `edit-body` | **1.79s** · 0.75x | 1.35s · 1.00x | 1.49s · 0.91x | +| `edit-comment` | **24.17s** · 1.09x | 26.35s · 1.00x | 31.28s · 0.84x | + +xlings `2026.8.13.1`,`modules-impl`,主机与载荷同上。原始报告: +`bench/results/xlings-3way-20260814/xlings-split-3way.json`。 + +* **在这个工作负载上,重构的收益盖过了任何引擎选择。** mcpp 的 `cold` 从 92.49s + 降到 27.59s(3.35x),而 cmake 自己的 cold 也从 119.46s 降到 50.13s(2.38x)。 + 把实现搬出接口单元,比换构建工具更值。 +* **`edit-body` 是 0.75x —— 这里 mcpp 比 cmake 慢**(1.79s 对 1.35s)。函数体在 + `.cpp` 里,只重编一个 object、没有级联,于是这个场景量的是**每次调用的固定 + 开销**而不是图推理 —— 和 `noop` 反映的是同一笔成本。在这条轴上 mcpp 没有优势 + 可言,数字也就这么写。 +* **`touch-hub` 仍然值:20.08x。** 比 combined 树的 54.96x 小,因为下游本来就 + 没剩多少可跳的了。 + +## 5. 可观测性：跑的时候看得见 + +harness 会把进度实时打到 **stderr**（逐行 flush），stdout 留给报告： + +``` +[ 12.3s] cmake/gcc/release/cold/xlings-2026.8.13.1/modules-impl configure +[ 15.1s] cmake/gcc/release/cold/xlings-2026.8.13.1/modules-impl seed build +[ 107.0s] cmake/gcc/release/cold/xlings-2026.8.13.1/modules-impl seed build exited 1 + | ld: undefined reference to `mbedtls_ssl_free' + | collect2: error: ld returned 1 exit status +``` + +两个刻意的设计： + +* **失败时直接打出子进程日志的尾部。** 只写 `see .../logs/cmake-cold.log` 在 CI + 上等于什么都没说 —— 那个文件跟着 runner 一起销毁了。 +* **每条 configure/build 都有超时**（`--timeout`，默认 1800 秒），超了就 kill 并 + 报 `TIMED OUT after Ns and was killed`。曾经有两个 job 各自卡在一个子进程里 + 25 分钟，日志一个字都没有。 + +--- + +## 8b. `bmi_schedule` 那个缺陷,以及它造出来的数字 + +**已于 2026-08-14 修复。** 记在这里是因为它藏起来的方式比修法更值得看,而且有三 +张表指向这一节。 + +**缺陷。** gcc 开 `bmi_schedule = "on"` 时,每个模块接口单元有两条 ninja 边:BMI +边在编译器发布 BMI 的那一刻就返回,object 边等同一个编译器跑完。而 object 边**唯 +一的输入是 BMI**。 + +那正是级联抑制**故意不动**的那个文件:新 BMI 与旧的等价时,mcpp 把旧文件连旧 +mtime 放回去 —— 这是 39 个导入者不必重建的原因,是对的。但 ninja 的 `restat` 随 +即会把「只因那个输出而脏」的边全部判成干净,object 边就是其中之一。它被跳过,链接 +也一起被跳过。 + +**改函数体不会改 GCC 的 BMI**(BMI 里没有函数体),所以这不是边角情况,而是最常见 +的那种。最小复现: + +```cpp +export module repro.leaf; +export int leaf_value() { return 1; } // 改成 42,重建 +``` + + Finished dev in 0.02s <- 报告成功,8 条边只跑了 3 条 + ./repro8b -> 1 <- 源码写的是 42 + +没有链接错误,没有任何诊断。分离出去的编译器在 0.2 秒后写出了正确的 object,而 +ninja 早已决定不链接它。在生成的 fixture 上,同一个跳过表现为 +`undefined reference to unit_19_value@fx.unit_19()` —— 那只是「符号原本就不存在」 +时的形态。此前只记了那一种形态,这正是它被当成「只在 fixture 上出现」整整一周的 +原因。 + +**修法**只是图的形状:object 边改为以**源码**为输入,BMI 降为隐式输入以维持顺序。 +「这个单元自己的 object」与「到导入者的级联」是两个问题,现在是两条不同的边。 + +**它对已发布数字做了什么。** 被跳过的 object 边和链接是构建欠下的工作,所以那一列 +量的东西比一次构建少。`bmi_schedule=on` 的两个「瞬时」格子在这部分工作回来之后翻了 +一倍: + +| 场景 | 已发布 | 修复后重测 | | +|---|---|---|---| +| `cold` | 35.43s | 36.36s | 不变 | +| `noop` | 0.16s | 0.16s | 不变 | +| `touch-hub` | 0.22s | **0.44s** | 当时没跑完 | +| `edit-body` | 30.17s | 30.48s | 不变 | +| `edit-comment` | 0.18s | **0.44s** | 当时没跑完 | + +头条结论保住了 —— `cold` 与 `edit-body` 做的是真实工作,仍然是 2.5x 和 2.8x。没保 +住的是「这个调度到处都有用」:在级联本来就被跳过的两行上,它现在明显比默认**更 +慢**,这才是诚实的形状。 + +原始数据:`bench/results/schedule-refix-20260814/`。`bench/src/main.cpp` 里的 +不变量没能抓到它 —— 两条都只作用于 `cold`,而且 0.22s 对 0.16s 的 `noop` 本来也不 +异常。**这个缺口仍然开着。** 为它写过一条不变量(轮询引擎写入的那棵树,若引擎退出后 +还有文件落盘就判失败),但它在本套件自己的 touch-hub 流程里,**对仍带该缺陷的二进制 +不触发**。于是撤掉而不是留着:一条无法被证明能抓到目标的守卫,和没有守卫无法区分, +而本节的全部意义正是「看起来像覆盖的东西是最贵的那种错」。手工重建确实能观察到 +`mcpp build` 在 0.56s 返回、`cc1plus` 仍在跑、object 1.24s 后落盘,所以现象是真的; +缺的是一个能从 harness 内部看见它的判据。 + +--- + +## 5b. 已声明的不对称 + +这些去不掉,所以写出来而不是藏起来。完整清单见英文版 §5;下面是**读数字之前 +必须知道**的几条。 + +* **三条臂拿依赖的方式不同,所以它们的 `cold` 不是同一个量。** xlings 链接 + ftxui / libarchive / lua / mbedtls,而 mcpp 的 registry 是以**源码**分发它们的。 + cmake 与 bazel 两条臂自己编这些源码 —— 而且是按各包自己的 `.xpkg.lua` 编, + 文件清单与 mcpp 完全一致 —— 于是它们的 `cold` 里含着约 470 个依赖翻译单元。 + xmake 臂改为经 xrepo 声明(和 xlings 自己的 `xmake.lua` 一样),链接的是 + xrepo 早先构建好的库,`cold` 里**不含**它们。`xmake clean` 不会清 xrepo 的包 + 缓存,所以这不是首次运行的假象,而是稳定存在的**工作量差异**,不是引擎快慢。 + 跨引擎比 `cold`,请用**生成的 fixture**(它完全没有第三方依赖);在 xlings 上 + 请比增量场景,那里没有任何一条臂会重建依赖。 +* **`+schedule=on` 那条臂是同一个二进制,不是另一个引擎。** 它是被测工程 manifest + 里的一个键,由 harness 经 `MCPP_BMI_SCHEDULE` 打开,标签写作 + `mcpp@+schedule=on`,和默认配置放在同一组行里一起读。 +* **没有任何 fixture 写 `import std;`。** 各引擎在「能否、以及如何」构建 std 模块 + 上差异极大(CMake 需要一把随版本变化的实验性 UUID,meson 根本没有说法),这个 + 差异会淹没一切测量。fixture 一律经全局模块片段取标准库。**本套件测的是模块 + 机制,不是 std 模块支持。** +* **bazel 的 cold 不是一台冷机器。** 它在工作区之外保有常驻 server 和 action + 缓存;这里的 `clean` **刻意不是** `--expunge`,否则连工具链一起丢掉,测的就 + 变成了 provisioning。 + +## 6. 运行 + +```bash +# 构建 harness +cd bench && mcpp build --release + +# 拉取被测的锁定工程（子模块） +git submodule update --init + +# 一次 smoke +./target/*/*/bin/mbench --engines mcpp,cmake --variants modules \ + --scenarios cold,noop --preset smoke --compiler payload:gcc +``` + +CI 跑的格子清单见 [`matrix.json`](matrix.json)；每次 run 的报告作为 artifact +上传，命名 `bench---`。 + +**轮次:harness 自己默认每格 3 轮,CI 的自动触发是 1 轮。** 两者回答的不是同一个 +问题:一次 push 或 PR 问的是「这次改动有没有弄坏或挪动什么」,一个样本就够;三个 +样本会把两小时矩阵的大半花在没人读的离散度上。要写进表格的数字请**手动**取 —— +`workflow_dispatch` 里把 `runs` 设成 3 或更多,或本地 `--runs N`。本文引用的所有 +表格都标 `n=1`,正是因为它们就是这样取的(读法见英文版 §4a R2)。 + +引用任何数字之前，请先读英文版的 §4a（什么时候一个格子**不能**被拿来比较）与 +§5（已声明的不对称）。 diff --git a/bench/SPEC.md b/bench/SPEC.md new file mode 100644 index 00000000..5e7e9912 --- /dev/null +++ b/bench/SPEC.md @@ -0,0 +1,359 @@ +# Benchmark specification + +What this suite measures, what a cell is, and which cells the standard data set covers. + +The **cell list itself is not here** — it is [`matrix.json`](matrix.json), which +`bench/run-standard.sh` reads to plan a run. A matrix written down +twice is a matrix that disagrees with itself, and the disagreement is silent: +both copies keep looking right. + +`README.md` is the measurement contract (how a timing is taken, what is +deliberately not controlled). This file is the *shape* of the measurement. + +--- + +## 1. The axes + +A measurement is identified by six coordinates. Five of them are already the +result schema's `CellKey` (`bench/src/protocol.cppm`); the sixth is the host, +which the report records in its run facts. + +| axis | values | where it is chosen | +|---|---|---| +| **OS** | `linux` `macos` `windows` | the runner selects the cells for the machine it is on | +| **Toolchain** | `gcc` `clang` `msvc` | one cell each — `--compiler` | +| **Build tool** | `mcpp` `cmake` `xmake` `bazel` | swept inside a job — `--engines` | +| **Engine options** | `mcpp[schedule=on]` | an ARM of an engine, not a fifth engine — `--engines` | +| **Project** | `fixture` `mcpp-2026.8.11.3` `xlings-2026.8.11.2` `xlings-2026.8.13.1` | one cell each — `--project` | +| **Variant** | `headers` `modules` `modules-impl` | swept inside a job — `--variants` | +| **Scenario** | `cold` `noop` `touch-hub` `touch-leaf` `edit-body` `edit-comment` | swept inside a job — `--scenarios` | + +**One cell = one (OS, toolchain, project).** The remaining axes are swept +inside it, because they share a checkout, a toolchain install and a generated +fixture. Promoting them to jobs would multiply runner minutes without adding a +single measurement. + +**An engine may appear more than once in the same cell**, and a row is named by +what the binary reports rather than by the token that asked for it: + +| spec | row | +|---|---| +| `mcpp=` | `mcpp@2026.8.13.1` | +| `mcpp=` | `mcpp@2026.8.11.3` | +| `mcpp[schedule=on]=` | `mcpp@2026.8.13.1+schedule=on` | + +That is how "did this release get faster?" and "what does the opt-in key buy?" +are answered — by running each and letting them label themselves, never by one +arm standing in for another. `[...]` is an option list understood by the +registry, which rejects a spec whose option it does not know rather than +silently ignoring it; `run-standard.sh` gives the reference arm to the bare +token only, because the released binary predates the fix in README §8b and +measuring its scheduler would read as a regression in the feature. + +### Everything the number depends on is pinned + +Not a tidiness preference. Each of these was unpinned once, and each produced a +table that was measuring something other than what it said: + +| pinned | where | what it cost while it was loose | +|---|---|---| +| cmake, xmake, bazel | `matrix.json.tools` | runner images ship cmake 3.31.6, which lacks the CMake 4.0 `import std` key, so **every module cell failed to configure** | +| the compiler | `bench/src/toolchain.cppm` | engines got `command -v g++` = gcc 13.3.0 while mcpp used the registry's gcc 16.1 — cmake could not configure, xmake crashed gcc outright | +| the workloads | git submodules under `bench/projects/` | xlings was cloned from its default branch at run time (`--hub src/xlings.cppm` named a file that had stopped existing); **mcpp's own sources were the checkout**, so every commit on a branch changed the thing being measured | +| the reference mcpp | `matrix.json.reference_mcpp` | a report said how fast this branch is, never whether it got faster | + +`--compiler payload:gcc` / `payload:clang` is the spelling that delivers the +second row: it resolves to the driver **inside mcpp's own registry**, so every +engine is handed the same binary. That is the suite's fairness rule +(`resolve_cxx`) actually enforced rather than merely written down. + +**mcpp needed a second half of that fix.** It resolves its own toolchain from +the *measured project's* manifest and ignores `--compiler` entirely. For the +generated fixture that is harmless, because the harness writes that manifest — +but the real workloads are pinned submodules whose `[toolchain]` says +`gcc@16.1.0`, so on a clang cell cmake and xmake ran clang while mcpp quietly +ran gcc. The mcpp engine now translates the requested compiler into +`MCPP_TOOLCHAIN` (the side channel `--toolchain` uses), from the same version +constants `payload:` resolves against, so the driver the other engines get and +the toolchain mcpp is told to use cannot name different versions. + +### Two mcpp binaries, always + +`mcpp` in a cell's engine list expands to **two** engines: the mcpp built from +the checkout and `reference_mcpp` installed by xlings. Each labels itself from +the version it reports, so the rows never collapse — and the harness warns if +two binaries claim the same version, because then they silently would. + +> **Not covered by that column:** the split build schedule (`[build] bmi_schedule = +> "on"`) is opt-in until it has been verified on every platform, so both +> binaries run with it OFF. These numbers therefore do not include it; see +> `.agents/docs/2026-08-13-build-optimization-status.md` for its separately +> measured effect. + +### Why the toolchain is an axis and not a detail + +Because the answer changes with it, and not by a constant factor. On mcpp's own +sources the compiler alone is worth **2.5x** (gcc 81.8s → clang 32.6s), and the +engine-level optimisation on top of that is worth a *different* multiple on each +(gcc 2.30x, clang 1.78x). A suite that pinned one compiler would report one of +those two numbers as if it were the answer. + +`--compiler` is passed to **every** engine that accepts one. An engine left on +its host default turns the comparison into compiler-vs-compiler while still +being labelled engine-vs-engine — see `resolve_cxx` in +`bench/src/engines/engine.cppm`, where that rule is enforced. + +### Why the project is an axis + +`fixture` is generated and calibrated, so it isolates one variable at a time. +Real projects are the control that stops an engine change from being an artefact +of one graph shape: + +* **`fixture`** — synthetic, parameterised (`--preset`, `--units`, `--fanin`, + `--weight`). The only project where `headers` / `modules` / `modules-impl` + are all generated, so it is where the *variant* axis is a controlled variable. +* **`mcpp-2026.8.11.3`** (`a749e9f`) — 137 modules / 57k lines, one source + dependency, build descriptions for every engine under `projects/mcpp/`. + Pinned like everything else: the engine under test is the binary, and a + workload that moves with the branch makes two runs incomparable. +* **`xlings-2026.8.11.2`** and **`xlings-2026.8.13.1`** — 110 modules / 46k + lines, **different authors**. This is what separates "a faster build engine" + from "a faster benchmark target". + +### The two xlings pins are a code-style comparison + +They are the same project either side of one refactor: + +| project | shape | variant | +|---|---|---| +| `xlings-2026.8.11.2` (`b1563fe`) | 110 `.cppm` + **2** `.cpp` — each interface unit carries its own implementation | `modules` | +| `xlings-2026.8.13.1` (`f072075`) | 110 `.cppm` + **92** `.cpp` — implementations split out | `modules-impl` | + +Same module graph, same line count, opposite answers to "where does the code +live" — which is exactly the `modules` vs `modules-impl` axis the generated +fixture has, except on a real codebase written by people who were not thinking +about this benchmark. `--body` differs accordingly: editing an implementation +means the `.cpp` in the split style and the `.cppm` in the combined one. + +**One description serves both.** `projects/xlings/{CMakeLists.txt,xmake.lua}` +glob `src/**/*.{cppm,cpp}` — the same rule mcpp itself infers from — so neither +style needs its own file, an environment switch, or a branch. Globbing only +`src/main.cpp`, which is what they used to do, compiles the split style's +interfaces, links nothing, and still reports a time. + +A real project has exactly one form — its own — so a cell states which of the +two names it, and the harness never generates over the tree. + +--- + +## 2. cmake is the baseline + +Every ratio in every report is against cmake, and the harness defaults +`--baseline` to it rather than leaving it unset. + +Not an arbitrary pick: + +* it is the reference implementation of C++ module builds — P1689 scanning and + ninja `dyndep` are its design, and every other engine here implements *its* + protocol; +* it is present on every machine this suite runs on, so the ratio exists in + every cell; +* a reader already has a feel for it. An absolute second count means nothing + without knowing the runner; **"1.8x cmake" survives being read on a different + machine**, which is the only way these numbers travel. + +A run whose engine set omits cmake prints `(no successful 'cmake' cell here; +ratios omitted)` rather than a table of bare seconds — the one form of this data +that cannot be compared to anything. + +### A cell may override it, and the xlings cells do + +`matrix.json` lets a cell name its own `baseline`. The xlings cells normalise +against the released mcpp instead, because their cmake and xmake arms compile +every translation unit and then **stop at the link**: xlings pulls ftxui, +libarchive, lua and mbedtls in as *source* packages that mcpp compiles, so the +foreign arms want symbols nobody built (`undefined reference to mbedtls_*`). + +Both arms are kept anyway — a documented wall is data, and the day someone adds +`add_subdirectory` for those four the cell turns green by itself — but they are +listed in that cell's `allow_failed` so a known gap does not fail the run. The +guard requires a waiver to name an engine the cell actually has *and* to carry a +`KNOWN GAP` note, because a waived failure that says nothing is a hidden one. + +Overriding the baseline is what turns that cell from "a table of bare seconds" +into the comparison it can actually make: **mcpp against mcpp**, which is the +question a control target exists to answer. + +--- + +### meson is not an engine here + +meson 1.10.2 has no way to declare a translation unit to be a module +**interface**. Listing `.cppm` files as ordinary sources compiles them as plain +TUs and the first importer fails with `fatal error: module 'x' not found`, and +there is no `import std;` equivalent either. So every module cell was an +`unavailable` row with the same reason — one honest row and five empty ones per +report, which is noise rather than a comparison. It was removed: engine, +descriptions and fixture emitter. + +The day meson grows the feature, the diff is adding +`bench/src/engines/meson.cppm` back and one line in `registry.cppm`. + +## 3. A cell may be undefined, and it must say why + +Three outcomes are distinguishable in the result schema, and collapsing them is +the failure this suite is built to avoid: + +| status | meaning | +|---|---| +| `ok` | measured; `samples` present | +| `failed` | the engine ran and did not produce the artifact — a real finding | +| `unavailable` | the engine is not installed here | +| `skipped` | this engine cannot express this cell — `note` says what is missing | + +`note` is **required** whenever the status is not `ok`. "No number" and "zero +seconds" must never render the same way, and neither must "not installed" and +"cannot do this". + +The same rule applies one level up, to cells the standard set does not cover at all: +[`matrix.json`](matrix.json) carries an `excluded` list where every entry has a +`reason`, and the ones that say **KNOWN GAP** are meaningful cells that are +simply not wired up rather than cells that make no sense — written down so that +"not measured" cannot quietly become "not applicable". An exclusion may also +name an `engine`, which scopes a caveat to one COLUMN instead of removing the +job: the cell still runs, and its note says what to distrust. + +--- + +## 4. The scenarios, and what each one is for + +| scenario | perturbation | the question | +|---|---|---| +| `cold` | no build dir | full graph construction + every compile | +| `noop` | nothing | how cheap is "already up to date" | +| `touch-hub` | mtime bump on a widely-imported unit, **content unchanged** | can the engine prove the interface did not change? | +| `edit-comment` | a comment inserted into that same unit | the bytes *did* change but the interface did not — only an engine that compares the produced BMI avoids the cascade | +| `edit-body` | a real semantic edit inside a function body | the everyday loop — and whether a cascade is owed depends on **where the body lives**, not on the edit. See below. | +| `touch-leaf` | mtime bump on a unit nobody imports | recompile 1 + link | + +#### ⚠️ `edit-body` perturbs a DIFFERENT FILE in each variant, and the two ask +#### opposite questions + +| variant | file perturbed | what a correct engine does | +|---|---|---| +| `headers` | `unit_0.cpp` | recompile 1 + link — never a cascade | +| `modules` | `unit_0.cppm` (interface) | **may cascade**, see below | +| `modules-impl` | `unit_0_impl.cpp` (implementation unit) | recompile 1 + link — **never** a cascade | + +An implementation unit produces no BMI, so nothing can depend on it; the absence +of a cascade there is structural. An interface unit is the opposite: whether the +edit cascades depends on the compiler and on the edit. + +**The perturbation inserts a line.** Under GCC that shifts the recorded source +location of every declaration after the insertion point, which changes the BMI — +so the cascade follows from a changed BMI rather than from the edited body. The +same edit expressed as a same-line substitution does **not** cascade on GCC. +Under clang the interface cascades either way, because clang serialises +definitions into the BMI regardless of where in the file they appear. + +⚠️ **The generated fixture does not reproduce this**: its perturbed function is +the LAST declaration in `unit_0.cppm`, so nothing shifts and the BMI is unchanged +(measured: 0.94s, against 80.87s for the same scenario on the mcpp tree). Any +conclusion about a real project drawn from the fixture's `edit-body` is invalid +— which is what `--project` mode exists for. Full measurements in +[`.agents/docs/2026-08-15-module-edit-granularity.md`](../.agents/docs/2026-08-15-module-edit-granularity.md). + +`edit-comment` exists **separately from `edit-body`** on purpose: without the +split, an engine that skips comment-only rebuilds can be advertised as "12x +faster on edits", which is a claim about comments. + +#### `edit-comment` has two forms, and the report says which one ran + +The comment goes **inside the first function body**. A unit with no function +body — a `modules-impl` interface, or a hub that only declares — has nowhere to +put it, so it is appended at end of file instead. Those are different +perturbations: + +| form | what moves | BMI | expected result | +|---|---|---|---| +| `in-body` | every subsequent line in the file | **changes** — GCC records inline-body source locations | a cascade is CORRECT | +| `end-of-file` | nothing | unchanged | an engine comparing BMIs skips the cascade | + +Measured the same day, same engine, same compiler: `edit-comment` on mcpp's hub +(66 lines, no bodies → `end-of-file`) was **0.38s**, and on xlings' hub (566 +lines, 56 bodies → `in-body`) was **95.02s**. Side by side and without the form, +that reads as "the optimisation works on one project and not the other" — which +is not what happened. The generated fixture splits the same way, `modules` going +in-body and `modules-impl` end-of-file. + +So the form is written into the cell's `note` +(`… · perturbation: in-body`). Same rule as a non-`ok` status carrying its +reason: **a number whose meaning depends on an invisible choice is not a +measurement.** + +`edit-body` is the control that keeps the suite honest in the other direction — +where a cascade IS owed, no engine should be fast, and one that is has skipped +work it owed. + +#### But a body edit does not always owe a cascade, and that is the point + +Measured directly, GCC 16.1, comparing the BMI before and after: + +| what is edited | BMI | cascade | +|---|---|---| +| a free exported function's body, in the `.cppm` | **byte-identical** | not owed | +| a **member function of an exported class**, inline in the `.cppm` | **differs** | **owed** | +| a body in a separate `.cpp` implementation unit | **byte-identical** | **not owed** | + +A class's member function bodies are part of the class definition, which every +importer has to see, so they are serialised into the BMI. A free function's body +is not, and nothing in an implementation unit is. + +So "editing one function rebuilt forty modules" is not inherent to named modules +— it is a consequence of where the body was written. `mcpp`'s own +`src/version_req.cppm` is the first case (the perturbation lands in +`Version::str()`, a member of an exported class), which is why its `edit-body` +row is a near-full rebuild and why that is correct. + +**This is what the two xlings pins measure.** Moving the implementations out of +the interface units takes `edit-body` from 88.33s to **1.77s** on the same +project — ~50x, the largest single effect anywhere in this suite, and a code +style rather than an engine feature. + +Real projects run five of the six: `touch-leaf` needs a unit nobody imports +*and* a stable name for it, which a generated fixture has by construction and a +real tree does not. + +--- + +### Shared build descriptions + +`projects/common/` holds the parts every arm needs: `cmake/hermetic_payload.cmake` +and `xmake/payload.lua`. Both answer one question — "make this engine drive the +same process tree mcpp does" — and both are **one branch per compiler family**, +because that is what makes the toolchain a real axis rather than a label. + +They exist because the two projects had two copies of it, and the copies were +already diverging. Two copies of a toolchain definition is the worst place for a +copy: they drift by one flag and the benchmark reports the difference between the +two *descriptions* as an engine result. + +## 5. What the standard data set covers + +`bench/run-standard.sh` plans one run per entry in `matrix.json.cells` whose +`os` matches the machine it is on, and runs **every engine that cell lists** at +**3 samples** each. + +⚠️ **`allow_failed` is NOT consulted by the runner.** Those waivers were recorded +against failures on a shared CI runner, and at least two of them describe arms +that configure and generate perfectly well on a developer machine. Filtering by +them would carry a runner's limitation into local data and publish a smaller +comparison than the machine can make. They stay in the file as the record of +what broke where; the runner ignores them, and a failure here is a failure here. + +**There is no CI job for this suite.** There was, and 12 of its 32 foreign-engine +arms were waived — xmake had more arms waived than measured — while the job +reported success. `tests/e2e/230_bench_harness.sh` still runs on every PR and +checks that the suite builds and emits a valid report; the measuring is manual, +and belongs before a release and after any change claiming a performance effect. + diff --git a/bench/matrix.json b/bench/matrix.json new file mode 100644 index 00000000..babac27b --- /dev/null +++ b/bench/matrix.json @@ -0,0 +1,293 @@ +{ + "schema": 2, + "_comment": [ + "THE benchmark matrix. Read by .github/workflows/bench.yml to plan its jobs and", + "by tests/e2e/233_bench_matrix.sh to check this file against the harness's own", + "vocabulary. bench/SPEC.md explains the axes; it deliberately does not repeat", + "the cell list, because a matrix written down twice is a matrix that disagrees", + "with itself.", + "", + "A cell is one CI job. Inside it the harness sweeps every engine x variant x", + "scenario, so those axes are per-cell lists rather than more jobs: they share a", + "checkout, a toolchain install and a fixture, and splitting them would multiply", + "runner minutes without adding a single measurement.", + "meson is deliberately absent, not missing: meson 1.10.2 has no way to declare a translation unit to be a module INTERFACE, so every module cell was an `unavailable` row. An engine that cannot express the thing being measured is not a comparison point, and keeping it produced one honest row and five empty ones per report." + ], + "baseline": "cmake", + "_baseline_note": "Every ratio in every report is against cmake unless a cell overrides it. See SPEC.md S2.", + "tools": { + "_note": [ + "EVERY tool is installed through xlings at an exact version, on every runner.", + "Not a tidiness preference — the unpinned matrix measured something else:", + "the runner images carry cmake 3.31.6, which does not have the CMake 4.0", + "`import std` experimental key, so every module cell failed to configure;", + "and `xlings install xmake` resolved to 3.0.7 on the runner while a", + "developer box had 3.1.0. A version that varies per runner is a variable", + "the report does not record and the reader cannot see." + ], + "cmake": "4.4.2", + "xmake": "3.1.0", + "bazel": "9.2.0", + "gcc": "16.1.0", + "llvm": "22.1.8", + "llvm_windows": "20.1.7", + "_compiler_note": [ + "gcc/llvm are the versions bench/src/toolchain.cppm pins and mcpp itself", + "builds with. Every engine is handed THAT driver via `--compiler payload:*`,", + "not `command -v g++`. The runner's own gcc is 13.3.0: cmake cannot", + "configure C++23 modules with it, xmake crashes it with an internal", + "compiler error, and mcpp quietly used the registry payload anyway — so the", + "table read `48 failed` while claiming to compare build engines." + ], + "_cmake_note": "4.4.2. `CMAKE_EXPERIMENTAL_CXX_IMPORT_STD` is gated by a UUID that CHANGES WITH THE CMAKE VERSION, and it is compiled into the cmake binary rather than written in the Modules — 4.4's is f35a9ac6-8463-4d38-8eec-5d6008153e7d, 4.0's was a9e1cf81-9932-4810-974b-6eccaf14e457, and 4.4.2 rejects the old one with `CMAKE_EXPERIMENTAL_CXX_IMPORT_STD is set to incorrect value`. So the version here and the key in every description under bench/projects/ MUST move in the same commit. The key was not guessed: each UUID in the binary was offered to cmake in a throwaway project until one was accepted. ⚠️ Offering them to the REAL project proves nothing — its CMakeLists.txt `set()`s the key itself, which overrides the -D, so all seven candidates came back rejected including the one that works." + }, + "reference_mcpp": "2026.8.11.3", + "_reference_mcpp_note": [ + "The last RELEASED mcpp, installed by xlings and measured alongside the mcpp", + "built from this checkout. Without it a report says how fast this branch is", + "and not whether it got faster, which is the question a benchmark on a pull", + "request is being asked." + ], + "axes": { + "os": [ + "linux", + "macos", + "windows" + ], + "toolchain": [ + "gcc", + "clang", + "msvc" + ], + "engine": [ + "mcpp", + "cmake", + "xmake", + "bazel" + ], + "project": [ + "fixture", + "mcpp-2026.8.11.3", + "xlings-2026.8.11.2", + "xlings-2026.8.13.1" + ], + "variant": [ + "headers", + "modules", + "modules-impl" + ], + "scenario": [ + "cold", + "noop", + "touch-hub", + "touch-leaf", + "edit-body", + "edit-comment" + ] + }, + "cells": [ + { + "os": "linux", + "toolchain": "gcc", + "project": "fixture", + "engines": "mcpp,cmake,xmake", + "variants": "headers,modules,modules-impl", + "scenarios": "cold,noop,touch-hub,touch-leaf,edit-body,edit-comment", + "preset": "standard", + "note": "bazel omitted: its module support requires a clang driver (see the bazel engine's unsupported_reason)" + }, + { + "os": "linux", + "toolchain": "clang", + "project": "fixture", + "engines": "mcpp,cmake,xmake,bazel", + "variants": "headers,modules,modules-impl", + "scenarios": "cold,noop,touch-hub,touch-leaf,edit-body,edit-comment", + "preset": "standard" + }, + { + "os": "windows", + "toolchain": "clang", + "project": "fixture", + "engines": "mcpp,cmake,xmake,bazel", + "variants": "headers,modules,modules-impl", + "scenarios": "cold,noop,touch-hub,touch-leaf,edit-body,edit-comment", + "preset": "standard", + "allow_failed": "xmake,bazel", + "note": "KNOWN GAP, two of them, in two foreign tools on Windows. xmake: exits -1 (0xFFFFFFFF) from the seed build. It USED to be reported as `could not start the process`, which was the harness's own bug — Windows crash/abort statuses are large DWORDs that become negative ints, and `started()` inferred launch failure from the sign. That is fixed, so this is now honestly an exit code, and it is xmake's. bazel: `msvc_deps_scanner_wrapper_x64.bat failed ... The syntax of the command is incorrect` on the modules-impl variant — rules_cc's MSVC dependency scanner, not something this suite passes. A separate bazel failure on this cell WAS ours and is fixed: `--force_pic` is fatal on Windows (`supports_pic` is not enabled there), and dropping it recovered the headers and modules variants. cmake and mcpp stay measured here." + }, + { + "os": "windows", + "toolchain": "msvc", + "project": "fixture", + "engines": "mcpp,cmake,xmake", + "variants": "headers,modules,modules-impl", + "scenarios": "cold,noop,touch-hub,touch-leaf,edit-body,edit-comment", + "preset": "standard", + "note": "bazel omitted: same clang-driver requirement as the gcc cell" + }, + { + "os": "linux", + "toolchain": "gcc", + "project": "mcpp-2026.8.11.3", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform/platform.cppm", + "body": "src/version_req.cppm", + "note": "KNOWN GAP: cmake cannot GENERATE this project on the runner. It configures (the compiler probe passes) and then fails with `CMake Error: the \"CXX_MODULE_STD\" property ... requires that the \"__CMAKE::CXX23\" target exist, but it was not provided by the toolchain. Reason: Only `libstdc++` is supported`. Everything checkable from outside has been checked and every one of them AGREES with a developer box where the same arm configures, generates and builds: same cmake (the very same `xim-x-cmake/4.0.2` xlings payload, not merely the same version), same `xim-x-gcc/16.1.0`, `libstdc++.modules.json` present on the runner, both sources it names (`std.cc`, `std.compat.cc`) present on the runner, and the runner's exact flag shape (`-B/lib -L/lib`, the branch this machine does not normally take) reproduced locally via a fake MCPP_HOME with no subos — it works there. What is left is inside CMake's own detection. The arm's CMakeConfigureLog is now attached to the cell log on failure so the next look starts from CMake's own record rather than another hypothesis. BASELINE MOVED to the released mcpp for this cell, exactly as the xlings cells do: normalising against an engine that never produced a binary prints bare seconds under a heading that says `relative to`. xmake and both mcpp arms stay measured here. The published cmake numbers in bench/README were taken on a machine where this arm works.", + "buildfiles": "mcpp", + "allow_failed": "cmake", + "baseline": "2026.8.11.3" + }, + { + "os": "linux", + "toolchain": "clang", + "project": "mcpp-2026.8.11.3", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform/platform.cppm", + "body": "src/version_req.cppm", + "buildfiles": "mcpp", + "allow_failed": "xmake", + "note": "KNOWN GAP: xmake cannot build `import std` with the libc++ payload on this project. It fails during dependency scanning with `warning: std and std.compat modules not found!` followed by `error: missing std dependency for module <...>`, naming a different module each run. REPRODUCED LOCALLY and bounded — it is not a runner problem: the same failure occurs on a developer box with the same payload. Two fixes were tried and neither works: passing `--sdk=` explicitly (the flag xmake's own warning suggests, and the one that the windows/clang arm needed), and switching from the custom `mcpp-clang` toolchain to xmake's built-in `llvm` toolchain with that same `--sdk`. The payload layout is the standard one — `/share/libc++/v1/std.cppm` is present. The gcc arm of this same project is green, and the generated fixture is green with this same clang, so this is specific to xmake + libc++ + a real tree. cmake is NOT waived here and must stay measured. Unresolved, not impossible: whoever picks this up should start from why the fixture succeeds where the project does not." + }, + { + "os": "windows", + "toolchain": "clang", + "project": "mcpp-2026.8.11.3", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform/platform.cppm", + "body": "src/version_req.cppm", + "buildfiles": "mcpp", + "allow_failed": "cmake,xmake", + "note": "KNOWN GAP, TWO DIFFERENT ONES, and they must not be conflated. cmake: no `import std` on Windows — it stops inside project() at the CXX_MODULE_STD toolchain-support probe (CMakeTestCXXCompiler -> CMakeDetermineCompilerSupport), because the Windows payload is llvm@20.1.7 against the MSVC STL, which ships no std module cmake can build. xmake: does not start at all — `seed build: could not start the process (no log written)`, i.e. the program is not on the runner's PATH. That is an ENVIRONMENT problem and is fixable, unlike cmake's; it is waived here only so the mcpp arms keep running, and it used to be waived under cmake's reason, which is how it went unnoticed. Re-check both when the Windows payload or the tool install changes. The mcpp arms measure fine, so the cell is kept for them and the two arms are waived rather than dropped — a waived failure stays visible in the report, an excluded engine does not." + }, + { + "os": "linux", + "toolchain": "gcc", + "project": "xlings-2026.8.11.2", + "buildfiles": "xlings", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform.cppm", + "body": "src/platform.cppm", + "note": "the COMBINED code style: 110 .cppm + 2 .cpp, each interface unit carrying its own implementation. src/platform.cppm has 45 importers, which is what makes it the hub KNOWN GAP on this project, and the reason the baseline is overridden: ftxui / libarchive / lua / mbedtls arrive as SOURCE packages that mcpp compiles, so cmake and xmake compile every translation unit and then stop at the link with `undefined reference to mbedtls_*`. Both descriptions ship their own CMakeLists and `add_subdirectory` would finish it — it is work, not a wall. Until then the arms are kept (a documented wall is data) but waived, and the cell answers the question it can answer: mcpp against mcpp.", + "baseline": "2026.8.11.3", + "allow_failed": "cmake,xmake" + }, + { + "os": "linux", + "toolchain": "gcc", + "project": "xlings-2026.8.13.1", + "buildfiles": "xlings", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules-impl", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform.cppm", + "body": "src/platform.cpp", + "note": "the SPLIT code style: the same 110 interfaces with implementations moved into 92 .cpp. Paired with the cell above, this is the only measurement in the suite of what that refactor costs on a real codebase — note `body` is the .cpp here, because editing an implementation unit is the point KNOWN GAP on this project, and the reason the baseline is overridden: ftxui / libarchive / lua / mbedtls arrive as SOURCE packages that mcpp compiles, so cmake and xmake compile every translation unit and then stop at the link with `undefined reference to mbedtls_*`. Both descriptions ship their own CMakeLists and `add_subdirectory` would finish it — it is work, not a wall. Until then the arms are kept (a documented wall is data) but waived, and the cell answers the question it can answer: mcpp against mcpp.", + "baseline": "2026.8.11.3", + "allow_failed": "cmake,xmake" + }, + { + "os": "linux", + "toolchain": "clang", + "project": "xlings-2026.8.11.2", + "buildfiles": "xlings", + "engines": "mcpp,mcpp[schedule=on],cmake,xmake", + "variants": "modules", + "scenarios": "cold,noop,touch-hub,edit-body,edit-comment", + "hub": "src/platform.cppm", + "body": "src/platform.cppm", + "baseline": "2026.8.11.3", + "allow_failed": "cmake,xmake", + "note": "KNOWN GAP, TWO DIFFERENT ONES — and the reason the baseline is overridden. They must not be conflated: this note used to describe BOTH arms as 'compile everything, then stop at the link with undefined reference to mbedtls_*', and by the time anyone read the log neither arm was failing that way any more. cmake: FATAL_ERROR from the suite's own `xpkg_source_library.cmake:231` — a source package's manifest carries no `sources` list, so the description cannot name what to compile. That is OUR code and it is fixable; it just has not been fixed. xmake: xrepo cannot BUILD the dependency packages — `install ftxui v6.1.9 .. failed`, `install mcpplibs-tinyhttps 0.2.9 .. failed`, `install mcpplibs-capi-lua 0.0.3 .. failed`, each ending in `std and std.compat modules not found!`. Same libc++ std-module gap as the linux/clang mcpp cell, reproduced locally there, with `--sdk` and both the custom and built-in llvm toolchains tried. It is upstream, not ours. Both arms stay listed rather than excluded — a documented wall is data — and the cell answers what it can: mcpp against mcpp. The run prints `WAIVED AWAY ENTIRELY: cmake, xmake` so a green job says so on its face." + } + ], + "excluded": [ + { + "os": "macos", + "toolchain": "gcc", + "reason": "no gcc payload exists for macOS in mcpp's registry, and a Homebrew gcc would make the cell a comparison of distributions rather than of engines" + }, + { + "os": "windows", + "toolchain": "gcc", + "reason": "mcpp supports x86_64-windows-gnu, but the other four engines would each have to be pointed at an msys2 gcc that mcpp does not use. KNOWN GAP, not a decision: the cell is meaningful and is simply not wired up yet" + }, + { + "os": "linux", + "toolchain": "msvc", + "reason": "msvc is Windows-only" + }, + { + "os": "macos", + "toolchain": "msvc", + "reason": "msvc is Windows-only" + }, + { + "os": "windows", + "toolchain": "msvc", + "project": "mcpp-*", + "reason": "mcpp's own schedule policy reports msvc as unmeasured (src/build/schedule/policy.cppm), so the mcpp arm would be measuring a shape nobody has validated" + }, + { + "os": "windows", + "toolchain": "msvc", + "project": "xlings-*", + "reason": "same as the mcpp project on msvc: no validated schedule, so a real-project number would describe an unvalidated shape" + }, + { + "os": "windows", + "toolchain": "clang", + "project": "xlings-*", + "reason": "xlings has not been shown to build on Windows at all. KNOWN GAP: verify the plain build first, then add the cell — a bench cell that cannot build is not a measurement. The `xlings-*` prefix is deliberate: a bare `*` here would also claim the windows/clang fixture and mcpp cells, which do run" + }, + { + "os": "*", + "toolchain": "clang", + "project": "*", + "engine": "xmake", + "reason": "KNOWN GAP, xmake+clang only: xmake locates libc++'s std module through lib/libc++.modules.json, which mcpp's llvm payload does not ship (it has share/libc++/v1/std.cppm). xmake warns 'std and std.compat modules not found' and build.c++.modules.std degrades SILENTLY — the arm would then measure a project without `import std;` against ones with it. The cell still runs; read its note before quoting the number" + }, + { + "os": "macos", + "toolchain": "clang", + "project": "xlings-*", + "reason": "KNOWN GAP, and it is an mcpp defect rather than a benchmark one: on macOS mcpp cannot compile a dependency's `build.mcpp` helper — xlings depends on mcpplibs.xpkg, whose helper links but then dies at run time with `dyld: Symbol not found: __ZdaPv` (operator delete[]), i.e. it cannot find libc++. Same family as the helper self-containment work (glibc via rpath, musl and PE via -static) with macOS never covered. Recorded in .agents/docs/2026-08-13-build-optimization-status.md S9a; the cell returns when that is fixed, and its blast radius is every macOS project depending on a package that ships a build.mcpp" + }, + { + "os": "macos", + "toolchain": "clang", + "project": "*", + "reason": "KNOWN GAP, and an mcpp/xlings environment defect rather than a benchmark one. On macOS the registry's libc++ ends up on the dynamic loader's search path for EVERY child process, so Apple's own linker — which links against libc++ — resolves against the payload copy and aborts before linking: `dyld: Symbol not found: __ZdaPv, Referenced from: .../XcodeDefault.xctoolchain/usr/bin/ld, Expected in: .../registry/.../lib/libc++.1.0.dylib`. cmake, bazel and the reference mcpp all fail identically while the mcpp under test passes, which is what identifies it as environmental. Same symbol and mechanism as the build.mcpp helper failure in .agents/docs/2026-08-13-build-optimization-status.md S9a. Removing the payload libc++ flags on macOS did NOT fix it, so the contamination arrives through DYLD_* rather than through link flags. The cells return once that is understood — deliberately not guessed at from a machine that cannot reproduce it" + }, + { + "project": "mcpp-*", + "engine": "bazel", + "reason": "bazel cannot build this workload: it will not glob sources from outside its workspace, and `import std;` has no bazel spelling. bench/projects/mcpp/BUILD.bazel therefore declares no rules — and `bazel build //...` over a package with no rules EXITS 0 having compiled nothing, which the matrix published as `bazel cold 0.43s` beside mcpp's 12s and cmake's 94s. Removed here, and the bazel adapter now reports `unavailable` for a ruleless package so the next one cannot be published as a measurement." + }, + { + "os": "linux", + "toolchain": "clang", + "project": "xlings-2026.8.13.1", + "reason": "clang 22.1.8 crashes compiling this tree's `xlings.core.config` module — a compiler bug, not a build-engine result: `clang frontend command failed with exit code 139`, `PLEASE submit a bug report to llvm/llvm-project`, on obj/xlings/src/core/config.m.o. The SAME tree with the SAME description builds clean under gcc 16.1.0 (linux/gcc/xlings-2026.8.13.1 is green), so nothing here measures an engine. Excluded rather than waived because a crashing compiler produces no number to look at; re-add the cell when llvm ships a fix." + } + ], + "_workload_note": [ + "THE MEASURED SOURCES ARE PINNED, INCLUDING mcpp's OWN.", + "A benchmark has two halves: the engine under test and the workload it is", + "given. The engine is the binary, which is what moves between runs; the", + "workload must not. `--project $GITHUB_WORKSPACE` made mcpp's own tree the", + "workload, so every commit on a branch silently changed the thing being", + "measured and no two runs were comparable — the same defect the xlings clone", + "had, just harder to see because the drift was our own.", + "Both are git submodules under bench/projects/ now. Bumping one is a", + "deliberate, reviewable act that invalidates the previous ratios on purpose." + ] +} diff --git a/bench/mcpp.toml b/bench/mcpp.toml new file mode 100644 index 00000000..8d947e93 --- /dev/null +++ b/bench/mcpp.toml @@ -0,0 +1,17 @@ +[package] +name = "mbench" +version = "0.1.0" +description = "Build-engine benchmark harness — cross-platform, engine-agnostic, protocol-versioned" +license = "Apache-2.0" +authors = ["mcpp-community"] + +[build] +default-profile = "release" + +# The harness must build on a machine that has nothing but mcpp: it is the first +# thing that runs when measuring a fresh environment, so it takes no +# dependencies beyond `import std;`. +[toolchain] +default = "gcc@16.1.0" +macos = "llvm@22.1.8" +windows = "llvm@20.1.7" diff --git a/bench/projects/.gitignore b/bench/projects/.gitignore new file mode 100644 index 00000000..c6ed6745 --- /dev/null +++ b/bench/projects/.gitignore @@ -0,0 +1,35 @@ +# Engine scratch, written INTO the description directories at measurement time. +# +# Every foreign engine keeps state next to the description it was pointed at: +# xmake puts its resolved configuration in `.xmake/`, cmake and xmake put objects +# under `build/`. None of it belongs in the repository, and one of these files +# has already been committed by accident and would have done real damage: +# +# bench/projects/mcpp/.xmake/linux/x86_64/xmake.conf +# builddir = "mcpp-2026.8.11.3/build" +# __toolchains_linux_x86_64 = { "gcc", "cuda", "rust", ... } +# +# That `builddir` is the path-doubling bug this suite was fixed for, frozen into +# a file CI would have READ — reinstating the defect on every runner while the +# code that caused it was already gone. The toolchain list is whatever happened +# to be installed on one developer's machine. +# +# The root .gitignore's `/.xmake/` is anchored to the repository root and does +# not reach here, which is exactly how this slipped through. +.xmake/ +build/ + +# cmake, when driven by hand rather than by the harness (`--work` keeps the +# harness's own output out of the tree). +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake +compile_commands.json + +# bazel +bazel-*/ +MODULE.bazel.lock + +# bazel convenience symlinks — created in the package dir on every build, +# and pointing into ~/.cache/bazel, so they are never worth tracking. +bazel-* diff --git a/bench/projects/common/cmake/hermetic_payload.cmake b/bench/projects/common/cmake/hermetic_payload.cmake new file mode 100644 index 00000000..4488986f --- /dev/null +++ b/bench/projects/common/cmake/hermetic_payload.cmake @@ -0,0 +1,319 @@ +# Shared by every CMake arm in bench/projects/. +# +# WHY THIS IS SHARED. mcpp's arm and xlings' arm need exactly the same thing — +# "make cmake drive the same process tree mcpp does" — and they had two copies +# of it. The copies were already diverging (one had learned about the `std` +# module / CMAKE_CXX_FLAGS trap, the other had not), and a benchmark whose two +# arms are configured differently is measuring the difference between the two +# descriptions. +# +# WHAT IT IS FOR. mcpp resolves a compiler out of its own registry and always +# passes the rest of the payload explicitly. A bare compiler from that registry +# falls back to PATH for `as`/`ld` and to the host for headers, so the two arms +# would compile the same sources against different libc. Reproducing the payload +# here is what makes the wall-clock difference attributable to the build engine. +# +# THE COMPILER IS AN AXIS, so this is not one block of flags — it is one per +# family. Getting it wrong is not a build failure, it is a slower or faster +# number with no visible cause. +# +# Usage, BEFORE the first target: +# include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/hermetic_payload.cmake) +# bench_hermetic_payload() + +# Where mcpp keeps its packages. `MCPP_HOME` first, matching mcpp's own +# resolution order. +function(bench_registry_xpkgs out) + if(DEFINED ENV{MCPP_HOME}) + set(home "$ENV{MCPP_HOME}") + elseif(WIN32) + set(home "$ENV{USERPROFILE}/.mcpp") + else() + set(home "$ENV{HOME}/.mcpp") + endif() + set(${out} "${home}/registry/data/xpkgs" PARENT_SCOPE) +endfunction() + +# The newest unpacked version of a package, or "" — used only for payload +# components (binutils, glibc headers), never for a dependency whose version the +# manifest pins. "Newest directory wins" is fine for a toolchain payload and is +# NOT fine for a dependency: the registry holds several versions and picking the +# lexically-last one only happens to agree with the pin. +function(bench_newest_package xpkgs name out) + file(GLOB dirs "${xpkgs}/${name}/*") + set(${out} "" PARENT_SCOPE) + if(dirs) + list(SORT dirs) + list(GET dirs -1 newest) + set(${out} "${newest}" PARENT_SCOPE) + endif() +endfunction() + +function(bench_hermetic_payload) + bench_registry_xpkgs(xpkgs) + set(sysroot "") + if(DEFINED ENV{MCPP_HOME}) + set(sysroot "$ENV{MCPP_HOME}/registry/subos/default") + elseif(NOT WIN32) + set(sysroot "$ENV{HOME}/.mcpp/registry/subos/default") + endif() + + # A compiler from OUTSIDE the registry is the caller's explicit opt-in to the + # host world — the same rule mcpp's own hermetic link check applies. Adding a + # registry sysroot to a host g++ produces a mixed build that fails somewhere + # unrelated, so say nothing instead. + get_filename_component(cxx_real "${CMAKE_CXX_COMPILER}" REALPATH) + string(FIND "${cxx_real}" "xpkgs" xpkgs_pos) + if(xpkgs_pos EQUAL -1) + message(STATUS "bench: ${CMAKE_CXX_COMPILER_ID} compiler is outside mcpp's " + "registry — using it as-is (no payload flags)") + return() + endif() + + # ── MSVC ──────────────────────────────────────────────────────────────── + # There is no payload: mcpp uses the SYSTEM Visual Studio installation + # (`msvc@system`), reached through the VS environment rather than through + # flags. Both arms therefore already share it, and there is nothing to add. + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + return() + endif() + + # CMAKE_CXX_FLAGS, not add_compile_options(): CMake generates the `std` module + # target ITSELF, and directory-scope options do not reach it. Without this the + # std module compiles against whatever libc headers the compiler defaults to + # while every project unit compiles against the payload, and the build dies on + # a type that exists in both: + # + # error: conflicting type for imported declaration 'char _IO_FILE::_unused2 [20]' + # .../glibc-2.39/include/bits/types/struct_FILE.h:98 + # note: existing declaration 'char _IO_FILE::_unused2 [8]' + # .../registry/subos/default/usr/include/bits/types/struct_FILE.h:109 + # + # Two glibcs in one link, and the error names neither the flag nor the target + # that is wrong. + set(cxx "${CMAKE_CXX_FLAGS}") + set(cc "${CMAKE_C_FLAGS}") + set(ld "${CMAKE_EXE_LINKER_FLAGS}") + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # -B and --sysroot must reach BOTH compile and link: the driver spawns `as` + # from it at compile time and `ld` from it at link time. Adding it on one + # side only silently falls through to PATH. + # ⚠️ C FLAGS TOO, not just CXX. xlings pulls libarchive, lua and mbedtls in + # as C sources that this arm has to compile, and CMAKE_C_FLAGS is a separate + # variable — setting only the C++ one puts the C half of the build on the + # HOST's headers and libc while the C++ half uses the payload's. That is the + # same two-libc failure the CMAKE_CXX_FLAGS comment above describes, just + # arriving through a different door. + bench_newest_package("${xpkgs}" "xim-x-binutils" binutils) + if(binutils) + string(APPEND cxx " -B${binutils}/bin") + string(APPEND cc " -B${binutils}/bin") + string(APPEND ld " -B${binutils}/bin") + endif() + if(IS_DIRECTORY "${sysroot}") + string(APPEND cxx " --sysroot=${sysroot}") + string(APPEND cc " --sysroot=${sysroot}") + string(APPEND ld " --sysroot=${sysroot}") + endif() + + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) + # NOT APPLE for the reason bench/src/toolchain.cppm spells out: adding the + # registry's lib directory on macOS makes Apple's own `ld` resolve against + # the payload's libc++ and abort with `Symbol not found: __ZdaPv` before it + # links anything. The payload clang finds its own libc++, and cmake supplies + # -isysroot itself. + # Clang's payload is shaped differently and `--sysroot` is NOT the + # equivalent: mcpp drives clang with an explicit include chain instead + # (verified against a real mcpp build command). Passing gcc's --sysroot to + # clang here would be the mirror of the bug above — one arm on the payload + # libc, the other on the host's. + # + # The llvm root is derived from the COMPILER PATH, not globbed: the registry + # can hold 20.1.7 and 22.1.8 at once, and "newest wins" would silently + # compile against a different libc++ than the driver being measured. + get_filename_component(llvm_bin "${cxx_real}" DIRECTORY) + get_filename_component(llvm_root "${llvm_bin}" DIRECTORY) + if(IS_DIRECTORY "${llvm_root}/include/c++/v1") + string(APPEND cxx " --no-default-config -nostdinc++") + string(APPEND cxx " -isystem${llvm_root}/include/c++/v1") + # The per-triple directory carries __config_site; it is absent on some + # builds, so it is added only when present rather than unconditionally. + file(GLOB triple_inc "${llvm_root}/include/*/c++/v1") + foreach(d IN LISTS triple_inc) + string(APPEND cxx " -isystem${d}") + endforeach() + string(APPEND ld " -nostdlib++ -L${llvm_root}/lib -lc++ -lc++abi") + # The unwinder ships beside libc++ in this payload; without it the link + # fails on _Unwind_Resume, which reads as a missing exception runtime + # rather than as a missing -L. + if(EXISTS "${llvm_root}/lib/libunwind.so" OR EXISTS "${llvm_root}/lib/libunwind.a") + string(APPEND ld " -lunwind") + endif() + endif() + bench_newest_package("${xpkgs}" "xim-x-glibc" glibc) + if(glibc AND IS_DIRECTORY "${glibc}/include") + string(APPEND cxx " -isystem${glibc}/include") + string(APPEND cc " -isystem${glibc}/include") + endif() + bench_newest_package("${xpkgs}" "xim-x-linux-headers" uapi) + if(uapi AND IS_DIRECTORY "${uapi}/include") + string(APPEND cxx " -isystem${uapi}/include") + string(APPEND cc " -isystem${uapi}/include") + endif() + # macOS: a registry clang has no idea where the platform SDK is, and cmake + # only passes -isysroot automatically for AppleClang. Without it even the + # compiler-works probe fails to LINK, which is reported as "cmake could not + # configure" — observed on every macOS bench cell. + if(APPLE AND CMAKE_OSX_SYSROOT) + string(APPEND cxx " -isysroot ${CMAKE_OSX_SYSROOT}") + string(APPEND cc " -isysroot ${CMAKE_OSX_SYSROOT}") + string(APPEND ld " -isysroot ${CMAKE_OSX_SYSROOT}") + endif() + endif() + + set(CMAKE_CXX_FLAGS "${cxx}" PARENT_SCOPE) + set(CMAKE_C_FLAGS "${cc}" PARENT_SCOPE) + set(CMAKE_EXE_LINKER_FLAGS "${ld}" PARENT_SCOPE) + message(STATUS "bench: hermetic payload for ${CMAKE_CXX_COMPILER_ID} applied") +endfunction() + +# One source dependency out of the registry, added to `target` as its own +# CXX_MODULES file set. +# +# Its own set, because a CXX_MODULES set requires every file to live under one +# of its base directories and these sit in the registry, outside the tree. +# +# The VERSION IS PINNED BY THE CALLER, from the manifest — see the warning in +# bench_newest_package about why "newest wins" is wrong here. +function(bench_add_source_dep target name version) + bench_registry_xpkgs(xpkgs) + set(dir "${xpkgs}/${name}/${version}") + file(GLOB_RECURSE srcs CONFIGURE_DEPENDS "${dir}/*/src/*.cppm") + if(NOT srcs) + message(WARNING "dependency ${name} ${version} is not unpacked at ${dir}; " + "this build will not match mcpp's own") + return() + endif() + list(GET srcs 0 first) + get_filename_component(base "${first}" DIRECTORY) + # A file-set name may only contain letters, digits and underscores — package + # names like `mcpplibs.capi-x-lua` do not qualify, and CMake rejects them at + # configure time rather than mangling them. + string(REGEX REPLACE "[^A-Za-z0-9_]" "_" fsname "fs_${name}") + target_sources(${target} PRIVATE + FILE_SET "${fsname}" TYPE CXX_MODULES BASE_DIRS "${base}" FILES ${srcs}) + + # ⚠️ .cpp UNDER src/ TOO, as ORDINARY sources. A package's `src/` may hold + # MODULE IMPLEMENTATION UNITS — `module mcpplibs.capi.lua;` with no `export` + # — and those are not part of a CXX_MODULES file set (CMake rejects a + # non-interface unit there); they are plain sources that the scanner picks + # the module edge out of. mcpplibs.capi.lua 0.0.3 keeps all 428 lines of its + # Lua-C wrapper in one, and globbing only `*.cppm` dropped it: the arm then + # failed at the link on every `mcpplibs::capi::lua::*` symbol, naming the + # consumer rather than the file that was never compiled. + # + # `*/src/` and not the whole package: examples/ and tests/ carry their own + # main() and are not what mcpp compiles for a dependency. + file(GLOB_RECURSE impls CONFIGURE_DEPENDS "${dir}/*/src/*.cpp") + if(impls) + target_sources(${target} PRIVATE ${impls}) + endif() +endfunction() + +# ── The half that must run BEFORE project() ───────────────────────────────── +# +# `bench_hermetic_payload()` above keys off CMAKE_CXX_COMPILER_ID, which does +# not exist until project() has probed the compiler — and that probe is exactly +# what fails without these flags: +# +# [2/2] .../xim-x-gcc/16.1.0/bin/g++ ... -o cmTC_c87ee +# /usr/bin/ld: cannot find crt1.o: No such file or directory +# /usr/bin/ld: cannot find crti.o: No such file or directory +# +# CMake reports that as "The C++ compiler is not able to compile a simple test +# program", naming neither the sysroot nor the payload. It passed on developer +# boxes because a host crt1.o was findable there and did not on CI. +# +# So this one keys off the compiler PATH, which the caller already has from +# -DCMAKE_CXX_COMPILER. Same flags, same reasoning as the GNU branch above; +# a compiler outside the registry is left alone, exactly as there. +function(bench_hermetic_payload_preproject) + if(NOT CMAKE_CXX_COMPILER OR WIN32) + return() + endif() + get_filename_component(_real "${CMAKE_CXX_COMPILER}" REALPATH) + string(FIND "${_real}" "xpkgs" _pos) + if(_pos EQUAL -1) + return() + endif() + # clang carries its own include chain (see the Clang branch above) and does + # not need -B/--sysroot to link a test program; only the gcc payload does. + if(NOT _real MATCHES "g\\+\\+$" AND NOT _real MATCHES "gcc$") + return() + endif() + bench_registry_xpkgs(_xpkgs) + bench_newest_package("${_xpkgs}" "xim-x-binutils" _binutils) + set(_sysroot "${_xpkgs}/../subos/default") + set(_add "") + if(_binutils) + string(APPEND _add " -B${_binutils}/bin") + endif() + # ── The C runtime: FIND crt1.o, then ADD its directory. No --sysroot. ────── + # + # ⚠️ `--sysroot` does not ADD a search path, it REPLACES gcc's default one, so + # pointing it anywhere that lacks a libc removes the C runtime entirely: + # + # ld: cannot find crt1.o: No such file or directory + # ld: cannot find crti.o: No such file or directory + # ld: cannot find -lm: No such file or directory + # + # — a message about the C runtime that names neither the flag nor the cause. + # All five cmake cells of the linux/gcc bench died there, inside cmake's own + # compiler probe, before a line of the project was configured. + # + # TWO WRONG ANSWERS PRECEDED THIS ONE, and both looked right: + # * `IS_DIRECTORY "${_sysroot}"` — true on the runners, because that path is + # created whether or not anything was installed into it. + # * then "only pass it when crt1.o is under the sysroot" — correct as far as + # it went, and it changed nothing: on the runner that directory does not + # exist at all, so neither branch ran, no --sysroot was passed, and the + # payload gcc fell back to a built-in prefix that is not there either. + # Diagnosed from the CI log by the ABSENCE of both `--sysroot=` on the + # command line and the STATUS line the second branch would have printed. + # + # The stable anchor is the PACKAGE. `subos/default/lib/crt1.o` is a symlink + # into `xpkgs/xim-x-glibc//lib/`, and xpkgs is where the compiler itself + # was found, so it exists by construction. `-B` (startup files) and `-L` + # (`-lm`) ADD to the search rather than replacing it, which is the property + # this needed all along. + set(_crtdir "") + bench_newest_package("${_xpkgs}" "xim-x-glibc" _glibc) + foreach(_root "${_sysroot}" "${_glibc}") + if(_crtdir OR NOT _root) + continue() + endif() + foreach(_d lib lib64 usr/lib usr/lib64 usr/lib/x86_64-linux-gnu) + if(EXISTS "${_root}/${_d}/crt1.o") + set(_crtdir "${_root}/${_d}") + break() + endif() + endforeach() + endforeach() + if(_crtdir) + string(APPEND _add " -B${_crtdir} -L${_crtdir}") + else() + # Say so. Falling back to the host's C runtime is a different measurement + # from the one this file claims to set up, and the only way anyone notices + # is if it announces itself. + message(STATUS "bench: no crt1.o under ${_sysroot} or ${_glibc} — this arm " + "links against the host C runtime") + endif() + if(_add STREQUAL "") + return() + endif() + foreach(_v CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS) + set(${_v} "${${_v}}${_add}" PARENT_SCOPE) + endforeach() + message(STATUS "bench: hermetic payload applied before project()${_add}") +endfunction() diff --git a/bench/projects/common/xmake/payload.lua b/bench/projects/common/xmake/payload.lua new file mode 100644 index 00000000..7d79a25a --- /dev/null +++ b/bench/projects/common/xmake/payload.lua @@ -0,0 +1,278 @@ +-- Shared by every xmake arm in bench/projects/. +-- +-- WHY THIS IS SHARED. mcpp's arm and xlings' arm need the same thing — "make +-- xmake drive the same process tree mcpp does" — and had two copies of a +-- 60-line toolchain block. Two copies of a toolchain definition is the worst +-- place for a copy: they drift by one flag and the benchmark reports the +-- difference between the two DESCRIPTIONS as an engine result. +-- +-- THE COMPILER IS AN AXIS, so this defines one toolchain per family rather than +-- one block of flags: +-- +-- mcpp-gcc the registry's gcc + its binutils + the subos sysroot +-- mcpp-clang the registry's clang + its own include chain (NOT --sysroot; +-- see the comment there — handing clang gcc's payload puts the +-- two arms on different libc) +-- msvc the SYSTEM Visual Studio, which is what mcpp uses too +-- (`msvc@system`), so there is nothing to define +-- +-- ⚠️ TWO SCOPES, TWO DIFFERENT MISSING PIECES. xmake's DESCRIPTION scope (the +-- top level of an xmake.lua, and this file) has no `io`. `on_load`'s sandbox has +-- `io` but cannot see globals defined here. Both were hit, in that order, trying +-- to share the manifest reader — see the note inside bench_define_toolchains. +-- Anything that reads a file must live inside on_load; anything that only +-- touches `os`/`path` can live out here. +-- +-- Usage, from a project's xmake.lua: +-- includes("../common/xmake/payload.lua") +-- bench_define_toolchains(path_to_the_measured_tree_mcpp_toml) +-- ... +-- local tc = bench_pinned_toolchain(); if tc then set_toolchains(tc) end + +function bench_mcpp_home() + local home = os.getenv("MCPP_HOME") + if not home then + local base = os.getenv("HOME") or os.getenv("USERPROFILE") or "" + home = path.join(base, ".mcpp") + end + return home +end + +function bench_xpkgs() + return path.join(bench_mcpp_home(), "registry", "data", "xpkgs") +end + +function bench_sysroot() + return path.join(bench_mcpp_home(), "registry", "subos", "default") +end + +-- Newest unpacked version of a package. For PAYLOAD components only — never for +-- a dependency whose version the manifest pins. The registry holds several +-- versions, and "lexically last" agreeing with the pin is a coincidence; a +-- benchmark whose fairness rests on a coincidence is not a benchmark. +function bench_newest(name) + local base = path.join(bench_xpkgs(), name) + if not os.isdir(base) then return nil end + local dirs = os.dirs(path.join(base, "*")) + if #dirs == 0 then return nil end + table.sort(dirs) + return dirs[#dirs] +end + +-- The single subdirectory a source package unpacks into +-- (`mcpplibs-x-cmdline/0.0.2/cmdline-0.0.2/`). Discovered rather than composed +-- from `-`: `mcpplibs.capi-x-lua/0.0.3/` does not follow that +-- pattern, and a guess there finds nothing — which surfaces as a missing module +-- three files later, not as a missing path. +function bench_package_root(name, version) + local base = path.join(bench_xpkgs(), name, version) + if not os.isdir(base) then return nil end + local dirs = os.dirs(path.join(base, "*")) + if #dirs == 0 then return nil end + table.sort(dirs) + return dirs[1] +end + +-- Defines `mcpp-gcc` and `mcpp-clang` when their payloads are present. +-- +-- `manifest` is the MEASURED TREE's mcpp.toml. Its `[toolchain] default` pins +-- the exact version, narrowed inside on_load so both arms run the same binary by +-- construction rather than by luck of directory ordering. +function bench_define_toolchains(manifest) + local xpkgs = bench_xpkgs() + local sysroot = bench_sysroot() + local binutils = bench_newest("xim-x-binutils") + local gcc_dir = bench_newest("xim-x-gcc") + local llvm_dir = bench_newest("xim-x-llvm") + -- Resolved HERE and captured as upvalues, because on_load cannot call + -- bench_newest: its sandbox does not see this file's globals. + local glibc_dir = bench_newest("xim-x-glibc") + local uapi_dir = bench_newest("xim-x-linux-headers") + + -- ⚠️ THE MANIFEST READER IS WRITTEN OUT INSIDE EACH on_load, TWICE. + -- + -- Not an oversight — it is the only scope it can live in: + -- * defined out here, a Lua closure carries its DEFINITION environment, so + -- it resolves `io` against the description scope, where io is nil: + -- attempt to index a nil value (global 'io') + -- * defined as a global in this file, on_load's sandbox cannot see it: + -- attempt to call a nil value (global 'bench_manifest_toolchain') + -- Both were hit, in that order, trying to share it. Twelve lines twice + -- inside ONE file is a far weaker coupling than the 60-line toolchain block + -- that used to be copied across two project files — which is what this + -- module exists to remove. + + if gcc_dir and binutils then + toolchain("mcpp-gcc") + set_kind("standalone") + set_homepage("hermetic gcc payload resolved by mcpp") + set_toolset("cc", path.join(gcc_dir, "bin", "gcc")) + set_toolset("cxx", path.join(gcc_dir, "bin", "g++")) + set_toolset("ld", path.join(gcc_dir, "bin", "g++")) + set_toolset("sh", path.join(gcc_dir, "bin", "g++")) + set_toolset("ar", path.join(binutils, "bin", "ar")) + set_toolset("strip", path.join(binutils, "bin", "strip")) + -- `as` and `ranlib` are NOT optional once xrepo builds packages with + -- this toolchain. `-B/bin` only lets the DRIVER find them; + -- xmake resolves the assembler itself and stops the whole configure + -- with `cannot get program for as` — a message that names neither + -- the toolchain nor the package it was building (ftxui, here). + set_toolset("as", path.join(gcc_dir, "bin", "gcc")) + set_toolset("ranlib", path.join(binutils, "bin", "ranlib")) + set_toolset("nm", path.join(binutils, "bin", "nm")) + set_toolset("objcopy", path.join(binutils, "bin", "objcopy")) + on_load(function (toolchain) + local read_pin = function (m) + if not m or not os.isfile(m) then return nil end + local in_tc = false + for _, line in ipairs((io.readfile(m) or ""):split("\n", {plain = true})) do + local section = line:match("^%s*%[(.-)%]") + if section then in_tc = (section == "toolchain") end + if in_tc then + local f, v = line:match('^%s*default%s*=%s*"([%w_]+)@([%w%.%-]+)"') + if f and v then return f, v end + end + end + return nil + end + local fam, ver = read_pin(manifest) + if fam == "gcc" and ver then + local pinned = path.join(xpkgs, "xim-x-gcc", ver) + if os.isdir(pinned) then + toolchain:set("toolset", "cc", path.join(pinned, "bin", "gcc")) + for _, k in ipairs({"cxx", "ld", "sh"}) do + toolchain:set("toolset", k, path.join(pinned, "bin", "g++")) + end + else + utils.warning("mcpp.toml pins gcc@%s, absent from the registry; " + .. "benchmark comparability is void", ver) + end + end + -- -B must reach BOTH compile and link: the driver spawns `as` + -- from it at compile time and `ld` from it at link time. + -- Omitting it on either side silently falls through to PATH — + -- where, on a host with xlings shims, `as` can resolve to a + -- stale path and every compile dies. + toolchain:add("cxflags", "-B" .. path.join(binutils, "bin"), {force = true}) + toolchain:add("ldflags", "-B" .. path.join(binutils, "bin"), {force = true}) + if os.isdir(sysroot) then + toolchain:add("cxflags", "--sysroot=" .. sysroot, {force = true}) + toolchain:add("ldflags", "--sysroot=" .. sysroot, {force = true}) + end + end) + toolchain_end() + end + + if llvm_dir then + toolchain("mcpp-clang") + set_kind("standalone") + set_homepage("hermetic llvm payload resolved by mcpp") + set_toolset("cc", path.join(llvm_dir, "bin", "clang")) + set_toolset("cxx", path.join(llvm_dir, "bin", "clang++")) + set_toolset("ld", path.join(llvm_dir, "bin", "clang++")) + set_toolset("sh", path.join(llvm_dir, "bin", "clang++")) + set_toolset("ar", path.join(llvm_dir, "bin", "llvm-ar")) + set_toolset("strip", path.join(llvm_dir, "bin", "llvm-strip")) + -- Same reason as the gcc arm: xrepo package builds resolve these + -- programs through the toolchain, not through the driver. + set_toolset("as", path.join(llvm_dir, "bin", "clang")) + set_toolset("ranlib", path.join(llvm_dir, "bin", "llvm-ranlib")) + set_toolset("nm", path.join(llvm_dir, "bin", "llvm-nm")) + set_toolset("objcopy", path.join(llvm_dir, "bin", "llvm-objcopy")) + -- xmake finds libc++'s `std.cppm` through the SDK dir, and it reads + -- that at DESCRIPTION scope — setting it inside on_load is too late + -- and leaves `std and std.compat modules not found!`, after which + -- `build.c++.modules.std` degrades silently and the arm measures a + -- project that does not use `import std;` against one that does. + set_sdkdir(llvm_dir) + on_load(function (toolchain) + local read_pin = function (m) + if not m or not os.isfile(m) then return nil end + local in_tc = false + for _, line in ipairs((io.readfile(m) or ""):split("\n", {plain = true})) do + local section = line:match("^%s*%[(.-)%]") + if section then in_tc = (section == "toolchain") end + if in_tc then + local f, v = line:match('^%s*default%s*=%s*"([%w_]+)@([%w%.%-]+)"') + if f and v then return f, v end + end + end + return nil + end + local root = llvm_dir + local fam, ver = read_pin(manifest) + if fam == "llvm" and ver then + local pinned = path.join(xpkgs, "xim-x-llvm", ver) + if os.isdir(pinned) then + root = pinned + toolchain:set("toolset", "cc", path.join(pinned, "bin", "clang")) + for _, k in ipairs({"cxx", "ld", "sh"}) do + toolchain:set("toolset", k, path.join(pinned, "bin", "clang++")) + end + else + utils.warning("mcpp.toml pins llvm@%s, absent from the registry; " + .. "benchmark comparability is void", ver) + end + end + -- xmake locates libc++'s `std.cppm` through the LLVM SDK dir, + -- not through the include chain below. Without it: + -- warning: std and std.compat modules not found! + -- and `set_policy("build.c++.modules.std", true)` silently + -- degrades — the arm then measures a project that does not use + -- `import std;` against one that does. + toolchain:set("sdkdir", root) + + -- NOT --sysroot. mcpp drives clang with an explicit include + -- chain instead (verified against a real mcpp compile command), + -- and handing clang gcc's sysroot puts the two arms on different + -- libc — the same class of bug the CMake side documents, where + -- the error names a struct field in and neither the + -- flag nor the target that is wrong. + if os.isdir(path.join(root, "include", "c++", "v1")) then + toolchain:add("cxflags", "--no-default-config", "-nostdinc++", {force = true}) + toolchain:add("cxflags", "-isystem" .. path.join(root, "include", "c++", "v1"), + {force = true}) + -- The per-triple directory carries __config_site; it is + -- absent on some builds, so it is added only when present. + for _, d in ipairs(os.dirs(path.join(root, "include", "*", "c++", "v1"))) do + toolchain:add("cxflags", "-isystem" .. d, {force = true}) + end + toolchain:add("ldflags", "-nostdlib++", "-L" .. path.join(root, "lib"), + "-lc++", "-lc++abi", {force = true}) + end + if glibc_dir and os.isdir(path.join(glibc_dir, "include")) then + toolchain:add("cxflags", "-isystem" .. path.join(glibc_dir, "include"), + {force = true}) + end + if uapi_dir and os.isdir(path.join(uapi_dir, "include")) then + toolchain:add("cxflags", "-isystem" .. path.join(uapi_dir, "include"), + {force = true}) + end + end) + toolchain_end() + end +end + +-- Which toolchain a target should pin, given what the caller asked for. +-- +-- Description-scope safe: it reads no files (see the scope note at the top). +-- The FAMILY is decided here; the exact VERSION is narrowed in on_load. +-- +-- Returns nil when the caller named a non-payload toolchain — an unconditional +-- set_toolchains() SILENTLY OVERRIDES `xmake f --toolchain=llvm`: the benchmark +-- then reports a "clang" cell that was in fact compiled by g++, and the giveaway +-- is only that the number lands suspiciously close to the gcc one. Verify with +-- xmake show -t | grep 'compiler (cxx)' +-- +-- To measure the clang cell, ask for the payload by name: +-- xmake f --toolchain=mcpp-clang +function bench_pinned_toolchain() + local requested = get_config("toolchain") + if requested ~= nil and requested ~= "" then + if requested:startswith("mcpp-") then return requested end + return nil + end + if bench_newest("xim-x-gcc") then return "mcpp-gcc" end + if bench_newest("xim-x-llvm") then return "mcpp-clang" end + return nil +end diff --git a/bench/projects/mcpp/BUILD.bazel b/bench/projects/mcpp/BUILD.bazel new file mode 100644 index 00000000..5836eabf --- /dev/null +++ b/bench/projects/mcpp/BUILD.bazel @@ -0,0 +1,16 @@ +# See MODULE.bazel: this cannot build mcpp today. `import std;` has no bazel +# equivalent, and bazel will not glob sources from outside its workspace. +# +# The shape a working version would take is kept here so the gap is legible: +# +# cc_binary( +# name = "mcpp", +# srcs = ["src/main.cpp"], +# module_interfaces = glob(["src/**/*.cppm"]), +# includes = ["src/libs/json"], +# copts = ["-std=c++23"], +# # ...plus whatever declares `import std;`, which does not exist yet. +# ) +# +# built with: +# bazel build //:mcpp --experimental_cpp_modules --features=cpp_modules --force_pic diff --git a/bench/projects/mcpp/CMakeLists.txt b/bench/projects/mcpp/CMakeLists.txt new file mode 100644 index 00000000..d35288ef --- /dev/null +++ b/bench/projects/mcpp/CMakeLists.txt @@ -0,0 +1,260 @@ +# CMake build description for mcpp — a like-for-like counterpart to mcpp.toml +# and to the xmake.lua beside it. +# +# WHY THIS FILE EXISTS. The build-engine benchmark in bench/ uses cmake as its +# performance baseline, and until now cmake could only build the synthetic +# fixture. The interesting workload is mcpp itself: 139 module interface units, +# 57k lines, every one of them `import std;`. Without this file the real-project +# arm had no baseline to be measured against. +# +# FAIRNESS CONTRACT — all five must hold or the comparison means nothing: +# 1. same compiler binary — the harness passes -DCMAKE_CXX_COMPILER, and the +# payload's binutils + sysroot are added below +# 2. same language flags — -std=c++23, -O2 in release +# 3. same source set — src/**.cppm + src/main.cpp + the pinned +# mcpplibs.cmdline units +# 4. same link output kind — one binary, -static-libstdc++ +# 5. same standard library — `import std;`, not a header shim +# +# Usage (benchmark): +# cmake -G Ninja -S bench/projects/mcpp -B build-cmake \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_CXX_COMPILER=$HOME/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++ +# cmake --build build-cmake + +cmake_minimum_required(VERSION 3.30) + +# `import std;` is still behind an experimental gate whose key changes with the +# CMake version — this is the CMake 4.4 key. Set BEFORE project(), because the +# compiler-support probe that reads it runs during project(). +set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "f35a9ac6-8463-4d38-8eec-5d6008153e7d") + +# The DIALECT settings belong before project() for the same reason as the key +# above, and getting that wrong is not a style question — it is why this arm +# could not build at all. +# +# CMake synthesises its own target for the std module during the compiler probe +# inside project(). That target captures whatever CMAKE_CXX_EXTENSIONS says AT +# THAT MOMENT, and the default is ON. Set OFF afterwards, std.pcm is built as +# `gnu++23` while every real target compiles as `c++23`, and clang refuses the +# mismatch: +# +# error: GNU extensions was enabled in precompiled file 'std.pcm' +# but is currently disabled +# error: precompiled file 'std.pcm' cannot be loaded due to a configuration +# mismatch with the current compilation +# +# `import std;` then supplies nothing and the build dies in 19 places with +# `use of undeclared identifier 'std'` — an error that points at mcpp's sources +# and names neither std.pcm nor extensions. +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/hermetic_payload.cmake) +bench_hermetic_payload_preproject() + +project(mcpp CXX) +# Every mcpp module says `import std;`. This asks CMake to build the standard +# library module from the compiler's own libstdc++.modules.json, which the +# hermetic gcc payload ships. +set(CMAKE_CXX_MODULE_STD 1) + +# ── Say whether the file that makes the line above work is actually there ──── +# +# When it is not, CMake fails at GENERATE — after "Configuring done", so the log +# reads as a successful configure — with +# +# CMake Error in CMakeLists.txt: +# The "CXX_MODULE_STD" property on the target "mcpp" requires that the +# "__CMAKE::CXX23" target exist, but it was not provided by the toolchain. +# Reason: +# Only `libstdc++` is supported +# +# which names neither the file nor the directory it was looked for in, and reads +# like a statement about the standard library rather than about a missing +# manifest. That message stopped the linux/gcc cmake arm on CI while the same +# compiler and the same cmake 4.0.2 configured fine on a developer box, and +# there was no way to tell the two apart from the log. +# +# Not a check — this must not decide whether the build proceeds, because +# CMake's own detection is the authority and may look somewhere this does not. +# It exists so the next failure carries its cause. +get_filename_component(_mcpp_cxx_dir "${CMAKE_CXX_COMPILER}" DIRECTORY) +get_filename_component(_mcpp_cxx_prefix "${_mcpp_cxx_dir}" DIRECTORY) +set(_mcpp_std_json "") +foreach(_d lib64 lib lib/gcc) + file(GLOB_RECURSE _found "${_mcpp_cxx_prefix}/${_d}/libstdc++.modules.json") + if(_found) + list(GET _found 0 _mcpp_std_json) + break() + endif() +endforeach() +if(_mcpp_std_json) + message(STATUS "bench: std module manifest: ${_mcpp_std_json}") + # ...and whether the SOURCE it names is actually there. A present manifest + # pointing at an absent `std.cc` fails exactly like an absent manifest, and + # CMake's message ("Only `libstdc++` is supported") distinguishes neither. + # This is the last untested difference between a developer box, where the arm + # generates fine, and the runner, where the manifest IS present and generate + # still fails — so the next matrix decides it instead of another round of + # hypotheses. + get_filename_component(_mcpp_json_dir "${_mcpp_std_json}" DIRECTORY) + file(READ "${_mcpp_std_json}" _mcpp_json_text) + string(JSON _mcpp_mod_count ERROR_VARIABLE _mcpp_json_err + LENGTH "${_mcpp_json_text}" modules) + if(_mcpp_json_err) + message(STATUS "bench: could not read `modules` from the manifest: ${_mcpp_json_err}") + else() + math(EXPR _mcpp_last "${_mcpp_mod_count} - 1") + foreach(_i RANGE 0 ${_mcpp_last}) + string(JSON _mcpp_src ERROR_VARIABLE _e GET "${_mcpp_json_text}" + modules ${_i} source-path) + string(JSON _mcpp_name ERROR_VARIABLE _e2 GET "${_mcpp_json_text}" + modules ${_i} logical-name) + if(_mcpp_src) + get_filename_component(_mcpp_abs "${_mcpp_json_dir}/${_mcpp_src}" ABSOLUTE) + if(EXISTS "${_mcpp_abs}") + message(STATUS "bench: ${_mcpp_name} -> ${_mcpp_abs} (present)") + else() + message(STATUS "bench: ${_mcpp_name} -> ${_mcpp_abs} MISSING — the " + "manifest is there but its source is not, which fails " + "exactly like no manifest at all") + endif() + endif() + endforeach() + endif() +else() + message(STATUS "bench: NO libstdc++.modules.json under ${_mcpp_cxx_prefix} — " + "CMAKE_CXX_MODULE_STD cannot work and generate will fail with " + "\"__CMAKE::CXX23 ... not provided by the toolchain\"; the gcc " + "payload on this machine does not ship it") +endif() + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() + +# --------------------------------------------------------------------------- +# Where the tree is, and the hermetic payload that makes this a fair comparison. +# +# The payload logic is SHARED with the xlings arm — see +# ../common/cmake/hermetic_payload.cmake, including why it is one branch per +# compiler family rather than one block of flags, and why it must be applied +# through CMAKE_CXX_FLAGS. +# --------------------------------------------------------------------------- +# The tree is whatever the harness points at — BENCH_PROJECT_ROOT, exported for +# every `--project` run. It used to be derived from this file's own location +# (`../../..`, i.e. the checkout), and that made mcpp's WORKING TREE the +# workload: every commit on a branch silently changed the thing being measured, +# so no two runs on that branch were comparable to each other. +# +# A benchmark has two halves. The engine under test is the binary and is +# supposed to move; the workload is not. It is now the pinned submodule +# `mcpp-/` beside this file, exactly like the xlings arms. +# +# Resolved to an absolute path, because a FILE_SET's base directory and a +# relative glob disagree about what "here" means. +if(NOT MCPP_ROOT AND DEFINED ENV{BENCH_PROJECT_ROOT}) + set(MCPP_ROOT "$ENV{BENCH_PROJECT_ROOT}") +endif() +if(NOT MCPP_ROOT) + # Driving this by hand: default to the pinned workload rather than to the + # checkout, so the hand-run and the CI run measure the same sources. + file(GLOB pinned "${CMAKE_CURRENT_SOURCE_DIR}/mcpp-*") + if(pinned) + list(SORT pinned) + list(GET pinned -1 MCPP_ROOT) + endif() +endif() +get_filename_component(MCPP_ROOT "${MCPP_ROOT}" ABSOLUTE) +if(NOT MCPP_ROOT OR NOT EXISTS "${MCPP_ROOT}/mcpp.toml") + message(FATAL_ERROR + "no mcpp tree at '${MCPP_ROOT}': set -DMCPP_ROOT=, or let the bench " + "harness export BENCH_PROJECT_ROOT via --project. The pinned workload is the " + "submodule bench/projects/mcpp/mcpp-/ — run `git submodule update --init`.") +endif() + +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/hermetic_payload.cmake) +bench_hermetic_payload() +bench_registry_xpkgs(MCPP_XPKGS) + +# --------------------------------------------------------------------------- +# Source set — mcpp.toml's inferred glob `src/**/*.{cppm,cpp}`. mcpp infers +# kind=bin from src/main.cpp; CMake needs it spelled out. +# +# GLOB, not a hand-written list: the two arms must compile the same files even +# as the tree changes, and a list that drifts silently measures two different +# projects. CONFIGURE_DEPENDS re-globs on build so an added module is not missed. +# --------------------------------------------------------------------------- +file(GLOB_RECURSE MCPP_MODULES CONFIGURE_DEPENDS "${MCPP_ROOT}/src/*.cppm") +# .cpp is globbed for the same reason, even though mcpp has exactly one today: +# the xlings arm found this the hard way. Naming `src/main.cpp` by hand keeps +# working right up until implementations are split out of the interface units, +# and then it compiles the interfaces, links nothing, and still reports a time. +file(GLOB_RECURSE MCPP_SOURCES CONFIGURE_DEPENDS "${MCPP_ROOT}/src/*.cpp") +list(LENGTH MCPP_MODULES MCPP_MODULE_COUNT) +list(LENGTH MCPP_SOURCES MCPP_SOURCE_COUNT) +if(MCPP_MODULE_COUNT EQUAL 0) + message(FATAL_ERROR "no module interface units found under src/ — refusing to " + "build a project that is not mcpp") +endif() + +# mcpp.toml pins `mcpplibs.cmdline = "0.0.1"` EXACTLY. Newer versions are often +# also unpacked in the registry, so pin rather than take the newest: otherwise +# the two arms are not compiling the same code. +# +# mcpp stages prebuilt objects for this dependency out of its global build cache +# and cmake has no such cache, so cmake compiles the 3 units from source. That is +# a small handicap on cmake's cold build, and it is declared in the benchmark +# report rather than hidden. +set(MCPP_CMDLINE_VERSION "0.0.1") +set(MCPP_CMDLINE_SRC + "${MCPP_XPKGS}/mcpplibs-x-cmdline/${MCPP_CMDLINE_VERSION}/cmdline-${MCPP_CMDLINE_VERSION}/src") +if(IS_DIRECTORY "${MCPP_CMDLINE_SRC}") + file(GLOB MCPP_CMDLINE_MODULES CONFIGURE_DEPENDS "${MCPP_CMDLINE_SRC}/*.cppm") +else() + # FATAL, not a warning. Warning here builds mcpp WITHOUT three of its units + # and the failure lands at the link as `undefined reference to ...cmdline...`, + # naming a consumer rather than the missing package — and on CI it did exactly + # that. A description that cannot name the same sources mcpp compiled is not a + # comparison arm. (The xmake arm beside this one raises for the same reason.) + message(FATAL_ERROR + "mcpplibs.cmdline ${MCPP_CMDLINE_VERSION} is not unpacked at " + "${MCPP_CMDLINE_SRC} — build the tree with mcpp once first, so both " + "arms compile the same dependency sources. A cache hit does NOT unpack " + "them; bench.yml drops the cached package before the matrix for this.") +endif() + +add_executable(mcpp ${MCPP_SOURCES}) + +# FILE_SET CXX_MODULES is the only way CMake learns these are interface units. +# Listing them as ordinary sources compiles them as plain TUs and the link fails +# with missing module symbols. +target_sources(mcpp + PRIVATE + FILE_SET CXX_MODULES BASE_DIRS "${MCPP_ROOT}/src" FILES ${MCPP_MODULES} +) + +# The dependency's units need their OWN file set: a CXX_MODULES set requires +# every file to live under one of its base directories, which defaults to the +# project source dir, and these live in the registry outside the tree. +if(MCPP_CMDLINE_MODULES) + target_sources(mcpp + PRIVATE + FILE_SET mcpp_cmdline_modules + TYPE CXX_MODULES + BASE_DIRS "${MCPP_CMDLINE_SRC}" + FILES ${MCPP_CMDLINE_MODULES} + ) +endif() + +# mcpp.toml: include_dirs = ["src/libs/json"] — src/libs/json.cppm reaches for +# from its global module fragment. +target_include_directories(mcpp PRIVATE "${MCPP_ROOT}/src/libs/json") + +# mcpp.toml default: static_stdlib = true, so the binary is portable. +target_link_options(mcpp PRIVATE -static-libstdc++) + +message(STATUS "mcpp: ${MCPP_MODULE_COUNT} module interface units + " + "${MCPP_SOURCE_COUNT} .cpp") diff --git a/bench/projects/mcpp/MODULE.bazel b/bench/projects/mcpp/MODULE.bazel new file mode 100644 index 00000000..e12ea2ca --- /dev/null +++ b/bench/projects/mcpp/MODULE.bazel @@ -0,0 +1,38 @@ +# bazel module for mcpp — BEST EFFORT, AND IT DOES NOT BUILD. +# +# bazel 9.2.0 + rules_cc 0.2.22 CAN build C++20 named modules — measured, with +# `module_interfaces` plus --experimental_cpp_modules --features=cpp_modules, +# and clang (its ddi aggregator cannot parse GCC's P1689 output). For the +# synthetic fixture that is enough. For mcpp it is not, for two reasons: +# +# 1. `import std;` works, but only by hand — CORRECTED, an earlier version of +# this comment claimed it was impossible. bazel has no counterpart to +# CMake's CXX_MODULE_STD, and its modmap generator fails with +# ERROR: Module not found: std +# but libc++ ships the std module as ORDINARY SOURCE, so it can be built +# like any other interface unit. Measured working on bazel 9.2.0: +# +# cp $LLVM/share/libc++/v1/std.cppm . +# cp -r $LLVM/share/libc++/v1/std . # 110 .inc files it includes +# cc_binary( +# srcs = ["main.cpp"] + glob(["std/**"]), +# module_interfaces = ["std.cppm", "m.cppm"], # std FIRST +# copts = ["-std=c++23", "-Wno-reserved-module-identifier"], +# ) +# bazel build //:t --experimental_cpp_modules --features=cpp_modules --force_pic +# +# (No `includes` attribute: "." is rejected as the workspace root, and the +# .inc files sit beside std.cppm in the sandbox anyway.) +# +# 2. Workspace boundary. mcpp's sources live three directories up from here. +# bazel will not glob outside its workspace, so a working setup would have +# to put MODULE.bazel at the repository root — exactly what these files +# were moved out of the root to avoid. +# +# 3. THE ONE THAT DECIDES IT: bazel builds C++20 modules only with clang (its +# ddi aggregator cannot parse GCC's P1689 output). mcpp, cmake and xmake +# are measured here against gcc@16.1.0. A bazel column in a gcc table would +# violate fairness invariant I1 — same compiler binary for every engine — +# so bazel belongs in a separate clang-baselined table, not this one. +module(name = "mcpp", version = "2026.8.13.1") +bazel_dep(name = "rules_cc", version = "0.2.22") diff --git a/bench/projects/mcpp/mcpp-2026.8.11.3 b/bench/projects/mcpp/mcpp-2026.8.11.3 new file mode 160000 index 00000000..a749e9f7 --- /dev/null +++ b/bench/projects/mcpp/mcpp-2026.8.11.3 @@ -0,0 +1 @@ +Subproject commit a749e9f723f747b96e601b90c7535c6c92421a5c diff --git a/bench/projects/mcpp/xmake.lua b/bench/projects/mcpp/xmake.lua new file mode 100644 index 00000000..b847b521 --- /dev/null +++ b/bench/projects/mcpp/xmake.lua @@ -0,0 +1,205 @@ +-- xmake build description for mcpp — a like-for-like counterpart to mcpp.toml. +-- +-- Lives under bench/projects/mcpp/ rather than at the repository root: mcpp is +-- built by mcpp, and a second build description at the root is something a +-- contributor has to learn to ignore. It is used only by the benchmark. +-- +-- Why this file exists: it is the control arm of the build-engine benchmark in +-- tools/bench/. mcpp builds itself; this makes xmake build the exact same 137 +-- module interface units + src/main.cpp with the exact same compiler binary, so +-- any wall-clock difference is attributable to the build engine (graph shape, +-- scheduling, staleness model) and not to a different toolchain. +-- +-- Fairness contract (all four must hold or the comparison is meaningless): +-- 1. same compiler binary -- pinned below to the hermetic payload mcpp resolves +-- 2. same language flags -- -std=c++23 -fmodules -O2 (release) / -O0 -g (debug) +-- 3. same source set -- src/**.cppm + src/main.cpp + the cmdline dependency +-- 4. same link output kind -- one binary, -static-libstdc++ +-- +-- Usage (benchmark), from the repository root: +-- xmake f -P bench/projects/mcpp -y -m release --toolchain=mcpp-gcc +-- xmake build -P bench/projects/mcpp -j32 +-- Usage (plain host toolchain, no pinning): +-- xmake f -y -m release --pin_payload=n && xmake build + +set_project("mcpp") +set_xmakever("2.9.0") +set_languages("c++23") +add_rules("mode.debug", "mode.release") + +-- --------------------------------------------------------------------------- +-- The hermetic payload and the toolchain-per-family definitions are SHARED with +-- the xlings arm: ../common/xmake/payload.lua. They used to be a 60-line copy in +-- each file, which is the worst place for a copy — two toolchain definitions +-- drift by one flag and the benchmark reports the difference between the two +-- DESCRIPTIONS as an engine result. +-- --------------------------------------------------------------------------- +includes("../common/xmake/payload.lua") + +-- The tree this file builds is whatever the harness points at — +-- BENCH_PROJECT_ROOT, exported for every `--project` run. +-- +-- It used to be `os.scriptdir()/../../..`, i.e. the checkout, which made mcpp's +-- WORKING TREE the workload: every commit on a branch silently changed the thing +-- being measured, so no two runs on that branch were comparable. The engine +-- under test is the binary and is supposed to move; the workload is not. +-- +-- Driving this by hand falls back to the pinned submodule beside this file +-- rather than to the checkout, so a hand-run and a CI run measure the same +-- sources. +local MCPP_ROOT = os.getenv("BENCH_PROJECT_ROOT") or os.getenv("MCPP_ROOT") +if not MCPP_ROOT then + local pinned = os.dirs(path.join(os.scriptdir(), "mcpp-*")) + if pinned and #pinned > 0 then + table.sort(pinned) + MCPP_ROOT = pinned[#pinned] + end +end +if not MCPP_ROOT or not os.isfile(path.join(MCPP_ROOT, "mcpp.toml")) then + raise("no mcpp tree: set MCPP_ROOT=, or let the bench harness export " + .. "BENCH_PROJECT_ROOT via --project. The pinned workload is the " + .. "submodule bench/projects/mcpp/mcpp-/ — run " + .. "`git submodule update --init`.") +end +MCPP_ROOT = path.normalize(MCPP_ROOT) +local MCPP_MANIFEST = path.join(MCPP_ROOT, "mcpp.toml") + +-- Which version of mcpplibs.cmdline to compile is read from the measured tree's +-- own mcpp.lock, NOT hardcoded and NOT "the newest unpacked". +-- +-- * hardcoding it is what broke CI: this said "0.0.1" because mcpp.toml says +-- `mcpplibs.cmdline = "0.0.1"`, but that is a REQUIREMENT, not a resolution. +-- A developer box that had 0.0.1 unpacked from some earlier run worked; a +-- fresh runner had only what mcpp resolved, `bench_package_root` returned +-- nil, the `if` below quietly added no files, and the build died 137 units +-- later with `missing mcpplibs.cmdline dependency for module mcpp.cli` — +-- an error that names neither the version nor the registry. +-- * "the newest unpacked" would silently compile different sources than mcpp +-- did, which is the one thing a comparison arm may not do. +-- +-- The lockfile is the resolution mcpp itself performed on this exact tree, so +-- both arms compile the same code by construction. +-- Read inside on_load, not here: `io` is nil in xmake's DESCRIPTION scope, so a +-- reader written at this level dies with `attempt to index a nil value (global +-- 'io')` — the same trap ../common/xmake/payload.lua documents for its manifest +-- reader, walked into a second time. Both the lock path and the registry root +-- are captured as upvalues for the same reason: on_load's sandbox cannot see +-- this file's globals, so `bench_package_root` is not callable from in there. +local MCPP_LOCK = path.join(MCPP_ROOT, "mcpp.lock") +local XPKGS = bench_xpkgs() + +option("pin_payload") + set_default(true) + set_showmenu(true) + set_description("Pin the hermetic mcpp toolchain payload (required for a fair benchmark)") +option_end() + +bench_define_toolchains(MCPP_MANIFEST) + +-- --------------------------------------------------------------------------- +-- The one and only target: mcpp's CLI binary. +-- --------------------------------------------------------------------------- +target("mcpp") + set_kind("binary") + + -- Source set == mcpp.toml's inferred glob src/**/*.{cppm,cpp}. mcpp infers + -- kind=bin from src/main.cpp; xmake needs it spelled out. + add_files(path.join(MCPP_ROOT, "src/**.cppm")) + -- .cpp is globbed even though mcpp has exactly one today: naming it by + -- hand keeps working right until implementations are split out of the + -- interface units (which is what xlings did), and then this compiles + -- the interfaces, links nothing, and still reports a time. + add_files(path.join(MCPP_ROOT, "src/**.cpp")) + + -- mcpp.toml: include_dirs = ["src/libs/json"] — src/libs/json.cppm reaches + -- for from its global module fragment. + add_includedirs(path.join(MCPP_ROOT, "src/libs/json")) + + -- mcpp.toml: [dependencies] mcpplibs.cmdline = "0.0.1". + -- mcpp stages prebuilt objects for this out of its global build cache; xmake + -- has no such cache, so it compiles the 3 units from source. That is a ~1s + -- handicap on xmake's cold build and is called out in the benchmark report + -- rather than hidden. + -- + -- Absence is FATAL rather than skipped. Skipping produced a build missing + -- three units out of 140 that announced itself only as + -- `missing mcpplibs.cmdline dependency for module mcpp.cli.cmd_cache` — + -- naming a consumer instead of the cause. A description that cannot name the + -- same sources mcpp compiled is not a comparison arm. + on_load(function (target) + local ver + if os.isfile(MCPP_LOCK) then + local in_section = false + for _, line in ipairs((io.readfile(MCPP_LOCK) or ""):split("\n", {plain = true})) do + local section = line:match("^%s*%[(.-)%]") + if section then in_section = (section == 'package."mcpplibs.cmdline"') + elseif in_section then + ver = ver or line:match('^%s*version%s*=%s*"([^"]+)"') + end + end + end + if not ver then + raise("bench: cannot read mcpplibs.cmdline's resolved version from " .. MCPP_LOCK + .. " — the measured tree must carry the lockfile mcpp resolved it with") + end + local base = path.join(XPKGS, "mcpplibs-x-cmdline", ver) + local dirs = os.isdir(base) and os.dirs(path.join(base, "*")) or {} + table.sort(dirs) + + -- NAME THE DIRECTORY, DO NOT SEARCH FOR IT. + -- + -- `cmdline-` is the registry's layout and is exactly what the + -- cmake arm beside this one writes down — which is why cmake's five + -- cells were green in the same run where these five were red. Two arms + -- that must compile the SAME sources cannot locate them two ways. + -- + -- Searching was wrong twice over. `dirs[1]` after a sort is "whatever + -- happens to come first", and the registry keeps a tarball and a lock + -- beside the unpacked tree while mcpp writes partial directories during + -- unpacking. Widening it to "the first directory that has a src/" is no + -- better: renaming the real tree to `cmdline-0.0.1.hidden` to test the + -- error path made this file compile THAT instead, silently, and report + -- success. A backup directory is a plausible thing to find on a machine. + local canonical = path.join(base, "cmdline-" .. ver, "src") + local src = os.isdir(canonical) and canonical or nil + if not src then + -- ⚠️ SAY WHAT WAS LOOKED AT. The previous message named the version + -- and the registry root and stopped there, so a CI failure could not + -- be told apart from "the package is genuinely absent", "the version + -- came out wrong", or "the directory is there but holds something + -- else". Three different causes, one sentence, none of them + -- actionable without a runner to log into. + local found = #dirs > 0 and table.concat(dirs, ", ") or "(nothing)" + raise("bench: mcpplibs.cmdline " .. ver .. " has no unpacked source tree.\n" + .. " expected: " .. canonical .. "\n" + .. " directories under " .. base .. ": " .. found .. "\n" + .. " base exists: " .. tostring(os.isdir(base)) .. "\n" + .. " Build the tree with mcpp once first, so both arms compile the " + .. "same dependency sources. A cache hit does NOT unpack them.") + end + target:add("files", path.join(src, "*.cppm")) + end) + + set_policy("build.c++.modules", true) + set_policy("build.c++.modules.std", true) + + -- mcpp.toml default: static_stdlib = true (portable binary). + add_ldflags("-static-libstdc++", {force = true}) + + if is_mode("release") then + set_optimize("fastest") -- -O2, matching mcpp's release profile + set_symbols("hidden") + elseif is_mode("debug") then + set_optimize("none") -- -O0 -g, matching mcpp's dev profile + set_symbols("debug") + end + + -- Which toolchain, and the rule for when NOT to pin one, live in + -- ../common/xmake/payload.lua — an unconditional set_toolchains() here + -- silently overrides `xmake f --toolchain=...` and the benchmark reports a + -- cell compiled by the wrong compiler. + if has_config("pin_payload") then + local tc = bench_pinned_toolchain() + if tc then set_toolchains(tc) end + end +target_end() diff --git a/bench/projects/xlings/.bazelignore b/bench/projects/xlings/.bazelignore new file mode 100644 index 00000000..649e082c --- /dev/null +++ b/bench/projects/xlings/.bazelignore @@ -0,0 +1,15 @@ +# The pinned trees are consumed through @xlings_tree, which symlinks the ONE +# named by BENCH_PROJECT_ROOT. Left visible here they would also be part of this +# package, so `bazel build //...` would glob both of them into every target it +# found — and a cell that measures two builds and reports one is worse than no +# cell. Ignoring them is what makes "//... builds the tree under measurement" +# true rather than approximately true. +xlings-2026.8.11.2 +xlings-2026.8.13.1 + +# Output directories of the other arms. cmake's build/ and mcpp's target/ hold +# tens of thousands of files with no BUILD file among them; scanning them costs +# real time on every `bazel build //...`. +build +target +.xmake diff --git a/bench/projects/xlings/.bazelrc b/bench/projects/xlings/.bazelrc new file mode 100644 index 00000000..97759b89 --- /dev/null +++ b/bench/projects/xlings/.bazelrc @@ -0,0 +1,45 @@ +# Flags this arm cannot build without. bench/src/engines/bazel.cppm passes the +# first three itself; they are repeated here so that the command lines in +# MODULE.bazel and README.md work as written, and so a hand-run build measures +# the same thing the harness does. + +# Each one's absence is a different error, and none of them says "add this flag": +# without --experimental_cpp_modules: `attribute module_interfaces: requires +# --experimental_cpp_modules` +# without --features=cpp_modules: `the feature cpp_modules must be enabled` +build --experimental_cpp_modules +build --features=cpp_modules + +# cc_binary registers the ddi aggregation action for BOTH the PIC and non-PIC +# object sets but names its output .CXXModules.json without a pic +# suffix, so analysis dies before a single file compiles: +# Attempted action contains artifacts not in previous action: +# _objs/xlings/main.pic.ddi ... Outputs: are equal +# Forcing one object flavour leaves one action. PIC rather than +# --features=-supports_pic because it yields a PIE executable, which is what +# gcc and clang produce by default for every other engine in the table. +build --force_pic + +# ⚠️ --compilation_mode=opt DOES NOT BUILD WITHOUT THIS, and the harness always +# passes opt for the release profile. bazel's opt mode appends +# `-D_FORTIFY_SOURCE=1` after its own `-U_FORTIFY_SOURCE`, glibc then replaces +# the string/stdio/wchar functions with `__fortify_function` wrappers — which +# have INTERNAL LINKAGE — and libc++'s std module cannot re-export them: +# +# libcxx_module/std/cwchar.inc:30:14: error: using declaration referring to +# 'swprintf' with internal linkage cannot be exported +# note: target of using declaration +# .../xim-x-glibc/2.44/include/bits/wchar2.h:181:8 +# +# It reads as a libc++/glibc incompatibility and is a build flag. --copt lands +# after the compilation-mode flags, so the -U wins; no other engine in this +# table defines _FORTIFY_SOURCE either, so removing it is also what keeps the +# arms comparable rather than a local workaround. +build --copt=-U_FORTIFY_SOURCE + +# The autoconfigured toolchain appends `-lstdc++ -lm` to every link. This arm is +# libc++ (the payload clang's config file sets -stdlib=libc++), and linking the +# two standard libraries into one binary is a coin flip decided by link order: +# --as-needed happens to drop libstdc++ here, and "happens to" is not a +# guarantee anyone should rely on for a 46k-line binary. +build --repo_env=BAZEL_LINKLIBS=-lm diff --git a/bench/projects/xlings/BUILD.bazel b/bench/projects/xlings/BUILD.bazel new file mode 100644 index 00000000..6a1a076b --- /dev/null +++ b/bench/projects/xlings/BUILD.bazel @@ -0,0 +1,45 @@ +# The bazel arm of the xlings control target. See MODULE.bazel for why the two +# interesting things — the tree and its dependencies — are external repositories +# rather than globs, and mcpp_registry.bzl for how they are built. +# +# THIS PACKAGE DECLARES ONE RULE ON PURPOSE, and that one rule is the point of +# the file. It used to declare none: a comment-only BUILD.bazel makes +# `bazel build //...` exit 0 in 0.14s with `Found 0 targets`, and the harness +# published that as a 0.43s cold build in a column next to cmake's 94s. +# bench/src/engines/bazel.cppm now asks `bazel query kind(rule, //...)` before it +# will measure anything here, so an empty description reports a reason instead of +# a number — but the fix for "no rules" is a rule. +# +# The alias, rather than the cc_binary itself, because `module_interfaces` takes +# labels bazel can glob and the sources live in a submodule this package must not +# write into. @xlings_tree carries the generated cc_binary; see _XLINGS_BUILD in +# mcpp_registry.bzl for the source set and why it is not `src/main.cpp` alone. +# +# MEASURED, on bazel 9.2.0 + rules_cc 0.2.22 with the payload clang 22.1.8, +# `--compilation_mode=opt`, cold after `bazel clean`: +# +# tree TUs compiled cold `--version` +# xlings-2026.8.11.2 110 .cppm + 2 .cpp 46.4s xlings 2026.8.11.2 +# xlings-2026.8.13.1 110 .cppm + 92 .cpp 18.0s xlings 2026.8.13.1 +# +# plus the dependency set, whose counts are the check that the .xpkg.lua +# manifests are being read rather than the trees globbed: libarchive 127 (a glob +# is 132), lua 32 (a glob is 34, and the two extra each define `main`), mbedtls +# 108, xz 74, ftxui 73, zstd 26, zlib 15, bzip2 7, lz4 5, and 19 translation +# units across the four mcpplibs packages — 18 of them module interfaces, one of +# which is the generated mcpplibs.xpkg.lua_stdlib. +# +# bazel build //:xlings # the default pin +# BENCH_PROJECT_ROOT=$PWD/xlings-2026.8.11.2 bazel build //:xlings +# bazel run //:xlings -- --version +# +# The binary itself lands at +# bazel-bin/external/+xlings_tree+xlings_tree/xlings +# because that is the repository that owns the rule; `bazel run` above is the +# path-independent way to reach it. + +alias( + name = "xlings", + actual = "@xlings_tree//:xlings", + visibility = ["//visibility:public"], +) diff --git a/bench/projects/xlings/CMakeLists.txt b/bench/projects/xlings/CMakeLists.txt new file mode 100644 index 00000000..3ab1dac6 --- /dev/null +++ b/bench/projects/xlings/CMakeLists.txt @@ -0,0 +1,276 @@ +# CMake build description for xlings — the benchmark's independent control target. +# +# WHY THIS EXISTS. mcpp measuring its own build proves nothing about build +# performance in general: an engine change can be an artefact of one project's +# module graph. xlings is written by different people against a different +# structure (110 module interface units, 46k lines, 6 dependencies), so a result +# that reproduces here is a result about the engine rather than about mcpp. +# +# THE TREE IS PINNED, not vendored: the two code styles live beside this file as +# git submodules (`xlings-2026.8.11.2`, `xlings-2026.8.13.1`). The harness points +# this description at one of them through BENCH_PROJECT_ROOT; by hand: +# +# cmake -G Ninja -S bench/projects/xlings -B build-xlings \ +# -DXLINGS_ROOT=bench/projects/xlings/xlings-2026.8.13.1 \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_CXX_COMPILER=$HOME/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++ +# cmake --build build-xlings +# +# FAIRNESS CONTRACT — the same five as bench/projects/mcpp/CMakeLists.txt: +# same compiler binary, same language flags, same source set, same link output +# kind, same standard library (`import std;`, not a header shim). +# +# THIS ARM USED TO STOP AT THE LINK, and that was recorded as a "known gap". It +# was not a gap — it was unfinished work, and the note saying otherwise let it +# sit. xlings links ftxui, libarchive, lua and mbedtls, all of which arrive in +# mcpp's registry as SOURCE; this description found their headers, compiled all +# 110 units, and then failed on ~1371 undefined `archive_*` / `mbedtls_*` / +# `lua_*`. They are compiled here now and the arm links. +# +# Three things that had looked like boundaries and were not: +# +# * Transitive headers. Every one is unpacked in mcpp's registry, and they now +# arrive as the PUBLIC include dirs of the dependency targets below rather +# than as a hand-written walk of registry subdirectories. +# * The source packages themselves. Their vendored CMakeLists cannot be used +# (mbedtls 3.6.1 FATAL_ERRORs unconditionally on a submodule the registry +# tarball does not carry) and a glob compiles the wrong file set, but every +# package ships a `.xpkg.lua` naming exactly what mcpp compiles — so +# xpkg_source_library.cmake builds them from that. +# * `mcpplibs.xpkg.lua_stdlib`, which is GENERATED by that package's +# `build.mcpp` rather than checked in. It embeds eleven `.lua` files as +# strings — small and fully specified, so `embed_lua_stdlib.cmake` +# reproduces it. "mcpp runs a build program" is not by itself a boundary. + +cmake_minimum_required(VERSION 3.30) + +# `import std;` is still behind an experimental gate whose key changes with the +# CMake version — this is the CMake 4.4 key. Must be set BEFORE project(). +set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "f35a9ac6-8463-4d38-8eec-5d6008153e7d") + +# Before project() for the same reason as the key above: CMake builds the std +# module through a target it synthesises during the compiler probe inside +# project(), and that target captures CMAKE_CXX_EXTENSIONS as it stands right +# then — default ON. Setting OFF afterwards yields a `gnu++23` std.pcm that no +# `c++23` target can load, and the build fails with `use of undeclared +# identifier 'std'` pointing at the project's own sources. See the identical +# block in ../mcpp/CMakeLists.txt, where this was diagnosed. +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# C IS A SECOND LANGUAGE HERE — libarchive, lua, mbedtls and the five +# compression libraries under libarchive are C, and this arm compiles them from +# source (see the dependency section). `project(xlings CXX)` alone leaves +# CMAKE_C_COMPILER unset and every .c source without a rule. +# +# The harness passes ONLY -DCMAKE_CXX_COMPILER (bench/src/engines/cmake.cppm), +# so the C driver is derived from the C++ one — the same sibling-driver rule +# mcpp applies to route a .c source through the toolchain it was handed. Left +# to CMake, `project(… C …)` would search PATH and find the HOST cc, and the +# hermetic payload in ../common/cmake/hermetic_payload.cmake — which keys off +# CMAKE_CXX_COMPILER — would then point a host gcc at the registry's sysroot. +# That is the two-libc failure its own comment describes, arriving through the +# C half of the build. +if(NOT CMAKE_C_COMPILER AND CMAKE_CXX_COMPILER) + get_filename_component(_cxx_dir "${CMAKE_CXX_COMPILER}" DIRECTORY) + get_filename_component(_cxx_name "${CMAKE_CXX_COMPILER}" NAME) + string(REGEX REPLACE "clang\\+\\+" "clang" _cc_name "${_cxx_name}") + if(_cc_name STREQUAL _cxx_name) + string(REGEX REPLACE "g\\+\\+" "gcc" _cc_name "${_cxx_name}") + endif() + if(NOT _cc_name STREQUAL _cxx_name AND EXISTS "${_cxx_dir}/${_cc_name}") + set(CMAKE_C_COMPILER "${_cxx_dir}/${_cc_name}") + endif() +endif() + +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/hermetic_payload.cmake) +bench_hermetic_payload_preproject() + +project(xlings C CXX) +set(CMAKE_CXX_MODULE_STD 1) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() + +# --------------------------------------------------------------------------- +# Where the tree is. +# --------------------------------------------------------------------------- +# BENCH_PROJECT_ROOT is what the harness exports for every --project run, and it +# is the reason this description can be shared by both pinned trees. -DXLINGS_ROOT +# and the XLINGS_ROOT environment variable stay supported for driving it by hand. +if(NOT XLINGS_ROOT AND DEFINED ENV{BENCH_PROJECT_ROOT}) + set(XLINGS_ROOT "$ENV{BENCH_PROJECT_ROOT}") +endif() +if(NOT XLINGS_ROOT AND DEFINED ENV{XLINGS_ROOT}) + set(XLINGS_ROOT "$ENV{XLINGS_ROOT}") +endif() +if(NOT XLINGS_ROOT OR NOT EXISTS "${XLINGS_ROOT}/mcpp.toml") + message(FATAL_ERROR + "no xlings tree: set -DXLINGS_ROOT=, or let the bench harness export " + "BENCH_PROJECT_ROOT via --project. The pinned trees are the submodules " + "bench/projects/xlings/xlings-/ — run `git submodule update --init`.") +endif() + +# --------------------------------------------------------------------------- +# The hermetic payload — SHARED with the mcpp arm. See +# ../common/cmake/hermetic_payload.cmake, including why it is one branch per +# compiler family (gcc gets -B/--sysroot, clang gets an explicit include chain, +# msvc gets nothing because mcpp uses the system Visual Studio too). +# --------------------------------------------------------------------------- +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/hermetic_payload.cmake) +bench_hermetic_payload() +bench_registry_xpkgs(MCPP_XPKGS) + +# --------------------------------------------------------------------------- +# Source set — xlings' mcpp.toml infers `src/**/*.{cppm,cpp}` and names +# src/main.cpp as the binary's entry point. +# +# BOTH .cppm AND .cpp ARE GLOBBED, and that is what lets ONE description measure +# xlings' two code styles: +# +# 2026.8.11.2 110 .cppm + 2 .cpp — each interface unit carries its own +# implementation +# 2026.8.13.1 110 .cppm + 92 .cpp — interface and implementation split +# +# Same module graph, same 46k lines, opposite answers to "where does the code +# live" — which is exactly the `modules` vs `modules-impl` axis the generated +# fixture has, on a real tree. A glob spans both because it is the same rule +# mcpp itself infers from; branching on the style (or carrying two description +# files, or an env var) would be one more thing that can differ between the arms +# for a reason that is not the engine. +# +# It also matters that this is not `main.cpp` alone: against the split style +# that description compiles the interfaces, links nothing, and reports a number +# for a build that never happened. +# --------------------------------------------------------------------------- +file(GLOB_RECURSE XLINGS_MODULES CONFIGURE_DEPENDS "${XLINGS_ROOT}/src/*.cppm") +file(GLOB_RECURSE XLINGS_SOURCES CONFIGURE_DEPENDS "${XLINGS_ROOT}/src/*.cpp") +list(LENGTH XLINGS_MODULES XLINGS_MODULE_COUNT) +list(LENGTH XLINGS_SOURCES XLINGS_SOURCE_COUNT) +if(XLINGS_MODULE_COUNT EQUAL 0) + message(FATAL_ERROR "no module interface units under ${XLINGS_ROOT}/src") +endif() +if(XLINGS_SOURCE_COUNT EQUAL 0) + message(FATAL_ERROR "no .cpp under ${XLINGS_ROOT}/src — not even main.cpp; " + "is XLINGS_ROOT pointing at a checkout?") +endif() + +add_executable(xlings ${XLINGS_SOURCES}) +target_sources(xlings + PRIVATE FILE_SET CXX_MODULES BASE_DIRS "${XLINGS_ROOT}/src" FILES ${XLINGS_MODULES}) + +# `[build] include_dirs = ["src/libs/json"]` — src/libs/json.cppm reaches for +# from its global module fragment. +target_include_directories(xlings PRIVATE "${XLINGS_ROOT}/src/libs/json") +# `[build] cxxflags` +target_compile_definitions(xlings PRIVATE LIBARCHIVE_STATIC UNICODE _UNICODE) + +# --------------------------------------------------------------------------- +# Dependencies. +# +# The four mcpplibs packages ship SOURCE and are compiled from it — each needs +# its own FILE_SET, because a CXX_MODULES set requires every file to live under +# one of its base directories and these sit in the registry outside the tree. +# +# Versions are PINNED to xlings' mcpp.toml. Newer ones are usually also unpacked +# in the registry, and taking the newest would mean the two arms compile +# different code — a benchmark whose fairness rests on directory ordering is not +# a benchmark. +# +# mcpp stages prebuilt objects for these out of its global cache while cmake +# compiles them from source: a handicap on cmake's cold build, declared here +# rather than hidden. +# --------------------------------------------------------------------------- + +bench_add_source_dep(xlings mcpplibs-x-cmdline 0.0.2) +bench_add_source_dep(xlings mcpplibs-x-xpkg 0.0.57) + +# `mcpplibs.xpkg.lua_stdlib` is generated, not checked in — libxpkg's build.mcpp +# embeds eleven .lua files as strings. Reproduced here so both arms compile the +# same set of translation units; see embed_lua_stdlib.cmake, which DERIVES the +# set from the directory rather than carrying a copy of it — the copy drifted +# once, and the loss surfaced three files away in a consumer. +file(GLOB xpkg_vers "${MCPP_XPKGS}/mcpplibs-x-xpkg/0.0.57/*") +foreach(d IN LISTS xpkg_vers) + if(IS_DIRECTORY "${d}/src/lua-stdlib") + set(XPKG_PKG_ROOT "${d}") + endif() +endforeach() +if(XPKG_PKG_ROOT) + set(LUA_STDLIB_CPPM "${CMAKE_CURRENT_BINARY_DIR}/generated/xpkg-lua-stdlib.cppm") + file(GLOB_RECURSE LUA_STDLIB_SOURCES "${XPKG_PKG_ROOT}/src/lua-stdlib/*.lua") + add_custom_command( + OUTPUT "${LUA_STDLIB_CPPM}" + COMMAND "${CMAKE_COMMAND}" -DXPKG_ROOT=${XPKG_PKG_ROOT} -DOUT=${LUA_STDLIB_CPPM} + -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_lua_stdlib.cmake" + DEPENDS ${LUA_STDLIB_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/embed_lua_stdlib.cmake" + COMMENT "Embedding libxpkg's lua-stdlib") + target_sources(xlings PRIVATE + FILE_SET fs_lua_stdlib TYPE CXX_MODULES + BASE_DIRS "${CMAKE_CURRENT_BINARY_DIR}/generated" + FILES "${LUA_STDLIB_CPPM}") +endif() +bench_add_source_dep(xlings mcpplibs-x-tinyhttps 0.2.9) +bench_add_source_dep(xlings mcpplibs.capi-x-lua 0.0.3) + +# --------------------------------------------------------------------------- +# The C/C++ libraries xlings links against. +# +# THIS IS WHAT USED TO STOP THE ARM AT THE LINK. ftxui, libarchive, lua and +# mbedtls arrive in mcpp's registry as SOURCE, and mcpp compiles them — so this +# description found their headers, compiled all 110 units, and then failed with +# ~1371 undefined `archive_*` / `mbedtls_*` / `lua_*`. It was recorded as a +# known gap; it is not one, it was unfinished work. +# +# EACH IS BUILT FROM ITS OWN `.xpkg.lua`, not from the vendored CMakeLists and +# not from a glob — see xpkg_source_library.cmake for the two attempts that +# failed and why the manifest is the only description that makes this arm +# compile the same files mcpp does. +# +# THE LIST IS TRANSITIVE, and it is written out rather than discovered because +# the discovery is what mcpp's package manager does: xlings names 6 direct +# dependencies, and wiring the source ones in surfaced two more headers +# (`mbedtls/ssl.h` for tinyhttps, `lua.h` for capi.lua) and then libarchive's +# own five. Naming them keeps this description honest about what it is — a +# hand-maintained copy of a resolved dependency set, which is exactly why +# bench/projects/ carries a description only for trees this repository can keep +# correct. The VERSIONS are the pins, from each manifest's `deps`. +# +# Both engines compile these from source on a cold build, which is what makes +# the comparison fair. mcpp may serve them from its global build cache instead — +# that asymmetry is declared in ../../README.md §5 rather than hidden here. +# --------------------------------------------------------------------------- +include(${CMAKE_CURRENT_LIST_DIR}/xpkg_source_library.cmake) + +bench_add_xpkg_library(bench_ftxui compat-x-ftxui 6.1.9) +bench_add_xpkg_library(bench_libarchive compat-x-libarchive 3.8.7) +bench_add_xpkg_library(bench_mbedtls compat-x-mbedtls 3.6.1) +bench_add_xpkg_library(bench_lua compat-x-lua 5.4.7) + +# libarchive's manifest names five dependencies and its generated config header +# turns each of them ON (`#define HAVE_LIBZ 1`, HAVE_LIBLZMA, …). Those defines +# are not optional decoration: they are what makes archive_read_support_filter_* +# call into zlib/bzip2/lz4/zstd/liblzma. libbench_libarchive.a leaves 72 distinct +# such symbols undefined, and nothing in that link error points at the manifest +# that asked for them. +bench_add_xpkg_library(bench_zlib compat-x-zlib 1.3.2) +bench_add_xpkg_library(bench_bzip2 compat-x-bzip2 1.0.8) +bench_add_xpkg_library(bench_lz4 compat-x-lz4 1.10.0) +bench_add_xpkg_library(bench_zstd compat-x-zstd 1.5.7) +bench_add_xpkg_library(bench_xz compat-x-xz 5.8.3) +target_link_libraries(bench_libarchive + PUBLIC bench_zlib bench_bzip2 bench_lz4 bench_zstd bench_xz) + +# PUBLIC on the libraries above carries their include dirs here too, which is +# what replaced a hand-written walk of registry subdirectories: xlings' own +# units reach for and , and the mcpplibs +# module units compiled INTO this target reach for and . +target_link_libraries(xlings PRIVATE + bench_ftxui bench_libarchive bench_mbedtls bench_lua) + +target_link_options(xlings PRIVATE -static-libstdc++) + +message(STATUS "xlings: ${XLINGS_MODULE_COUNT} module interface units + " + "${XLINGS_SOURCE_COUNT} .cpp from ${XLINGS_ROOT}") diff --git a/bench/projects/xlings/MODULE.bazel b/bench/projects/xlings/MODULE.bazel new file mode 100644 index 00000000..c488a71a --- /dev/null +++ b/bench/projects/xlings/MODULE.bazel @@ -0,0 +1,61 @@ +# bazel module for xlings — the benchmark's independent control target. +# +# THIS FILE USED TO SAY "BEST EFFORT, AND IT DOES NOT BUILD", and the three +# reasons it gave were a workspace boundary, four source dependencies, and +# `import std;`. Two of them were real problems with a solution and one had +# already been solved and written down as unsolved. What made the note worse +# than useless is what it produced: BUILD.bazel declared no rules, so +# `bazel build //...` exited 0 in 0.14s having found 0 targets, and the harness +# published that as a 0.43s cold build next to cmake's 94s. +# +# What each of the three actually was: +# +# 1. WORKSPACE BOUNDARY — real, and solved by a repository rule. bazel will +# not glob outside its workspace, but a repo rule may name an absolute path +# and compute it from the environment, which is how @local_config_cc finds +# a compiler. `@xlings_tree` resolves BENCH_PROJECT_ROOT — the variable +# bench/src/main.cpp exports for every --project run, the same one +# CMakeLists.txt and xmake.lua read — so one description serves both pinned +# trees and a cell builds the tree it claims to measure. +# +# 2. FOUR SOURCE DEPENDENCIES — nine, transitively, and also real. Same +# mechanism: `@mcpp_deps` reaches into ~/.mcpp/registry and reads each +# package's `.xpkg.lua` for the exact sources, include dirs and cflags mcpp +# compiles it with. Not BCR: the point of this arm is that both engines +# compile the SAME code, and a bazel_dep would pin whatever version the +# registry happens to carry. +# +# 3. `import std;` — NOT A BOUNDARY, and the old note conceded as much three +# lines after asserting it. bazel has no CXX_MODULE_STD and its modmap +# generator fails with `Module not found: std`, but libc++ ships the std +# module as ordinary source, so `@mcpp_deps//:std` compiles it as a +# `module_interfaces` unit like any other and every dependent imports it. +# +# WHAT IS STILL TRUE: this arm is CLANG-ONLY. bazel's ddi aggregator cannot +# parse GCC's P1689 output (`aggregate-ddi: "Invalid JSON string"`), so a bazel +# column in the gcc table would violate fairness invariant I1 — same compiler +# binary for every engine. bench/src/engines/bazel.cppm refuses the cell rather +# than measuring a different compiler, and _libcxx_root() in mcpp_registry.bzl +# fails loudly if $CC is not a clang that ships std.cppm. +# +# bazel build //:xlings # the default pin +# BENCH_PROJECT_ROOT=$PWD/xlings-2026.8.11.2 bazel build //:xlings +# +# (the flags this needs live in .bazelrc, so both of those work as written) +module(name = "xlings", version = "0.0.0") + +# NOT 0.1.x: `module_interfaces` does not exist there, and the attribute error +# reads as a typo rather than as a version floor. +bazel_dep(name = "rules_cc", version = "0.2.22") + +mcpp_deps = use_repo_rule("//:mcpp_registry.bzl", "mcpp_deps") +xlings_tree = use_repo_rule("//:mcpp_registry.bzl", "xlings_tree") + +mcpp_deps(name = "mcpp_deps") + +# The default is the newer pin, so a bare `bazel build //...` builds something +# real. Which tree that is belongs in the diff, not in a shell profile. +xlings_tree( + name = "xlings_tree", + default_tree = "xlings-2026.8.13.1", +) diff --git a/bench/projects/xlings/README.md b/bench/projects/xlings/README.md new file mode 100644 index 00000000..9b406fb6 --- /dev/null +++ b/bench/projects/xlings/README.md @@ -0,0 +1,155 @@ +# `xlings` — the independent control target + +mcpp measuring its own build proves nothing about **build performance in +general**: an optimisation can be an artefact of one project's module graph, and +"make the benchmark's target faster" is not an optimisation at all. A second +project, written by different people against a different structure, is what +separates the two. + +`xlings` fits: **110 module interface units, 46k lines**, every one of them +`import std;`, and it already carries an `mcpp.toml` — so mcpp builds it with no +adaptation, which is exactly what makes it a fair control rather than a +purpose-built fixture. + +## Pinned as submodules — and why that replaced "not vendored" + +This directory used to say *"there is no copy of xlings here, on purpose: a +vendored snapshot rots"*, and CI cloned the default branch at run time. + +**The reasoning was right and the implementation did the opposite of it.** A +target cloned from a moving branch does not merely rot, it rots *invisibly*: +`--hub src/xlings.cppm` went on naming a file that had stopped existing, so +every xlings cell reported `skipped`, every xlings job reported success, and +nobody had a reason to look. Drift was not prevented — it was made unobservable. + +A submodule is a **pin**, not a snapshot. The commit is in the diff, it is +reviewed like any other change, bumping it is a deliberate act with a +before/after, and `tests/e2e/233_bench_matrix.sh` can check that each `hub` and +`body` still exists in the tree CI will actually measure. + +```bash +git submodule update --init # get both pinned trees +``` + +| directory | version | commit | shape | variant | +|---|---|---|---|---| +| `xlings-2026.8.11.2` | 2026.8.11.2 | `b1563fe` | 110 `.cppm` + **2** `.cpp` | `modules` | +| `xlings-2026.8.13.1` | 2026.8.13.1 | `f072075` | 110 `.cppm` + **92** `.cpp` | `modules-impl` | + +### Two pins, because the code style is the measurement + +They are the same project either side of one refactor — `f072075` moved the +implementations out of the interface units. Same module graph, same 46k lines, +opposite answers to "where does the code live". That is the `modules` vs +`modules-impl` axis the generated fixture has, except here it was done by people +who were not thinking about this benchmark, which is the entire value of it. + +`--body` follows the style: the `.cpp` in the split tree, the `.cppm` in the +combined one. Editing an implementation is the point, and in the combined style +the implementation *is* the interface unit — which is why the two are expected +to behave differently, and why measuring both is the only way to say by how much. + +**One description serves both.** `CMakeLists.txt` and `xmake.lua` here glob +`src/**/*.{cppm,cpp}` — the same rule mcpp infers from — so neither style needs +its own file, an environment switch, or a branch. They used to name +`src/main.cpp` alone, which against the split tree compiles 110 interfaces, +links nothing, and still reports a time. + +```bash +bench --project bench/projects/xlings/xlings-2026.8.13.1 \ + --buildfiles bench/projects/xlings \ + --engines mcpp=,mcpp --compiler payload:gcc \ + --scenarios cold,noop --hub src/platform.cppm --body src/platform.cpp +``` + +`--hub src/platform.cppm` is the hub because it has the most importers (45 in +the combined tree, 54 in the split one). + +## What it has shown so far + +The split module schedule (`bmi_schedule = "on"`, see +`.agents/docs/2026-08-13-build-performance-architecture.md` L2) reproduces on +both projects, with a *larger* effect on the one that was not used to develop it: + +| project | modules / lines | `bmi_schedule=off` | `bmi_schedule=on` | ratio | +|---|---|---|---|---| +| mcpp | 138 / 57k | 79.9s | **34.80s** | **2.30x** | +| **xlings** | 110 / 46k | 112.92s | **33.41s** | **3.38x** | + +Both are no-ops on a second build — mcpp 0.21s, xlings 10.77s where the whole +10.77s is dependency resolution and `.ninja_log` grows by **zero edges**. That +distinction matters: a schedule whose depfile target is wrong looks exactly like +success while recompiling everything, and counting re-run edges is the only +check that catches it. + +**Declared asymmetry**: in the run above, the `off` arm compiled +`mcpplibs.xpkg` (5 units) while the `on` arm hit the dependency cache. Five +units against an 80-second difference does not move the conclusion, but it is +recorded rather than smoothed over. + +## The foreign build descriptions, and where each stops + +| engine | file | status | +|---|---|---| +| cmake | [`CMakeLists.txt`](CMakeLists.txt) | **complete** — configures, compiles all 110 units plus the nine source packages (467 C/C++ TUs), links a binary that runs | +| xmake | [`xmake.lua`](xmake.lua) | **in progress** — declares its dependencies through xrepo the way xlings' own xmake.lua does; blocked on mcpplibs-index#16. Shares the toolchain definitions in [`../common/xmake/payload.lua`](../common/xmake/payload.lua) | +| bazel | [`MODULE.bazel`](MODULE.bazel) | **complete** — `@xlings_tree` reaches the pinned tree through a repository rule (the workspace boundary is crossed, not worked around), `@mcpp_deps` builds the same 13 packages out of their `.xpkg.lua`, and `@mcpp_deps//:std` compiles libc++'s own `std.cppm` so `import std;` resolves. Verified on both trees | +| meson | — | removed from the suite entirely: meson cannot declare a module interface unit at all (see `../../SPEC.md`) | + +Both working arms take the compiler as a parameter, so the **toolchain is a real +axis here** and not a label: gcc gets `-B` + `--sysroot`, clang gets its +own include chain (handing clang gcc's sysroot puts the two arms on different +libc), msvc gets nothing because mcpp uses the system Visual Studio too. That +logic is shared with the mcpp arm rather than copied — see +[`../common/`](../common/). + +## Where the cmake arm used to stop + +`CMakeLists.txt` here used to configure, find all 110 module interface units, +compile every one of them — and then fail at the link with ~1371 undefined +`archive_*` / `mbedtls_*` / `lua_*`. That was written down as a *"known gap"*, +and the note is what let it sit: it was not a boundary, it was unfinished work. +The arm now links a binary that answers `xlings --version`. + +Three things looked like boundaries and were not: + +* **Transitive headers.** All of them are unpacked in mcpp's registry — `mbedtls` + via mcpplibs `tinyhttps`, `lua` via `capi.lua`. They arrive as the `PUBLIC` + include directories of the dependency targets rather than as a hand-written + walk of registry subdirectories. +* **A generated module.** `mcpplibs.xpkg.lua_stdlib` is not checked in; libxpkg's + `build.mcpp` produces it. But all it does is embed eleven `.lua` files as + strings, so [`embed_lua_stdlib.cmake`](embed_lua_stdlib.cmake) reproduces it. + *"mcpp runs a build program"* is not by itself a boundary. +* **The source dependencies.** `ftxui`, `libarchive`, `lua` and `mbedtls` arrive + as **source** and mcpp compiles them, so the link asked for symbols nobody had + built here. Neither obvious answer works: `add_subdirectory` on the vendored + CMakeLists drags in test suites libarchive cannot even configure without, and + mbedtls 3.6.1 `FATAL_ERROR`s **unconditionally** on a `framework/` submodule + the registry tarball does not carry — no option disables it. A glob of the + unpacked tree compiles the wrong set (`libarchive/*.c` is 132 files where mcpp + compiles 127; `lua/src/*.c` is 34 where it compiles 32, the two extra being + `lua.c` and `luac.c`, each with its own `main()`). + + What does work is that **every package in mcpp's registry ships a `.xpkg.lua` + naming exactly the sources, include dirs and cflags mcpp compiles it with**. + [`xpkg_source_library.cmake`](xpkg_source_library.cmake) reads that, so both + engines compile the same 127 files with the same defines, and a pattern that + resolves to nothing is a configure error rather than a quietly smaller build. + +⚠️ The copied module list in the generator **already drifted once**: a first +regex caught ten of eleven entries, and the failure surfaced three files away as +`error: 'base64_lua' is not a member of ...detail`. The generator now fails on a +missing `.lua` rather than trusting the list. + +**The xlings arm therefore does both jobs.** mcpp against mcpp (two releases, or +two code styles) is what a control target is for — it answers *"does this engine +change hold on a codebase nobody tuned it for?"*, and that question does not need +a second engine. And because the cmake description now links, the same tree also +carries a cross-engine comparison on a project with six dependencies, four of +them compiled from source by both engines. + +⚠️ **One asymmetry is real and is declared rather than smoothed over**: on a cold +build cmake compiles all nine source packages (467 C/C++ translation units) +while mcpp may serve them from its global dependency cache. See `../../README.md` +§5. diff --git a/bench/projects/xlings/embed_lua_stdlib.cmake b/bench/projects/xlings/embed_lua_stdlib.cmake new file mode 100644 index 00000000..44257a21 --- /dev/null +++ b/bench/projects/xlings/embed_lua_stdlib.cmake @@ -0,0 +1,76 @@ +# Reproduce `mcpplibs.xpkg.lua_stdlib` for the foreign build arms. +# +# That module is not a checked-in file: the xpkg package generates it at build +# time with a `build.mcpp` program. What the program does, though, is small and +# fully specified — it embeds every `.lua` under `src/lua-stdlib/` as a string +# named after the file — so a foreign build system CAN reproduce it, and "mcpp +# runs a build program" is not by itself a boundary. Reproducing it is what +# keeps the cross-engine comparison honest: both arms then compile the same set +# of translation units. +# +# ⚠️ THIS USED TO CARRY A COPIED LIST OF THE ELEVEN MODULES, AND THE COPY DRIFTED. +# A first pass extracted ten of eleven (a regex that missed `base64_lua`), and +# the failure was not "list incomplete" but +# +# xpkg-executor.cppm:585 error: 'base64_lua' is not a member of ...detail +# +# i.e. it surfaced in a consumer, three files away from the cause. The list is +# now DERIVED — every `.lua` under `src/lua-stdlib`, variable name = basename + +# `_lua` — which is verifiably the same rule build.mcpp applies (checked against +# libxpkg 0.0.57: 11 files, 11 embeddings, names identical). A rule cannot drift +# from itself; a copy of its output can, and did. +# +# `bench/projects/xlings/xmake.lua` implements the same rule in Lua, because +# xmake cannot run a CMake script portably. That is two implementations of one +# RULE, which is a different and much weaker coupling than two copies of one +# LIST — the rule is one line long and its failure is loud (see below). +# +# Usage (from add_custom_command): +# cmake -DXPKG_ROOT= -DOUT= -P embed_lua_stdlib.cmake + +if(NOT XPKG_ROOT OR NOT OUT) + message(FATAL_ERROR "embed_lua_stdlib.cmake needs -DXPKG_ROOT= and -DOUT=") +endif() + +# Every .lua under src/lua-stdlib, in a stable order. GLOB is normally the wrong +# tool for a build input — it hides an added file until someone reconfigures — +# but here the SET IS THE CONTRACT: build.mcpp embeds whatever is in that +# directory, so globbing is not an approximation of the list, it is the list. +file(GLOB_RECURSE LUA_FILES "${XPKG_ROOT}/src/lua-stdlib/*.lua") +list(SORT LUA_FILES) +if(NOT LUA_FILES) + message(FATAL_ERROR + "no .lua under ${XPKG_ROOT}/src/lua-stdlib — either the package layout " + "changed or XPKG_ROOT points at the wrong directory. Emitting an empty " + "module would fail three files away, in a consumer.") +endif() + +# Bracket syntax, not a quoted string: a quoted CMake string needs `\;` for a +# literal semicolon, and that backslash reaches the generated C++ verbatim — +# `error: stray '\' in program` on every line of the module preamble. +set(text [[// Generated by bench/projects/xlings/embed_lua_stdlib.cmake — do not edit. +// Mirrors what libxpkg's build.mcpp produces; edit the .lua sources. +module; +export module mcpplibs.xpkg.lua_stdlib; +import std; + +export namespace mcpplibs::xpkg::detail { + +]]) + +foreach(src IN LISTS LUA_FILES) + get_filename_component(stem "${src}" NAME_WE) + set(var "${stem}_lua") + file(READ "${src}" body) + # A C++ raw string literal, so nothing in the Lua needs escaping. The + # delimiter is one no Lua file contains; if that ever stops being true the + # generated file will not compile, which is the loud failure we want. + string(APPEND text "inline const std::string_view ${var} = R\"XLUA(${body})XLUA\"") + string(APPEND text ";\n\n") +endforeach() + +string(APPEND text "} // namespace mcpplibs::xpkg::detail\n") + +get_filename_component(outdir "${OUT}" DIRECTORY) +file(MAKE_DIRECTORY "${outdir}") +file(WRITE "${OUT}" "${text}") diff --git a/bench/projects/xlings/mcpp_registry.bzl b/bench/projects/xlings/mcpp_registry.bzl new file mode 100644 index 00000000..077bbb48 --- /dev/null +++ b/bench/projects/xlings/mcpp_registry.bzl @@ -0,0 +1,772 @@ +# Repository rules that let bazel reach the two things it cannot glob: the +# pinned xlings tree and mcpp's package registry. +# +# WHY ANY OF THIS EXISTS. bazel will not read sources from outside its +# workspace, and the xlings arm needs two kinds of outside: +# +# 1. THE TREE UNDER MEASUREMENT. The harness names it with --project and +# exports BENCH_PROJECT_ROOT (bench/src/main.cpp), exactly as the cmake and +# xmake arms consume it. A `bazel build //...` must build THAT tree and no +# other: declaring both pinned submodules as ordinary targets in this +# package would make every cell measure two builds and report one. +# +# 2. THE DEPENDENCIES, which mcpp resolves into ~/.mcpp/registry as SOURCE. +# Nine C/C++ libraries and four C++23 module packages, none of them in this +# repository, none of them with a bazel description upstream. +# +# Both are `repository_rule`s because that is the one place bazel lets you name +# an absolute path and compute it from the environment. Everything downstream is +# ordinary cc_library/cc_binary. +# +# ⚠️ THE SOURCE LISTS ARE DERIVED, NOT COPIED. Every package in mcpp's registry +# carries a `.xpkg.lua` manifest naming the EXACT files, include dirs and flags +# mcpp compiles it with, and this file parses that manifest instead of restating +# it. Globbing would be wrong and quietly so: +# +# libarchive-3.8.7/libarchive/*.c 132 files; the manifest names 127 +# lua-5.4.7/src/*.c 34 files; the manifest names 32 +# +# and the two extra lua files are `lua.c` and `luac.c`, each with its own +# `main()` — a glob links the interpreter into xlings and fails on a duplicate +# symbol, or worse, does not. The same reasoning as +# bench/projects/xlings/embed_lua_stdlib.cmake: a rule cannot drift from itself, +# a copy of its output can. + +# --------------------------------------------------------------------------- +# A Lua-table reader, just big enough for `.xpkg.lua`. +# +# These manifests are hand-written Lua with line comments, single- and +# double-quoted strings and nested tables. Anything that scans them has to skip +# comments BEFORE it looks at quotes, because the comments contain apostrophes +# ("-- xmake's mbedtls package") and a scanner that sees `'` first reads the +# rest of the file as one string and silently returns nothing. +# --------------------------------------------------------------------------- + +_IDENT = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" + +def _skip_ws(s, i): + for _ in range(len(s) + 1): + if i >= len(s): + return i + if s[i] == " " or s[i] == "\t" or s[i] == "\n" or s[i] == "\r": + i += 1 + elif s[i:i + 2] == "--": + nl = s.find("\n", i) + i = len(s) if nl < 0 else nl + 1 + else: + return i + return i + +def _skip_string(s, i, where): + quote = s[i] + i += 1 + for _ in range(len(s) + 1): + if i >= len(s): + fail("unterminated string in " + where) + if s[i] == "\\": + i += 2 + elif s[i] == quote: + return i + 1 + else: + i += 1 + fail("unterminated string in " + where) + +def _match_table(s, start, where): + """s[start] must be '{'. Returns the index just past the matching '}'.""" + depth = 0 + i = start + for _ in range(len(s) + 1): + if i >= len(s): + fail("unbalanced table in " + where) + if s[i:i + 2] == "--": + nl = s.find("\n", i) + i = len(s) if nl < 0 else nl + 1 + elif s[i] == '"' or s[i] == "'": + i = _skip_string(s, i, where) + elif s[i] == "{": + depth += 1 + i += 1 + elif s[i] == "}": + depth -= 1 + i += 1 + if depth == 0: + return i + else: + i += 1 + fail("unbalanced table in " + where) + +def _find_value(s, key, where): + """Index of the value of `key = ...`, or -1. Comments and strings skipped.""" + i = 0 + for _ in range(len(s) + 1): + if i >= len(s): + return -1 + if s[i:i + 2] == "--": + nl = s.find("\n", i) + i = len(s) if nl < 0 else nl + 1 + continue + if s[i] == '"' or s[i] == "'": + i = _skip_string(s, i, where) + continue + if s[i:i + len(key)] == key: + end = i + len(key) + before_ok = i == 0 or s[i - 1] not in _IDENT + after_ok = end >= len(s) or s[end] not in _IDENT + if before_ok and after_ok: + after = _skip_ws(s, end) + if after < len(s) and s[after] == "=" and s[after + 1:after + 2] != "=": + return _skip_ws(s, after + 1) + i += 1 + return -1 + +def _unescape(s): + out = "" + i = 0 + for _ in range(len(s) + 1): + if i >= len(s): + break + if s[i] == "\\" and i + 1 < len(s): + c = s[i + 1] + out += "\n" if c == "n" else ("\t" if c == "t" else c) + i += 2 + else: + out += s[i] + i += 1 + return out + +def _string_list(s, start, where): + """Every string literal directly inside the table at s[start].""" + end = _match_table(s, start, where) + body = s[start + 1:end - 1] + out = [] + i = 0 + for _ in range(len(body) + 1): + if i >= len(body): + break + if body[i:i + 2] == "--": + nl = body.find("\n", i) + i = len(body) if nl < 0 else nl + 1 + elif body[i] == '"': + j = _skip_string(body, i, where) + out.append(_unescape(body[i + 1:j - 1])) + i = j + else: + i += 1 + return out + +def _field_strings(seg, key, where): + at = _find_value(seg, key, where) + if at < 0 or seg[at] != "{": + return [] + return _string_list(seg, at, where) + +def _field_scalar(seg, key, where): + at = _find_value(seg, key, where) + if at < 0 or seg[at] != '"': + return "" + end = _skip_string(seg, at, where) + return _unescape(seg[at + 1:end - 1]) + +def _read_manifest(rctx, verdir, inner, label): + """The `mcpp = { ... }` segment of a package's .xpkg.lua, or None. + + None means the descriptor says nothing about how to compile the package — + either it has no `mcpp` key at all (tinyhttps, xpkg) or it points at the + package's own manifest (`mcpp = "*/mcpp.toml"`). Both mean the same thing: + the package ships an mcpp.toml and mcpp reads its source set off the default + convention, src/**/*.{cppm,cpp}. + + That mcpp.toml is checked to EXIST rather than assumed, because "no build + information anywhere" and "build information mcpp reads from a file this + parser does not" look identical from here and produce very different builds. + """ + path = rctx.path(verdir + "/.xpkg.lua") + if not path.exists: + fail("no .xpkg.lua for {}: mcpp's registry has not unpacked {}. Run a ".format(label, verdir) + + "`mcpp build` of the xlings tree first; this arm builds what mcpp resolved, " + + "it does not resolve packages itself.") + + # watch = "no": the registry is outside the workspace and bazel refuses to + # watch paths it does not own. The cost is that a registry change needs + # `bazel fetch --force`; the alternative is not fetching at all. + text = rctx.read(path, watch = "no") + at = _find_value(text, "mcpp", label) + if at >= 0 and text[at] == "{": + return text[at:_match_table(text, at, label)] + if not rctx.path(verdir + "/" + inner + "/mcpp.toml").exists: + fail("{}: .xpkg.lua carries no inline `mcpp = {{...}}` segment and the package ".format(label) + + "ships no mcpp.toml either, so nothing here says which files to compile") + return None + +# --------------------------------------------------------------------------- +# Path shapes in the registry. +# --------------------------------------------------------------------------- + +def _xpkgs_root(rctx): + home = rctx.os.environ.get("MCPP_HOME") + if not home: + userhome = rctx.os.environ.get("HOME") or rctx.os.environ.get("USERPROFILE") + if not userhome: + fail("neither MCPP_HOME nor HOME is set; cannot find mcpp's registry") + home = userhome + "/.mcpp" + return home + "/registry/data/xpkgs" + +def _inner_dir(rctx, verdir, label): + """What `*` means in a manifest path. + + Every package is a tarball unpacked one level below the version directory — + `compat-x-ftxui/6.1.9/FTXUI-6.1.9/` — and the manifests write that wrap layer + as a leading `*/`. `include_dirs` needs a concrete directory (bazel's + `includes` attribute takes literal paths, not patterns), so it is resolved + here rather than left as a glob: the version directory holds exactly one + subdirectory besides mcpp's own `mcpp_generated/`. + """ + dirs = [ + p.basename + for p in rctx.path(verdir).readdir() + if p.is_dir and p.basename != "mcpp_generated" + ] + if len(dirs) != 1: + fail("{}: expected one unpacked tree under {}, found {}".format(label, verdir, dirs)) + return dirs[0] + +# bazel's own marker files, which the repo rule must not import from a source +# tarball. See _link_package. +_PACKAGE_MARKERS = ["BUILD", "BUILD.bazel", "REPO.bazel", "MODULE.bazel", "WORKSPACE", "WORKSPACE.bazel"] + +def _link_package(rctx, verdir, inner, name): + """Mirror one registry package into this repo, minus bazel's marker files. + + ⚠️ THE INNER TREE IS LINKED CHILD BY CHILD, AND THAT IS THE WHOLE POINT. + zlib and FTXUI both ship an upstream BUILD.bazel — both are in the Bazel + Central Registry — and one `rctx.symlink(verdir, name)` brings it along. + A stray BUILD file makes that directory a bazel PACKAGE, which is invisible + everywhere except in the errors it causes somewhere else: + + glob pattern 'ftxui/FTXUI-6.1.9/src/ftxui/**/*.cpp' didn't match anything + Label '...//:zlib/zlib-1.3.2/adler32.c' is invalid because + '...//zlib/zlib-1.3.2' is a subpackage + + Neither message names a BUILD file. Deferring to the vendored descriptions + instead is not an option: they compile a different source set with different + flags than mcpp does, and "both engines compile the same code" is the only + thing that makes this arm a measurement. + + Only DIRECTORIES are linked at the version level: it also holds the source + tarball the package was unpacked from, and a 30 MB archive bazel has to + stat and hash is not an input to anything. + """ + for entry in rctx.path(verdir).readdir(): + if entry.is_dir and entry.basename != inner: + rctx.symlink(entry, name + "/" + entry.basename) + for entry in rctx.path(verdir + "/" + inner).readdir(): + if entry.basename not in _PACKAGE_MARKERS: + rctx.symlink(entry, name + "/" + inner + "/" + entry.basename) + +# --------------------------------------------------------------------------- +# Manifest paths -> bazel labels. +# --------------------------------------------------------------------------- + +def _resolve_star(pattern, inner): + """A leading `*` segment is the unpacked tree; other wildcards stay globs.""" + if pattern == "*": + return inner + if pattern.startswith("*/"): + return inner + pattern[1:] + return pattern + +def _split_sources(rctx, verdir, prefix, inner, patterns, label): + """(literal labels, glob patterns, glob exclusions). + + Patterns with no wildcard become literal file names and are checked to + exist HERE, at fetch time, with the package that is missing them named. A + glob would drop them silently and the build would fail hundreds of files + later on an undefined symbol. + """ + literals = [] + globs = [] + excludes = [] + for pattern in patterns: + negated = pattern.startswith("!") + rel = _resolve_star(pattern[1:] if negated else pattern, inner) + if negated: + excludes.append(prefix + "/" + rel) + elif "*" in rel: + globs.append(prefix + "/" + rel) + else: + if not rctx.path(verdir + "/" + rel).exists: + fail("{}: .xpkg.lua names {} but it is not in the unpacked tree".format(label, rel)) + literals.append(prefix + "/" + rel) + return literals, globs, excludes + +def _copts(cflags, standard): + """mcpp's cflags are shell words, and so are bazel's copts. + + ⚠️ THE BACKSLASHES IN libarchive's + `-DPLATFORM_CONFIG_H=\\"mcpp_libarchive_config.h\\"` ARE LOAD-BEARING AND MUST + SURVIVE. `copts` is documented as subject to "Bourne shell tokenization", so + an unescaped `"` is eaten exactly as a shell would eat it and the compiler + receives + + -DPLATFORM_CONFIG_H=mcpp_libarchive_config.h + + Nothing complains. `#include PLATFORM_CONFIG_H` then finds no header, every + HAVE_* stays undefined, and the build fails 60 files later with 20 errors of + the form `call to undeclared library function 'strcmp'` in + archive_write_set_format_zip.c — a file that has nothing to do with it. The + "helpful" unescaping this function used to do is what produced that. + + Splitting on spaces is redundant with the same tokenization, and kept + because it makes `-include mcpp_lua_platform_config.h` visibly two flags + rather than one that bazel happens to split. + """ + out = [] + if standard: + out.append("-std=" + standard) + for flag in cflags: + for piece in flag.split(" "): + if piece: + out.append(piece) + return out + +# --------------------------------------------------------------------------- +# The dependency set, in xlings' own resolution order. +# +# VERSIONS ARE PINNED FROM xlings' mcpp.toml AND ITS TRANSITIVE MANIFESTS, never +# "the newest directory in the registry": the registry holds several versions at +# once and taking the last one would mean the bazel arm compiles different code +# than the mcpp arm it is being compared against. +# +# The deps edges below are the one thing NOT derived from the manifests: mcpp +# resolves `compat.zlib = "1.3.2"` to a package, bazel needs a label, and the +# mapping between the two is this file. They are checked by the link. +# --------------------------------------------------------------------------- + +_C_LIBS = [ + # name xpkg version language deps + ("zlib", "compat-x-zlib", "1.3.2", "c", []), + ("bzip2", "compat-x-bzip2", "1.0.8", "c", []), + ("lz4", "compat-x-lz4", "1.10.0", "c", []), + ("zstd", "compat-x-zstd", "1.5.7", "c", []), + ("xz", "compat-x-xz", "5.8.3", "c", []), + # libarchive's own manifest names these five; its generated config header + # sets HAVE_LIBZ/HAVE_LIBLZ4/... unconditionally, so they are link-time + # requirements rather than options. + ("libarchive", "compat-x-libarchive", "3.8.7", "c", ["zlib", "bzip2", "lz4", "zstd", "xz"]), + ("lua", "compat-x-lua", "5.4.7", "c", []), + ("mbedtls", "compat-x-mbedtls", "3.6.1", "c", []), + ("ftxui", "compat-x-ftxui", "6.1.9", "c++", []), +] + +# lz4hc.c does `#include "lz4.c"` under LZ4_COMMONDEFS_ONLY to pull in the +# static helpers. Both files are compiled in their own right, so lz4.c is a +# source AND a textual header — bazel needs it named in both places or the +# compile of lz4hc.c fails with `'lz4.c' file not found`, since a sibling source +# is not an input to a compile action. +_TEXTUAL_SOURCES = { + "lz4": ["lib/lz4.c", "lib/xxhash.c"], +} + +# The C++23 module packages. `deps` here are bazel labels in this same repo; +# the module graph itself (who imports whom) is discovered by bazel's ddi +# scanner, so only the package-level edges have to be written down. +_MODULE_LIBS = [ + ("cmdline", "mcpplibs-x-cmdline", "0.0.2", ["std"]), + ("capi_lua", "mcpplibs.capi-x-lua", "0.0.3", ["std", "lua"]), + ("tinyhttps", "mcpplibs-x-tinyhttps", "0.2.9", ["std", "mbedtls"]), + ("xpkg", "mcpplibs-x-xpkg", "0.0.57", ["std", "capi_lua"]), +] + +_HDR_PATTERNS = ["*.h", "*.hh", "*.hpp", "*.hxx", "*.inc", "*.def", "*.ipp"] + +def _lit(values): + # Escaped, because the values are compiler flags and one of them is + # libarchive's -DPLATFORM_CONFIG_H="mcpp_libarchive_config.h". Emitted raw it + # closes the Starlark string early and the generated BUILD file fails to + # parse with `syntax error at 'mcpp_libarchive_config': expected ]` — an + # error that names the flag's contents and nothing about quoting. + return "[" + ", ".join([ + '"{}"'.format(v.replace("\\", "\\\\").replace('"', '\\"')) + for v in values + ]) + "]" + +def _glob(patterns, exclude = [], allow_empty = False): + out = "glob({}".format(_lit(patterns)) + if exclude: + out += ", exclude = {}".format(_lit(exclude)) + if allow_empty: + out += ", allow_empty = True" + return out + ")" + +def _join(*parts): + """`a + b` over the non-empty pieces, so the generated file has no `[] + `.""" + return " + ".join([p for p in parts if p]) or "[]" + +def _hdrs_glob(prefix): + return _glob([prefix + "/**/" + p for p in _HDR_PATTERNS], allow_empty = True) + +# --------------------------------------------------------------------------- +# @mcpp_deps — everything xlings links that is not xlings. +# --------------------------------------------------------------------------- + +def _libcxx_root(rctx, xpkgs): + """The libc++ that goes with the compiler being measured. + + DERIVED FROM $CC, not globbed. The registry can hold llvm 20.1.7 and 22.1.8 + at once; compiling libc++'s std.cppm out of one of them while the driver + ships the headers of the other is the two-standard-libraries failure that + ../common/cmake/hermetic_payload.cmake documents, and it surfaces as an + error inside rather than as a version mismatch. + """ + cc = rctx.os.environ.get("CC", "") + if cc: + root = cc.rsplit("/bin/", 1)[0] if "/bin/" in cc else "" + if root and rctx.path(root + "/share/libc++/v1/std.cppm").exists: + return root + fail( + "CC={} has no libc++ std module at /share/libc++/v1/std.cppm ".format(cc) + + "(CC must be an absolute path to a /bin/ compiler).\n" + + "bazel has no counterpart to CMake's CXX_MODULE_STD: `import std;` is " + + "supplied here by compiling libc++'s own std.cppm, so the compiler must " + + "be a clang that ships it. bazel also cannot build modules with GCC at " + + "all (its ddi aggregator rejects GCC's P1689 output), which is why this " + + "arm is clang-only.", + ) + llvm = rctx.path(xpkgs + "/xim-x-llvm") + if llvm.exists: + versions = sorted([p.basename for p in llvm.readdir() if p.is_dir]) + for version in reversed(versions): + if rctx.path(xpkgs + "/xim-x-llvm/" + version + "/share/libc++/v1/std.cppm").exists: + return xpkgs + "/xim-x-llvm/" + version + fail("no clang with a libc++ std module in {}; set CC to one".format(xpkgs)) + +def _c_lib_rule(rctx, xpkgs, name, xpkg, version, language, deps): + verdir = "{}/{}/{}".format(xpkgs, xpkg, version) + label = "{} {}".format(xpkg, version) + inner = _inner_dir(rctx, verdir, label) + segment = _read_manifest(rctx, verdir, inner, label) + if segment == None: + fail("{}: expected an inline `mcpp = {{...}}` manifest naming its sources".format(label)) + _link_package(rctx, verdir, inner, name) + + sources = _field_strings(segment, "sources", label) + if not sources: + fail("{}: .xpkg.lua names no sources".format(label)) + literals, globs, excludes = _split_sources(rctx, verdir, name, inner, sources, label) + + includes = [name + "/" + _resolve_star(d, inner) for d in _field_strings(segment, "include_dirs", label)] + + # `c_standard = "c11"` and `language = "c++23"` are already the -std spelling, + # so they are used verbatim rather than looked up in a table. A table that + # does not know a value has to choose between failing and defaulting, and the + # default is the dangerous half: compiling lua at the compiler's default + # standard instead of the manifest's c99 is a build that works and is not the + # one mcpp ran. + key = "language" if language == "c++" else "c_standard" + standard = _field_scalar(segment, key, label) + if not standard: + fail("{}: .xpkg.lua names no `{}`, so this arm would compile it at the ".format(label, key) + + "compiler's default standard rather than mcpp's") + copts = _copts(_field_strings(segment, "cflags", label), standard) + + # `-x c` because bazel's autoconfigured toolchain has ONE compiler for both + # languages and this arm has to point it at clang++ (the C driver's config + # file carries no libc++ include chain, so C++ would compile against the + # host's libstdc++ headers and die in std.cppm on a missing <__config>). + # clang++ then reads a .c file as C++, which libarchive does not survive. + # mcpp has the same split and solves it by spawning the sibling `clang`. + if language == "c": + copts = ["-x", "c"] + copts + + if excludes and not globs: + fail("{}: exclusion patterns with no glob to exclude from".format(label)) + srcs = _join(_lit(literals) if literals else "", _glob(globs, excludes) if globs else "") + + textual = _TEXTUAL_SOURCES.get(name, []) + rule = [ + "cc_library(", + ' name = "{}",'.format(name), + " srcs = {},".format(srcs), + " hdrs = {},".format(_hdrs_glob(name)), + ] + if textual: + for rel in textual: + if not rctx.path("{}/{}/{}".format(verdir, inner, rel)).exists: + fail("{}: _TEXTUAL_SOURCES names {} and the tree has no such file".format(label, rel)) + rule.append(" textual_hdrs = {},".format( + _lit(["{}/{}/{}".format(name, inner, rel) for rel in textual]), + )) + rule += [ + " includes = {},".format(_lit(includes)), + " copts = {},".format(_lit(copts)), + " deps = {},".format(_lit([":" + d for d in deps])), + " linkstatic = True,", + ")", + ] + return "\n".join(rule) + +def _module_lib_rule(rctx, xpkgs, name, xpkg, version, deps, extra_interfaces = []): + verdir = "{}/{}/{}".format(xpkgs, xpkg, version) + label = "{} {}".format(xpkg, version) + inner = _inner_dir(rctx, verdir, label) + segment = _read_manifest(rctx, verdir, inner, label) + _link_package(rctx, verdir, inner, name) + + # Four packages, THREE manifest shapes, and the difference is not cosmetic: + # capi.lua names its two files outright ("*/src/capi/lua.cppm", ".cpp") + # cmdline names a pattern ("*/src/**/*.cppm") + # xpkg, tinyhttps carry `mcpp = "*/mcpp.toml"`, deferring to the + # package's own manifest, whose source set is mcpp's default + # convention (src/**/*.{cppm,cpp}). + # Interface units and implementation units go to different attributes, so + # every shape has to be sorted by extension rather than by position. + base = "{}/{}/src".format(name, inner) + if segment: + sources = _field_strings(segment, "sources", label) + literals, globs, excludes = _split_sources(rctx, verdir, name, inner, sources, label) + if excludes: + fail("{}: exclusions in a module package are not handled".format(label)) + includes = [name + "/" + _resolve_star(d, inner) for d in _field_strings(segment, "include_dirs", label)] or [base] + for pattern in globs: + if not pattern.endswith(".cppm") and not pattern.endswith(".cpp"): + fail("{}: source pattern {} names no extension, so it cannot be sorted into ".format(label, pattern) + + "module_interfaces vs srcs") + else: + literals = [] + globs = [base + "/**/*.cppm", base + "/**/*.cpp"] + includes = [base] + + interfaces = [f for f in literals if f.endswith(".cppm")] + interface_globs = [p for p in globs if p.endswith(".cppm")] + srcs_literals = [f for f in literals if not f.endswith(".cppm")] + src_globs = [p for p in globs if not p.endswith(".cppm")] + + named = interfaces + extra_interfaces + interface_expr = _join( + _lit(named) if named else "", + _glob(interface_globs) if interface_globs else "", + ) + + # allow_empty on the IMPLEMENTATION units only: a module package with no + # .cpp is ordinary (tinyhttps and xpkg are interface-only), a module package + # with no .cppm is a resolution that went wrong. + srcs = _join( + _lit(srcs_literals) if srcs_literals else "", + _glob(src_globs, allow_empty = True) if src_globs else "", + ) + + return "\n".join([ + "cc_library(", + ' name = "{}",'.format(name), + " srcs = {},".format(srcs), + " hdrs = {},".format(_hdrs_glob(name)), + " module_interfaces = {},".format(interface_expr), + " includes = {},".format(_lit(includes)), + ' copts = ["-std=c++23"],', + # See :module_sources — a clang BMI re-opens the sources it was built + # from, so every compile that loads one needs them staged. + ' additional_compiler_inputs = [":module_sources"],', + " deps = {},".format(_lit([":" + d for d in deps])), + " linkstatic = True,", + ")", + ]) + +# Every C++ source a BMI in this repo was built from. +# +# ⚠️ NOT A CONVENIENCE. A clang BMI records the absolute-ish paths of its input +# files and re-opens them whenever a dependent loads it, so a compile that +# imports `mcpplibs.xpkg` needs xpkg's .cppm on disk even though it only reads +# the .pcm. bazel stages declared inputs and nothing else, and the modmap +# declares .pcm files, so the compile dies with +# fatal error: cannot open file '.../xpkg.cppm': No such file or directory +# naming a file that is right there in the execroot. Every target that can load +# a BMI from this repo lists this filegroup. +def _module_sources_rule(module_names): + return "\n".join([ + "filegroup(", + ' name = "module_sources",', + " srcs = {} + [\":xpkg_lua_stdlib\"],".format( + _glob([n + "/**/*.cppm" for n in module_names] + ["libcxx_module/**"], allow_empty = True), + ), + ")", + ]) + +# `mcpplibs.xpkg.lua_stdlib` is not a checked-in file: libxpkg's build.mcpp +# generates it, embedding every .lua under src/lua-stdlib as a string_view named +# after the file. That is small and fully specified, so "mcpp runs a build +# program" is not by itself a boundary — a genrule reproduces it, and both arms +# then compile the same set of translation units. +# +# The list is DERIVED (a glob of the directory, which IS the contract build.mcpp +# implements) rather than copied. embed_lua_stdlib.cmake records what a copy +# cost: a regex that caught ten of eleven files failed three files away as +# `error: 'base64_lua' is not a member of ...detail`. +_LUA_STDLIB_GENRULE = ''' +genrule( + name = "xpkg_lua_stdlib", + srcs = {srcs}, + outs = ["generated/xpkg-lua-stdlib.cppm"], + cmd = """ +set -eu +{{ + echo '// Generated by bench/projects/xlings/mcpp_registry.bzl - do not edit.' + echo '// Mirrors what libxpkg build.mcpp produces; edit the .lua sources.' + echo 'module;' + echo 'export module mcpplibs.xpkg.lua_stdlib;' + echo 'import std;' + echo '' + echo 'export namespace mcpplibs::xpkg::detail {{' + echo '' + for f in $(SRCS); do + stem=$$(basename "$$f" .lua) + printf 'inline const std::string_view %s_lua = R"XLUA(' "$$stem" + cat "$$f" + printf ')XLUA";\\n\\n' + done + echo '}} // namespace mcpplibs::xpkg::detail' +}} > $@ +""", +) +''' + +_STD_RULE = ''' +# `import std;` HAS NO BAZEL SPELLING. There is no counterpart to CMake's +# CXX_MODULE_STD, and bazel's own modmap generator fails with +# ERROR: Module not found: std +# What makes it work anyway is that libc++ ships the std module as ORDINARY +# SOURCE, so it compiles like any other interface unit. The 110 .inc files it +# textually includes have to be inputs too, and they resolve relative to +# std.cppm's own directory, which is why they are listed rather than reached +# through `includes`. +# +# -Wno-reserved-module-identifier: naming a module `std` is reserved to the +# implementation, and libc++ is the implementation. +cc_library( + name = "std", + srcs = glob(["libcxx_module/std/**"]), + module_interfaces = ["libcxx_module/std.cppm"], + copts = ["-std=c++23", "-Wno-reserved-module-identifier"], + linkstatic = True, +) +''' + +def _mcpp_deps_impl(rctx): + xpkgs = _xpkgs_root(rctx) + if not rctx.path(xpkgs).exists: + fail("mcpp's registry is not at {} — set MCPP_HOME".format(xpkgs)) + + rctx.symlink(_libcxx_root(rctx, xpkgs) + "/share/libc++/v1", "libcxx_module") + + parts = [ + "# GENERATED by //:mcpp_registry.bzl from the .xpkg.lua manifests in", + "# mcpp's registry. Do not edit; edit the rule.", + 'load("@rules_cc//cc:defs.bzl", "cc_library")', + '', + 'package(default_visibility = ["//visibility:public"])', + _STD_RULE, + ] + for name, xpkg, version, language, deps in _C_LIBS: + parts.append(_c_lib_rule(rctx, xpkgs, name, xpkg, version, language, deps)) + + lua_stdlib_verdir = None + for name, xpkg, version, deps in _MODULE_LIBS: + extra = [] + if name == "xpkg": + lua_stdlib_verdir = "{}/{}/{}".format(xpkgs, xpkg, version) + extra = [":xpkg_lua_stdlib"] + parts.append(_module_lib_rule(rctx, xpkgs, name, xpkg, version, deps, extra)) + + inner = _inner_dir(rctx, lua_stdlib_verdir, "mcpplibs-x-xpkg") + stdlib_dir = "xpkg/{}/src/lua-stdlib".format(inner) + if not rctx.path(lua_stdlib_verdir + "/" + inner + "/src/lua-stdlib").exists: + fail("libxpkg has no src/lua-stdlib; mcpplibs.xpkg.lua_stdlib cannot be reproduced") + parts.append(_LUA_STDLIB_GENRULE.format( + srcs = _glob([stdlib_dir + "/**/*.lua"]), + )) + parts.append(_module_sources_rule([name for name, _, _, _ in _MODULE_LIBS])) + + rctx.file("BUILD.bazel", "\n".join(parts) + "\n") + +mcpp_deps = repository_rule( + implementation = _mcpp_deps_impl, + doc = "xlings' dependency set, compiled from the source mcpp resolved into ~/.mcpp/registry.", + environ = ["MCPP_HOME", "HOME", "USERPROFILE", "CC"], +) + +# --------------------------------------------------------------------------- +# @xlings_tree — the tree under measurement. +# --------------------------------------------------------------------------- + +_XLINGS_BUILD = '''# GENERATED by //:mcpp_registry.bzl for {root} +load("@rules_cc//cc:defs.bzl", "cc_binary") + +# BOTH .cppm AND .cpp, which is what lets one description measure xlings' two +# code styles: +# +# 2026.8.11.2 110 .cppm + 2 .cpp implementation inside the interface unit +# 2026.8.13.1 110 .cppm + 92 .cpp interface and implementation split +# +# `srcs = ["src/main.cpp"]` alone is the trap: against the split tree it +# compiles 110 interfaces, links nothing, and still reports a time. Same rule +# mcpp infers from its own manifest, same rule ../CMakeLists.txt globs. +cc_binary( + name = "xlings", + srcs = glob(["src/**/*.cpp"]) + glob(["src/**/*.h", "src/**/*.hpp"], allow_empty = True), + module_interfaces = glob(["src/**/*.cppm"]), + # ⚠️ THE INTERFACE SOURCES ARE ALSO INPUTS TO EVERY COMPILE, and they have to + # be said twice. A clang BMI records the paths of the sources it was built + # from and re-opens them when a dependent loads it, so compiling the module + # implementation unit src/runtime/event_stream.cpp loads event_stream's BMI, + # which loads cancellation's BMI, which reaches for + # fatal error: cannot open file '.../src/runtime/cancellation.cppm' + # bazel stages only declared inputs, and the modmap declares .pcm files. + # + # THE SANDBOX IS WHAT MAKES THIS VISIBLE, not what makes it wrong: + # `--spawn_strategy=local` builds and links this target clean, because the + # execroot happens to have every source in it. That is an undeclared + # dependency either way, and the one form of it a benchmark cannot tolerate — + # it decides whether a cell builds at all. + additional_compiler_inputs = glob(["src/**/*.cppm"]) + ["@mcpp_deps//:module_sources"], + # [build] include_dirs — src/libs/json.cppm reaches for from its + # global module fragment. + includes = ["src/libs/json"], + # [build] cxxflags + defines = ["LIBARCHIVE_STATIC", "UNICODE", "_UNICODE"], + copts = ["-std=c++23"], + deps = [ + "@mcpp_deps//:std", + "@mcpp_deps//:cmdline", + "@mcpp_deps//:capi_lua", + "@mcpp_deps//:tinyhttps", + "@mcpp_deps//:xpkg", + "@mcpp_deps//:ftxui", + "@mcpp_deps//:libarchive", + ], + visibility = ["//visibility:public"], +) +''' + +def _xlings_tree_impl(rctx): + # BENCH_PROJECT_ROOT is what bench/src/main.cpp exports for every --project + # run; the default is the newer pin so a bare `bazel build //...` in this + # directory still builds something real. + root = rctx.os.environ.get("BENCH_PROJECT_ROOT") + if not root: + root = str(rctx.workspace_root) + "/" + rctx.attr.default_tree + if not rctx.path(root + "/mcpp.toml").exists: + fail( + "no xlings tree at {}: BENCH_PROJECT_ROOT must name a checkout with a ".format(root) + + "mcpp.toml. The pinned trees are the submodules " + + "bench/projects/xlings/xlings-/ — run `git submodule update --init`.", + ) + if not rctx.path(root + "/src").exists: + fail("{} has no src/".format(root)) + rctx.symlink(root + "/src", "src") + rctx.file("BUILD.bazel", _XLINGS_BUILD.format(root = root)) + +xlings_tree = repository_rule( + implementation = _xlings_tree_impl, + doc = "The pinned xlings checkout named by --project / BENCH_PROJECT_ROOT.", + attrs = {"default_tree": attr.string(mandatory = True)}, + environ = ["BENCH_PROJECT_ROOT"], +) diff --git a/bench/projects/xlings/packages/libarchive.lua b/bench/projects/xlings/packages/libarchive.lua new file mode 100644 index 00000000..54d3b3a0 --- /dev/null +++ b/bench/projects/xlings/packages/libarchive.lua @@ -0,0 +1,85 @@ +-- libarchive-xlings — reused verbatim in shape from xlings' own +-- `xmake/packages/libarchive.lua` (openxlings/xlings @ bb27e43), which is where +-- xlings keeps it. Kept as a local package definition here for the same reason +-- it is local there: it overrides a third-party package for this project's +-- needs, so it does not belong in mcpplibs-index. +-- +-- WHY THE OVERRIDE IS NEEDED AT ALL. xmake-repo's `libarchive` configures with +-- libarchive's own defaults, and under the hermetic payload toolchain that +-- stops the install dead: +-- +-- CMake Error at CMakeLists.txt:1349 (MESSAGE): +-- libgcc not found. +-- +-- `-DENABLE_LibGCC=OFF` below is the line that fixes it. The rest of the OFFs +-- are the tools and test suites xlings never links — the same class of problem +-- the cmake arm hit from the other direction, where libarchive's test suite +-- could not even configure. +-- +-- UPSTREAM'S OWN NOTE, kept because it is not obvious: the dependency list uses +-- `xz` rather than `lzma`, because libarchive probes via `find_package(LibLZMA)` +-- and that resolves to xz-utils' liblzma, not the 7-Zip LZMA SDK. With the wrong +-- one it silently falls back to fork-exec for `.tar.xz`, which is a correctness +-- difference, not a packaging preference. +package("libarchive-xlings") + + set_base("libarchive") + + add_versions("3.8.7", "4b787cca6697a95c7725e45293c973c208cbdc71ae2279f30ef09f52472b9166") + add_versions("3.8.6", "213269b05aac957c98f6e944774bb438d0bd168a2ec60b9e4f8d92035925821c") + + add_deps("cmake") + -- openssl is in the list even though `-DENABLE_OPENSSL=OFF` is passed + -- below: `set_base("libarchive")` inherits the upstream package's openssl + -- dependency, so `-lssl -lcrypto` reach libarchive's own link line anyway. + -- Declaring it HERE is what puts openssl's lib dir on that line — a + -- project-level `add_requires("openssl")` installs the package but does not + -- reach into this package's build, and libarchive then fails with + -- ld: cannot find -lssl: No such file or directory + -- because the host ships libssl.so.3 without a `libssl.so` dev symlink. + add_deps("zlib", "bzip2", "lz4", "zstd", "xz", "openssl") + + if is_plat("windows") then + add_syslinks("advapi32") + end + + on_install("windows", "linux", "macosx", function (package) + local configs = { + "-DENABLE_TEST=OFF", + "-DENABLE_CAT=OFF", + "-DENABLE_TAR=OFF", + "-DENABLE_CPIO=OFF", + "-DENABLE_OPENSSL=OFF", + "-DENABLE_PCREPOSIX=OFF", + "-DENABLE_LibGCC=OFF", + "-DENABLE_CNG=OFF", + "-DENABLE_ICONV=OFF", + "-DENABLE_ACL=OFF", + "-DENABLE_EXPAT=OFF", + "-DENABLE_LIBXML2=OFF", + "-DENABLE_LIBB2=OFF", + "-DENABLE_ZLIB=ON", + "-DENABLE_BZip2=ON", + "-DENABLE_LZ4=ON", + "-DENABLE_ZSTD=ON", + "-DENABLE_LZMA=ON", + -- NO `-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=OFF` HERE. It looks like the + -- right way to stop libarchive preferring the host's shared zlib/bz2 + -- over the static ones xrepo built, and it does — along with + -- everything else libarchive legitimately probes for, so the package + -- then fails to link at all. The runtime path is solved on the + -- consumer side instead (see xmake.lua), not by blinding configure. + } + table.insert(configs, "-DCMAKE_BUILD_TYPE=" .. (package:debug() and "Debug" or "Release")) + table.insert(configs, "-DBUILD_SHARED_LIBS=" .. (package:config("shared") and "ON" or "OFF")) + if not package:config("shared") then + package:add("defines", "LIBARCHIVE_STATIC") + end + import("package.tools.cmake").install(package, configs) + end) + + on_test(function (package) + assert(package:has_cfuncs("archive_version_number", {includes = "archive.h"})) + end) + +package_end() diff --git a/bench/projects/xlings/xlings-2026.8.11.2 b/bench/projects/xlings/xlings-2026.8.11.2 new file mode 160000 index 00000000..b1563feb --- /dev/null +++ b/bench/projects/xlings/xlings-2026.8.11.2 @@ -0,0 +1 @@ +Subproject commit b1563feb17f0b14b280cc10e909a65101d6ebf5b diff --git a/bench/projects/xlings/xlings-2026.8.13.1 b/bench/projects/xlings/xlings-2026.8.13.1 new file mode 160000 index 00000000..f0720758 --- /dev/null +++ b/bench/projects/xlings/xlings-2026.8.13.1 @@ -0,0 +1 @@ +Subproject commit f07207584f3e321d7a00d3279e131e96b847b740 diff --git a/bench/projects/xlings/xmake.lua b/bench/projects/xlings/xmake.lua new file mode 100644 index 00000000..e087b2b2 --- /dev/null +++ b/bench/projects/xlings/xmake.lua @@ -0,0 +1,187 @@ +-- xmake build description for xlings — the benchmark's independent control target. +-- +-- Counterpart to CMakeLists.txt here, and to bench/projects/mcpp/xmake.lua. The +-- fairness contract is the same five: same compiler binary, same language +-- flags, same source set, same link output kind, same standard library +-- (`import std;`, not a header shim). +-- +-- THE TREE IS NOT VENDORED, so unlike bench/projects/mcpp/xmake.lua this cannot +-- derive its root from os.scriptdir(). Point it at a checkout: +-- +-- XLINGS_ROOT=/path/to/xlings xmake f -P bench/projects/xlings -y -m release +-- XLINGS_ROOT=/path/to/xlings xmake build -P bench/projects/xlings -j32 +-- +-- Record the commit with the numbers; the published ones are from `b1563fe`. +-- +-- ⚠️ SAME STATUS AS THE CMAKE ARM: every translation unit compiles; the LINK +-- does not. ftxui / libarchive / lua / mbedtls arrive as SOURCE and mcpp +-- compiles them, so the link wants symbols nobody built here. That is ordinary +-- work, not a wall — and it is the same gap in both foreign arms, which is why +-- the xlings numbers are quoted as mcpp-vs-mcpp (see README.md). + +set_project("xlings") +set_xmakever("2.9.0") +set_languages("c++23") +add_rules("mode.debug", "mode.release") + +-- The hermetic payload and the toolchain-per-family definitions are SHARED with +-- the mcpp arm: ../common/xmake/payload.lua. +includes("../common/xmake/payload.lua") + +-- BENCH_PROJECT_ROOT is what the harness exports for every --project run, and +-- it is why one description serves both pinned trees. XLINGS_ROOT stays +-- supported for driving this by hand. +local XLINGS_ROOT = os.getenv("BENCH_PROJECT_ROOT") or os.getenv("XLINGS_ROOT") +local XLINGS_MANIFEST = XLINGS_ROOT and path.join(XLINGS_ROOT, "mcpp.toml") + +option("pin_payload") + set_default(true) + set_showmenu(true) + set_description("Pin the hermetic mcpp toolchain payload (required for a fair benchmark)") +option_end() + +bench_define_toolchains(XLINGS_MANIFEST) + +-- ── Everything the helpers answer is resolved HERE, at description scope ────── +-- +-- ⚠️ THE HELPERS ARE NOT REACHABLE FROM on_load/before_build. xmake runs those +-- callbacks in a sandbox that does not carry an include()'d file's globals, so +-- `bench_package_root(...)` inside one fails with +-- +-- error: attempt to call a nil value (global 'bench_package_root') +-- +-- and the whole xlings/xmake arm reported `configure exited 255` — in CI, for +-- every cell, behind a green check. Lua closures capture their upvalues +-- lexically, so resolving to LOCALS here and letting the target read those is +-- both the fix and the shape bench/projects/mcpp/xmake.lua already used. +-- ── Dependencies, declared the way xlings itself declares them ─────────────── +-- +-- Shaped after openxlings/xlings@bb27e43's own xmake.lua: `add_requires` for +-- every dependency, `add_packages` on the target. Compiling them out of mcpp's +-- registry by hand — the previous shape here — went wrong in five separate ways +-- (source lists that a glob gets wrong, `!` exclusions that `target:add` ignores, +-- escaped quotes in cflags, `*`-prefixed paths resolving to the filesystem root, +-- and `io` being nil in description scope) before producing a binary. This is +-- what an xmake user would actually write. +-- +-- The index needed three newer versions than it carried (mcpplibs-xpkg 0.0.57, +-- capi-lua 0.0.3, tinyhttps 0.2.9); they were added there rather than overridden +-- here, because every xmake user of these libraries needs them, not just this +-- benchmark. mcpplibs-index also had to learn to SUPPLY a build description for +-- mcpplibs-xpkg: libxpkg moved to mcpp, so its 0.0.57 tarball ships `mcpp.toml` +-- and no xmake.lua at all. +-- +-- (mcpplibs/mcpplibs-index#14, merged.) MCPPLIBS_INDEX still overrides the URL, +-- which is how the next version bump gets tested against a checkout before it +-- is published. +includes("packages/libarchive.lua") + +-- EVERY dependency static, transitively. Without this the binary builds and +-- then cannot start: +-- error while loading shared libraries: libbz2.so.1.0: cannot open +-- shared object file: No such file or directory +-- bzip2 arrives through libarchive and xrepo built it shared, so the link +-- succeeded against a .so that is not on any runtime path. It also matters for +-- the comparison: mcpp produces a self-contained binary here (the arm passes +-- `-static-libstdc++` below), so an arm that leaves its dependencies dynamic is +-- not producing the same artifact. +add_requireconfs("**", {configs = {shared = false}}) +add_repositories("mcpplibs-index " .. + (os.getenv("MCPPLIBS_INDEX") or "https://github.com/mcpplibs/mcpplibs-index.git")) + +add_requires("cmdline 0.0.2") +add_requires("mcpplibs-capi-lua 0.0.3") +add_requires("mcpplibs-tinyhttps 0.2.9") +add_requires("mcpplibs-xpkg 0.0.57") +add_requires("ftxui 6.1.9") +-- libarchive's compression backends, declared EXPLICITLY exactly as xlings' own +-- xmake.lua declares them. Leaving them to libarchive-xlings' `add_deps` is not +-- enough: zlib, bzip2 and lz4 were then never installed at all +-- (`~/.xmake/packages/b/` empty), so `-lz -lbz2 -llz4` fell through to the +-- HOST's shared libraries. The link succeeded and the product could not start — +-- error while loading shared libraries: libz.so.1 +-- because the payload's private loader does not search /usr/lib. +add_requires("zlib", {system = false}) +add_requires("bzip2", {system = false}) +add_requires("lz4", {system = false}) +add_requires("zstd", {system = false}) +add_requires("xz", {system = false}) +-- openssl too, even though the override passes `-DENABLE_OPENSSL=OFF`: +-- `set_base("libarchive")` inherits the upstream package's openssl dependency, +-- so `-lssl -lcrypto` reach the link line regardless. The host has +-- libssl.so.3 but no `libssl.so` development symlink, so they resolve to +-- nothing and libarchive itself fails to build: +-- ld: cannot find -lssl: No such file or directory +add_requires("openssl", {system = false}) +-- libarchive-xlings, not plain libarchive: xmake-repo's build stops at +-- `CMake Error at CMakeLists.txt:1349 (MESSAGE): libgcc not found.` under the +-- payload toolchain. The override is xlings' own (see packages/libarchive.lua). +add_requires("libarchive-xlings 3.8.7") + + +target("xlings") + set_kind("binary") + + if not XLINGS_ROOT or not os.isfile(XLINGS_MANIFEST) then + raise("no xlings tree: set XLINGS_ROOT=, or let the bench harness " + .. "export BENCH_PROJECT_ROOT via --project. The pinned trees are " + .. "the submodules bench/projects/xlings/xlings-/ — run " + .. "`git submodule update --init`.") + end + + -- Source set == xlings' mcpp.toml inferred glob src/**/*.{cppm,cpp}; mcpp + -- infers kind=bin from src/main.cpp, xmake needs it spelled out. + -- + -- BOTH extensions, which is what lets ONE description measure xlings' two + -- code styles: 2026.8.11.2 has 110 .cppm + 2 .cpp (implementation inside + -- each interface unit), 2026.8.13.1 has 110 .cppm + 92 .cpp (split out). + -- Globbing `src/main.cpp` alone would compile the interfaces of the split + -- style, link nothing, and still report a number. Same note in CMakeLists.txt. + add_files(path.join(XLINGS_ROOT, "src/**.cppm")) + add_files(path.join(XLINGS_ROOT, "src/**.cpp")) + + -- Every dependency comes through xrepo, exactly as xlings' own xmake.lua + -- does. `mcpplibs-xpkg` brings the generated `mcpplibs.xpkg.lua_stdlib` + -- module with it — that generation lives in the PACKAGE now + -- (xrepo/packages/m/mcpplibs-xpkg/xmake.lua), which is where libxpkg keeps + -- it too, instead of being re-implemented against the registry here. + add_packages("cmdline", "mcpplibs-capi-lua", "mcpplibs-tinyhttps", + "mcpplibs-xpkg", "ftxui", "libarchive-xlings", + "zlib", "bzip2", "lz4", "zstd", "xz") + + -- `[build] include_dirs = ["src/libs/json"]` — src/libs/json.cppm reaches + -- for from its global module fragment. + add_includedirs(path.join(XLINGS_ROOT, "src/libs/json")) + -- `[build] cxxflags` + add_defines("LIBARCHIVE_STATIC", "UNICODE", "_UNICODE") + + + -- `mcpplibs.xpkg.lua_stdlib` is GENERATED by libxpkg's build.mcpp rather + -- than checked in: it embeds every .lua under src/lua-stdlib as a string + -- named after the file. Same RULE as embed_lua_stdlib.cmake (which the cmake + -- arm runs) — deliberately not the same list, because a copied list here + -- already drifted once and the failure landed three files away, in a + -- consumer, as `'base64_lua' is not a member of ...detail`. + -- + -- on_load, NOT before_build. The file existing early is only half of what is + + set_policy("build.c++.modules", true) + set_policy("build.c++.modules.std", true) + + add_ldflags("-static-libstdc++", {force = true}) + + if is_mode("release") then + set_optimize("fastest") + set_symbols("hidden") + elseif is_mode("debug") then + set_optimize("none") + set_symbols("debug") + end + + -- Which toolchain, and the rule for when NOT to pin one, live in + -- ../common/xmake/payload.lua. + if has_config("pin_payload") then + local tc = bench_pinned_toolchain() + if tc then set_toolchains(tc) end + end +target_end() diff --git a/bench/projects/xlings/xpkg_source_library.cmake b/bench/projects/xlings/xpkg_source_library.cmake new file mode 100644 index 00000000..a7617907 --- /dev/null +++ b/bench/projects/xlings/xpkg_source_library.cmake @@ -0,0 +1,288 @@ +# Build a registry package from its OWN manifest, for the cmake arm. +# +# WHY THIS EXISTS. xlings links ftxui, libarchive, lua and mbedtls, and mcpp's +# registry ships all four as SOURCE — there is no prebuilt .a to point at. The +# cmake arm therefore has to compile them, and the first two attempts at that +# both failed: +# +# * add_subdirectory() on the vendored CMakeLists. libarchive builds its test +# suite unconditionally enough that configure dies in +# `FILE STRINGS ... test_read_format_cab_skip_malformed.c cannot be read` +# (its own switch is ENABLE_TEST, singular — BUILD_TESTING/ENABLE_TESTING +# are ignored), and mbedtls 3.6.1 has an UNCONDITIONAL FATAL_ERROR at +# CMakeLists.txt:304 when `framework/CMakeLists.txt` is absent. The registry +# tarball has no `framework/` submodule, so that path can never work — it is +# not gated by any option. lua ships no CMakeLists at all. +# +# * A glob of the unpacked tree. `libarchive/*.c` is 132 files where mcpp +# compiles 127, and `lua/src/*.c` is 34 where mcpp compiles 32 — the two +# extra being lua.c and luac.c, the interpreter and the bytecode compiler, +# each with its own main(). Linking either into xlings is a duplicate-symbol +# error, and the five extra libarchive files are the ones its own configure +# decides against. +# +# So the source set is READ OUT OF THE MANIFEST. Every package in mcpp's +# registry carries `.xpkg.lua` beside its unpacked tree, and its `mcpp = { … }` +# table is the exact `sources` / `include_dirs` / `cflags` mcpp itself compiles +# the package with. Reading it is what makes this a fair arm: both engines then +# compile the same 127 files with the same defines. A copied list would be a +# fifth place the same decision lives, and bench/projects/xlings/README.md +# already records what happened the last time this repo copied a generated list +# (embed_lua_stdlib.cmake, which dropped one of eleven modules and surfaced the +# loss three files away, in a consumer). +# +# WHAT THIS DOES NOT DO. It is not a Lua interpreter — it reads four flat +# string-list fields out of one table whose shape every Form B manifest shares. +# Anything it cannot find is a FATAL_ERROR rather than an empty list, because +# the failure mode being avoided here is precisely "compiled a subset and +# reported a build time for it". + +# --------------------------------------------------------------------------- +# Manifest reading +# --------------------------------------------------------------------------- + +# The text of a package's `mcpp = { … }` table, and of the sub-table for the +# platform being built, if it has one. +function(_bench_xpkg_manifest verdir out_main out_plat) + set(manifest "${verdir}/.xpkg.lua") + if(NOT EXISTS "${manifest}") + message(FATAL_ERROR + "bench: no .xpkg.lua at ${verdir}. That file is what says which sources " + "mcpp compiles this package from; without it this arm would guess.") + endif() + file(READ "${manifest}" text) + + # `\n mcpp = {` — matching the four-space indent is what tells the + # package-level table apart from one NAMED IN A COMMENT at column 0, which + # mcpplibs-x-cmdline's manifest opens with (`-- Form B (inline mcpp = {…})`). + string(FIND "${text}" "\n mcpp = {" pos) + if(pos EQUAL -1) + message(FATAL_ERROR + "bench: ${manifest} has no `mcpp = {` table (a Form A descriptor, which " + "defers to an upstream mcpp.toml). This helper only reads Form B.") + endif() + string(SUBSTRING "${text}" ${pos} -1 seg) + + # Platform sub-tables (`windows = { cxxflags = … }`) sit at the end of the + # table and MUST be cut off the main one, not just skipped: ftxui has no + # top-level cxxflags and a windows one, so a search over the whole table + # finds the Windows flags and applies -DUNICODE on Linux. The current + # platform's table is taken out first, by name. + if(WIN32) + set(plat_key "windows") + elseif(APPLE) + set(plat_key "macosx") + else() + set(plat_key "linux") + endif() + set(plat "") + string(FIND "${seg}" "\n ${plat_key} = {" ppos) + if(NOT ppos EQUAL -1) + string(SUBSTRING "${seg}" ${ppos} -1 plat) + endif() + set(cut -1) + foreach(k linux macosx windows) + string(FIND "${seg}" "\n ${k} = {" p) + if(NOT p EQUAL -1) + if(cut EQUAL -1 OR p LESS cut) + set(cut ${p}) + endif() + endif() + endforeach() + if(NOT cut EQUAL -1) + string(SUBSTRING "${seg}" 0 ${cut} seg) + endif() + + set(${out_main} "${seg}" PARENT_SCOPE) + set(${out_plat} "${plat}" PARENT_SCOPE) +endfunction() + +# One `field = { "a", "b" }` list out of a manifest table, as a CMake list. +# Missing field -> empty, which the callers check where emptiness is wrong. +function(_bench_xpkg_field seg field out) + set(result "") + if("${seg}" MATCHES "[\r\n][ \t]*${field}[ \t]*=[ \t]*{([^}]*)}") + # Lua string literals, escapes included: "([^"\]|\.)*" + string(REGEX MATCHALL "\"([^\"\\\\]|\\\\.)*\"" quoted "${CMAKE_MATCH_1}") + foreach(q IN LISTS quoted) + string(REGEX REPLACE "^\"" "" v "${q}") + string(REGEX REPLACE "\"$" "" v "${v}") + # TWO un-escaping passes, and both are real. The manifest holds a Lua + # string whose VALUE is a shell token: libarchive's + # "-DPLATFORM_CONFIG_H=\\\"mcpp_libarchive_config.h\\\"" + # is the Lua spelling of -DPLATFORM_CONFIG_H=\"…\" which mcpp splices + # into a command line, where the shell strips the backslashes and the + # macro ends up as the quoted string `#include PLATFORM_CONFIG_H` needs. + # CMake does its own shell-escaping of compile options, so what it must + # be handed is the SHELL-LEVEL value — one pass short and the macro + # expands to \"mcpp_libarchive_config.h\", which fails as + # `#include` with a stray backslash rather than as a bad flag. + string(REGEX REPLACE "\\\\(.)" "\\1" v "${v}") + string(REGEX REPLACE "\\\\(.)" "\\1" v "${v}") + list(APPEND result "${v}") + endforeach() + endif() + set(${out} "${result}" PARENT_SCOPE) +endfunction() + +# One `field = "value"` scalar (c_standard, language) out of a manifest table. +function(_bench_xpkg_scalar seg field out) + set(result "") + if("${seg}" MATCHES "[\r\n][ \t]*${field}[ \t]*=[ \t]*\"([^\"]*)\"") + set(result "${CMAKE_MATCH_1}") + endif() + set(${out} "${result}" PARENT_SCOPE) +endfunction() + +# --------------------------------------------------------------------------- +# Path patterns +# --------------------------------------------------------------------------- +# Manifest paths are relative to the VERSION directory, and a leading `*` +# absorbs the tarball's wrap layer (`compat-x-lua/5.4.7/lua-5.4.7/…`). A bare +# path is the version directory itself — that is where `mcpp_generated/` holds +# the config headers mcpp materialises from the manifest's `generated_files` +# (mcpp_libarchive_config.h, mcpp_lua_platform_config.h, mcpp_zlib_config.h). +# `!` prefixes an exclusion (ftxui's *_test.cpp / *_fuzzer.cpp, zstd's +# zstd_trace.c). + +function(_bench_xpkg_expand verdir patterns out) + set(files "") + foreach(p IN LISTS patterns) + if(p MATCHES "\\*\\*") + # CMake has no `**`: GLOB_RECURSE already recurses below the matched + # directory, so `a/**/*.cpp` is spelled `a/*.cpp` there. + string(REPLACE "/**/" "/" p "${p}") + file(GLOB_RECURSE hit "${verdir}/${p}") + list(APPEND files ${hit}) + else() + file(GLOB hit "${verdir}/${p}") + list(APPEND files ${hit}) + endif() + endforeach() + if(files) + list(REMOVE_DUPLICATES files) + endif() + set(${out} "${files}" PARENT_SCOPE) +endfunction() + +# Resolve a manifest pattern list against the unpacked tree. +# +# A pattern that matches NOTHING is fatal. The whole point of reading the +# manifest is that the arms compile the same files; a pattern that quietly +# resolves to zero would put this arm back where the glob was, one translation +# unit short and reporting a time for it. +function(_bench_xpkg_resolve verdir what patterns out) + set(keep "") + set(drop "") + foreach(p IN LISTS patterns) + if(p MATCHES "^!(.+)$") + list(APPEND drop "${CMAKE_MATCH_1}") + else() + _bench_xpkg_expand("${verdir}" "${p}" hit) + if(NOT hit) + message(FATAL_ERROR + "bench: ${what} pattern '${p}' matched nothing under ${verdir}. The " + "package is unpacked but its manifest and its tree disagree.") + endif() + list(APPEND keep ${hit}) + endif() + endforeach() + _bench_xpkg_expand("${verdir}" "${drop}" dropped) + if(dropped) + list(REMOVE_ITEM keep ${dropped}) + endif() + list(REMOVE_DUPLICATES keep) + list(SORT keep) + set(${out} "${keep}" PARENT_SCOPE) +endfunction() + +# --------------------------------------------------------------------------- +# The target +# --------------------------------------------------------------------------- +# STATIC, because that is the link kind mcpp produces for a `kind = "lib"` +# package and the fairness contract in CMakeLists.txt covers the output kind. +# +# The VERSION IS PASSED IN, pinned by the caller from xlings' resolved +# dependency set. The registry holds several versions of some packages and +# "newest wins" would have the two arms compile different code. +function(bench_add_xpkg_library target pkg version) + bench_registry_xpkgs(xpkgs) + set(verdir "${xpkgs}/${pkg}/${version}") + if(NOT IS_DIRECTORY "${verdir}") + message(FATAL_ERROR + "bench: ${pkg} ${version} is not unpacked at ${verdir}. Run the mcpp arm " + "once (or `mcpp build` in the xlings tree) to populate the registry.") + endif() + + _bench_xpkg_manifest("${verdir}" seg plat) + _bench_xpkg_field("${seg}" sources pat_srcs) + _bench_xpkg_field("${seg}" include_dirs pat_incs) + _bench_xpkg_field("${seg}" cflags cflags) + _bench_xpkg_field("${seg}" cxxflags cxxflags) + _bench_xpkg_scalar("${seg}" c_standard c_standard) + if(plat) + _bench_xpkg_field("${plat}" cflags plat_cflags) + _bench_xpkg_field("${plat}" cxxflags plat_cxxflags) + list(APPEND cflags ${plat_cflags}) + list(APPEND cxxflags ${plat_cxxflags}) + endif() + + if(NOT pat_srcs) + message(FATAL_ERROR "bench: ${pkg}'s manifest has no `sources` list") + endif() + _bench_xpkg_resolve("${verdir}" "${pkg} sources" "${pat_srcs}" srcs) + + # include_dirs are globs too, and `"*"` (libarchive, zlib) resolves to + # everything beside the tree — tarballs included. Directories only. + _bench_xpkg_expand("${verdir}" "${pat_incs}" inc_hits) + set(incs "") + foreach(d IN LISTS inc_hits) + if(IS_DIRECTORY "${d}") + list(APPEND incs "${d}") + endif() + endforeach() + + add_library(${target} STATIC ${srcs}) + + # PUBLIC: mcpp propagates a package's include dirs to whatever imports it, + # and xlings' own units reach for , , and + # directly. This is what replaced a hand-written list of registry + # subdirectories in CMakeLists.txt. + target_include_directories(${target} PUBLIC ${incs}) + + # PRIVATE: these are how the package compiles ITSELF (`-include + # mcpp_zlib_config.h`, `-DZSTD_DISABLE_ASM=1`). The defines a CONSUMER needs + # — LIBARCHIVE_STATIC — are in xlings' own mcpp.toml and set on that target. + # + # Split on spaces first: `-include mcpp_zlib_config.h` is one manifest string + # but two argv words, and passed whole it reaches the compiler as a single + # quoted argument that gcc reports as an unrecognised option. + foreach(f IN LISTS cflags) + string(REPLACE " " ";" toks "${f}") + foreach(t IN LISTS toks) + target_compile_options(${target} PRIVATE "$<$:${t}>") + endforeach() + endforeach() + foreach(f IN LISTS cxxflags) + string(REPLACE " " ";" toks "${f}") + foreach(t IN LISTS toks) + target_compile_options(${target} PRIVATE "$<$:${t}>") + endforeach() + endforeach() + + if(c_standard AND c_standard MATCHES "c([0-9]+)") + # mcpp emits `-std=` verbatim, so extensions OFF — `gnu11` and + # `c11` are not the same dialect and these packages were configured for the + # strict one (which is why their manifests define _GNU_SOURCE by hand). + set_target_properties(${target} PROPERTIES + C_STANDARD ${CMAKE_MATCH_1} C_STANDARD_REQUIRED ON C_EXTENSIONS OFF) + endif() + + # None of these packages is a module (`import_std = false`, no `modules` + # key), and scanning them for module dependencies would add a scan of ~400 + # translation units to every cold build that mcpp does not pay. + set_target_properties(${target} PROPERTIES CXX_SCAN_FOR_MODULES OFF) + + list(LENGTH srcs n) + message(STATUS "bench: ${pkg} ${version} — ${n} sources from .xpkg.lua") +endfunction() diff --git a/bench/proto-bmi-release/README.md b/bench/proto-bmi-release/README.md new file mode 100644 index 00000000..8baa52a8 --- /dev/null +++ b/bench/proto-bmi-release/README.md @@ -0,0 +1,74 @@ +> **SUPERSEDED — read `.agents/docs/2026-08-12-cold-build-optimization-plan.md` +> appendix A instead.** That appendix re-derives the same proposal from direct +> measurements on mcpp's own 80 s cold build, and one thing it establishes here +> matters: this prototype's headline number was distorted by the detached child +> inheriting ninja's pipe (hazard 1 below), so any figure quoted from a run of +> these scripts is suspect. What survives intact is the hazard list. +> +> The corrected evidence: the BMI is byte-identical to the finished one and +> already usable at **15–39% (median ~22%)** of each compile — verified three +> ways (size stabilises, `cmp` against the finished file, and a real downstream +> importer compiles against the snapshot). Projected headroom **80 s → 25–35 s**. + +# Prototype: release importers at BMI-flush, not at compiler exit + +A throwaway, measurable prototype of the largest optimisation identified in +`.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md`. It is not +production code — it exists so the proposal rests on a measurement instead of a +simulation, and so the two implementation hazards below are on record before +anyone builds the real thing. + +## What it does + +`split_graph.py` mechanically rewrites mcpp's generated `build.ninja` so each +module interface unit becomes two edges driven by **one** compiler process: + +``` +build gcm.cache/X.gcm : cxx_module_bmi src/X.cppm | X.ddi.dd # exits at BMI rename +build obj/X.m.o : cxx_module_obj gcm.cache/X.gcm # waits for codegen +``` + +Importers already depend on `gcm.cache/X.gcm` (dyndep emits exactly that), so +nothing downstream needs rewriting — those dependencies simply become satisfiable +about 4x earlier. The link edge still waits for every object. + +## Measured on this repo + +``` +baseline (edge-complete release) 77.42 s +split (BMI-flush release) 36.56 s 2.12x, identical 19,347,008 B binary +``` + +Both arms cap concurrent compilers at `nproc`. Total CPU work is unchanged. + +```bash +bench/proto-bmi-release/run_proto.sh +``` + +## Two hazards this prototype exists to document + +**1. The detached compiler must not inherit the build system's stdout/stderr.** +ninja ends an edge at pipe EOF, not at direct-child exit. Leave the pipe +inherited and the early exit is invisible: every BMI edge logs the *full* compile +duration and the arm silently measures the baseline. The first run here did +exactly that (BMI edges median 2018 ms == full compiles) and looked like "the +idea does not work". Redirect the child's streams to a file and replay them from +phase 2, or compiler diagnostics vanish. + +**2. ninja's `-j` must be far larger than the compiler cap.** +Once detached, a compiler no longer holds a ninja slot, so concurrency is bounded +by the semaphore instead. With `-j` equal to the cap, ninja's slots fill with +edges that are only sleeping — blocked on the semaphore or waiting for codegen — +and the ready frontier starves. The first run used `-j32` with a cap of 32 and +came out at 78.99 s, *slower* than baseline. The measurement above uses `-j` = 6x +the cap. + +## Known limitations (fine for cold-build timing, not for production) + +- Drops the `-MMD` depfile plumbing, so header dependencies of global module + fragments are not tracked → valid for cold builds, not incremental correctness. +- Polls the filesystem every 5 ms and implements the semaphore with `mkdir` + tokens; a real implementation should use the GCC module-mapper protocol + (`MODULE-COMPILED` is the ready signal) or proper job control. +- No cleanup of detached compilers on SIGINT. Production needs process groups — + on Windows, a Job Object. diff --git a/bench/proto-bmi-release/bmi_release.sh b/bench/proto-bmi-release/bmi_release.sh new file mode 100755 index 00000000..83a19875 --- /dev/null +++ b/bench/proto-bmi-release/bmi_release.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Phase 1 of a two-edge module compile: start the compiler, return as soon as the +# BMI is on disk, and leave codegen running in the background. +# +# bmi_release.sh -- +# +# GCC writes the CMI to `~` and rename()s it into place (verified by strace), +# so the file is atomically complete-or-absent: seeing it appear is sufficient — +# no stability polling, no locking. +# +# FAIRNESS: once the compiler is detached it is invisible to ninja's -j +# accounting, so live compilers would no longer be bounded by the job count and +# the A/B against the unsplit graph would be comparing different amounts of +# hardware. A counting semaphore (atomic `mkdir` tokens) is therefore held from +# compiler start to compiler exit, capping concurrent compilers at MCPP_BMI_JOBS. +# No deadlock is possible: a token holder never waits on another token. +set -u + +SEMDIR="${MCPP_BMI_SEM:-.bmisem}" +JOBS="${MCPP_BMI_JOBS:-$(nproc)}" + +sem_acquire() { + mkdir -p "$SEMDIR" + while :; do + for i in $(seq 1 "$JOBS"); do + if mkdir "$SEMDIR/$i" 2>/dev/null; then printf '%s' "$SEMDIR/$i"; return 0; fi + done + sleep 0.005 + done +} + +slot="$1"; bmi="$2"; shift 2 +[ "${1:-}" = "--" ] && shift + +mkdir -p "$(dirname "$slot")" +rm -f "$slot.rc" "$bmi" + +tok=$(sem_acquire) + +# Detach: this script exits at BMI-flush while the compiler finishes codegen. +# +# THE TRAP: the background compiler inherits this process's stdout/stderr, which +# are ninja's pipes. ninja finishes an edge when the pipe reaches EOF, NOT when +# its direct child exits — so an inherited pipe makes the early exit completely +# invisible and every BMI edge is logged with the FULL compile duration. That is +# exactly what the first run of this prototype produced (BMI edges median 2018 ms +# == full compiles) and it read as "the idea does not work". +# Redirect the child's streams to a log; phase 2 replays it so diagnostics are +# still reported, attributed to the object edge. +( "$@" >"$slot.log" 2>&1 /dev/null + echo "$rc" > "$slot.rc.tmp"; mv "$slot.rc.tmp" "$slot.rc" ) >/dev/null 2>&1 "$slot.pid" + +while :; do + [ -f "$bmi" ] && exit 0 # BMI landed -> importers may proceed + if [ -f "$slot.rc" ]; then # compiler finished without producing one + rc=$(cat "$slot.rc") + [ "$rc" = "0" ] && exit 0 # e.g. an implementation unit: no BMI by design + exit "$rc" # real failure: fail this edge now + fi + sleep 0.005 +done diff --git a/bench/proto-bmi-release/bmi_wait.sh b/bench/proto-bmi-release/bmi_wait.sh new file mode 100755 index 00000000..578a53d6 --- /dev/null +++ b/bench/proto-bmi-release/bmi_wait.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Phase 2 of a two-edge module compile: block until the compiler started by +# bmi_release.sh has finished, and propagate its exit status. +# +# bmi_wait.sh +# +# This edge holds a ninja job slot while it waits, which is deliberate: it keeps +# the number of live compilers bounded by -j even though the compiler is no +# longer this script's child. +set -u +slot="$1"; obj="${2:-}" + +while [ ! -f "$slot.rc" ]; do sleep 0.01; done +rc=$(cat "$slot.rc") +# Replay whatever the detached compiler said. Its streams were redirected to a +# file so that phase 1 could exit early without holding ninja's pipe open; if +# they were not replayed here, warnings and errors would vanish silently. +[ -s "$slot.log" ] && cat "$slot.log" >&2 +if [ "$rc" != "0" ]; then exit "$rc"; fi +if [ -n "$obj" ] && [ ! -f "$obj" ]; then + echo "bmi_wait: compiler reported success but $obj is missing" >&2 + exit 1 +fi +exit 0 diff --git a/bench/proto-bmi-release/run_proto.sh b/bench/proto-bmi-release/run_proto.sh new file mode 100755 index 00000000..1ecdbe45 --- /dev/null +++ b/bench/proto-bmi-release/run_proto.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Measured A/B for the "BMI-release scheduling" proposal. +# +# Same build directory, same compiler, same flags, same job cap, same source set. +# The ONLY difference is the shape of the ninja graph: +# baseline : one edge per module; importers wait for the compiler to EXIT +# split : two edges per module; importers wait for the BMI to LAND +# One compiler process per module in both arms — total CPU work is identical. +set -u +PROTO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# bench/proto-bmi-release -> two levels up is the repository root. +R="$(cd "$PROTO/../.." && pwd)" +NINJA=$(command -v ninja) +J=${J:-$(nproc)} + +# Regenerate build.ninja first: the benchmark matrix may have left the tree in a +# state where the newest build dir belongs to a different fingerprint. +( cd "$R" && mcpp build >/dev/null 2>&1 ) + +BD=$(ls -td $R/target/x86_64-linux-gnu/*/ | while read -r d; do [ -f "$d/build.ninja" ] && echo "$d" && break; done) +[ -z "$BD" ] && { echo "no build dir with build.ninja"; exit 1; } +echo "build dir : $BD" +echo "ninja : $NINJA -j$J" + +cd "$BD" || exit 1 +python3 "$PROTO/split_graph.py" build.ninja build-split.ninja "$PROTO" || exit 1 + +wipe() { rm -rf obj gcm.cache slots .bmisem bin .ninja_deps; } + +# In the split arm, CPU parallelism is capped by the SEMAPHORE (MCPP_BMI_JOBS), +# not by ninja's -j: once a compiler is detached it no longer occupies a ninja +# slot. ninja's -j must therefore be much LARGER than the compiler cap, or its +# slots fill with edges that are merely sleeping — blocked on the semaphore or +# waiting for codegen — and the ready frontier starves. With -j equal to the cap +# the schedule degenerates to the baseline, which is what the first attempt here +# measured. Both arms still run at most $J compilers at once. +run() { # name ninjafile ninja_jobs + wipe + local s e + s=$(date +%s.%N) + MCPP_BMI_JOBS=$J MCPP_BMI_SEM=.bmisem "$NINJA" -f "$2" -j"$3" > "/tmp/proto_$1.log" 2>&1 + local rc=$? + e=$(date +%s.%N) + printf '%-10s rc=%d wall=%.2fs ninja -j%-4s compilers<=%s binary=%s\n' \ + "$1" "$rc" "$(echo "$e-$s" | bc)" "$3" "$J" \ + "$([ -f bin/mcpp ] && stat -c %s bin/mcpp || echo MISSING)" + [ $rc -ne 0 ] && tail -15 "/tmp/proto_$1.log" + return $rc +} + +echo +echo "=== arm 1: baseline graph (edge-complete release) ===" +run baseline build.ninja "$J" + +echo +echo "=== arm 2: split graph (BMI-flush release) ===" +run split build-split.ninja "$((J * 6))" + +echo +echo "--- sanity: were BMI edges actually short-lived? ---" +python3 - <<'PY' +rows=[] +for line in open('.ninja_log'): + if line.startswith('#'): continue + p=line.rstrip('\n').split('\t') + if len(p)>=5: rows.append((int(p[0]),int(p[1]),p[3])) +start=0 +for i in range(1,len(rows)): + if rows[i][1] BMI median must be a FRACTION of the OBJ median, or phase 1 is not") +print(" exiting early and the arm is measuring the baseline in disguise.") +PY + +echo +echo "=== verify the split build produced a WORKING binary ===" +./bin/mcpp --version || echo "BINARY BROKEN" + +echo +echo "=== leftover detached compilers? (must be 0) ===" +pgrep -c cc1plus || echo 0 diff --git a/bench/proto-bmi-release/split_graph.py b/bench/proto-bmi-release/split_graph.py new file mode 100644 index 00000000..0d3273a4 --- /dev/null +++ b/bench/proto-bmi-release/split_graph.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Rewrite an mcpp-generated build.ninja so importers are released at BMI-flush +instead of at compiler exit. Prototype for the "BMI-release scheduling" proposal. + +Per module interface unit: + + build obj/X.m.o | gcm.cache/X.gcm : cxx_module src/X.cppm | X.ddi.dd + dyndep = X.ddi.dd + +becomes two edges driven by ONE compiler process: + + build gcm.cache/X.gcm : cxx_module_bmi src/X.cppm | X.ddi.dd # exits at BMI rename + dyndep = X.ddi.dd + build obj/X.m.o : cxx_module_obj gcm.cache/X.gcm # waits for codegen + +Nothing downstream needs rewriting: dyndep already makes importers depend on +gcm.cache/X.gcm, so they simply become satisfiable ~4x earlier. The link edge +depends on obj/*.m.o and still waits for every object. + +The one non-obvious rewrite: dyndep attaches its module deps to whatever the +scan recorded as `-fdeps-target`, which is `obj/X.m.o`. If left alone, the +imports would gate the edge that merely WAITS, while the edge that actually +COMPILES would start with no BMIs present. + +Repointing that target needs care, because mcpp's cxx_scan rule spends +`$compile_target` TWICE — once as `-fdeps-target=` and once as `-o`. Changing +the shared variable would aim the preprocessor's `-o` at the BMI path and risk +truncating it. So the rule is rewritten to read `-fdeps-target=$deps_target`, +and only that new variable is repointed; `-o $compile_target` is left alone. + +Total CPU work is unchanged: still exactly one `g++ -c` per module. + +Limitation (fine for cold-build timing, not for incremental correctness): the +prototype drops the `-MMD` depfile plumbing, so header dependencies of global +module fragments are not tracked. + +Usage: split_graph.py +""" +import re, sys, os + + +def main(): + src, dst, proto = sys.argv[1], sys.argv[2], os.path.abspath(sys.argv[3]) + lines = open(src).read().split('\n') + + edge_re = re.compile(r'^build (\S+) \| (\S+) : cxx_module (\S+)(.*)$') + scan_re = re.compile(r'^build (\S+) : cxx_scan (\S+)(.*)$') + + # pass 1: obj -> gcm, so the scan edges can be repointed + gcm_of_obj = {} + for line in lines: + m = edge_re.match(line) + if m: + gcm_of_obj[m.group(1)] = m.group(2) + + rules = f'''rule cxx_module_bmi + command = {proto}/bmi_release.sh $slot $out -- $cxx $local_includes $cxxflags $unit_cxxflags -x c++ -c $in -o $obj_out + description = BMI $out + restat = 1 + +rule cxx_module_obj + command = {proto}/bmi_wait.sh $slot $out + description = OBJ $out + restat = 1 + +''' + + out, i, n_split, n_scan = [], 0, 0, 0 + while i < len(lines): + line = lines[i] + + if line.startswith('rule cxx_object'): + out.append(rules.rstrip('\n')) + out.append('') + out.append(line) + i += 1 + continue + + # rewrite the scan RULE so -fdeps-target reads its own variable + if line.strip().startswith('command =') and '-fdeps-format=p1689r5' in line: + out.append(line.replace('-fdeps-target=$compile_target', + '-fdeps-target=$deps_target')) + i += 1 + continue + + m = scan_re.match(line) + if m: + out.append(line) + i += 1 + extra = None + while i < len(lines) and lines[i].startswith(' '): + p = lines[i] + key = p.strip().split('=')[0].strip() + if key == 'compile_target': + tgt = p.split('=', 1)[1].strip() + # -o keeps pointing at the object; only the dyndep target moves + extra = f' deps_target = {gcm_of_obj.get(tgt, tgt)}' + if tgt in gcm_of_obj: + n_scan += 1 + out.append(p) + i += 1 + if extra: + out.append(extra) + continue + + m = edge_re.match(line) + if m: + obj, gcm, source, tail = m.groups() + props, j = [], i + 1 + while j < len(lines) and lines[j].startswith(' '): + props.append(lines[j]) + j += 1 + keep = [p for p in props if not p.strip().startswith('bmi_out')] + slot = 'slots/' + gcm.replace('/', '_') + out.append(f'build {gcm} : cxx_module_bmi {source}{tail}') + out.extend(keep) + out.append(f' obj_out = {obj}') + out.append(f' slot = {slot}') + out.append(f'build {obj} : cxx_module_obj {gcm}') + out.append(f' slot = {slot}') + n_split += 1 + i = j + continue + + out.append(line) + i += 1 + + open(dst, 'w').write('\n'.join(out)) + print(f'split {n_split} module edges, repointed {n_scan} scan targets -> {dst}') + + +if __name__ == '__main__': + main() diff --git a/bench/results/README.md b/bench/results/README.md new file mode 100644 index 00000000..386abf71 --- /dev/null +++ b/bench/results/README.md @@ -0,0 +1,57 @@ +# `bench/results/` — one directory per measurement run + +A run is a directory, not a filename prefix. The flat layout this replaces put +27 files from three unrelated runs side by side, sorted by tool name rather than +by run, so telling which JSON belonged to which table meant decoding timestamps. + +Each directory holds its own `report.md` and the raw files that report was +written from, named by what varies **within** the run (host, compiler) rather +than repeating what the directory already says. + +## Which of these is the standard data + +**`standard---/` is the standard data set.** It is the only +directory produced by `bench/run-standard.sh`, the only one taken at 3 samples +per cell, and the only one the README tables quote. Everything else is history — +kept because a claim that cannot be checked against the run that produced it is +not a measurement, and deleted history cannot be checked at all. + +⚠️ **Every directory below the standard one was taken under conditions that no +longer hold.** Read them as a record of what was measured then, not as data +about mcpp now: + +| taken with | what changed since | +|---|---| +| **cmake 4.0.2** | the pin is 4.4.2; its `import std` gate key is different, so those descriptions would not even configure today | +| **n=1** | the standard set is n=3; a single sample has no dispersion, which is why each of those tables carries a "do not compare the digits" caveat | +| **CI runners** | measured a shared 2-core machine — the same tree took 243s there and 79s on a developer box | +| **`bmi_schedule=on` before the §8b fix** | `touch-hub` and `edit-comment` in those columns were timing a build that had not finished; see ../README.md §8b | + +| run | what it measures | +|---|---| +| [`five-way-20260812/`](five-way-20260812/) | six engines × three source forms × six scenarios, on a **generated fixture**. cmake is the baseline. Two compilers, one file each. | +| [`mcpp-self-20260813/`](mcpp-self-20260813/) | the same scenarios on the **real project** — mcpp building itself, 138 module interface units. cmake is the baseline. | +| [`pinned-workloads-20260813/`](pinned-workloads-20260813/) | **the first run in which everything that moves a number is pinned** — tools, compiler, measured sources, reference mcpp. mcpp building itself five ways, and xlings in two code styles. Earlier runs are not comparable to it. | +| [`hyperfine-20260812/`](hyperfine-20260812/) | the earlier one-off mcpp-vs-xmake runs, driven by hyperfine before the harness existed. Superseded by the two above; kept because `NOTES.md` records how those numbers were taken. | + +**Read the reports, not the JSON.** The raw files are what makes a claim +checkable, but a number in them means nothing without the run's declared +asymmetries — those live in the report and in [`../README.md`](../README.md) §5. + +**Generate the tables, do not type them.** + +```bash +bench/tools/report.py /*.json --baseline cmake +``` + +The tables in these reports used to be transcribed from harness output by hand, +and transcription is the one error this suite cannot catch: a mistyped headline +number is indistinguishable from a measured one, and no test will ever fail. The +generator also enforces two things a person forgets — a non-`ok` cell renders as +its status rather than as a blank or a zero, and a group whose cells used +different **perturbation forms** gets a footnote saying so. + +**Before comparing anything across runs**, apply the validity rules in +[`../README.md`](../README.md) §4a: a cell within 2x of its own engine's `noop` +is measuring process startup, and absolute seconds do not carry between hosts — +only ratios within one table do. diff --git a/bench/results/five-way-20260812/linux-x86_64-clang.json b/bench/results/five-way-20260812/linux-x86_64-clang.json new file mode 100644 index 00000000..708e6fba --- /dev/null +++ b/bench/results/five-way-20260812/linux-x86_64-clang.json @@ -0,0 +1,1600 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-12T15:55:32Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.476, + "min_s": 0.456, + "max_s": 0.495, + "samples": [0.495, 0.456] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.186, + "min_s": 0.176, + "max_s": 0.195, + "samples": [0.195, 0.176] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.454, + "min_s": 0.454, + "max_s": 0.454, + "samples": [0.454, 0.454] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.352, + "min_s": 0.336, + "max_s": 0.367, + "samples": [0.367, 0.336] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.358, + "min_s": 0.350, + "max_s": 0.367, + "samples": [0.350, 0.367] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.443, + "min_s": 0.439, + "max_s": 0.447, + "samples": [0.447, 0.439] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 2.647, + "min_s": 2.605, + "max_s": 2.689, + "samples": [2.689, 2.605] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.184, + "min_s": 0.174, + "max_s": 0.194, + "samples": [0.174, 0.194] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.350, + "min_s": 0.330, + "max_s": 0.371, + "samples": [0.371, 0.330] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.358, + "min_s": 0.356, + "max_s": 0.360, + "samples": [0.356, 0.360] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.523, + "min_s": 0.510, + "max_s": 0.535, + "samples": [0.510, 0.535] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.528, + "min_s": 0.517, + "max_s": 0.539, + "samples": [0.517, 0.539] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 2.274, + "min_s": 2.254, + "max_s": 2.294, + "samples": [2.294, 2.254] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.197, + "min_s": 0.192, + "max_s": 0.201, + "samples": [0.192, 0.201] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.361, + "min_s": 0.354, + "max_s": 0.367, + "samples": [0.367, 0.354] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.356, + "min_s": 0.355, + "max_s": 0.356, + "samples": [0.356, 0.355] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.357, + "min_s": 0.342, + "max_s": 0.371, + "samples": [0.371, 0.342] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.505, + "min_s": 0.490, + "max_s": 0.520, + "samples": [0.490, 0.520] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.413, + "min_s": 0.406, + "max_s": 0.421, + "samples": [0.406, 0.421] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.173, + "min_s": 0.171, + "max_s": 0.174, + "samples": [0.171, 0.174] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.380, + "min_s": 0.366, + "max_s": 0.394, + "samples": [0.394, 0.366] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.303, + "min_s": 0.282, + "max_s": 0.323, + "samples": [0.323, 0.282] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.313, + "min_s": 0.290, + "max_s": 0.336, + "samples": [0.290, 0.336] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.396, + "min_s": 0.387, + "max_s": 0.405, + "samples": [0.405, 0.387] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 2.495, + "min_s": 2.472, + "max_s": 2.519, + "samples": [2.472, 2.519] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.176, + "min_s": 0.167, + "max_s": 0.185, + "samples": [0.167, 0.185] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.275, + "min_s": 0.274, + "max_s": 0.277, + "samples": [0.274, 0.277] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.294, + "min_s": 0.279, + "max_s": 0.308, + "samples": [0.308, 0.279] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.456, + "min_s": 0.455, + "max_s": 0.456, + "samples": [0.456, 0.455] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.432, + "min_s": 0.414, + "max_s": 0.451, + "samples": [0.451, 0.414] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 2.187, + "min_s": 2.178, + "max_s": 2.197, + "samples": [2.178, 2.197] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.184, + "min_s": 0.179, + "max_s": 0.189, + "samples": [0.179, 0.189] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.278, + "min_s": 0.268, + "max_s": 0.288, + "samples": [0.288, 0.268] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.323, + "min_s": 0.320, + "max_s": 0.326, + "samples": [0.326, 0.320] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.308, + "min_s": 0.295, + "max_s": 0.320, + "samples": [0.295, 0.320] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.438, + "min_s": 0.430, + "max_s": 0.446, + "samples": [0.446, 0.430] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.026, + "min_s": 2.022, + "max_s": 2.029, + "samples": [2.029, 2.022] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.334, + "min_s": 0.326, + "max_s": 0.342, + "samples": [0.326, 0.342] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.546, + "min_s": 0.540, + "max_s": 0.553, + "samples": [0.553, 0.540] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.415, + "min_s": 0.413, + "max_s": 0.417, + "samples": [0.413, 0.417] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.412, + "min_s": 0.407, + "max_s": 0.417, + "samples": [0.417, 0.407] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.541, + "min_s": 0.531, + "max_s": 0.551, + "samples": [0.531, 0.551] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 3.999, + "min_s": 3.991, + "max_s": 4.008, + "samples": [3.991, 4.008] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.323, + "min_s": 0.321, + "max_s": 0.326, + "samples": [0.326, 0.321] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.665, + "min_s": 2.658, + "max_s": 2.672, + "samples": [2.672, 2.658] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.459, + "min_s": 0.449, + "max_s": 0.469, + "samples": [0.469, 0.449] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.620, + "min_s": 2.619, + "max_s": 2.621, + "samples": [2.619, 2.621] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.656, + "min_s": 2.626, + "max_s": 2.685, + "samples": [2.626, 2.685] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 3.965, + "min_s": 3.964, + "max_s": 3.967, + "samples": [3.964, 3.967] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.334, + "min_s": 0.332, + "max_s": 0.337, + "samples": [0.337, 0.332] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.350, + "min_s": 2.319, + "max_s": 2.381, + "samples": [2.319, 2.381] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.485, + "min_s": 0.478, + "max_s": 0.491, + "samples": [0.478, 0.491] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.420, + "min_s": 0.419, + "max_s": 0.420, + "samples": [0.420, 0.419] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 2.331, + "min_s": 2.307, + "max_s": 2.355, + "samples": [2.307, 2.355] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 2.676, + "min_s": 2.660, + "max_s": 2.692, + "samples": [2.660, 2.692] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.289, + "min_s": 0.288, + "max_s": 0.289, + "samples": [0.288, 0.289] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.759, + "min_s": 0.291, + "max_s": 1.227, + "samples": [1.227, 0.291] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.568, + "min_s": 0.283, + "max_s": 0.852, + "samples": [0.852, 0.283] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.249, + "min_s": 1.242, + "max_s": 1.255, + "samples": [1.242, 1.255] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.763, + "min_s": 0.298, + "max_s": 1.229, + "samples": [1.229, 0.298] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 13.193, + "min_s": 13.161, + "max_s": 13.225, + "samples": [13.161, 13.225] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.319, + "min_s": 0.319, + "max_s": 0.319, + "samples": [0.319, 0.319] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 12.762, + "min_s": 12.529, + "max_s": 12.996, + "samples": [12.529, 12.996] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.410, + "min_s": 1.409, + "max_s": 1.412, + "samples": [1.412, 1.409] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 12.682, + "min_s": 12.557, + "max_s": 12.807, + "samples": [12.807, 12.557] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 12.564, + "min_s": 12.505, + "max_s": 12.623, + "samples": [12.505, 12.623] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 13.401, + "min_s": 13.333, + "max_s": 13.469, + "samples": [13.469, 13.333] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.349, + "min_s": 0.347, + "max_s": 0.351, + "samples": [0.351, 0.347] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 12.641, + "min_s": 12.638, + "max_s": 12.644, + "samples": [12.638, 12.644] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.489, + "min_s": 1.475, + "max_s": 1.503, + "samples": [1.503, 1.475] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.575, + "min_s": 0.369, + "max_s": 0.780, + "samples": [0.780, 0.369] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 12.682, + "min_s": 12.677, + "max_s": 12.687, + "samples": [12.677, 12.687] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 2.551, + "min_s": 2.551, + "max_s": 2.551, + "samples": [2.551, 2.551] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.630, + "min_s": 0.621, + "max_s": 0.638, + "samples": [0.621, 0.638] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.817, + "min_s": 0.812, + "max_s": 0.822, + "samples": [0.812, 0.822] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.685, + "min_s": 0.684, + "max_s": 0.685, + "samples": [0.684, 0.685] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.693, + "min_s": 0.678, + "max_s": 0.707, + "samples": [0.707, 0.678] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.809, + "min_s": 0.804, + "max_s": 0.815, + "samples": [0.815, 0.804] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.667, + "min_s": 0.634, + "max_s": 0.701, + "samples": [0.701, 0.634] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.206, + "min_s": 0.204, + "max_s": 0.209, + "samples": [0.209, 0.204] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.212, + "min_s": 0.210, + "max_s": 0.213, + "samples": [0.210, 0.213] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.204, + "min_s": 0.201, + "max_s": 0.208, + "samples": [0.208, 0.201] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.273, + "min_s": 0.272, + "max_s": 0.273, + "samples": [0.272, 0.273] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.422, + "min_s": 0.407, + "max_s": 0.437, + "samples": [0.407, 0.437] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 3.188, + "min_s": 3.146, + "max_s": 3.230, + "samples": [3.230, 3.146] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.203, + "min_s": 0.202, + "max_s": 0.204, + "samples": [0.202, 0.204] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.225, + "min_s": 0.217, + "max_s": 0.234, + "samples": [0.234, 0.217] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.215, + "min_s": 0.213, + "max_s": 0.217, + "samples": [0.217, 0.213] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 2.837, + "min_s": 2.800, + "max_s": 2.874, + "samples": [2.874, 2.800] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 2.842, + "min_s": 2.836, + "max_s": 2.848, + "samples": [2.836, 2.848] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 2.816, + "min_s": 2.757, + "max_s": 2.874, + "samples": [2.874, 2.757] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.209, + "min_s": 0.204, + "max_s": 0.213, + "samples": [0.213, 0.204] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.214, + "min_s": 0.207, + "max_s": 0.220, + "samples": [0.220, 0.207] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.210, + "min_s": 0.206, + "max_s": 0.215, + "samples": [0.206, 0.215] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.323, + "min_s": 0.312, + "max_s": 0.335, + "samples": [0.312, 0.335] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 2.442, + "min_s": 2.423, + "max_s": 2.462, + "samples": [2.423, 2.462] + } + ] +} diff --git a/bench/results/five-way-20260812/linux-x86_64-gcc.json b/bench/results/five-way-20260812/linux-x86_64-gcc.json new file mode 100644 index 00000000..324501bf --- /dev/null +++ b/bench/results/five-way-20260812/linux-x86_64-gcc.json @@ -0,0 +1,1564 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-12T15:44:29Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.567, + "min_s": 0.556, + "max_s": 0.578, + "samples": [0.556, 0.578] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.167, + "min_s": 0.157, + "max_s": 0.177, + "samples": [0.157, 0.177] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.538, + "min_s": 0.529, + "max_s": 0.547, + "samples": [0.529, 0.547] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.371, + "min_s": 0.369, + "max_s": 0.373, + "samples": [0.373, 0.369] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.370, + "min_s": 0.364, + "max_s": 0.375, + "samples": [0.364, 0.375] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.549, + "min_s": 0.528, + "max_s": 0.569, + "samples": [0.569, 0.528] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.606, + "min_s": 3.581, + "max_s": 3.631, + "samples": [3.581, 3.631] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.155, + "min_s": 0.153, + "max_s": 0.158, + "samples": [0.158, 0.153] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.609, + "min_s": 3.586, + "max_s": 3.631, + "samples": [3.631, 3.586] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.387, + "min_s": 0.334, + "max_s": 0.439, + "samples": [0.439, 0.334] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.651, + "min_s": 3.618, + "max_s": 3.684, + "samples": [3.618, 3.684] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.672, + "min_s": 3.641, + "max_s": 3.703, + "samples": [3.703, 3.641] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.357, + "min_s": 3.356, + "max_s": 3.358, + "samples": [3.356, 3.358] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.169, + "min_s": 0.163, + "max_s": 0.174, + "samples": [0.174, 0.163] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.337, + "min_s": 3.310, + "max_s": 3.363, + "samples": [3.310, 3.363] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.409, + "min_s": 0.370, + "max_s": 0.447, + "samples": [0.370, 0.447] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.370, + "min_s": 0.369, + "max_s": 0.372, + "samples": [0.372, 0.369] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 3.298, + "min_s": 3.292, + "max_s": 3.303, + "samples": [3.303, 3.292] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.508, + "min_s": 0.504, + "max_s": 0.512, + "samples": [0.512, 0.504] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.149, + "min_s": 0.148, + "max_s": 0.149, + "samples": [0.149, 0.148] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.470, + "min_s": 0.469, + "max_s": 0.471, + "samples": [0.469, 0.471] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.287, + "min_s": 0.279, + "max_s": 0.295, + "samples": [0.295, 0.279] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.292, + "min_s": 0.290, + "max_s": 0.293, + "samples": [0.290, 0.293] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.458, + "min_s": 0.451, + "max_s": 0.465, + "samples": [0.451, 0.465] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 3.530, + "min_s": 3.528, + "max_s": 3.532, + "samples": [3.528, 3.532] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.142, + "min_s": 0.139, + "max_s": 0.145, + "samples": [0.139, 0.145] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.291, + "min_s": 0.288, + "max_s": 0.295, + "samples": [0.295, 0.288] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.304, + "min_s": 0.304, + "max_s": 0.305, + "samples": [0.305, 0.304] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.291, + "min_s": 0.275, + "max_s": 0.307, + "samples": [0.275, 0.307] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.296, + "min_s": 0.294, + "max_s": 0.299, + "samples": [0.299, 0.294] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 3.253, + "min_s": 3.232, + "max_s": 3.273, + "samples": [3.273, 3.232] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.154, + "min_s": 0.151, + "max_s": 0.158, + "samples": [0.151, 0.158] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.296, + "min_s": 0.293, + "max_s": 0.299, + "samples": [0.293, 0.299] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.294, + "min_s": 0.292, + "max_s": 0.295, + "samples": [0.295, 0.292] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.309, + "min_s": 0.307, + "max_s": 0.311, + "samples": [0.307, 0.311] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.305, + "min_s": 0.297, + "max_s": 0.312, + "samples": [0.312, 0.297] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 4.068, + "min_s": 4.059, + "max_s": 4.078, + "samples": [4.059, 4.078] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.331, + "min_s": 0.330, + "max_s": 0.333, + "samples": [0.333, 0.330] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 1.332, + "min_s": 1.315, + "max_s": 1.348, + "samples": [1.315, 1.348] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.792, + "min_s": 0.784, + "max_s": 0.800, + "samples": [0.800, 0.784] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.794, + "min_s": 0.790, + "max_s": 0.798, + "samples": [0.790, 0.798] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 1.353, + "min_s": 1.349, + "max_s": 1.357, + "samples": [1.349, 1.357] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 13.055, + "min_s": 13.037, + "max_s": 13.072, + "samples": [13.037, 13.072] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.336, + "min_s": 0.332, + "max_s": 0.339, + "samples": [0.339, 0.332] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 10.316, + "min_s": 10.289, + "max_s": 10.343, + "samples": [10.343, 10.289] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.987, + "min_s": 0.979, + "max_s": 0.995, + "samples": [0.979, 0.995] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 10.292, + "min_s": 10.252, + "max_s": 10.331, + "samples": [10.331, 10.252] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 10.313, + "min_s": 10.305, + "max_s": 10.321, + "samples": [10.305, 10.321] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 12.796, + "min_s": 12.782, + "max_s": 12.810, + "samples": [12.810, 12.782] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.333, + "min_s": 0.329, + "max_s": 0.337, + "samples": [0.337, 0.329] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 10.008, + "min_s": 9.967, + "max_s": 10.048, + "samples": [10.048, 9.967] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 1.049, + "min_s": 1.034, + "max_s": 1.064, + "samples": [1.064, 1.034] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.793, + "min_s": 0.788, + "max_s": 0.799, + "samples": [0.799, 0.788] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 10.036, + "min_s": 10.032, + "max_s": 10.039, + "samples": [10.039, 10.032] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 2.711, + "min_s": 2.678, + "max_s": 2.743, + "samples": [2.678, 2.743] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.286, + "min_s": 0.281, + "max_s": 0.292, + "samples": [0.292, 0.281] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.263, + "min_s": 1.263, + "max_s": 1.264, + "samples": [1.263, 1.264] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.693, + "min_s": 0.509, + "max_s": 0.877, + "samples": [0.877, 0.509] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.087, + "min_s": 0.907, + "max_s": 1.268, + "samples": [1.268, 0.907] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.071, + "min_s": 0.901, + "max_s": 1.240, + "samples": [1.240, 0.901] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.455, + "min_s": 11.445, + "max_s": 11.466, + "samples": [11.445, 11.466] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.317, + "min_s": 0.316, + "max_s": 0.318, + "samples": [0.316, 0.318] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.129, + "min_s": 11.103, + "max_s": 11.156, + "samples": [11.156, 11.103] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.157, + "min_s": 1.150, + "max_s": 1.164, + "samples": [1.164, 1.150] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.154, + "min_s": 11.134, + "max_s": 11.175, + "samples": [11.134, 11.175] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 10.546, + "min_s": 10.529, + "max_s": 10.563, + "samples": [10.529, 10.563] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.975, + "min_s": 11.940, + "max_s": 12.010, + "samples": [12.010, 11.940] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.357, + "min_s": 0.354, + "max_s": 0.360, + "samples": [0.360, 0.354] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.546, + "min_s": 11.528, + "max_s": 11.565, + "samples": [11.565, 11.528] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 1.262, + "min_s": 1.254, + "max_s": 1.269, + "samples": [1.254, 1.269] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.833, + "min_s": 0.641, + "max_s": 1.024, + "samples": [1.024, 0.641] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 11.567, + "min_s": 11.541, + "max_s": 11.593, + "samples": [11.541, 11.593] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 4.843, + "min_s": 4.832, + "max_s": 4.855, + "samples": [4.855, 4.832] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 0.644, + "min_s": 0.627, + "max_s": 0.661, + "samples": [0.661, 0.627] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 1.565, + "min_s": 1.562, + "max_s": 1.569, + "samples": [1.562, 1.569] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 1.048, + "min_s": 1.046, + "max_s": 1.051, + "samples": [1.051, 1.046] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 1.033, + "min_s": 1.033, + "max_s": 1.033, + "samples": [1.033, 1.033] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "meson 1.10.2 + ninja", + "runs": 2, + "median_s": 1.537, + "min_s": 1.534, + "max_s": 1.540, + "samples": [1.540, 1.534] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "meson", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "meson 1.10.2 does not build C++20 named modules (measured: \"module 'fx.a' not found\"; no attribute declares an interface unit)", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.796, + "min_s": 0.787, + "max_s": 0.805, + "samples": [0.787, 0.805] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.204, + "min_s": 0.203, + "max_s": 0.205, + "samples": [0.205, 0.203] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.210, + "min_s": 0.207, + "max_s": 0.212, + "samples": [0.212, 0.207] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.204, + "min_s": 0.203, + "max_s": 0.204, + "samples": [0.204, 0.203] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.301, + "min_s": 0.295, + "max_s": 0.307, + "samples": [0.307, 0.295] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 2, + "median_s": 0.506, + "min_s": 0.506, + "max_s": 0.506, + "samples": [0.506, 0.506] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + }, + { + "engine": "bazel", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-40x3", + "variant": "modules-impl", + "status": "unavailable", + "note": "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); re-run with --compiler to measure this cell", + "runs": 0, + "samples": [] + } + ] +} diff --git a/bench/results/five-way-20260812/report.md b/bench/results/five-way-20260812/report.md new file mode 100644 index 00000000..2a266d7b --- /dev/null +++ b/bench/results/five-way-20260812/report.md @@ -0,0 +1,203 @@ +# Five-way engine comparison — 2026-08-12 + +**mcpp (old) · mcpp (new) · cmake · xmake · meson · bazel**, on the same fixture, +with the same compiler binary, across two compilers. + +**cmake is the performance baseline.** Every cell shows the median wall time and +its ratio to cmake in the same row, so `0.26x` reads "took 26% of what cmake took" +and `4.66x` reads "took 4.66 times as long". + +| | | +|---|---| +| host | Linux x86_64 · 13th Gen Intel Core i9-13900K · 32 logical / 24 physical (heterogeneous) · 64 GiB | +| fixture | generated, **40 units / fan-in 3 / weight 6**, medians of 2 runs | +| compilers | `gcc@16.1.0` and `llvm@22.1.8`, both hermetic mcpp payloads, pinned into every engine | +| engines | mcpp 2026.8.11.3 (previous release) and 2026.8.12.1 (this PR), cmake 4.0.2, xmake v3.0.7+HEAD.77d94ad, meson 1.10.2, bazel 9.2.0 + rules_cc 0.2.22 — each recorded in the result file by the engine itself, not asserted here | +| raw | [`linux-x86_64-gcc.json`](linux-x86_64-gcc.json), [`linux-x86_64-clang.json`](linux-x86_64-clang.json) | + +Reproduce: + +``` +bench --engines mcpp=,mcpp=,cmake,xmake,meson,bazel \ + --compiler \ + --units 40 --fanin 3 --weight 6 --runs 2 --baseline cmake +``` + +--- + +#### gcc@16.1.0 + +**`headers`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 0.57s · 0.14x | 0.51s · 0.12x | **4.07s** · 1.00x | 2.71s · 0.67x | 4.84s · 1.19x | 0.80s · 0.20x | +| `noop` | 0.17s · 0.50x | 0.15s · 0.45x | **0.33s** · 1.00x | 0.29s · 0.86x | 0.64s · 1.95x | 0.20s · 0.62x | +| `touch-leaf` | 0.37s · 0.47x | 0.29s · 0.36x | **0.79s** · 1.00x | 0.69s · 0.87x | 1.05s · 1.32x | 0.20s · 0.26x | +| `edit-body` | 0.37s · 0.47x | 0.29s · 0.37x | **0.79s** · 1.00x | 1.09s · 1.37x | 1.03s · 1.30x | 0.30s · 0.38x | +| `edit-comment` | 0.55s · 0.41x | 0.46s · 0.34x | **1.35s** · 1.00x | 1.07s · 0.79x | 1.54s · 1.14x | 0.51s · 0.37x | +| `touch-hub` | 0.54s · 0.40x | 0.47s · 0.35x | **1.33s** · 1.00x | 1.26s · 0.95x | 1.56s · 1.17x | 0.21s · 0.16x | + +**`modules`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 3.61s · 0.28x | 3.53s · 0.27x | **13.05s** · 1.00x | 11.46s · 0.88x | _unavailable_ | _unavailable_ | +| `noop` | 0.15s · 0.46x | 0.14s · 0.42x | **0.34s** · 1.00x | 0.32s · 0.94x | _unavailable_ | _unavailable_ | +| `touch-leaf` | 0.39s · 0.39x | 0.30s · 0.31x | **0.99s** · 1.00x | 1.16s · 1.17x | _unavailable_ | _unavailable_ | +| `edit-body` | 3.65s · 0.35x | 0.29s · 0.03x | **10.29s** · 1.00x | 11.15s · 1.08x | _unavailable_ | _unavailable_ | +| `edit-comment` | 3.67s · 0.36x | 0.30s · 0.03x | **10.31s** · 1.00x | 10.55s · 1.02x | _unavailable_ | _unavailable_ | +| `touch-hub` | 3.61s · 0.35x | 0.29s · 0.03x | **10.32s** · 1.00x | 11.13s · 1.08x | _unavailable_ | _unavailable_ | + +**`modules-impl`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 3.36s · 0.26x | 3.25s · 0.25x | **12.80s** · 1.00x | 11.97s · 0.94x | _unavailable_ | _unavailable_ | +| `noop` | 0.17s · 0.51x | 0.15s · 0.46x | **0.33s** · 1.00x | 0.36s · 1.07x | _unavailable_ | _unavailable_ | +| `touch-leaf` | 0.41s · 0.39x | 0.29s · 0.28x | **1.05s** · 1.00x | 1.26s · 1.20x | _unavailable_ | _unavailable_ | +| `edit-body` | 0.37s · 0.47x | 0.31s · 0.39x | **0.79s** · 1.00x | 0.83s · 1.05x | _unavailable_ | _unavailable_ | +| `edit-comment` | 3.30s · 0.33x | 0.30s · 0.03x | **10.04s** · 1.00x | 11.57s · 1.15x | _unavailable_ | _unavailable_ | +| `touch-hub` | 3.34s · 0.33x | 0.30s · 0.03x | **10.01s** · 1.00x | 11.55s · 1.15x | _unavailable_ | _unavailable_ | + + +#### llvm@22.1.8 + +**`headers`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 0.48s · 0.23x | 0.41s · 0.20x | **2.03s** · 1.00x | 2.68s · 1.32x | 2.55s · 1.26x | 0.67s · 0.33x | +| `noop` | 0.19s · 0.56x | 0.17s · 0.52x | **0.33s** · 1.00x | 0.29s · 0.87x | 0.63s · 1.89x | 0.21s · 0.62x | +| `touch-leaf` | 0.35s · 0.85x | 0.30s · 0.73x | **0.41s** · 1.00x | 0.57s · 1.37x | 0.69s · 1.65x | 0.20s · 0.49x | +| `edit-body` | 0.36s · 0.87x | 0.31s · 0.76x | **0.41s** · 1.00x | 1.25s · 3.03x | 0.69s · 1.68x | 0.27s · 0.66x | +| `edit-comment` | 0.44s · 0.82x | 0.40s · 0.73x | **0.54s** · 1.00x | 0.76s · 1.41x | 0.81s · 1.50x | 0.42s · 0.78x | +| `touch-hub` | 0.45s · 0.83x | 0.38s · 0.70x | **0.55s** · 1.00x | 0.76s · 1.39x | 0.82s · 1.50x | 0.21s · 0.39x | + +**`modules`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 2.65s · 0.66x | 2.50s · 0.62x | **4.00s** · 1.00x | 13.19s · 3.30x | _unavailable_ | 3.19s · 0.80x | +| `noop` | 0.18s · 0.57x | 0.18s · 0.54x | **0.32s** · 1.00x | 0.32s · 0.99x | _unavailable_ | 0.20s · 0.63x | +| `touch-leaf` | 0.36s · 0.78x | 0.29s · 0.64x | **0.46s** · 1.00x | 1.41s · 3.07x | _unavailable_ | 0.21s · 0.47x | +| `edit-body` | 0.52s · 0.20x | 0.46s · 0.17x | **2.62s** · 1.00x | 12.68s · 4.84x | _unavailable_ | 2.84s · 1.08x | +| `edit-comment` | 0.53s · 0.20x | 0.43s · 0.16x | **2.66s** · 1.00x | 12.56s · 4.73x | _unavailable_ | 2.84s · 1.07x | +| `touch-hub` | 0.35s · 0.13x | 0.28s · 0.10x | **2.67s** · 1.00x | 12.76s · 4.79x | _unavailable_ | 0.23s · 0.08x | + +**`modules-impl`** + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | meson | bazel | +|---|---|---|---|---|---|---| +| `cold` | 2.27s · 0.57x | 2.19s · 0.55x | **3.96s** · 1.00x | 13.40s · 3.38x | _unavailable_ | 2.82s · 0.71x | +| `noop` | 0.20s · 0.59x | 0.18s · 0.55x | **0.33s** · 1.00x | 0.35s · 1.04x | _unavailable_ | 0.21s · 0.63x | +| `touch-leaf` | 0.36s · 0.73x | 0.32s · 0.67x | **0.48s** · 1.00x | 1.49s · 3.07x | _unavailable_ | 0.21s · 0.43x | +| `edit-body` | 0.36s · 0.85x | 0.31s · 0.73x | **0.42s** · 1.00x | 0.57s · 1.37x | _unavailable_ | 0.32s · 0.77x | +| `edit-comment` | 0.51s · 0.22x | 0.44s · 0.19x | **2.33s** · 1.00x | 12.68s · 5.44x | _unavailable_ | 2.44s · 1.05x | +| `touch-hub` | 0.36s · 0.15x | 0.28s · 0.12x | **2.35s** · 1.00x | 12.64s · 5.38x | _unavailable_ | 0.21s · 0.09x | + + +--- + +## What the numbers say + +### 1. The module cascade is the whole story, and it is avoidable + +Under gcc, a change to the most-imported interface unit costs **cmake 10.3s and +xmake 11.2s** — they rebuild 39 downstream units. mcpp 2026.8.12.1 costs +**0.29s**, because it compares the BMI the compiler just produced against the +previous one and, when they are equivalent, puts the old file back so ninja's +`restat` sees no change. + +That is a **35x** gap against the baseline, and a **12.5x** gap against mcpp's own +previous release — which had the same mechanism but compared bytes, and GCC +writes `buildtime:`/`localtime:` stamps into every BMI, so no two BMIs were ever +byte-equal and the suppression never once fired. + +### 2. Under gcc, editing a function body need not cascade at all + +`edit-body` inserts a real `volatile` statement: the object file genuinely +changes. It still costs mcpp 0.29s, and this is **correct, not a missed +rebuild**: GCC 16.1 does not encode the body of an exported non-template +function into the BMI. Verified by compiling the same unit twice with and +without the edit and diffing the two BMIs — the only differing bytes are the +seconds digit of the embedded timestamps, at the same offsets a *control* pair +(identical source, compiled twice) differs at. + +cmake and xmake pay the full cascade for that edit anyway, because they decide +from the BMI's mtime rather than its content. + +### 3. Clang changes who wins, and by how much + +Clang's BMIs are stamp-free, so mcpp's *previous* release already avoided the +cascade there (0.52s vs 0.46s — the new mechanism adds nothing under clang). +What clang changes is everyone else: + +* cmake's module cold build drops from 13.06s to 4.00s — **3.3x faster than + itself**, the single largest effect in the entire matrix. It is a compiler + effect, not a build-system one. +* xmake goes the other way: 11.46s → 13.19s cold, and **every** incremental + module scenario costs ~12.6s. It rebuilds the world on any module change under + either compiler. +* bazel becomes able to build modules at all (see below) and lands at 3.19s + cold — 0.80x cmake. + +### 4. bazel builds C++20 modules — with clang only + +`rules_cc` 0.2.22 has a `module_interfaces` attribute, and with +`--experimental_cpp_modules --features=cpp_modules` bazel builds and runs a +module program. With **gcc** it fails in bazel's own scanner: + +``` +aggregate-ddi failed: ... what(): Invalid JSON string +``` + +so its ddi aggregator cannot parse GCC's P1689 output. Module cells are reported +`unavailable` **with that measurement** in the gcc table rather than as a slow +number. + +bazel's `touch-hub` of 0.21s (0.08x) is real but is not the same achievement as +mcpp's: bazel hashes content, so an mtime bump with unchanged bytes is a +no-op by construction. On `edit-body`, where the bytes do change, bazel pays +2.84s — a full cascade, like cmake. + +### 5. meson cannot build named modules at all + +meson 1.10.2 has no attribute that declares an interface unit; the build fails +with `fatal error: module 'fx.a' not found` under both compilers. Its headers +columns are real and it is consistently the slowest engine there (1.2–1.9x cmake). + +### 6. Where mcpp does *not* win + +* **Module cold builds under clang**: 2.50s vs cmake's 4.00s is 0.62x — a real + lead, but far from the 0.26x it holds under gcc. Cold module builds are + latency-bound on the BMI chain, and no scheduler beats that (see + `.agents/docs/2026-08-12-modular-build-performance-deep-analysis.md`). +* **`headers` incremental under clang**: 0.30–0.39s against cmake's 0.41–0.54s. + At this scale process startup dominates and the engines are within noise of + each other; bazel is faster still (0.20–0.29s). +* **`noop`**: 0.14–0.20s everywhere except meson. Nobody is meaningfully ahead. + +--- + +## Reading caveats + +* **This fixture is small.** 40 units at weight 6 build in seconds; the absolute + numbers are not a prediction for a large codebase. The *ratios* between engines + on the same row are what carry over, and the cascade ratios grow with unit + count, not shrink. +* **mcpp resolves its own toolchain.** The generated `mcpp.toml` pins the same + family the harness hands every other engine (`gcc@16.1.0` for the gcc table, + `llvm@22.1.8` for the clang table), so this is not a compiler comparison in + disguise. It is pinned in the manifest rather than passed on the command line + because `mcpp build` has no toolchain flag, and the comparison must run against + *released* binaries that would not have one anyway. +* **bazel's cold is not a cold machine.** `clean` here is deliberately not + `--expunge`, which would also discard the downloaded toolchain and turn the + measurement into provisioning. Its module builds also pass `--force_pic` — see + `bench/README.md` §5 for why analysis fails without it. +* **No fixture says `import std;`.** Engines differ wildly in std-module support + and that difference would dominate everything else. This measures module + machinery. +* Two runs per cell. Enough to catch a gross outlier, not enough for a confidence + interval — none is reported. diff --git a/bench/results/hyperfine-20260812/NOTES.md b/bench/results/hyperfine-20260812/NOTES.md new file mode 100644 index 00000000..435eed51 --- /dev/null +++ b/bench/results/hyperfine-20260812/NOTES.md @@ -0,0 +1,76 @@ +# Result provenance + +> **These files predate protocol v1.** They were produced by the one-off bash + +> hyperfine harness that `bench/` replaced, and are kept as reference data for the +> 2026-08-12 analysis — the TSVs have no `status` column, which is precisely the +> gap that let a failed cell be written as `0.000` (see below). New runs emit the +> versioned JSON described in `bench/README.md` §6; do not merge the two formats. + +Raw `hyperfine` JSON and the per-run TSV land here. Read this before quoting a +number out of them. + +## Host (all runs below) + +| | | +|---|---| +| CPU | Intel i9-13900K — **8 P-core + 16 E-core, 32 threads** (heterogeneous: do not read "32 cores" as 32 equal cores) | +| RAM | 62 GB | +| Kernel | Linux 6.8 | +| Compiler | GCC 16.1.0, mcpp hermetic payload (`~/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0`) | +| Engines | mcpp 2026.8.11.3 · xmake v3.0.7+HEAD.77d94ad | +| Project | mcpp itself — 137 `.cppm` + 1 `.cpp`, 56 555 LOC | + +## `matrix-20260812-104244.tsv` + +Valid: all five `mcpp` rows, plus `xmake` `cold`, `noop`, `touch-hub`. + +**Void: the `xmake` `edit-body` and `touch-main` rows.** They read `0.000`, which +is not a measurement — the build failed on every run and an early version of +`run.sh` tested `[[ -f json ]]` instead of `[[ -s json ]]`, so an empty +hyperfine export was formatted as a zero. The cause was self-inflicted: `xmake.lua` +was edited *while the matrix was running*, and the edit read a file from xmake's +description scope, where `io` is nil (`attempt to index a nil value (global 'io')`). +Both bugs are fixed — `run.sh` now records `FAILED`, and `xmake.lua` reads the +manifest inside `on_load`. Those two cells were re-measured; see the newer TSV. + +Two lessons worth keeping: +1. Never edit the build description of a benchmark that is mid-flight. +2. A benchmark harness must not be able to emit a number when the thing it was + timing did not run. + +## `matrix-20260812-112142.tsv` + +Valid: `mcpp / clang / release / cold` = **32.076 s**. + +**Void: the `xmake / clang / release / cold` row (90.233 s).** `xmake.lua` called +`set_toolchains("mcpp-gcc")` unconditionally, which silently overrode +`xmake f --toolchain=llvm`; that cell was compiled by **g++**, not clang. The tell +was that it landed within noise of the gcc cell (88.942 s). Fixed: the pin is now +skipped when the caller requested a toolchain. Always confirm with + +```bash +xmake show -t mcpp | grep 'compiler (cxx)' +``` + +With the override fixed, xmake *does* select `clang++ 22.1.8` — but the build then +fails outright: + +``` +error: missing std dependency for module mcpp.cli.cmd_build +warning: std and std.compat modules not found! maybe try to add --sdk= +``` + +even with `--sdk=`, and even though that payload does ship +`lib/x86_64-unknown-linux-gnu/libc++.modules.json` and +`share/libc++/v1/std.cppm`. xmake v3.0.7 does not discover libc++'s std module +from this layout. mcpp does not depend on that discovery — it precompiles +`std.pcm` itself. **The xmake/clang cell is therefore unmeasured, not slow.** + +## Declared asymmetry: the `std` module + +mcpp stages a prebuilt `std.gcm` (31.5 MB) out of `~/.mcpp/build-cache/v1/`; +xmake compiles `std` from libstdc++ sources. Both end up with byte-comparable +artifacts (31 458 736 B vs 31 458 752 B). Because **every** module imports `std`, +that compile sits on xmake's critical path and mcpp's cold-build advantage is +partly a *caching* advantage, not a *scheduling* one. See the analysis doc for +the measured size of that head start before attributing the cold-build delta. diff --git a/bench/results/hyperfine-20260812/matrix-20260812-104244.tsv b/bench/results/hyperfine-20260812/matrix-20260812-104244.tsv new file mode 100644 index 00000000..011bceb3 --- /dev/null +++ b/bench/results/hyperfine-20260812/matrix-20260812-104244.tsv @@ -0,0 +1,11 @@ +engine compiler profile scenario median_s min_s max_s runs +mcpp gcc release cold 77.551 77.244 77.970 3 +mcpp gcc release noop 0.430 0.428 0.448 3 +mcpp gcc release touch-hub 73.790 73.602 73.919 3 +mcpp gcc release edit-body 47.753 47.522 47.811 3 +mcpp gcc release touch-main 5.401 5.338 5.754 3 +xmake gcc release cold 88.942 88.279 88.968 3 +xmake gcc release noop 0.381 0.374 0.384 3 +xmake gcc release touch-hub 81.700 80.927 82.221 3 +xmake gcc release edit-body 0.000 0.000 0.000 3 +xmake gcc release touch-main 0.000 0.000 0.000 3 diff --git a/bench/results/hyperfine-20260812/matrix-20260812-110709.tsv b/bench/results/hyperfine-20260812/matrix-20260812-110709.tsv new file mode 100644 index 00000000..db9b924e --- /dev/null +++ b/bench/results/hyperfine-20260812/matrix-20260812-110709.tsv @@ -0,0 +1,2 @@ +engine compiler profile scenario median_s min_s max_s runs +xmake gcc release edit-body 52.193 52.038 52.526 3 diff --git a/bench/results/hyperfine-20260812/matrix-20260812-110946.tsv b/bench/results/hyperfine-20260812/matrix-20260812-110946.tsv new file mode 100644 index 00000000..b83ae874 --- /dev/null +++ b/bench/results/hyperfine-20260812/matrix-20260812-110946.tsv @@ -0,0 +1,2 @@ +engine compiler profile scenario median_s min_s max_s runs +xmake gcc release touch-main 5.277 0.532 5.288 3 diff --git a/bench/results/hyperfine-20260812/matrix-20260812-112142.tsv b/bench/results/hyperfine-20260812/matrix-20260812-112142.tsv new file mode 100644 index 00000000..b5025f95 --- /dev/null +++ b/bench/results/hyperfine-20260812/matrix-20260812-112142.tsv @@ -0,0 +1,3 @@ +engine compiler profile scenario median_s min_s max_s runs +mcpp clang release cold 32.076 32.003 32.149 2 +xmake clang release cold 90.233 90.219 90.248 2 diff --git a/bench/results/hyperfine-20260812/matrix-20260812-114125.tsv b/bench/results/hyperfine-20260812/matrix-20260812-114125.tsv new file mode 100644 index 00000000..eab882db --- /dev/null +++ b/bench/results/hyperfine-20260812/matrix-20260812-114125.tsv @@ -0,0 +1,3 @@ +engine compiler profile scenario median_s min_s max_s runs +mcpp gcc debug cold 44.225 43.860 44.591 2 +xmake gcc debug cold 46.297 46.257 46.338 2 diff --git a/bench/results/hyperfine-20260812/mcpp-clang-release-cold-20260812-112142.json b/bench/results/hyperfine-20260812/mcpp-clang-release-cold-20260812-112142.json new file mode 100644 index 00000000..b39ba8ad --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-clang-release-cold-20260812-112142.json @@ -0,0 +1,22 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 32.07636324756, + "stddev": 0.10329186686265025, + "median": 32.07636324756, + "user": 115.66690496, + "system": 6.18005896, + "min": 32.003324868060005, + "max": 32.14940162706, + "times": [ + 32.003324868060005, + 32.14940162706 + ], + "exit_codes": [ + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-debug-cold-20260812-114125.json b/bench/results/hyperfine-20260812/mcpp-gcc-debug-cold-20260812-114125.json new file mode 100644 index 00000000..e6049385 --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-debug-cold-20260812-114125.json @@ -0,0 +1,22 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --dev", + "mean": 44.22504332058, + "stddev": 0.51686951422911, + "median": 44.22504332058, + "user": 160.26381633999998, + "system": 18.52578548, + "min": 43.859561382079995, + "max": 44.590525259079996, + "times": [ + 43.859561382079995, + 44.590525259079996 + ], + "exit_codes": [ + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-release-cold-20260812-104244.json b/bench/results/hyperfine-20260812/mcpp-gcc-release-cold-20260812-104244.json new file mode 100644 index 00000000..7f046ee1 --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-release-cold-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 77.58809723248667, + "stddev": 0.3647750861532106, + "median": 77.55061916382, + "user": 286.9025155266667, + "system": 15.796073299999998, + "min": 77.24350802782, + "max": 77.97016450582001, + "times": [ + 77.55061916382, + 77.97016450582001, + 77.24350802782 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-release-edit-body-20260812-104244.json b/bench/results/hyperfine-20260812/mcpp-gcc-release-edit-body-20260812-104244.json new file mode 100644 index 00000000..ee28e339 --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-release-edit-body-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 47.69535751190667, + "stddev": 0.15295179921445068, + "median": 47.75266863124, + "user": 99.13558640666668, + "system": 4.293043553333334, + "min": 47.52202704224, + "max": 47.811376862239996, + "times": [ + 47.52202704224, + 47.75266863124, + 47.811376862239996 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-release-noop-20260812-104244.json b/bench/results/hyperfine-20260812/mcpp-gcc-release-noop-20260812-104244.json new file mode 100644 index 00000000..54edc590 --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-release-noop-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 0.4352798993733334, + "stddev": 0.010957615616486657, + "median": 0.43000388804000006, + "user": 0.38728746000000003, + "system": 0.04728526666666666, + "min": 0.42795838104000006, + "max": 0.44787742904000005, + "times": [ + 0.42795838104000006, + 0.43000388804000006, + 0.44787742904000005 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-hub-20260812-104244.json b/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-hub-20260812-104244.json new file mode 100644 index 00000000..172517d4 --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-hub-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 73.77027963598, + "stddev": 0.1597509066891337, + "median": 73.78993318698001, + "user": 227.29512307999997, + "system": 10.390090766666667, + "min": 73.60161125498, + "max": 73.91929446598, + "times": [ + 73.78993318698001, + 73.91929446598, + 73.60161125498 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-main-20260812-104244.json b/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-main-20260812-104244.json new file mode 100644 index 00000000..1ec5f4cf --- /dev/null +++ b/bench/results/hyperfine-20260812/mcpp-gcc-release-touch-main-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && mcpp build --release", + "mean": 5.498000105013333, + "stddev": 0.22413979130805034, + "median": 5.40148036268, + "user": 5.13607398, + "system": 0.36021838666666667, + "min": 5.33828978468, + "max": 5.754230167679999, + "times": [ + 5.754230167679999, + 5.40148036268, + 5.33828978468 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-clang-release-cold-20260812-112142.json b/bench/results/hyperfine-20260812/xmake-clang-release-cold-20260812-112142.json new file mode 100644 index 00000000..65011e83 --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-clang-release-cold-20260812-112142.json @@ -0,0 +1,22 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 90.23348003891999, + "stddev": 0.020491174580007758, + "median": 90.23348003891999, + "user": 326.44624239999996, + "system": 16.18380444, + "min": 90.21899059041999, + "max": 90.24796948742, + "times": [ + 90.24796948742, + 90.21899059041999 + ], + "exit_codes": [ + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-debug-cold-20260812-114125.json b/bench/results/hyperfine-20260812/xmake-gcc-debug-cold-20260812-114125.json new file mode 100644 index 00000000..82e6f5f6 --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-debug-cold-20260812-114125.json @@ -0,0 +1,22 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 46.29744415179999, + "stddev": 0.0577741058167317, + "median": 46.29744415179999, + "user": 167.15905125999998, + "system": 18.39386518, + "min": 46.2565916898, + "max": 46.3382966138, + "times": [ + 46.2565916898, + 46.3382966138 + ], + "exit_codes": [ + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-cold-20260812-104244.json b/bench/results/hyperfine-20260812/xmake-gcc-release-cold-20260812-104244.json new file mode 100644 index 00000000..9e588d21 --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-release-cold-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 88.72970099721333, + "stddev": 0.39065526676808404, + "median": 88.94198277587999, + "user": 324.7123754, + "system": 16.166824253333335, + "min": 88.27886622588, + "max": 88.96825398988, + "times": [ + 88.96825398988, + 88.94198277587999, + 88.27886622588 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-edit-body-20260812-104244.json b/bench/results/hyperfine-20260812/xmake-gcc-release-edit-body-20260812-104244.json new file mode 100644 index 00000000..e69de29b diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-edit-body-20260812-110709.json b/bench/results/hyperfine-20260812/xmake-gcc-release-edit-body-20260812-110709.json new file mode 100644 index 00000000..9791b0c9 --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-release-edit-body-20260812-110709.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 52.25233713404666, + "stddev": 0.24897905061143294, + "median": 52.193040510379994, + "user": 109.27093788, + "system": 4.2777480599999995, + "min": 52.038359707379996, + "max": 52.52561118438, + "times": [ + 52.193040510379994, + 52.52561118438, + 52.038359707379996 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-noop-20260812-104244.json b/bench/results/hyperfine-20260812/xmake-gcc-release-noop-20260812-104244.json new file mode 100644 index 00000000..7b8bba5c --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-release-noop-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 0.37954382739333337, + "stddev": 0.005064813871963533, + "median": 0.38089023506, + "user": 0.32198313333333334, + "system": 0.04366906, + "min": 0.37394185806, + "max": 0.38379938906, + "times": [ + 0.38089023506, + 0.37394185806, + 0.38379938906 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-touch-hub-20260812-104244.json b/bench/results/hyperfine-20260812/xmake-gcc-release-touch-hub-20260812-104244.json new file mode 100644 index 00000000..edc3debe --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-release-touch-hub-20260812-104244.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 81.61594661257334, + "stddev": 0.6509197495428294, + "median": 81.69964002124, + "user": 249.84436750666666, + "system": 10.477282126666665, + "min": 80.92722814324, + "max": 82.22097167324, + "times": [ + 80.92722814324, + 81.69964002124, + 82.22097167324 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-touch-main-20260812-104244.json b/bench/results/hyperfine-20260812/xmake-gcc-release-touch-main-20260812-104244.json new file mode 100644 index 00000000..e69de29b diff --git a/bench/results/hyperfine-20260812/xmake-gcc-release-touch-main-20260812-110946.json b/bench/results/hyperfine-20260812/xmake-gcc-release-touch-main-20260812-110946.json new file mode 100644 index 00000000..a3002926 --- /dev/null +++ b/bench/results/hyperfine-20260812/xmake-gcc-release-touch-main-20260812-110946.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "command": "cd '/home/speak/workspace/github/mcpp-community/mcpp' && xmake build -j32", + "mean": 3.6989989001666665, + "stddev": 2.7426382272323924, + "median": 5.2770528315, + "user": 3.4384828533333334, + "system": 0.24469848000000002, + "min": 0.5320792145000001, + "max": 5.2878646545, + "times": [ + 5.2878646545, + 0.5320792145000001, + 5.2770528315 + ], + "exit_codes": [ + 0, + 0, + 0 + ] + } + ] +} diff --git a/bench/results/mcpp-self-20260813/linux-x86_64-gcc.json b/bench/results/mcpp-self-20260813/linux-x86_64-gcc.json new file mode 100644 index 00000000..25b3bf90 --- /dev/null +++ b/bench/results/mcpp-self-20260813/linux-x86_64-gcc.json @@ -0,0 +1,376 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-12T17:30:54Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 80.489, + "min_s": 80.344, + "max_s": 80.633, + "samples": [80.633, 80.344] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 0.284, + "min_s": 0.277, + "max_s": 0.292, + "samples": [0.277, 0.292] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 76.504, + "min_s": 76.177, + "max_s": 76.831, + "samples": [76.831, 76.177] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 17.390, + "min_s": 17.267, + "max_s": 17.513, + "samples": [17.513, 17.267] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 18.297, + "min_s": 18.257, + "max_s": 18.336, + "samples": [18.336, 18.257] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 2, + "median_s": 76.500, + "min_s": 76.130, + "max_s": 76.870, + "samples": [76.130, 76.870] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 82.866, + "min_s": 82.072, + "max_s": 83.661, + "samples": [83.661, 82.072] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.200, + "min_s": 0.200, + "max_s": 0.201, + "samples": [0.201, 0.200] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.441, + "min_s": 0.431, + "max_s": 0.450, + "samples": [0.431, 0.450] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 2.141, + "min_s": 2.141, + "max_s": 2.142, + "samples": [2.142, 2.141] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 18.293, + "min_s": 18.172, + "max_s": 18.413, + "samples": [18.172, 18.413] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 2, + "median_s": 0.457, + "min_s": 0.456, + "max_s": 0.457, + "samples": [0.456, 0.457] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 94.534, + "min_s": 93.878, + "max_s": 95.190, + "samples": [93.878, 95.190] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 0.344, + "min_s": 0.328, + "max_s": 0.360, + "samples": [0.360, 0.328] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 84.526, + "min_s": 84.101, + "max_s": 84.950, + "samples": [84.950, 84.101] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 18.057, + "min_s": 18.029, + "max_s": 18.086, + "samples": [18.086, 18.029] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 19.645, + "min_s": 19.422, + "max_s": 19.869, + "samples": [19.422, 19.869] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 2, + "median_s": 85.034, + "min_s": 84.760, + "max_s": 85.309, + "samples": [85.309, 84.760] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 94.626, + "min_s": 94.502, + "max_s": 94.751, + "samples": [94.751, 94.502] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 0.380, + "min_s": 0.379, + "max_s": 0.380, + "samples": [0.380, 0.379] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 83.647, + "min_s": 83.618, + "max_s": 83.675, + "samples": [83.618, 83.675] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 18.474, + "min_s": 18.390, + "max_s": 18.559, + "samples": [18.559, 18.390] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 19.974, + "min_s": 19.884, + "max_s": 20.063, + "samples": [19.884, 20.063] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": ".", + "variant": "native", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 2, + "median_s": 84.685, + "min_s": 83.660, + "max_s": 85.710, + "samples": [83.660, 85.710] + } + ] +} diff --git a/bench/results/mcpp-self-20260813/report.md b/bench/results/mcpp-self-20260813/report.md new file mode 100644 index 00000000..3c4cdbf7 --- /dev/null +++ b/bench/results/mcpp-self-20260813/report.md @@ -0,0 +1,94 @@ +# Building mcpp with four engines — 2026-08-13 + +The **real project**, not a fixture: 138 module interface units, 57k lines, every +one of them `import std;`. cmake is the baseline; every cell shows the median +wall time and its ratio to cmake in the same row. + +| | | +|---|---| +| host | Linux x86_64 · 13th Gen Intel Core i9-13900K · 32 logical / 24 physical · 64 GiB | +| workload | mcpp itself, measured in place (`--project .`) | +| compiler | `gcc@16.1.0`, the hermetic mcpp payload, pinned into every engine | +| engines | mcpp 2026.8.11.3 and 2026.8.12.1, cmake 4.0.2 + ninja, xmake v3.0.7+HEAD | +| build files | `bench/projects/mcpp/` (`--buildfiles`), so nothing foreign sits at the repo root | +| perturbed | hub `src/platform/platform.cppm` (46 importers) · leaf `src/pm/publisher.cppm` (0) · body `src/build/stage.cppm` | +| runs | 2 per cell | +| raw | [`linux-x86_64-gcc.json`](linux-x86_64-gcc.json) | + +| scenario | mcpp@2026.8.11.3 | mcpp@2026.8.12.1 | cmake | xmake | +|---|---|---|---|---| +| `cold` | 80.49s · 0.85x | 82.87s · 0.88x | **94.53s** · 1.00x | 94.63s · 1.00x | +| `noop` | 0.28s · 0.83x | 0.20s · 0.58x | **0.34s** · 1.00x | 0.38s · 1.10x | +| `touch-leaf` | 17.39s · 0.96x | 2.14s · 0.12x | **18.06s** · 1.00x | 18.47s · 1.02x | +| `edit-body` | 18.30s · 0.93x | 18.29s · 0.93x | **19.64s** · 1.00x | 19.97s · 1.02x | +| `edit-comment` | 76.50s · 0.90x | 0.46s · 0.01x | **85.03s** · 1.00x | 84.69s · 1.00x | +| `touch-hub` | 76.50s · 0.91x | 0.44s · 0.01x | **84.53s** · 1.00x | 83.65s · 0.99x | + +--- + +## What this says + +### 1. On cold builds, all four engines are within 15% of each other + +80.5–94.6s. The synthetic fixture put mcpp at **0.26x** cmake; here it is +**0.85x**. Anyone quoting the fixture ratio as mcpp's cold-build advantage is +quoting an artefact of a workload whose units cost 0.09s each. + +The reason nobody wins is structural: mcpp's cold build is **100% critical +path** (79.73s of a 79.79s makespan, average parallelism 3.94x of 32 hardware +threads). Every engine walks the same 26-deep chain of module interfaces, so +scheduling cannot help and neither can more cores. See +`.agents/docs/2026-08-12-cold-build-optimization-plan.md` for the measured +headroom (BMI is complete at ~22% of each compile; the other 78% is code +generation nobody downstream needs). + +### 2. On the scenarios that dominate a working day, the gap is ~190x + +Touching the most-imported unit — `mcpp.platform`, 46 importers — costs cmake +**84.53s** and xmake **83.65s**: they rebuild the world because the BMI's mtime +moved. mcpp 2026.8.12.1 costs **0.44s**, because it compares the BMI the +compiler just produced against the previous one and puts the old file back when +they are equivalent. + +That is **192x** against the baseline and **174x** against mcpp's own previous +release, which had the same mechanism and never once fired: GCC stamps a wall +clock into every BMI, so the byte compare it used could never report "unchanged". + +`edit-comment` — a real content change that leaves the interface alone — is the +same story at 185x. + +### 3. `edit-body` shows no gain, and that is the point + +18.29s for both mcpp releases, 19.64s for cmake. Editing a function body in +`src/build/stage.cppm` genuinely changes that unit's BMI, so the cascade is +**correct** and every engine pays it. A mechanism that made this row fast too +would be skipping rebuilds it must not skip. + +The row is in the table for exactly that reason: it is the control that +distinguishes "avoids unnecessary work" from "avoids work". + +### 4. `touch-leaf` costs 17–18s for everyone but the new mcpp + +A unit nobody imports still takes 17.4s to rebuild under cmake, xmake and the +previous mcpp — because it is one of the fat ones, and its own compile is that +expensive. 2.14s for mcpp 2026.8.12.1 is the same BMI-equivalence check firing +one level down. + +--- + +## Caveats + +* **Single host, two runs per cell.** Under §4a's dispersion rule the cold rows + (80–95s, spreads under 2%) are solid; `noop` at 0.20–0.38s is within 2x of the + engines' own floor and should be read as "all four are instant", not a ranking. +* **cmake and xmake compile `mcpplibs.cmdline` from source; mcpp stages it from + its global cache.** Three units, ~1s. It is declared here rather than hidden, + and it does not move any conclusion above. +* **No bazel column.** bazel builds C++20 modules only with clang — its ddi + aggregator cannot parse GCC's P1689 output — so including it here would break + the "same compiler binary" invariant. `import std;` itself is *not* the + blocker: libc++ ships the std module as ordinary source and bazel builds it + fine (recipe in `bench/projects/mcpp/MODULE.bazel`). A bazel column needs a + clang-baselined table. +* **No meson column.** meson 1.10.2 has no way to declare a module interface + unit, and no `import std;` story. diff --git a/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-5way.json b/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-5way.json new file mode 100644 index 00000000..d2e3730b --- /dev/null +++ b/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-5way.json @@ -0,0 +1,391 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T06:04:30Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 79.538, + "min_s": 79.538, + "max_s": 79.538, + "samples": [79.538] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.401, + "min_s": 0.401, + "max_s": 0.401, + "samples": [0.401] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 76.241, + "min_s": 76.241, + "max_s": 76.241, + "samples": [76.241] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.382, + "min_s": 0.382, + "max_s": 0.382, + "samples": [0.382] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 35.426, + "min_s": 35.426, + "max_s": 35.426, + "samples": [35.426] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.221, + "samples": [0.221] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 30.166, + "min_s": 30.166, + "max_s": 30.166, + "samples": [30.166] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.181, + "samples": [0.181] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 79.459, + "min_s": 79.459, + "max_s": 79.459, + "samples": [79.459] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 0.341, + "min_s": 0.341, + "max_s": 0.341, + "samples": [0.341] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 76.528, + "min_s": 76.528, + "max_s": 76.528, + "samples": [76.528] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 77.330, + "min_s": 77.330, + "max_s": 77.330, + "samples": [77.330] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 75.692, + "min_s": 75.692, + "max_s": 75.692, + "samples": [75.692] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 92.325, + "min_s": 92.325, + "max_s": 92.325, + "samples": [92.325] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 0.282, + "min_s": 0.282, + "max_s": 0.282, + "samples": [0.282] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 83.387, + "min_s": 83.387, + "max_s": 83.387, + "samples": [83.387] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 85.639, + "min_s": 85.639, + "max_s": 85.639, + "samples": [85.639] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 82.960, + "min_s": 82.960, + "max_s": 82.960, + "samples": [82.960] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 0.603, + "min_s": 0.603, + "max_s": 0.603, + "samples": [0.603] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 0.341, + "min_s": 0.341, + "max_s": 0.341, + "samples": [0.341] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 82.790, + "min_s": 82.790, + "max_s": 82.790, + "samples": [82.790] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 96.060, + "min_s": 96.060, + "max_s": 96.060, + "samples": [96.060] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 83.207, + "min_s": 83.207, + "max_s": 83.207, + "samples": [83.207] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-xmake-refixed.json b/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-xmake-refixed.json new file mode 100644 index 00000000..24d6919e --- /dev/null +++ b/bench/results/pinned-workloads-20260813/mcpp-linux-gcc-xmake-refixed.json @@ -0,0 +1,91 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T06:44:18Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 90.295, + "min_s": 90.295, + "max_s": 90.295, + "samples": [90.295] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 0.382, + "min_s": 0.382, + "max_s": 0.382, + "samples": [0.382] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 82.075, + "min_s": 82.075, + "max_s": 82.075, + "samples": [82.075] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 1, + "median_s": 84.607, + "min_s": 84.607, + "max_s": 84.607, + "samples": [84.607] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.0.7+HEAD.77d94ad, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 1, + "median_s": 82.730, + "min_s": 82.730, + "max_s": 82.730, + "samples": [82.730] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/report.md b/bench/results/pinned-workloads-20260813/report.md new file mode 100644 index 00000000..94edeee5 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/report.md @@ -0,0 +1,133 @@ +# Pinned workloads — 2026-08-13 + +The first run of this suite in which **everything that moves a number is pinned**: +the tool versions, the compiler, the measured sources, and the reference mcpp. +Previous runs are not comparable to this one, and the reason is not subtle — see +[`../../SPEC.md`](../../SPEC.md) §1 and the audit note at the bottom. + +| | | +|---|---| +| host | Linux x86_64 · 13th Gen Intel Core i9-13900K · 32 logical / 24 physical (heterogeneous) | +| compiler | `gcc 16.1.0`, mcpp's own payload, handed to **every** engine (`--compiler payload:gcc`) | +| tools | cmake 4.0.2 + ninja, xmake 3.0.7 — *local versions; CI pins 4.4.2 / 3.1.0 / 9.2.0* | +| workloads | `mcpp-2026.8.11.3` (`a749e9f`, 137 modules) · `xlings-2026.8.11.2` (`b1563fe`) · `xlings-2026.8.13.1` (`f072075`) — all git submodules | +| repetitions | **n=1**, except `xlings-split-cold-n3` | + +Regenerate any table below from the raw files rather than transcribing it: + +```bash +bench/tools/report.py bench/results/pinned-workloads-20260813/mcpp-linux-gcc-5way.json +``` + +--- + +## 1. mcpp building mcpp — five arms + +Ratios against cmake. `+bmi_schedule=on` is the **same binary** with the opt-in +BMI schedule enabled via `MCPP_BMI_SCHEDULE`. + +| scenario | `mcpp@2026.8.11.3` | `mcpp@2026.8.13.1` | `+bmi_schedule=on` | `cmake` | `xmake` | +|---|---|---|---|---|---| +| `cold` | 79.46s · 0.86x | 79.54s · 0.86x | **35.43s · 0.38x** | **92.33s** · 1.00x | 90.30s · 0.98x | +| `noop` | 0.34s · 1.21x | 0.16s · 0.57x | 0.16s · 0.57x | **0.28s** · 1.00x | 0.38s · 1.36x | +| `touch-hub` | 76.53s · 0.92x | **0.40s · 0.005x** | **0.22s · 0.003x** | **83.39s** · 1.00x | 82.07s · 0.98x | +| `edit-body` | 77.33s · 0.90x | 76.24s · 0.89x | **30.17s · 0.35x** | **85.64s** · 1.00x | 84.61s · 0.99x | +| `edit-comment` | 75.69s · 0.91x | **0.38s · 0.005x** | **0.18s · 0.002x** | **82.96s** · 1.00x | 82.73s · 1.00x | + +* **Nobody wins `cold`, and that is correct.** All within 15%. mcpp's cold build + is 100% critical path — 79.7s of a 79.8s makespan, average parallelism 3.94 of + 32 threads — so every engine walks the same 26-deep interface chain. +* **The cold lever is the setting, not the release.** 79.46 → 79.54 between + releases is nothing; `bmi_schedule = "on"` takes it to 35.43s (2.24x). +* **`touch-hub` / `edit-comment` are ~190x**, and they sit 2.5x / 2.4x above + mcpp's own `noop` — just past R1's floor, so read them as two orders of + magnitude, not as three digits. +* **`edit-comment` here is the `end-of-file` form**: mcpp's hub has no function + body, so nothing shifts. See §3. + +⚠️ **The `xmake` column is from a separate run** (`…-xmake-refixed.json`). In the +five-arm file its `cold` reads **0.60s** — invalid. xmake normalises `--buildir` +to a path relative to `-P` and resolves it against the process cwd, so `clean()` +had been removing a directory nothing ever wrote to. Fixed, re-measured, and the +harness now refuses a `cold` that is under 2x its own `noop`. + +## 2. xlings — two code styles, mcpp against mcpp + +The same project either side of one refactor. cmake and xmake are absent because +their arms stop at the link here (SPEC.md §2), so the baseline is the released +mcpp. + +| scenario | combined `2026.8.11.2` old → new | split `2026.8.13.1` old → new | what the split buys | +|---|---|---|---| +| `cold` | 97.01s → 92.48s | 30.33s → 29.78s ⁽ⁿ⁼³⁾ | **3.11x** | +| `noop` | 1.55s → 0.72s | 1.62s → 0.76s | — | +| `touch-hub` | 89.39s → **1.76s** (50.6x) | 24.87s → **1.30s** (19.1x) | 1.35x | +| `edit-body` | 89.46s → 88.33s | 2.73s → **1.77s** | **49.96x** | +| `edit-comment` | 95.40s → 95.02s | 25.09s → 25.29s | 3.76x | + +* Splitting implementations out of the interface units is worth **3.1x cold** and + **~50x on an edit**. A code style, not an engine feature — and the largest + single effect anywhere in this suite. + +* `touch-hub` reproduces the engine result on a codebase nobody tuned for it. + +⚠️ **The `cold` row was nearly published as a 23% regression.** At n=1 it read +`29.13s → 35.88s`. At n=3 it is `30.33s → 29.78s`, marginally faster: the single +pair had caught the new arm near the old arm's max. The old arm's spread is +**19.1%** — a hair under the 20% that §4a R2 calls noisy. + +### 2b. …and the opt-in schedule on top of it + +| tree | scenario | default | `+bmi_schedule=on` | | +|---|---|---|---|---| +| combined | `cold` | 92.95s | **43.26s** | **2.15x** | +| combined | `edit-body` | 91.66s | **30.19s** | **3.04x** | +| split | `cold` | 27.62s | 29.72s | 0.93x | +| split | `edit-body` | 1.79s | 1.79s | 1.00x | + +**The two levers overlap, and the code style is the bigger one.** The schedule +lets importers start as soon as a BMI exists, so it only helps where there is a +cascade to overlap. Splitting the implementations removes the cascade instead, +after which the schedule has nothing left to win — and costs a little on `cold`. + +Raw: `xlings-combined-schedule-linux-gcc.json`, `xlings-split-schedule-linux-gcc.json`. + +## 3. The finding that only a second project could produce + +`edit-comment` is **199x on mcpp's own tree and 1.00x on xlings**. Not an +optimisation that works sometimes: + +| hub | lines | `) {` anchors | perturbation form | result | +|---|---|---|---|---| +| mcpp `src/platform/platform.cppm` | 66 | **0** | `end-of-file` — nothing shifts | BMI unchanged, cascade skipped | +| xlings `src/platform.cppm` | 566 | **56** | `in-body` — every later line shifts | BMI changes, **cascade is owed** | + +mcpp measuring itself could never have seen this, because its hub happens to +have no function bodies at all. The form is now recorded in every cell's `note`. + +**On the mechanism, only what was measured.** Directly comparing BMIs before and +after an edit, on GCC 16.1: + +| edit | BMI | +|---|---| +| a free exported function's body, in the `.cppm` | **byte-identical** | +| a **member function of an exported class**, inline in the `.cppm` | **differs** | +| a body in a separate `.cpp` implementation unit | **byte-identical** | + +A class's member function bodies are part of the class definition every importer +must see, so they are serialised; a free function's body is not, and nothing in +an implementation unit is. What that does *not* settle is how much of the xlings +`in-body` result is the serialised entity and how much is the line-number shift +the insertion causes — both are present there, and the suite does not currently +separate them. Stated rather than guessed. + +--- + +## What changed about the suite itself before these numbers could be trusted + +The previous matrix reported success while measuring almost nothing: one job was +**6 ok / 48 failed / 18 unavailable**, and every xlings job had zero +measurements. Six independent causes, every one of them a failure that looked +like a success. `.agents/docs/2026-08-13-build-optimization-status.md` §7 has the +full list; the assertions added as a result are in +[`../../SPEC.md`](../../SPEC.md) §3 and `tests/e2e/233_bench_matrix.sh`. diff --git a/bench/results/pinned-workloads-20260813/xlings-combined-linux-gcc.json b/bench/results/pinned-workloads-20260813/xlings-combined-linux-gcc.json new file mode 100644 index 00000000..b8eb6f91 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/xlings-combined-linux-gcc.json @@ -0,0 +1,166 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T05:39:25Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 92.485, + "min_s": 92.485, + "max_s": 92.485, + "samples": [92.485] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 0.722, + "min_s": 0.722, + "max_s": 0.722, + "samples": [0.722] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 1.765, + "min_s": 1.765, + "max_s": 1.765, + "samples": [1.765] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 88.335, + "min_s": 88.335, + "max_s": 88.335, + "samples": [88.335] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 95.018, + "min_s": 95.018, + "max_s": 95.018, + "samples": [95.018] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 97.007, + "min_s": 97.007, + "max_s": 97.007, + "samples": [97.007] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 1.546, + "min_s": 1.546, + "max_s": 1.546, + "samples": [1.546] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 89.395, + "min_s": 89.395, + "max_s": 89.395, + "samples": [89.395] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 89.461, + "min_s": 89.461, + "max_s": 89.461, + "samples": [89.461] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 95.397, + "min_s": 95.397, + "max_s": 95.397, + "samples": [95.397] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/xlings-combined-schedule-linux-gcc.json b/bench/results/pinned-workloads-20260813/xlings-combined-schedule-linux-gcc.json new file mode 100644 index 00000000..c48c6f92 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/xlings-combined-schedule-linux-gcc.json @@ -0,0 +1,76 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T07:05:05Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 92.951, + "min_s": 92.951, + "max_s": 92.951, + "samples": [92.951] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 91.661, + "min_s": 91.661, + "max_s": 91.661, + "samples": [91.661] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 43.263, + "min_s": 43.263, + "max_s": 43.263, + "samples": [43.263] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 30.191, + "min_s": 30.191, + "max_s": 30.191, + "samples": [30.191] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/xlings-split-cold-n3-linux-gcc.json b/bench/results/pinned-workloads-20260813/xlings-split-cold-n3-linux-gcc.json new file mode 100644 index 00000000..bd08c905 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/xlings-split-cold-n3-linux-gcc.json @@ -0,0 +1,46 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T06:51:28Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 29.779, + "min_s": 28.651, + "max_s": 30.042, + "samples": [30.042, 29.779, 28.651] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 30.329, + "min_s": 29.922, + "max_s": 35.724, + "samples": [35.724, 30.329, 29.922] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/xlings-split-linux-gcc.json b/bench/results/pinned-workloads-20260813/xlings-split-linux-gcc.json new file mode 100644 index 00000000..438778a0 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/xlings-split-linux-gcc.json @@ -0,0 +1,166 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T05:55:13Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 35.882, + "min_s": 35.882, + "max_s": 35.882, + "samples": [35.882] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 0.763, + "min_s": 0.763, + "max_s": 0.763, + "samples": [0.763] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 1.305, + "min_s": 1.305, + "max_s": 1.305, + "samples": [1.305] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 1.768, + "min_s": 1.768, + "max_s": 1.768, + "samples": [1.768] + }, + { + "engine": "mcpp@2026.8.12.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.12.1", + "runs": 1, + "median_s": 25.291, + "min_s": 25.291, + "max_s": 25.291, + "samples": [25.291] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 29.128, + "min_s": 29.128, + "max_s": 29.128, + "samples": [29.128] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 1.625, + "min_s": 1.625, + "max_s": 1.625, + "samples": [1.625] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 24.870, + "min_s": 24.870, + "max_s": 24.870, + "samples": [24.870] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 2.732, + "min_s": 2.732, + "max_s": 2.732, + "samples": [2.732] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 1, + "median_s": 25.090, + "min_s": 25.090, + "max_s": 25.090, + "samples": [25.090] + } + ] +} diff --git a/bench/results/pinned-workloads-20260813/xlings-split-schedule-linux-gcc.json b/bench/results/pinned-workloads-20260813/xlings-split-schedule-linux-gcc.json new file mode 100644 index 00000000..d5733f70 --- /dev/null +++ b/bench/results/pinned-workloads-20260813/xlings-split-schedule-linux-gcc.json @@ -0,0 +1,76 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T07:11:42Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 27.619, + "min_s": 27.619, + "max_s": 27.619, + "samples": [27.619] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 1.786, + "min_s": 1.786, + "max_s": 1.786, + "samples": [1.786] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 29.717, + "min_s": 29.717, + "max_s": 29.717, + "samples": [29.717] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 1.787, + "min_s": 1.787, + "max_s": 1.787, + "samples": [1.787] + } + ] +} diff --git a/bench/results/schedule-refix-20260814/mcpp-linux-gcc-schedule-refixed.json b/bench/results/schedule-refix-20260814/mcpp-linux-gcc-schedule-refixed.json new file mode 100644 index 00000000..12f06ac3 --- /dev/null +++ b/bench/results/schedule-refix-20260814/mcpp-linux-gcc-schedule-refixed.json @@ -0,0 +1,166 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-14T02:31:18Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 36.359, + "min_s": 36.359, + "max_s": 36.359, + "samples": [36.359] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.442, + "min_s": 0.442, + "max_s": 0.442, + "samples": [0.442] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 30.481, + "min_s": 30.481, + "max_s": 30.481, + "samples": [30.481] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 1, + "median_s": 0.442, + "min_s": 0.442, + "max_s": 0.442, + "samples": [0.442] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 79.501, + "min_s": 79.501, + "max_s": 79.501, + "samples": [79.501] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.402, + "min_s": 0.402, + "max_s": 0.402, + "samples": [0.402] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 77.037, + "min_s": 77.037, + "max_s": 77.037, + "samples": [77.037] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "native", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 1, + "median_s": 0.382, + "min_s": 0.382, + "max_s": 0.382, + "samples": [0.382] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/README.md b/bench/results/standard-20260814-linux-x86_64/README.md new file mode 100644 index 00000000..2a264ba7 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/README.md @@ -0,0 +1,75 @@ +# standard-20260814-linux-x86_64 + +The Linux standard data set. **696 measured samples**, 3 per cell, produced by +`bash bench/run-standard.sh` on one machine in one sitting. + +| | | +|---|---| +| host | Linux x86_64 · i9-13900K (24 physical / 32 logical, heterogeneous) | +| compilers | gcc 16.1.0, clang/libc++ 22.1.8 — both mcpp payloads, handed to **every** engine | +| foreign engines | cmake 4.4.2, xmake 3.1.0, bazel 9.2.0 | +| reference mcpp | 2026.8.11.3 (released) | +| **mcpp under test** | **built from `8b579fa`** — see the note below | +| started | 2026-08-15 (UTC date stamp `20260814`) | + +## ⚠️ The commit is recorded HERE and not in the JSON + +Every engine labels itself from `--version`, and mcpp's version is a **date**: +every commit on a branch reports `2026.8.13.1`. So these reports can say which +*release* they measured but not which *build* of it — and the numbers here move +with single commits. + +`mbench --under-test` now records it, and `run-standard.sh` passes +`git rev-parse --short HEAD`. **These files predate that field.** The commit +above was recovered from the binary's mtime against `git log -- src/`, which is +weaker evidence than a recorded field; every later run states it in the JSON. + +## Coverage + +`-` means **not measured**. It never means "not applicable" and never means 0. + +| toolchain | project | cells | outcome | +|---|---|---|---| +| gcc | `fixture` | 72 | 72 ok | +| clang | `fixture` | 90 | 90 ok | +| gcc | `mcpp-2026.8.11.3` | 25 | 25 ok | +| clang | `mcpp-2026.8.11.3` | 25 | 20 ok, 5 failed (xmake) | +| gcc | `xlings-2026.8.11.2` | 25 | 25 ok | +| gcc | `xlings-2026.8.13.1` | - | not measured — the run was stopped after five of seven cells | +| clang | `xlings-2026.8.11.2` | - | not measured — same | + +`bash bench/run-standard.sh --resume` fills the last two in without repeating +any of the 696 samples above. + +## The one failure + +`clang / mcpp-2026.8.11.3 / xmake` — all five scenarios, `seed build exited +255`. Root cause reproduced by hand and filed upstream as +[#424](https://github.com/mcpp-community/mcpp/issues/424): xmake's default shape +on clang requires a **full** BMI, and publishing one to importers makes clang +22.1.8 fail on a downstream translation unit that uses a *narrow* +`std::format` string. Not waived with `allow_failed` — a reproduced failure is a +finding, and hiding it is what this suite exists to stop. + +## The one declared outlier + +`gcc / xlings-2026.8.11.2 / mcpp@2026.8.13.1+schedule=on / touch-hub` was +measured as `[1.77, 20.82, 1.79]` — a 1066% spread around a 1.79s median. + +Re-measured immediately afterwards at 8 samples: +`[1.79, 1.79, 1.79, 1.79, 1.78, 1.79, 1.81, 1.79]` — 1% spread, no outlier. So +the 20.82s was machine noise, not an intermittent failure of the cascade +suppression. That probe is +[`probe-touch-hub-outlier-8-samples.json`](probe-touch-hub-outlier-8-samples.json). + +**The published cell is left exactly as measured.** Splicing a second run's +samples into a first run's report is the one thing this suite must never do; the +probe is evidence *about* the number, not a replacement for it. + +## Files + +| file | | +|---|---| +| `-.json` | the report — one per cell | +| `-.log` | the harness's own output for that cell | +| `probe-touch-hub-outlier-8-samples.json` | the follow-up above; **not part of the standard set** | diff --git a/bench/results/standard-20260814-linux-x86_64/clang-fixture.json b/bench/results/standard-20260814-linux-x86_64/clang-fixture.json new file mode 100644 index 00000000..d6fa7a75 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/clang-fixture.json @@ -0,0 +1,1366 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-14T23:12:21Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.185, + "min_s": 1.184, + "max_s": 1.204, + "samples": [1.185, 1.204, 1.184] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.181, + "samples": [0.181, 0.181, 0.181] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.124, + "min_s": 1.104, + "max_s": 1.144, + "samples": [1.144, 1.104, 1.124] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.724, + "min_s": 0.682, + "max_s": 0.724, + "samples": [0.724, 0.724, 0.682] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 0.683, + "min_s": 0.663, + "max_s": 0.683, + "samples": [0.683, 0.683, 0.663] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 1.164, + "min_s": 1.124, + "max_s": 1.186, + "samples": [1.186, 1.164, 1.124] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 9.774, + "min_s": 9.734, + "max_s": 9.796, + "samples": [9.734, 9.774, 9.796] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.201, + "min_s": 0.201, + "max_s": 0.261, + "samples": [0.261, 0.201, 0.201] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.844, + "min_s": 0.843, + "max_s": 0.845, + "samples": [0.844, 0.845, 0.843] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.763, + "min_s": 0.742, + "max_s": 0.782, + "samples": [0.742, 0.763, 0.782] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 10.160, + "min_s": 10.024, + "max_s": 10.189, + "samples": [10.160, 10.024, 10.189] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 10.053, + "min_s": 10.017, + "max_s": 10.157, + "samples": [10.053, 10.157, 10.017] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 7.271, + "min_s": 7.245, + "max_s": 7.402, + "samples": [7.271, 7.402, 7.245] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.221, + "samples": [0.181, 0.221, 0.181] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.542, + "min_s": 0.542, + "max_s": 0.542, + "samples": [0.542, 0.542, 0.542] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.602, + "min_s": 0.602, + "max_s": 0.602, + "samples": [0.602, 0.602, 0.602] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 0.682, + "min_s": 0.662, + "max_s": 0.683, + "samples": [0.662, 0.682, 0.683] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 7.430, + "min_s": 7.415, + "max_s": 7.646, + "samples": [7.415, 7.430, 7.646] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.265, + "min_s": 1.264, + "max_s": 1.305, + "samples": [1.265, 1.264, 1.305] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.181, + "samples": [0.181, 0.181, 0.181] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.224, + "min_s": 1.204, + "max_s": 1.245, + "samples": [1.204, 1.245, 1.224] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.763, + "min_s": 0.745, + "max_s": 0.785, + "samples": [0.763, 0.785, 0.745] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 0.745, + "min_s": 0.726, + "max_s": 0.763, + "samples": [0.726, 0.745, 0.763] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: end-of-file", + "runs": 3, + "median_s": 1.265, + "min_s": 1.226, + "max_s": 1.405, + "samples": [1.226, 1.265, 1.405] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 9.894, + "min_s": 9.790, + "max_s": 9.976, + "samples": [9.976, 9.790, 9.894] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.241, + "samples": [0.181, 0.181, 0.241] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.923, + "min_s": 0.863, + "max_s": 1.025, + "samples": [0.863, 0.923, 1.025] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.823, + "min_s": 0.804, + "max_s": 0.843, + "samples": [0.804, 0.843, 0.823] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 10.014, + "min_s": 9.970, + "max_s": 10.036, + "samples": [10.014, 10.036, 9.970] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 10.016, + "min_s": 9.953, + "max_s": 10.105, + "samples": [9.953, 10.105, 10.016] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 7.554, + "min_s": 7.509, + "max_s": 7.772, + "samples": [7.554, 7.509, 7.772] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.201, + "samples": [0.181, 0.201, 0.181] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.602, + "min_s": 0.582, + "max_s": 0.602, + "samples": [0.582, 0.602, 0.602] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.662, + "min_s": 0.662, + "max_s": 0.682, + "samples": [0.682, 0.662, 0.662] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 0.762, + "min_s": 0.742, + "max_s": 0.763, + "samples": [0.763, 0.742, 0.762] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: end-of-file", + "runs": 3, + "median_s": 7.407, + "min_s": 7.391, + "max_s": 7.726, + "samples": [7.726, 7.391, 7.407] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 3.913, + "min_s": 3.874, + "max_s": 3.973, + "samples": [3.913, 3.973, 3.874] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.362, + "min_s": 0.362, + "max_s": 0.363, + "samples": [0.363, 0.362, 0.362] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.526, + "min_s": 1.505, + "max_s": 1.546, + "samples": [1.505, 1.546, 1.526] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.024, + "min_s": 1.003, + "max_s": 1.024, + "samples": [1.003, 1.024, 1.024] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 1.044, + "min_s": 1.024, + "max_s": 1.064, + "samples": [1.064, 1.044, 1.024] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 1.505, + "min_s": 1.505, + "max_s": 1.689, + "samples": [1.505, 1.505, 1.689] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 12.716, + "min_s": 12.645, + "max_s": 13.086, + "samples": [13.086, 12.716, 12.645] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.362, + "min_s": 0.361, + "max_s": 0.363, + "samples": [0.361, 0.363, 0.362] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 10.056, + "min_s": 9.997, + "max_s": 10.083, + "samples": [10.056, 9.997, 10.083] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.144, + "min_s": 1.144, + "max_s": 1.164, + "samples": [1.144, 1.144, 1.164] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 10.071, + "min_s": 9.982, + "max_s": 10.221, + "samples": [10.221, 9.982, 10.071] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 10.052, + "min_s": 10.016, + "max_s": 10.118, + "samples": [10.052, 10.118, 10.016] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 10.097, + "min_s": 10.037, + "max_s": 10.156, + "samples": [10.156, 10.037, 10.097] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.361, + "min_s": 0.361, + "max_s": 0.362, + "samples": [0.362, 0.361, 0.361] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 7.544, + "min_s": 7.508, + "max_s": 7.644, + "samples": [7.544, 7.508, 7.644] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.404, + "min_s": 1.385, + "max_s": 1.405, + "samples": [1.404, 1.405, 1.385] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 1.044, + "min_s": 1.024, + "max_s": 1.044, + "samples": [1.024, 1.044, 1.044] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 7.569, + "min_s": 7.512, + "max_s": 7.625, + "samples": [7.569, 7.512, 7.625] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 1.546, + "min_s": 1.526, + "max_s": 1.746, + "samples": [1.546, 1.746, 1.526] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.301, + "min_s": 0.301, + "max_s": 0.301, + "samples": [0.301, 0.301, 0.301] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 1.224, + "min_s": 0.341, + "max_s": 1.248, + "samples": [1.248, 0.341, 1.224] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.763, + "min_s": 0.321, + "max_s": 0.783, + "samples": [0.783, 0.321, 0.763] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 0.763, + "min_s": 0.301, + "max_s": 0.783, + "samples": [0.763, 0.301, 0.783] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 3, + "median_s": 1.244, + "min_s": 0.804, + "max_s": 1.264, + "samples": [1.264, 0.804, 1.244] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 26.772, + "min_s": 26.695, + "max_s": 27.153, + "samples": [27.153, 26.772, 26.695] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.322, + "min_s": 0.322, + "max_s": 0.341, + "samples": [0.322, 0.341, 0.322] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 26.999, + "min_s": 26.425, + "max_s": 27.241, + "samples": [26.425, 26.999, 27.241] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 3.029, + "min_s": 3.029, + "max_s": 3.110, + "samples": [3.110, 3.029, 3.029] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 26.922, + "min_s": 26.681, + "max_s": 27.082, + "samples": [26.922, 26.681, 27.082] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 26.848, + "min_s": 26.554, + "max_s": 26.859, + "samples": [26.848, 26.554, 26.859] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 28.002, + "min_s": 27.996, + "max_s": 28.257, + "samples": [27.996, 28.257, 28.002] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.341, + "min_s": 0.341, + "max_s": 0.341, + "samples": [0.341, 0.341, 0.341] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 27.509, + "min_s": 27.270, + "max_s": 27.575, + "samples": [27.575, 27.270, 27.509] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 3.391, + "min_s": 3.351, + "max_s": 3.471, + "samples": [3.391, 3.351, 3.471] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 0.342, + "min_s": 0.341, + "max_s": 0.864, + "samples": [0.864, 0.342, 0.341] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 3, + "median_s": 27.619, + "min_s": 27.582, + "max_s": 27.962, + "samples": [27.619, 27.962, 27.582] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 1.411, + "min_s": 1.367, + "max_s": 1.689, + "samples": [1.689, 1.367, 1.411] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.241, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.241, 0.221, 0.241] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.241, + "min_s": 0.241, + "max_s": 0.241, + "samples": [0.241, 0.241, 0.241] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.221, + "samples": [0.221, 0.221, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: in-body", + "runs": 3, + "median_s": 0.702, + "min_s": 0.702, + "max_s": 0.703, + "samples": [0.703, 0.702, 0.702] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: end-of-file", + "runs": 3, + "median_s": 1.204, + "min_s": 1.185, + "max_s": 1.224, + "samples": [1.204, 1.224, 1.185] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 10.594, + "min_s": 10.554, + "max_s": 10.660, + "samples": [10.660, 10.594, 10.554] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.241, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.241, 0.221, 0.241] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.221, + "samples": [0.221, 0.221, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.221, 0.241, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: in-body", + "runs": 3, + "median_s": 10.454, + "min_s": 10.374, + "max_s": 10.457, + "samples": [10.454, 10.374, 10.457] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: in-body", + "runs": 3, + "median_s": 10.302, + "min_s": 10.297, + "max_s": 10.414, + "samples": [10.297, 10.414, 10.302] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 8.311, + "min_s": 8.209, + "max_s": 8.390, + "samples": [8.311, 8.390, 8.209] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.241, 0.221, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.241, 0.221, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge)", + "runs": 3, + "median_s": 0.221, + "min_s": 0.221, + "max_s": 0.241, + "samples": [0.221, 0.241, 0.221] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: in-body", + "runs": 3, + "median_s": 0.723, + "min_s": 0.722, + "max_s": 0.743, + "samples": [0.723, 0.743, 0.722] + }, + { + "engine": "bazel", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "bazel 9.2.0 (cold excludes server start; clean is not --expunge) · perturbation: end-of-file", + "runs": 3, + "median_s": 7.905, + "min_s": 7.745, + "max_s": 8.267, + "samples": [7.745, 7.905, 8.267] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/clang-fixture.log b/bench/results/standard-20260814-linux-x86_64/clang-fixture.log new file mode 100644 index 00000000..4d55c597 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/clang-fixture.log @@ -0,0 +1,676 @@ +payload: payload:clang → /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++ +run id : f46f8e3b (fresh) +host : linux x86_64 · 13th Gen Intel(R) Core(TM) i9-13900K · 32 logical / 24 physical (heterogeneous) +fixture: 20 units, fanin 3, weight 4 + +[ 0.0s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers configure +[ 0.0s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers seed build +[ 1.3s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers run 1/3 +[ 2.5s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers run 2/3 +[ 3.7s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/cold/synth-20x3/headers 1.18s (min 1.18 / max 1.20, n=3) +[ 4.9s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers configure +[ 4.9s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers seed build +[ 5.1s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers run 1/3 +[ 5.3s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers run 2/3 +[ 5.5s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/noop/synth-20x3/headers 0.18s (min 0.18 / max 0.18, n=3) +[ 5.6s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers configure +[ 5.6s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers seed build +[ 5.8s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers run 1/3 +[ 7.0s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers run 2/3 +[ 8.1s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/headers 1.12s (min 1.10 / max 1.14, n=3) +[ 9.2s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers configure +[ 9.2s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers seed build +[ 9.4s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers run 1/3 +[ 10.1s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers run 2/3 +[ 10.8s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/headers 0.72s (min 0.68 / max 0.72, n=3) +[ 11.5s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers configure +[ 11.5s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers seed build +[ 11.7s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers run 1/3 +[ 12.4s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers run 2/3 +[ 13.1s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/headers 0.68s (min 0.66 / max 0.68, n=3) +[ 13.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers configure +[ 13.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers seed build +[ 14.4s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers run 1/3 +[ 15.6s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers run 2/3 +[ 16.8s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers run 3/3 +mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/headers 1.16s (min 1.12 / max 1.19, n=3) +[ 17.9s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules configure +[ 17.9s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules seed build +[ 27.7s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules run 1/3 +[ 37.5s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules run 2/3 +[ 47.2s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules 9.77s (min 9.73 / max 9.80, n=3) +[ 57.1s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules configure +[ 57.1s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules seed build +[ 57.2s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules run 1/3 +[ 57.5s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules run 2/3 +[ 57.7s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules 0.20s (min 0.20 / max 0.26, n=3) +[ 57.9s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules configure +[ 57.9s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules seed build +[ 58.1s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules run 1/3 +[ 58.9s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules run 2/3 +[ 59.8s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules 0.84s (min 0.84 / max 0.85, n=3) +[ 60.6s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules configure +[ 60.6s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules seed build +[ 60.8s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules run 1/3 +[ 61.5s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules run 2/3 +[ 62.3s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules 0.76s (min 0.74 / max 0.78, n=3) +[ 63.1s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules configure +[ 63.1s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules seed build +[ 63.3s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules run 1/3 +[ 73.4s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules run 2/3 +[ 83.5s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules 10.16s (min 10.02 / max 10.19, n=3) +[ 93.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules configure +[ 93.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules seed build +[ 103.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules run 1/3 +[ 113.7s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules run 2/3 +[ 123.9s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules 10.05s (min 10.02 / max 10.16, n=3) +[ 133.9s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl configure +[ 133.9s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl seed build +[ 141.1s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl run 1/3 +[ 148.4s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl run 2/3 +[ 155.8s] mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/cold/synth-20x3/modules-impl 7.27s (min 7.24 / max 7.40, n=3) +[ 163.0s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl configure +[ 163.0s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl seed build +[ 163.2s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl run 1/3 +[ 163.4s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl run 2/3 +[ 163.7s] mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/noop/synth-20x3/modules-impl 0.18s (min 0.18 / max 0.22, n=3) +[ 163.8s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl configure +[ 163.8s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl seed build +[ 164.0s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 164.6s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 165.1s] mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/touch-hub/synth-20x3/modules-impl 0.54s (min 0.54 / max 0.54, n=3) +[ 165.6s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl configure +[ 165.6s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl seed build +[ 165.8s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 166.4s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 167.0s] mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/touch-leaf/synth-20x3/modules-impl 0.60s (min 0.60 / max 0.60, n=3) +[ 167.6s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl configure +[ 167.6s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl seed build +[ 167.8s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 168.5s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 169.2s] mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/edit-body/synth-20x3/modules-impl 0.68s (min 0.66 / max 0.68, n=3) +[ 169.8s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl configure +[ 169.8s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl seed build +[ 170.5s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 177.9s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 185.4s] mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.13.1/clang/release/edit-comment/synth-20x3/modules-impl 7.43s (min 7.42 / max 7.65, n=3) +[ 193.0s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers configure +[ 193.0s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers seed build +[ 194.3s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers run 1/3 +[ 195.6s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers run 2/3 +[ 196.9s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/cold/synth-20x3/headers 1.26s (min 1.26 / max 1.30, n=3) +[ 198.2s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers configure +[ 198.2s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers seed build +[ 198.4s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers run 1/3 +[ 198.5s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers run 2/3 +[ 198.7s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/noop/synth-20x3/headers 0.18s (min 0.18 / max 0.18, n=3) +[ 198.9s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers configure +[ 198.9s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers seed build +[ 199.1s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers run 1/3 +[ 200.3s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers run 2/3 +[ 201.5s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/headers 1.22s (min 1.20 / max 1.24, n=3) +[ 202.8s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers configure +[ 202.8s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers seed build +[ 203.0s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers run 1/3 +[ 203.7s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers run 2/3 +[ 204.5s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/headers 0.76s (min 0.75 / max 0.78, n=3) +[ 205.3s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers configure +[ 205.3s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers seed build +[ 205.4s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers run 1/3 +[ 206.2s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers run 2/3 +[ 206.9s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/headers 0.75s (min 0.73 / max 0.76, n=3) +[ 207.7s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers configure +[ 207.7s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers seed build +[ 208.4s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers run 1/3 +[ 209.7s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers run 2/3 +[ 210.9s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers run 3/3 +mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/headers 1.26s (min 1.23 / max 1.41, n=3) +[ 212.3s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules configure +[ 212.3s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules seed build +[ 222.2s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules run 1/3 +[ 232.2s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules run 2/3 +[ 242.0s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules 9.89s (min 9.79 / max 9.98, n=3) +[ 251.9s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules configure +[ 251.9s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules seed build +[ 252.1s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules run 1/3 +[ 252.2s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules run 2/3 +[ 252.4s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules 0.18s (min 0.18 / max 0.24, n=3) +[ 252.7s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules configure +[ 252.7s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules seed build +[ 252.8s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules run 1/3 +[ 253.7s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules run 2/3 +[ 254.6s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules 0.92s (min 0.86 / max 1.02, n=3) +[ 255.7s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules configure +[ 255.7s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules seed build +[ 255.8s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules run 1/3 +[ 256.6s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules run 2/3 +[ 257.5s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules 0.82s (min 0.80 / max 0.84, n=3) +[ 258.3s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules configure +[ 258.3s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules seed build +[ 258.5s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules run 1/3 +[ 268.5s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules run 2/3 +[ 278.6s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules 10.01s (min 9.97 / max 10.04, n=3) +[ 288.5s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules configure +[ 288.5s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules seed build +[ 298.4s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules run 1/3 +[ 308.4s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules run 2/3 +[ 318.5s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules 10.02s (min 9.95 / max 10.11, n=3) +[ 328.5s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl configure +[ 328.5s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl seed build +[ 336.0s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl run 1/3 +[ 343.5s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl run 2/3 +[ 351.1s] mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/cold/synth-20x3/modules-impl 7.55s (min 7.51 / max 7.77, n=3) +[ 358.8s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl configure +[ 358.8s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl seed build +[ 359.0s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl run 1/3 +[ 359.2s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl run 2/3 +[ 359.4s] mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/noop/synth-20x3/modules-impl 0.18s (min 0.18 / max 0.20, n=3) +[ 359.6s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl configure +[ 359.6s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl seed build +[ 359.8s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 360.4s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 361.0s] mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/touch-hub/synth-20x3/modules-impl 0.60s (min 0.58 / max 0.60, n=3) +[ 361.6s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl configure +[ 361.6s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl seed build +[ 361.8s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 362.4s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 363.1s] mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/touch-leaf/synth-20x3/modules-impl 0.66s (min 0.66 / max 0.68, n=3) +[ 363.8s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl configure +[ 363.8s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl seed build +[ 364.0s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 364.7s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 365.5s] mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/edit-body/synth-20x3/modules-impl 0.76s (min 0.74 / max 0.76, n=3) +[ 366.2s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl configure +[ 366.2s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl seed build +[ 367.0s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 374.7s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 382.1s] mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/clang/release/edit-comment/synth-20x3/modules-impl 7.41s (min 7.39 / max 7.73, n=3) +[ 390.3s] cmake/clang/release/cold/synth-20x3/headers configure +[ 392.9s] cmake/clang/release/cold/synth-20x3/headers seed build +[ 394.4s] cmake/clang/release/cold/synth-20x3/headers run 1/3 +[ 398.6s] cmake/clang/release/cold/synth-20x3/headers run 2/3 +[ 402.9s] cmake/clang/release/cold/synth-20x3/headers run 3/3 +cmake/clang/release/cold/synth-20x3/headers 3.91s (min 3.87 / max 3.97, n=3) +[ 407.5s] cmake/clang/release/noop/synth-20x3/headers configure +[ 408.1s] cmake/clang/release/noop/synth-20x3/headers seed build +[ 408.4s] cmake/clang/release/noop/synth-20x3/headers run 1/3 +[ 409.1s] cmake/clang/release/noop/synth-20x3/headers run 2/3 +[ 409.8s] cmake/clang/release/noop/synth-20x3/headers run 3/3 +cmake/clang/release/noop/synth-20x3/headers 0.36s (min 0.36 / max 0.36, n=3) +[ 410.9s] cmake/clang/release/touch-hub/synth-20x3/headers configure +[ 411.4s] cmake/clang/release/touch-hub/synth-20x3/headers seed build +[ 411.8s] cmake/clang/release/touch-hub/synth-20x3/headers run 1/3 +[ 413.7s] cmake/clang/release/touch-hub/synth-20x3/headers run 2/3 +[ 415.5s] cmake/clang/release/touch-hub/synth-20x3/headers run 3/3 +cmake/clang/release/touch-hub/synth-20x3/headers 1.53s (min 1.51 / max 1.55, n=3) +[ 417.9s] cmake/clang/release/touch-leaf/synth-20x3/headers configure +[ 418.4s] cmake/clang/release/touch-leaf/synth-20x3/headers seed build +[ 418.8s] cmake/clang/release/touch-leaf/synth-20x3/headers run 1/3 +[ 420.2s] cmake/clang/release/touch-leaf/synth-20x3/headers run 2/3 +[ 421.5s] cmake/clang/release/touch-leaf/synth-20x3/headers run 3/3 +cmake/clang/release/touch-leaf/synth-20x3/headers 1.02s (min 1.00 / max 1.02, n=3) +[ 423.2s] cmake/clang/release/edit-body/synth-20x3/headers configure +[ 423.8s] cmake/clang/release/edit-body/synth-20x3/headers seed build +[ 424.1s] cmake/clang/release/edit-body/synth-20x3/headers run 1/3 +[ 425.6s] cmake/clang/release/edit-body/synth-20x3/headers run 2/3 +[ 427.0s] cmake/clang/release/edit-body/synth-20x3/headers run 3/3 +cmake/clang/release/edit-body/synth-20x3/headers 1.04s (min 1.02 / max 1.06, n=3) +[ 428.7s] cmake/clang/release/edit-comment/synth-20x3/headers configure +[ 429.2s] cmake/clang/release/edit-comment/synth-20x3/headers seed build +[ 430.3s] cmake/clang/release/edit-comment/synth-20x3/headers run 1/3 +[ 432.1s] cmake/clang/release/edit-comment/synth-20x3/headers run 2/3 +[ 434.0s] cmake/clang/release/edit-comment/synth-20x3/headers run 3/3 +cmake/clang/release/edit-comment/synth-20x3/headers 1.51s (min 1.50 / max 1.69, n=3) +[ 436.9s] cmake/clang/release/cold/synth-20x3/modules configure +[ 439.3s] cmake/clang/release/cold/synth-20x3/modules seed build +[ 449.5s] cmake/clang/release/cold/synth-20x3/modules run 1/3 +[ 462.9s] cmake/clang/release/cold/synth-20x3/modules run 2/3 +[ 476.0s] cmake/clang/release/cold/synth-20x3/modules run 3/3 +cmake/clang/release/cold/synth-20x3/modules 12.72s (min 12.64 / max 13.09, n=3) +[ 489.4s] cmake/clang/release/noop/synth-20x3/modules configure +[ 489.9s] cmake/clang/release/noop/synth-20x3/modules seed build +[ 490.3s] cmake/clang/release/noop/synth-20x3/modules run 1/3 +[ 491.0s] cmake/clang/release/noop/synth-20x3/modules run 2/3 +[ 491.7s] cmake/clang/release/noop/synth-20x3/modules run 3/3 +cmake/clang/release/noop/synth-20x3/modules 0.36s (min 0.36 / max 0.36, n=3) +[ 492.8s] cmake/clang/release/touch-hub/synth-20x3/modules configure +[ 493.3s] cmake/clang/release/touch-hub/synth-20x3/modules seed build +[ 493.7s] cmake/clang/release/touch-hub/synth-20x3/modules run 1/3 +[ 504.1s] cmake/clang/release/touch-hub/synth-20x3/modules run 2/3 +[ 514.5s] cmake/clang/release/touch-hub/synth-20x3/modules run 3/3 +cmake/clang/release/touch-hub/synth-20x3/modules 10.06s (min 10.00 / max 10.08, n=3) +[ 525.2s] cmake/clang/release/touch-leaf/synth-20x3/modules configure +[ 525.8s] cmake/clang/release/touch-leaf/synth-20x3/modules seed build +[ 526.2s] cmake/clang/release/touch-leaf/synth-20x3/modules run 1/3 +[ 527.7s] cmake/clang/release/touch-leaf/synth-20x3/modules run 2/3 +[ 529.2s] cmake/clang/release/touch-leaf/synth-20x3/modules run 3/3 +cmake/clang/release/touch-leaf/synth-20x3/modules 1.14s (min 1.14 / max 1.16, n=3) +[ 531.0s] cmake/clang/release/edit-body/synth-20x3/modules configure +[ 531.6s] cmake/clang/release/edit-body/synth-20x3/modules seed build +[ 531.9s] cmake/clang/release/edit-body/synth-20x3/modules run 1/3 +[ 542.5s] cmake/clang/release/edit-body/synth-20x3/modules run 2/3 +[ 552.8s] cmake/clang/release/edit-body/synth-20x3/modules run 3/3 +cmake/clang/release/edit-body/synth-20x3/modules 10.07s (min 9.98 / max 10.22, n=3) +[ 563.6s] cmake/clang/release/edit-comment/synth-20x3/modules configure +[ 564.1s] cmake/clang/release/edit-comment/synth-20x3/modules seed build +[ 574.1s] cmake/clang/release/edit-comment/synth-20x3/modules run 1/3 +[ 584.5s] cmake/clang/release/edit-comment/synth-20x3/modules run 2/3 +[ 595.0s] cmake/clang/release/edit-comment/synth-20x3/modules run 3/3 +cmake/clang/release/edit-comment/synth-20x3/modules 10.05s (min 10.02 / max 10.12, n=3) +[ 606.1s] cmake/clang/release/cold/synth-20x3/modules-impl configure +[ 608.5s] cmake/clang/release/cold/synth-20x3/modules-impl seed build +[ 616.2s] cmake/clang/release/cold/synth-20x3/modules-impl run 1/3 +[ 626.7s] cmake/clang/release/cold/synth-20x3/modules-impl run 2/3 +[ 637.1s] cmake/clang/release/cold/synth-20x3/modules-impl run 3/3 +cmake/clang/release/cold/synth-20x3/modules-impl 10.10s (min 10.04 / max 10.16, n=3) +[ 647.9s] cmake/clang/release/noop/synth-20x3/modules-impl configure +[ 648.4s] cmake/clang/release/noop/synth-20x3/modules-impl seed build +[ 648.8s] cmake/clang/release/noop/synth-20x3/modules-impl run 1/3 +[ 649.5s] cmake/clang/release/noop/synth-20x3/modules-impl run 2/3 +[ 650.2s] cmake/clang/release/noop/synth-20x3/modules-impl run 3/3 +cmake/clang/release/noop/synth-20x3/modules-impl 0.36s (min 0.36 / max 0.36, n=3) +[ 651.3s] cmake/clang/release/touch-hub/synth-20x3/modules-impl configure +[ 651.8s] cmake/clang/release/touch-hub/synth-20x3/modules-impl seed build +[ 652.2s] cmake/clang/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 660.1s] cmake/clang/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 668.0s] cmake/clang/release/touch-hub/synth-20x3/modules-impl run 3/3 +cmake/clang/release/touch-hub/synth-20x3/modules-impl 7.54s (min 7.51 / max 7.64, n=3) +[ 676.3s] cmake/clang/release/touch-leaf/synth-20x3/modules-impl configure +[ 676.8s] cmake/clang/release/touch-leaf/synth-20x3/modules-impl seed build +[ 677.2s] cmake/clang/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 679.0s] cmake/clang/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 680.7s] cmake/clang/release/touch-leaf/synth-20x3/modules-impl run 3/3 +cmake/clang/release/touch-leaf/synth-20x3/modules-impl 1.40s (min 1.39 / max 1.41, n=3) +[ 682.8s] cmake/clang/release/edit-body/synth-20x3/modules-impl configure +[ 683.3s] cmake/clang/release/edit-body/synth-20x3/modules-impl seed build +[ 683.7s] cmake/clang/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 685.1s] cmake/clang/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 686.5s] cmake/clang/release/edit-body/synth-20x3/modules-impl run 3/3 +cmake/clang/release/edit-body/synth-20x3/modules-impl 1.04s (min 1.02 / max 1.04, n=3) +[ 688.2s] cmake/clang/release/edit-comment/synth-20x3/modules-impl configure +[ 688.7s] cmake/clang/release/edit-comment/synth-20x3/modules-impl seed build +[ 689.8s] cmake/clang/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 697.7s] cmake/clang/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 705.5s] cmake/clang/release/edit-comment/synth-20x3/modules-impl run 3/3 +cmake/clang/release/edit-comment/synth-20x3/modules-impl 7.57s (min 7.51 / max 7.62, n=3) +[ 713.9s] xmake/clang/release/cold/synth-20x3/headers configure +[ 714.3s] xmake/clang/release/cold/synth-20x3/headers seed build +[ 716.1s] xmake/clang/release/cold/synth-20x3/headers run 1/3 +[ 717.9s] xmake/clang/release/cold/synth-20x3/headers run 2/3 +[ 719.9s] xmake/clang/release/cold/synth-20x3/headers run 3/3 +xmake/clang/release/cold/synth-20x3/headers 1.55s (min 1.53 / max 1.75, n=3) +[ 721.8s] xmake/clang/release/noop/synth-20x3/headers configure +[ 722.1s] xmake/clang/release/noop/synth-20x3/headers seed build +[ 722.4s] xmake/clang/release/noop/synth-20x3/headers run 1/3 +[ 722.9s] xmake/clang/release/noop/synth-20x3/headers run 2/3 +[ 723.4s] xmake/clang/release/noop/synth-20x3/headers run 3/3 +xmake/clang/release/noop/synth-20x3/headers 0.30s (min 0.30 / max 0.30, n=3) +[ 724.1s] xmake/clang/release/touch-hub/synth-20x3/headers configure +[ 724.4s] xmake/clang/release/touch-hub/synth-20x3/headers seed build +[ 724.7s] xmake/clang/release/touch-hub/synth-20x3/headers run 1/3 +[ 726.2s] xmake/clang/release/touch-hub/synth-20x3/headers run 2/3 +[ 726.7s] xmake/clang/release/touch-hub/synth-20x3/headers run 3/3 +xmake/clang/release/touch-hub/synth-20x3/headers 1.22s (min 0.34 / max 1.25, n=3) +[ 728.3s] xmake/clang/release/touch-leaf/synth-20x3/headers configure +[ 728.6s] xmake/clang/release/touch-leaf/synth-20x3/headers seed build +[ 728.9s] xmake/clang/release/touch-leaf/synth-20x3/headers run 1/3 +[ 729.9s] xmake/clang/release/touch-leaf/synth-20x3/headers run 2/3 +[ 730.4s] xmake/clang/release/touch-leaf/synth-20x3/headers run 3/3 +xmake/clang/release/touch-leaf/synth-20x3/headers 0.76s (min 0.32 / max 0.78, n=3) +[ 731.6s] xmake/clang/release/edit-body/synth-20x3/headers configure +[ 731.9s] xmake/clang/release/edit-body/synth-20x3/headers seed build +[ 732.2s] xmake/clang/release/edit-body/synth-20x3/headers run 1/3 +[ 733.2s] xmake/clang/release/edit-body/synth-20x3/headers run 2/3 +[ 733.7s] xmake/clang/release/edit-body/synth-20x3/headers run 3/3 +xmake/clang/release/edit-body/synth-20x3/headers 0.76s (min 0.30 / max 0.78, n=3) +[ 734.9s] xmake/clang/release/edit-comment/synth-20x3/headers configure +[ 735.2s] xmake/clang/release/edit-comment/synth-20x3/headers seed build +[ 735.5s] xmake/clang/release/edit-comment/synth-20x3/headers run 1/3 +[ 736.9s] xmake/clang/release/edit-comment/synth-20x3/headers run 2/3 +[ 738.0s] xmake/clang/release/edit-comment/synth-20x3/headers run 3/3 +xmake/clang/release/edit-comment/synth-20x3/headers 1.24s (min 0.80 / max 1.26, n=3) +[ 739.8s] xmake/clang/release/cold/synth-20x3/modules configure +[ 740.2s] xmake/clang/release/cold/synth-20x3/modules seed build +[ 767.5s] xmake/clang/release/cold/synth-20x3/modules run 1/3 +[ 794.9s] xmake/clang/release/cold/synth-20x3/modules run 2/3 +[ 821.9s] xmake/clang/release/cold/synth-20x3/modules run 3/3 +xmake/clang/release/cold/synth-20x3/modules 26.77s (min 26.69 / max 27.15, n=3) +[ 849.1s] xmake/clang/release/noop/synth-20x3/modules configure +[ 849.3s] xmake/clang/release/noop/synth-20x3/modules seed build +[ 849.7s] xmake/clang/release/noop/synth-20x3/modules run 1/3 +[ 850.2s] xmake/clang/release/noop/synth-20x3/modules run 2/3 +[ 850.8s] xmake/clang/release/noop/synth-20x3/modules run 3/3 +xmake/clang/release/noop/synth-20x3/modules 0.32s (min 0.32 / max 0.34, n=3) +[ 851.5s] xmake/clang/release/touch-hub/synth-20x3/modules configure +[ 851.8s] xmake/clang/release/touch-hub/synth-20x3/modules seed build +[ 852.1s] xmake/clang/release/touch-hub/synth-20x3/modules run 1/3 +[ 878.7s] xmake/clang/release/touch-hub/synth-20x3/modules run 2/3 +[ 905.9s] xmake/clang/release/touch-hub/synth-20x3/modules run 3/3 +xmake/clang/release/touch-hub/synth-20x3/modules 27.00s (min 26.43 / max 27.24, n=3) +[ 933.6s] xmake/clang/release/touch-leaf/synth-20x3/modules configure +[ 933.9s] xmake/clang/release/touch-leaf/synth-20x3/modules seed build +[ 934.2s] xmake/clang/release/touch-leaf/synth-20x3/modules run 1/3 +[ 937.5s] xmake/clang/release/touch-leaf/synth-20x3/modules run 2/3 +[ 940.8s] xmake/clang/release/touch-leaf/synth-20x3/modules run 3/3 +xmake/clang/release/touch-leaf/synth-20x3/modules 3.03s (min 3.03 / max 3.11, n=3) +[ 944.2s] xmake/clang/release/edit-body/synth-20x3/modules configure +[ 944.5s] xmake/clang/release/edit-body/synth-20x3/modules seed build +[ 944.8s] xmake/clang/release/edit-body/synth-20x3/modules run 1/3 +[ 972.0s] xmake/clang/release/edit-body/synth-20x3/modules run 2/3 +[ 998.8s] xmake/clang/release/edit-body/synth-20x3/modules run 3/3 +xmake/clang/release/edit-body/synth-20x3/modules 26.92s (min 26.68 / max 27.08, n=3) +[ 1026.4s] xmake/clang/release/edit-comment/synth-20x3/modules configure +[ 1026.7s] xmake/clang/release/edit-comment/synth-20x3/modules seed build +[ 1053.9s] xmake/clang/release/edit-comment/synth-20x3/modules run 1/3 +[ 1080.9s] xmake/clang/release/edit-comment/synth-20x3/modules run 2/3 +[ 1107.7s] xmake/clang/release/edit-comment/synth-20x3/modules run 3/3 +xmake/clang/release/edit-comment/synth-20x3/modules 26.85s (min 26.55 / max 26.86, n=3) +[ 1135.2s] xmake/clang/release/cold/synth-20x3/modules-impl configure +[ 1135.5s] xmake/clang/release/cold/synth-20x3/modules-impl seed build +[ 1163.9s] xmake/clang/release/cold/synth-20x3/modules-impl run 1/3 +[ 1192.1s] xmake/clang/release/cold/synth-20x3/modules-impl run 2/3 +[ 1220.6s] xmake/clang/release/cold/synth-20x3/modules-impl run 3/3 +xmake/clang/release/cold/synth-20x3/modules-impl 28.00s (min 28.00 / max 28.26, n=3) +[ 1249.1s] xmake/clang/release/noop/synth-20x3/modules-impl configure +[ 1249.3s] xmake/clang/release/noop/synth-20x3/modules-impl seed build +[ 1249.7s] xmake/clang/release/noop/synth-20x3/modules-impl run 1/3 +[ 1250.2s] xmake/clang/release/noop/synth-20x3/modules-impl run 2/3 +[ 1250.8s] xmake/clang/release/noop/synth-20x3/modules-impl run 3/3 +xmake/clang/release/noop/synth-20x3/modules-impl 0.34s (min 0.34 / max 0.34, n=3) +[ 1251.5s] xmake/clang/release/touch-hub/synth-20x3/modules-impl configure +[ 1251.8s] xmake/clang/release/touch-hub/synth-20x3/modules-impl seed build +[ 1252.1s] xmake/clang/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 1279.9s] xmake/clang/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 1307.4s] xmake/clang/release/touch-hub/synth-20x3/modules-impl run 3/3 +xmake/clang/release/touch-hub/synth-20x3/modules-impl 27.51s (min 27.27 / max 27.57, n=3) +[ 1335.3s] xmake/clang/release/touch-leaf/synth-20x3/modules-impl configure +[ 1335.6s] xmake/clang/release/touch-leaf/synth-20x3/modules-impl seed build +[ 1335.9s] xmake/clang/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 1339.5s] xmake/clang/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 1343.1s] xmake/clang/release/touch-leaf/synth-20x3/modules-impl run 3/3 +xmake/clang/release/touch-leaf/synth-20x3/modules-impl 3.39s (min 3.35 / max 3.47, n=3) +[ 1347.0s] xmake/clang/release/edit-body/synth-20x3/modules-impl configure +[ 1347.2s] xmake/clang/release/edit-body/synth-20x3/modules-impl seed build +[ 1347.6s] xmake/clang/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 1348.6s] xmake/clang/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 1349.2s] xmake/clang/release/edit-body/synth-20x3/modules-impl run 3/3 +xmake/clang/release/edit-body/synth-20x3/modules-impl 0.34s (min 0.34 / max 0.86, n=3) +[ 1349.9s] xmake/clang/release/edit-comment/synth-20x3/modules-impl configure +[ 1350.2s] xmake/clang/release/edit-comment/synth-20x3/modules-impl seed build +[ 1351.1s] xmake/clang/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 1378.9s] xmake/clang/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 1407.1s] xmake/clang/release/edit-comment/synth-20x3/modules-impl run 3/3 +xmake/clang/release/edit-comment/synth-20x3/modules-impl 27.62s (min 27.58 / max 27.96, n=3) +[ 1449.1s] bazel/clang/release/cold/synth-20x3/headers configure +[ 1449.1s] bazel/clang/release/cold/synth-20x3/headers seed build +[ 1452.3s] bazel/clang/release/cold/synth-20x3/headers run 1/3 +[ 1454.4s] bazel/clang/release/cold/synth-20x3/headers run 2/3 +[ 1456.3s] bazel/clang/release/cold/synth-20x3/headers run 3/3 +bazel/clang/release/cold/synth-20x3/headers 1.41s (min 1.37 / max 1.69, n=3) +[ 1458.6s] bazel/clang/release/noop/synth-20x3/headers configure +[ 1458.6s] bazel/clang/release/noop/synth-20x3/headers seed build +[ 1458.9s] bazel/clang/release/noop/synth-20x3/headers run 1/3 +[ 1459.3s] bazel/clang/release/noop/synth-20x3/headers run 2/3 +[ 1459.7s] bazel/clang/release/noop/synth-20x3/headers run 3/3 +bazel/clang/release/noop/synth-20x3/headers 0.24s (min 0.22 / max 0.24, n=3) +[ 1460.5s] bazel/clang/release/touch-hub/synth-20x3/headers configure +[ 1460.5s] bazel/clang/release/touch-hub/synth-20x3/headers seed build +[ 1460.7s] bazel/clang/release/touch-hub/synth-20x3/headers run 1/3 +[ 1461.1s] bazel/clang/release/touch-hub/synth-20x3/headers run 2/3 +[ 1461.6s] bazel/clang/release/touch-hub/synth-20x3/headers run 3/3 +bazel/clang/release/touch-hub/synth-20x3/headers 0.24s (min 0.24 / max 0.24, n=3) +[ 1462.4s] bazel/clang/release/touch-leaf/synth-20x3/headers configure +[ 1462.4s] bazel/clang/release/touch-leaf/synth-20x3/headers seed build +[ 1462.6s] bazel/clang/release/touch-leaf/synth-20x3/headers run 1/3 +[ 1463.0s] bazel/clang/release/touch-leaf/synth-20x3/headers run 2/3 +[ 1463.4s] bazel/clang/release/touch-leaf/synth-20x3/headers run 3/3 +bazel/clang/release/touch-leaf/synth-20x3/headers 0.22s (min 0.22 / max 0.22, n=3) +[ 1464.2s] bazel/clang/release/edit-body/synth-20x3/headers configure +[ 1464.2s] bazel/clang/release/edit-body/synth-20x3/headers seed build +[ 1464.4s] bazel/clang/release/edit-body/synth-20x3/headers run 1/3 +[ 1465.3s] bazel/clang/release/edit-body/synth-20x3/headers run 2/3 +[ 1466.2s] bazel/clang/release/edit-body/synth-20x3/headers run 3/3 +bazel/clang/release/edit-body/synth-20x3/headers 0.70s (min 0.70 / max 0.70, n=3) +[ 1467.4s] bazel/clang/release/edit-comment/synth-20x3/headers configure +[ 1467.4s] bazel/clang/release/edit-comment/synth-20x3/headers seed build +[ 1468.1s] bazel/clang/release/edit-comment/synth-20x3/headers run 1/3 +[ 1469.5s] bazel/clang/release/edit-comment/synth-20x3/headers run 2/3 +[ 1470.9s] bazel/clang/release/edit-comment/synth-20x3/headers run 3/3 +bazel/clang/release/edit-comment/synth-20x3/headers 1.20s (min 1.18 / max 1.22, n=3) +[ 1485.8s] bazel/clang/release/cold/synth-20x3/modules configure +[ 1485.8s] bazel/clang/release/cold/synth-20x3/modules seed build +[ 1498.4s] bazel/clang/release/cold/synth-20x3/modules run 1/3 +[ 1509.5s] bazel/clang/release/cold/synth-20x3/modules run 2/3 +[ 1520.6s] bazel/clang/release/cold/synth-20x3/modules run 3/3 +bazel/clang/release/cold/synth-20x3/modules 10.59s (min 10.55 / max 10.66, n=3) +[ 1532.0s] bazel/clang/release/noop/synth-20x3/modules configure +[ 1532.0s] bazel/clang/release/noop/synth-20x3/modules seed build +[ 1532.2s] bazel/clang/release/noop/synth-20x3/modules run 1/3 +[ 1532.6s] bazel/clang/release/noop/synth-20x3/modules run 2/3 +[ 1533.0s] bazel/clang/release/noop/synth-20x3/modules run 3/3 +bazel/clang/release/noop/synth-20x3/modules 0.24s (min 0.22 / max 0.24, n=3) +[ 1533.8s] bazel/clang/release/touch-hub/synth-20x3/modules configure +[ 1533.8s] bazel/clang/release/touch-hub/synth-20x3/modules seed build +[ 1534.1s] bazel/clang/release/touch-hub/synth-20x3/modules run 1/3 +[ 1534.5s] bazel/clang/release/touch-hub/synth-20x3/modules run 2/3 +[ 1534.8s] bazel/clang/release/touch-hub/synth-20x3/modules run 3/3 +bazel/clang/release/touch-hub/synth-20x3/modules 0.22s (min 0.22 / max 0.22, n=3) +[ 1535.6s] bazel/clang/release/touch-leaf/synth-20x3/modules configure +[ 1535.6s] bazel/clang/release/touch-leaf/synth-20x3/modules seed build +[ 1535.9s] bazel/clang/release/touch-leaf/synth-20x3/modules run 1/3 +[ 1536.3s] bazel/clang/release/touch-leaf/synth-20x3/modules run 2/3 +[ 1536.7s] bazel/clang/release/touch-leaf/synth-20x3/modules run 3/3 +bazel/clang/release/touch-leaf/synth-20x3/modules 0.22s (min 0.22 / max 0.24, n=3) +[ 1537.4s] bazel/clang/release/edit-body/synth-20x3/modules configure +[ 1537.4s] bazel/clang/release/edit-body/synth-20x3/modules seed build +[ 1537.7s] bazel/clang/release/edit-body/synth-20x3/modules run 1/3 +[ 1548.3s] bazel/clang/release/edit-body/synth-20x3/modules run 2/3 +[ 1558.9s] bazel/clang/release/edit-body/synth-20x3/modules run 3/3 +bazel/clang/release/edit-body/synth-20x3/modules 10.45s (min 10.37 / max 10.46, n=3) +[ 1569.9s] bazel/clang/release/edit-comment/synth-20x3/modules configure +[ 1569.9s] bazel/clang/release/edit-comment/synth-20x3/modules seed build +[ 1580.1s] bazel/clang/release/edit-comment/synth-20x3/modules run 1/3 +[ 1590.6s] bazel/clang/release/edit-comment/synth-20x3/modules run 2/3 +[ 1601.2s] bazel/clang/release/edit-comment/synth-20x3/modules run 3/3 +bazel/clang/release/edit-comment/synth-20x3/modules 10.30s (min 10.30 / max 10.41, n=3) +[ 1624.6s] bazel/clang/release/cold/synth-20x3/modules-impl configure +[ 1624.6s] bazel/clang/release/cold/synth-20x3/modules-impl seed build +[ 1634.5s] bazel/clang/release/cold/synth-20x3/modules-impl run 1/3 +[ 1643.3s] bazel/clang/release/cold/synth-20x3/modules-impl run 2/3 +[ 1652.3s] bazel/clang/release/cold/synth-20x3/modules-impl run 3/3 +bazel/clang/release/cold/synth-20x3/modules-impl 8.31s (min 8.21 / max 8.39, n=3) +[ 1661.4s] bazel/clang/release/noop/synth-20x3/modules-impl configure +[ 1661.4s] bazel/clang/release/noop/synth-20x3/modules-impl seed build +[ 1661.6s] bazel/clang/release/noop/synth-20x3/modules-impl run 1/3 +[ 1662.0s] bazel/clang/release/noop/synth-20x3/modules-impl run 2/3 +[ 1662.4s] bazel/clang/release/noop/synth-20x3/modules-impl run 3/3 +bazel/clang/release/noop/synth-20x3/modules-impl 0.22s (min 0.22 / max 0.24, n=3) +[ 1663.2s] bazel/clang/release/touch-hub/synth-20x3/modules-impl configure +[ 1663.2s] bazel/clang/release/touch-hub/synth-20x3/modules-impl seed build +[ 1663.5s] bazel/clang/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 1663.9s] bazel/clang/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 1664.3s] bazel/clang/release/touch-hub/synth-20x3/modules-impl run 3/3 +bazel/clang/release/touch-hub/synth-20x3/modules-impl 0.22s (min 0.22 / max 0.24, n=3) +[ 1665.0s] bazel/clang/release/touch-leaf/synth-20x3/modules-impl configure +[ 1665.0s] bazel/clang/release/touch-leaf/synth-20x3/modules-impl seed build +[ 1665.3s] bazel/clang/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 1665.7s] bazel/clang/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 1666.1s] bazel/clang/release/touch-leaf/synth-20x3/modules-impl run 3/3 +bazel/clang/release/touch-leaf/synth-20x3/modules-impl 0.22s (min 0.22 / max 0.24, n=3) +[ 1666.9s] bazel/clang/release/edit-body/synth-20x3/modules-impl configure +[ 1666.9s] bazel/clang/release/edit-body/synth-20x3/modules-impl seed build +[ 1667.1s] bazel/clang/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 1668.0s] bazel/clang/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 1668.9s] bazel/clang/release/edit-body/synth-20x3/modules-impl run 3/3 +bazel/clang/release/edit-body/synth-20x3/modules-impl 0.72s (min 0.72 / max 0.74, n=3) +[ 1670.2s] bazel/clang/release/edit-comment/synth-20x3/modules-impl configure +[ 1670.2s] bazel/clang/release/edit-comment/synth-20x3/modules-impl seed build +[ 1671.0s] bazel/clang/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 1678.9s] bazel/clang/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 1687.0s] bazel/clang/release/edit-comment/synth-20x3/modules-impl run 3/3 +bazel/clang/release/edit-comment/synth-20x3/modules-impl 7.91s (min 7.75 / max 8.27, n=3) + +=== relative to cmake (>1.00 = slower than the baseline) === + +-- headers / cold -- + mcpp@2026.8.13.1 1.18s 0.30x + mcpp@2026.8.11.3 1.26s 0.32x + cmake 3.91s 1.00x <- baseline + xmake 1.55s 0.39x + bazel 1.41s 0.36x + +-- headers / noop -- + mcpp@2026.8.13.1 0.18s 0.50x + mcpp@2026.8.11.3 0.18s 0.50x + cmake 0.36s 1.00x <- baseline + xmake 0.30s 0.83x + bazel 0.24s 0.67x + +-- headers / touch-hub -- + mcpp@2026.8.13.1 1.12s 0.74x + mcpp@2026.8.11.3 1.22s 0.80x + cmake 1.53s 1.00x <- baseline + xmake 1.22s 0.80x + bazel 0.24s 0.16x + +-- headers / touch-leaf -- + mcpp@2026.8.13.1 0.72s 0.71x + mcpp@2026.8.11.3 0.76s 0.75x + cmake 1.02s 1.00x <- baseline + xmake 0.76s 0.75x + bazel 0.22s 0.22x + +-- headers / edit-body -- + mcpp@2026.8.13.1 0.68s 0.65x + mcpp@2026.8.11.3 0.75s 0.71x + cmake 1.04s 1.00x <- baseline + xmake 0.76s 0.73x + bazel 0.70s 0.67x + +-- headers / edit-comment -- + mcpp@2026.8.13.1 1.16s 0.77x + mcpp@2026.8.11.3 1.26s 0.84x + cmake 1.51s 1.00x <- baseline + xmake 1.24s 0.83x + bazel 1.20s 0.80x + +-- modules / cold -- + mcpp@2026.8.13.1 9.77s 0.77x + mcpp@2026.8.11.3 9.89s 0.78x + cmake 12.72s 1.00x <- baseline + xmake 26.77s 2.11x + bazel 10.59s 0.83x + +-- modules / noop -- + mcpp@2026.8.13.1 0.20s 0.55x + mcpp@2026.8.11.3 0.18s 0.50x + cmake 0.36s 1.00x <- baseline + xmake 0.32s 0.89x + bazel 0.24s 0.67x + +-- modules / touch-hub -- + mcpp@2026.8.13.1 0.84s 0.08x + mcpp@2026.8.11.3 0.92s 0.09x + cmake 10.06s 1.00x <- baseline + xmake 27.00s 2.68x + bazel 0.22s 0.02x + +-- modules / touch-leaf -- + mcpp@2026.8.13.1 0.76s 0.67x + mcpp@2026.8.11.3 0.82s 0.72x + cmake 1.14s 1.00x <- baseline + xmake 3.03s 2.65x + bazel 0.22s 0.19x + +-- modules / edit-body -- + mcpp@2026.8.13.1 10.16s 1.01x + mcpp@2026.8.11.3 10.01s 0.99x + cmake 10.07s 1.00x <- baseline + xmake 26.92s 2.67x + bazel 10.45s 1.04x + +-- modules / edit-comment -- + mcpp@2026.8.13.1 10.05s 1.00x + mcpp@2026.8.11.3 10.02s 1.00x + cmake 10.05s 1.00x <- baseline + xmake 26.85s 2.67x + bazel 10.30s 1.02x + +-- modules-impl / cold -- + mcpp@2026.8.13.1 7.27s 0.72x + mcpp@2026.8.11.3 7.55s 0.75x + cmake 10.10s 1.00x <- baseline + xmake 28.00s 2.77x + bazel 8.31s 0.82x + +-- modules-impl / noop -- + mcpp@2026.8.13.1 0.18s 0.50x + mcpp@2026.8.11.3 0.18s 0.50x + cmake 0.36s 1.00x <- baseline + xmake 0.34s 0.94x + bazel 0.22s 0.61x + +-- modules-impl / touch-hub -- + mcpp@2026.8.13.1 0.54s 0.07x + mcpp@2026.8.11.3 0.60s 0.08x + cmake 7.54s 1.00x <- baseline + xmake 27.51s 3.65x + bazel 0.22s 0.03x + +-- modules-impl / touch-leaf -- + mcpp@2026.8.13.1 0.60s 0.43x + mcpp@2026.8.11.3 0.66s 0.47x + cmake 1.40s 1.00x <- baseline + xmake 3.39s 2.41x + bazel 0.22s 0.16x + +-- modules-impl / edit-body -- + mcpp@2026.8.13.1 0.68s 0.65x + mcpp@2026.8.11.3 0.76s 0.73x + cmake 1.04s 1.00x <- baseline + xmake 0.34s 0.33x + bazel 0.72s 0.69x + +-- modules-impl / edit-comment -- + mcpp@2026.8.13.1 7.43s 0.98x + mcpp@2026.8.11.3 7.41s 0.98x + cmake 7.57s 1.00x <- baseline + xmake 27.62s 3.65x + bazel 7.91s 1.04x + +report : /home/speak/workspace/github/mcpp-community/mcpp/bench/results/standard-20260814-linux-x86_64/clang-fixture.json +cells : 90 ok, 0 failed, 0 not applicable diff --git a/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.json b/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.json new file mode 100644 index 00000000..8fcd4814 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.json @@ -0,0 +1,376 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-15T00:55:48Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 32.058, + "min_s": 31.934, + "max_s": 32.225, + "samples": [32.225, 31.934, 32.058] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.161, 0.161, 0.181] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.302, + "min_s": 0.282, + "max_s": 0.321, + "samples": [0.282, 0.321, 0.302] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 30.682, + "min_s": 30.566, + "max_s": 30.800, + "samples": [30.682, 30.566, 30.800] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 24.788, + "min_s": 24.706, + "max_s": 24.889, + "samples": [24.889, 24.706, 24.788] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 78.698, + "min_s": 78.690, + "max_s": 79.253, + "samples": [78.690, 79.253, 78.698] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.241, + "min_s": 0.241, + "max_s": 0.241, + "samples": [0.241, 0.241, 0.241] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 74.123, + "min_s": 73.437, + "max_s": 74.463, + "samples": [74.123, 73.437, 74.463] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 75.777, + "min_s": 75.490, + "max_s": 75.781, + "samples": [75.490, 75.781, 75.777] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: end-of-file", + "runs": 3, + "median_s": 74.139, + "min_s": 73.958, + "max_s": 74.796, + "samples": [74.796, 74.139, 73.958] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 18.143, + "min_s": 17.981, + "max_s": 18.324, + "samples": [17.981, 18.324, 18.143] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.161, + "max_s": 0.201, + "samples": [0.201, 0.181, 0.161] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.281, + "min_s": 0.261, + "max_s": 0.282, + "samples": [0.282, 0.281, 0.261] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 14.640, + "min_s": 14.446, + "max_s": 14.774, + "samples": [14.774, 14.446, 14.640] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 13.506, + "min_s": 13.423, + "max_s": 14.334, + "samples": [13.423, 14.334, 13.506] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 35.199, + "min_s": 35.133, + "max_s": 35.539, + "samples": [35.539, 35.133, 35.199] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.321, + "min_s": 0.301, + "max_s": 0.322, + "samples": [0.322, 0.301, 0.321] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 31.215, + "min_s": 30.841, + "max_s": 31.374, + "samples": [30.841, 31.374, 31.215] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 31.147, + "min_s": 31.132, + "max_s": 31.166, + "samples": [31.147, 31.166, 31.132] + }, + { + "engine": "cmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 31.237, + "min_s": 31.160, + "max_s": 31.385, + "samples": [31.160, 31.385, 31.237] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "failed", + "note": "seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-cold.log)", + "runs": 0, + "samples": [] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "failed", + "note": "seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-noop.log)", + "runs": 0, + "samples": [] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "failed", + "note": "seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-touch-hub.log)", + "runs": 0, + "samples": [] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "failed", + "note": "seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-body.log)", + "runs": 0, + "samples": [] + }, + { + "engine": "xmake", + "compiler": "clang", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "failed", + "note": "seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-comment.log)", + "runs": 0, + "samples": [] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.log b/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.log new file mode 100644 index 00000000..87f0982c --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.log @@ -0,0 +1,311 @@ +payload: payload:clang → /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++ +run id : 7b78b5de (fresh) +host : linux x86_64 · 13th Gen Intel(R) Core(TM) i9-13900K · 32 logical / 24 physical (heterogeneous) +project: /home/speak/workspace/github/mcpp-community/mcpp/bench/projects/mcpp/mcpp-2026.8.11.3 (measured in place) + +[ 0.0s] mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules configure +[ 0.0s] mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules seed build +[ 32.2s] mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 64.5s] mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 96.5s] mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/cold/mcpp-2026.8.11.3/modules 32.06s (min 31.93 / max 32.23, n=3) +[ 128.6s] mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules configure +[ 128.6s] mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules seed build +[ 128.7s] mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 128.9s] mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 129.1s] mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/noop/mcpp-2026.8.11.3/modules 0.16s (min 0.16 / max 0.18, n=3) +[ 129.2s] mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 129.2s] mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 129.4s] mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 129.7s] mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 130.0s] mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/touch-hub/mcpp-2026.8.11.3/modules 0.30s (min 0.28 / max 0.32, n=3) +[ 130.3s] mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 130.3s] mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 130.5s] mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 161.2s] mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 191.7s] mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/edit-body/mcpp-2026.8.11.3/modules 30.68s (min 30.57 / max 30.80, n=3) +[ 222.5s] mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 222.5s] mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 253.3s] mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 278.2s] mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 302.9s] mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/clang/release/edit-comment/mcpp-2026.8.11.3/modules 24.79s (min 24.71 / max 24.89, n=3) +[ 327.7s] mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules configure +[ 327.7s] mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules seed build +[ 406.9s] mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 485.6s] mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 564.9s] mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/cold/mcpp-2026.8.11.3/modules 78.70s (min 78.69 / max 79.25, n=3) +[ 643.6s] mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules configure +[ 643.6s] mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules seed build +[ 643.8s] mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 644.1s] mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 644.3s] mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/noop/mcpp-2026.8.11.3/modules 0.24s (min 0.24 / max 0.24, n=3) +[ 644.6s] mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 644.6s] mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 644.8s] mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 718.9s] mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 792.4s] mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/touch-hub/mcpp-2026.8.11.3/modules 74.12s (min 73.44 / max 74.46, n=3) +[ 866.8s] mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 866.8s] mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 867.1s] mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 942.6s] mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 1018.4s] mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/edit-body/mcpp-2026.8.11.3/modules 75.78s (min 75.49 / max 75.78, n=3) +[ 1094.1s] mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 1094.1s] mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 1170.1s] mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 1244.9s] mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 1319.1s] mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/clang/release/edit-comment/mcpp-2026.8.11.3/modules 74.14s (min 73.96 / max 74.80, n=3) +[ 1393.0s] mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules configure +[ 1393.0s] mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules seed build +[ 1411.1s] mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 1429.1s] mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 1447.5s] mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/clang/release/cold/mcpp-2026.8.11.3/modules 18.14s (min 17.98 / max 18.32, n=3) +[ 1465.7s] mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules configure +[ 1465.7s] mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules seed build +[ 1465.9s] mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 1466.1s] mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 1466.3s] mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/clang/release/noop/mcpp-2026.8.11.3/modules 0.18s (min 0.16 / max 0.20, n=3) +[ 1466.4s] mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 1466.4s] mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 1466.6s] mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 1466.9s] mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 1467.2s] mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/clang/release/touch-hub/mcpp-2026.8.11.3/modules 0.28s (min 0.26 / max 0.28, n=3) +[ 1467.4s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 1467.4s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 1467.6s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 1482.4s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 1496.8s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/clang/release/edit-body/mcpp-2026.8.11.3/modules 14.64s (min 14.45 / max 14.77, n=3) +[ 1511.5s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 1511.5s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 1525.9s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 1539.3s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 1553.7s] mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/clang/release/edit-comment/mcpp-2026.8.11.3/modules 13.51s (min 13.42 / max 14.33, n=3) +[ 1568.0s] cmake/clang/release/cold/mcpp-2026.8.11.3/modules configure +[ 1569.4s] cmake/clang/release/cold/mcpp-2026.8.11.3/modules seed build +[ 1603.4s] cmake/clang/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 1639.3s] cmake/clang/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 1674.8s] cmake/clang/release/cold/mcpp-2026.8.11.3/modules run 3/3 +cmake/clang/release/cold/mcpp-2026.8.11.3/modules 35.20s (min 35.13 / max 35.54, n=3) +[ 1710.8s] cmake/clang/release/noop/mcpp-2026.8.11.3/modules configure +[ 1711.3s] cmake/clang/release/noop/mcpp-2026.8.11.3/modules seed build +[ 1711.7s] cmake/clang/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 1712.3s] cmake/clang/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 1713.0s] cmake/clang/release/noop/mcpp-2026.8.11.3/modules run 3/3 +cmake/clang/release/noop/mcpp-2026.8.11.3/modules 0.32s (min 0.30 / max 0.32, n=3) +[ 1714.0s] cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 1714.5s] cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 1714.9s] cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 1746.0s] cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 1777.8s] cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +cmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules 31.21s (min 30.84 / max 31.37, n=3) +[ 1809.7s] cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 1810.2s] cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 1810.5s] cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 1842.0s] cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 1873.5s] cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +cmake/clang/release/edit-body/mcpp-2026.8.11.3/modules 31.15s (min 31.13 / max 31.17, n=3) +[ 1905.3s] cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 1905.9s] cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 1937.0s] cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 1968.5s] cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 2000.3s] cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +cmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules 31.24s (min 31.16 / max 31.38, n=3) +[ 2032.3s] xmake/clang/release/cold/mcpp-2026.8.11.3/modules configure +[ 2032.6s] xmake/clang/release/cold/mcpp-2026.8.11.3/modules seed build +[ 2036.9s] xmake/clang/release/cold/mcpp-2026.8.11.3/modules seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-cold.log) +[ 2036.9s] xmake/clang/release/cold/mcpp-2026.8.11.3/modules --- error lines from xmake-cold.log --- + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ +[ 2036.9s] xmake/clang/release/cold/mcpp-2026.8.11.3/modules --- last lines of xmake-cold.log --- + | [ 28%]:  compiling.module.bmi.release mcpp.toolchain.clang + | [ 28%]:  compiling.module.bmi.release mcpp.pm.publisher + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:98:62: note: while substituting into a lambda expression here + | 98 | __parse_ = [](basic_format_parse_context<_CharT>& __ctx) { + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:393:25: note: in instantiation of function template specialization 'std::__format::__compile_time_handle::__enable>' requested here + | 393 | __handle.template __enable<_Tp>(); + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:389:99: note: while substituting into a lambda expression here + | 389 | static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] { + | | ^ + | mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm:140:17: note: in instantiation of template class 'std::__1::basic_format_string, std::__1::basic_string, std::__1::basic_string>' requested here + | 140 | "{{\"namespace\":\"{}\",\"name\":\"{}\",\"error\":\"{}\"}}", + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/formatter_string.h:139:70: note: default constructor of 'formatter, wchar_t>' is implicitly deleted because base class '__disabled_formatter' has a deleted default constructor + | 139 | struct formatter, wchar_t> : __disabled_formatter {}; + | > in mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm +xmake/clang/release/cold/mcpp-2026.8.11.3/modules failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-cold.log) +[ 2037.1s] xmake/clang/release/noop/mcpp-2026.8.11.3/modules configure +[ 2037.4s] xmake/clang/release/noop/mcpp-2026.8.11.3/modules seed build +[ 2041.2s] xmake/clang/release/noop/mcpp-2026.8.11.3/modules seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-noop.log) +[ 2041.2s] xmake/clang/release/noop/mcpp-2026.8.11.3/modules --- error lines from xmake-noop.log --- + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ +[ 2041.2s] xmake/clang/release/noop/mcpp-2026.8.11.3/modules --- last lines of xmake-noop.log --- + | [ 30%]:  compiling.module.bmi.release mcpp.toolchain.stdmod + | [ 30%]:  compiling.module.bmi.release mcpp.pm.publisher + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:98:62: note: while substituting into a lambda expression here + | 98 | __parse_ = [](basic_format_parse_context<_CharT>& __ctx) { + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:393:25: note: in instantiation of function template specialization 'std::__format::__compile_time_handle::__enable>' requested here + | 393 | __handle.template __enable<_Tp>(); + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:389:99: note: while substituting into a lambda expression here + | 389 | static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] { + | | ^ + | mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm:140:17: note: in instantiation of template class 'std::__1::basic_format_string, std::__1::basic_string, std::__1::basic_string>' requested here + | 140 | "{{\"namespace\":\"{}\",\"name\":\"{}\",\"error\":\"{}\"}}", + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/formatter_string.h:139:70: note: default constructor of 'formatter, wchar_t>' is implicitly deleted because base class '__disabled_formatter' has a deleted default constructor + | 139 | struct formatter, wchar_t> : __disabled_formatter {}; + | > in mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm +xmake/clang/release/noop/mcpp-2026.8.11.3/modules failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-noop.log) +[ 2041.4s] xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 2041.8s] xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 2045.3s] xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-touch-hub.log) +[ 2045.3s] xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules --- error lines from xmake-touch-hub.log --- + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ +[ 2045.3s] xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules --- last lines of xmake-touch-hub.log --- + | [ 28%]:  compiling.module.bmi.release mcpp.toolchain.detect + | [ 28%]:  compiling.module.bmi.release mcpp.toolchain.registry + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:98:62: note: while substituting into a lambda expression here + | 98 | __parse_ = [](basic_format_parse_context<_CharT>& __ctx) { + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:393:25: note: in instantiation of function template specialization 'std::__format::__compile_time_handle::__enable>' requested here + | 393 | __handle.template __enable<_Tp>(); + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:389:99: note: while substituting into a lambda expression here + | 389 | static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] { + | | ^ + | mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm:140:17: note: in instantiation of template class 'std::__1::basic_format_string, std::__1::basic_string, std::__1::basic_string>' requested here + | 140 | "{{\"namespace\":\"{}\",\"name\":\"{}\",\"error\":\"{}\"}}", + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/formatter_string.h:139:70: note: default constructor of 'formatter, wchar_t>' is implicitly deleted because base class '__disabled_formatter' has a deleted default constructor + | 139 | struct formatter, wchar_t> : __disabled_formatter {}; + | > in mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm +xmake/clang/release/touch-hub/mcpp-2026.8.11.3/modules failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-touch-hub.log) +[ 2045.5s] xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 2045.7s] xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 2049.3s] xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-body.log) +[ 2049.3s] xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules --- error lines from xmake-edit-body.log --- + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ +[ 2049.3s] xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules --- last lines of xmake-edit-body.log --- + | [ 30%]:  compiling.module.bmi.release mcpp.build.tool_store + | [ 30%]:  compiling.module.bmi.release mcpp.build.hermetic + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:98:62: note: while substituting into a lambda expression here + | 98 | __parse_ = [](basic_format_parse_context<_CharT>& __ctx) { + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:393:25: note: in instantiation of function template specialization 'std::__format::__compile_time_handle::__enable>' requested here + | 393 | __handle.template __enable<_Tp>(); + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:389:99: note: while substituting into a lambda expression here + | 389 | static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] { + | | ^ + | mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm:140:17: note: in instantiation of template class 'std::__1::basic_format_string, std::__1::basic_string, std::__1::basic_string>' requested here + | 140 | "{{\"namespace\":\"{}\",\"name\":\"{}\",\"error\":\"{}\"}}", + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/formatter_string.h:139:70: note: default constructor of 'formatter, wchar_t>' is implicitly deleted because base class '__disabled_formatter' has a deleted default constructor + | 139 | struct formatter, wchar_t> : __disabled_formatter {}; + | > in mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm +xmake/clang/release/edit-body/mcpp-2026.8.11.3/modules failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-body.log) +[ 2049.5s] xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 2049.8s] xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 2053.4s] xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-comment.log) +[ 2053.4s] xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules --- error lines from xmake-edit-comment.log --- + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ +[ 2053.4s] xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules --- last lines of xmake-edit-comment.log --- + | [ 29%]:  compiling.module.bmi.release mcpp.toolchain.hostflags + | [ 30%]:  compiling.module.bmi.release mcpp.pm.publisher + | error: /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:99:30: error: call to implicitly-deleted default constructor of 'formatter, std::__1::allocator>, wchar_t>' + | 99 | formatter<_Tp, _CharT> __f; + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:98:62: note: while substituting into a lambda expression here + | 98 | __parse_ = [](basic_format_parse_context<_CharT>& __ctx) { + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:393:25: note: in instantiation of function template specialization 'std::__format::__compile_time_handle::__enable>' requested here + | 393 | __handle.template __enable<_Tp>(); + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/format_functions.h:389:99: note: while substituting into a lambda expression here + | 389 | static constexpr array<__format::__compile_time_handle<_CharT>, sizeof...(_Args)> __handles_{[] { + | | ^ + | mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm:140:17: note: in instantiation of template class 'std::__1::basic_format_string, std::__1::basic_string, std::__1::basic_string>' requested here + | 140 | "{{\"namespace\":\"{}\",\"name\":\"{}\",\"error\":\"{}\"}}", + | | ^ + | /home/speak/.mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/include/c++/v1/__format/formatter_string.h:139:70: note: default constructor of 'formatter, wchar_t>' is implicitly deleted because base class '__disabled_formatter' has a deleted default constructor + | 139 | struct formatter, wchar_t> : __disabled_formatter {}; + | > in mcpp-2026.8.11.3/src/cli/cmd_xpkg.cppm +xmake/clang/release/edit-comment/mcpp-2026.8.11.3/modules failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-comment.log) + +=== relative to cmake (>1.00 = slower than the baseline) === + +-- modules / cold -- + mcpp@2026.8.13.1 32.06s 0.91x + mcpp@2026.8.11.3 78.70s 2.24x + mcpp@2026.8.13.1+schedule=on 18.14s 0.52x + cmake 35.20s 1.00x <- baseline + xmake failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-cold.log) + +-- modules / noop -- + mcpp@2026.8.13.1 0.16s 0.50x + mcpp@2026.8.11.3 0.24s 0.75x + mcpp@2026.8.13.1+schedule=on 0.18s 0.56x + cmake 0.32s 1.00x <- baseline + xmake failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-noop.log) + +-- modules / touch-hub -- + mcpp@2026.8.13.1 0.30s 0.01x + mcpp@2026.8.11.3 74.12s 2.37x + mcpp@2026.8.13.1+schedule=on 0.28s 0.01x + cmake 31.21s 1.00x <- baseline + xmake failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-touch-hub.log) + +-- modules / edit-body -- + mcpp@2026.8.13.1 30.68s 0.99x + mcpp@2026.8.11.3 75.78s 2.43x + mcpp@2026.8.13.1+schedule=on 14.64s 0.47x + cmake 31.15s 1.00x <- baseline + xmake failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-body.log) + +-- modules / edit-comment -- + mcpp@2026.8.13.1 24.79s 0.79x + mcpp@2026.8.11.3 74.14s 2.37x + mcpp@2026.8.13.1+schedule=on 13.51s 0.43x + cmake 31.24s 1.00x <- baseline + xmake failed seed build exited 255 (see /tmp/bench-standard-20260814-1267933/logs/xmake-edit-comment.log) + +report : /home/speak/workspace/github/mcpp-community/mcpp/bench/results/standard-20260814-linux-x86_64/clang-mcpp-2026.8.11.3.json +cells : 20 ok, 5 failed, 0 not applicable +bench: 5 cell(s) FAILED — the engine ran and produced no artifact. Each one's reason and log tail are above. diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-fixture.json b/bench/results/standard-20260814-linux-x86_64/gcc-fixture.json new file mode 100644 index 00000000..08f17c4f --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-fixture.json @@ -0,0 +1,1096 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-14T22:50:56Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.307, + "min_s": 1.285, + "max_s": 1.327, + "samples": [1.285, 1.307, 1.327] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.181, 0.161, 0.161] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.305, + "min_s": 1.305, + "max_s": 1.351, + "samples": [1.351, 1.305, 1.305] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.783, + "min_s": 0.782, + "max_s": 0.783, + "samples": [0.783, 0.782, 0.783] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.784, + "min_s": 0.783, + "max_s": 0.803, + "samples": [0.784, 0.783, 0.803] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.264, + "min_s": 1.244, + "max_s": 1.285, + "samples": [1.244, 1.264, 1.285] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 18.511, + "min_s": 17.918, + "max_s": 18.703, + "samples": [18.703, 18.511, 17.918] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.181, 0.181, 0.161] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.927, + "min_s": 0.906, + "max_s": 0.944, + "samples": [0.944, 0.906, 0.927] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.404, + "min_s": 1.388, + "max_s": 1.405, + "samples": [1.388, 1.405, 1.404] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.943, + "min_s": 0.923, + "max_s": 0.963, + "samples": [0.963, 0.943, 0.923] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.923, + "min_s": 0.923, + "max_s": 0.923, + "samples": [0.923, 0.923, 0.923] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 5.346, + "min_s": 5.301, + "max_s": 5.359, + "samples": [5.359, 5.346, 5.301] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.161, 0.181, 0.181] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.482, + "min_s": 0.482, + "max_s": 0.502, + "samples": [0.482, 0.502, 0.482] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.482, + "min_s": 0.462, + "max_s": 0.482, + "samples": [0.462, 0.482, 0.482] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.784, + "min_s": 0.764, + "max_s": 0.803, + "samples": [0.803, 0.764, 0.784] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.482, + "min_s": 0.482, + "max_s": 0.522, + "samples": [0.482, 0.482, 0.522] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.386, + "min_s": 1.345, + "max_s": 1.486, + "samples": [1.386, 1.345, 1.486] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161, 0.161, 0.161] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.426, + "min_s": 1.366, + "max_s": 1.508, + "samples": [1.366, 1.426, 1.508] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.863, + "min_s": 0.843, + "max_s": 0.903, + "samples": [0.903, 0.843, 0.863] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.864, + "min_s": 0.863, + "max_s": 0.864, + "samples": [0.863, 0.864, 0.864] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.345, + "min_s": 1.305, + "max_s": 1.385, + "samples": [1.345, 1.305, 1.385] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 18.114, + "min_s": 17.958, + "max_s": 18.313, + "samples": [17.958, 18.313, 18.114] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.181, 0.161, 0.161] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 18.110, + "min_s": 18.068, + "max_s": 18.239, + "samples": [18.239, 18.110, 18.068] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.465, + "min_s": 1.465, + "max_s": 1.467, + "samples": [1.465, 1.465, 1.467] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 18.149, + "min_s": 18.080, + "max_s": 18.157, + "samples": [18.157, 18.149, 18.080] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 18.194, + "min_s": 18.186, + "max_s": 18.290, + "samples": [18.290, 18.186, 18.194] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 5.469, + "min_s": 5.461, + "max_s": 5.631, + "samples": [5.469, 5.461, 5.631] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.181, + "min_s": 0.181, + "max_s": 0.201, + "samples": [0.181, 0.181, 0.201] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 5.489, + "min_s": 5.466, + "max_s": 5.501, + "samples": [5.489, 5.466, 5.501] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.085, + "min_s": 1.084, + "max_s": 1.105, + "samples": [1.084, 1.085, 1.105] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 0.883, + "min_s": 0.864, + "max_s": 0.884, + "samples": [0.883, 0.864, 0.884] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: end-of-file", + "runs": 3, + "median_s": 6.100, + "min_s": 5.883, + "max_s": 6.708, + "samples": [5.883, 6.708, 6.100] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 3.619, + "min_s": 3.614, + "max_s": 3.754, + "samples": [3.754, 3.619, 3.614] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.361, + "min_s": 0.361, + "max_s": 0.361, + "samples": [0.361, 0.361, 0.361] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.927, + "min_s": 1.926, + "max_s": 2.067, + "samples": [1.927, 1.926, 2.067] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.184, + "min_s": 1.164, + "max_s": 1.204, + "samples": [1.184, 1.164, 1.204] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 1.184, + "min_s": 1.165, + "max_s": 1.204, + "samples": [1.184, 1.165, 1.204] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 1.868, + "min_s": 1.867, + "max_s": 1.889, + "samples": [1.889, 1.867, 1.868] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 24.264, + "min_s": 24.220, + "max_s": 25.186, + "samples": [24.220, 24.264, 25.186] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.381, + "min_s": 0.381, + "max_s": 0.382, + "samples": [0.382, 0.381, 0.381] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 22.244, + "min_s": 22.195, + "max_s": 22.389, + "samples": [22.244, 22.195, 22.389] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.809, + "min_s": 1.789, + "max_s": 1.849, + "samples": [1.849, 1.809, 1.789] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 22.257, + "min_s": 22.212, + "max_s": 22.355, + "samples": [22.212, 22.257, 22.355] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 22.353, + "min_s": 22.216, + "max_s": 22.359, + "samples": [22.359, 22.353, 22.216] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 7.713, + "min_s": 7.654, + "max_s": 7.728, + "samples": [7.654, 7.728, 7.713] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.362, + "min_s": 0.362, + "max_s": 0.363, + "samples": [0.362, 0.362, 0.363] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 5.966, + "min_s": 5.948, + "max_s": 6.021, + "samples": [5.948, 6.021, 5.966] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 1.405, + "min_s": 1.385, + "max_s": 1.406, + "samples": [1.406, 1.405, 1.385] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 1.165, + "min_s": 1.164, + "max_s": 1.204, + "samples": [1.164, 1.165, 1.204] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 6.022, + "min_s": 5.984, + "max_s": 6.023, + "samples": [5.984, 6.023, 6.022] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 2.609, + "min_s": 2.549, + "max_s": 2.650, + "samples": [2.609, 2.650, 2.549] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.301, + "min_s": 0.301, + "max_s": 0.301, + "samples": [0.301, 0.301, 0.301] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 2.167, + "min_s": 1.987, + "max_s": 2.249, + "samples": [2.249, 1.987, 2.167] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 1.425, + "min_s": 0.301, + "max_s": 1.445, + "samples": [1.445, 0.301, 1.425] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 1.405, + "min_s": 0.301, + "max_s": 1.426, + "samples": [1.426, 1.405, 0.301] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "headers", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 3, + "median_s": 2.295, + "min_s": 1.928, + "max_s": 2.430, + "samples": [2.430, 1.928, 2.295] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 23.302, + "min_s": 23.276, + "max_s": 23.432, + "samples": [23.432, 23.302, 23.276] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.321, + "min_s": 0.321, + "max_s": 0.321, + "samples": [0.321, 0.321, 0.321] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 22.753, + "min_s": 22.745, + "max_s": 22.799, + "samples": [22.799, 22.745, 22.753] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 1.926, + "min_s": 1.926, + "max_s": 2.006, + "samples": [2.006, 1.926, 1.926] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 22.770, + "min_s": 22.712, + "max_s": 22.799, + "samples": [22.799, 22.712, 22.770] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 22.574, + "min_s": 22.436, + "max_s": 22.675, + "samples": [22.574, 22.436, 22.675] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 7.146, + "min_s": 7.111, + "max_s": 7.168, + "samples": [7.168, 7.146, 7.111] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.341, + "min_s": 0.322, + "max_s": 0.342, + "samples": [0.341, 0.342, 0.322] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 6.723, + "min_s": 6.702, + "max_s": 6.744, + "samples": [6.702, 6.744, 6.723] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-leaf", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 1.569, + "min_s": 1.568, + "max_s": 1.585, + "samples": [1.568, 1.569, 1.585] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 1.265, + "min_s": 0.382, + "max_s": 1.326, + "samples": [1.326, 0.382, 1.265] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "synth-20x3", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 3, + "median_s": 6.706, + "min_s": 6.641, + "max_s": 6.844, + "samples": [6.706, 6.844, 6.641] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-fixture.log b/bench/results/standard-20260814-linux-x86_64/gcc-fixture.log new file mode 100644 index 00000000..3141968b --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-fixture.log @@ -0,0 +1,550 @@ +payload: payload:gcc → /home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++ +run id : 4e58a816 (resuming, 73 unit(s) recorded) +host : linux x86_64 · 13th Gen Intel(R) Core(TM) i9-13900K · 32 logical / 24 physical (heterogeneous) +fixture: 20 units, fanin 3, weight 4 + +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers configure +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers seed build +[ 1.3s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers run 1/3 — already recorded, skipping +[ 1.3s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers run 2/3 — already recorded, skipping +[ 1.3s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/headers 1.31s (min 1.28 / max 1.33, n=3) +[ 1.3s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers configure +[ 1.3s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers seed build +[ 1.4s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers run 1/3 — already recorded, skipping +[ 1.4s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers run 2/3 — already recorded, skipping +[ 1.4s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/headers 0.16s (min 0.16 / max 0.18, n=3) +[ 1.4s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers configure +[ 1.4s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers seed build +[ 1.6s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers run 1/3 — already recorded, skipping +[ 1.6s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers run 2/3 — already recorded, skipping +[ 1.6s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/headers 1.30s (min 1.30 / max 1.35, n=3) +[ 1.6s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers configure +[ 1.6s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers seed build +[ 1.8s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers run 1/3 — already recorded, skipping +[ 1.8s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers run 2/3 — already recorded, skipping +[ 1.8s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/headers 0.78s (min 0.78 / max 0.78, n=3) +[ 1.8s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers configure +[ 1.8s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers seed build +[ 1.9s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers run 1/3 — already recorded, skipping +[ 1.9s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers run 2/3 — already recorded, skipping +[ 1.9s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/headers 0.78s (min 0.78 / max 0.80, n=3) +[ 1.9s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers configure +[ 1.9s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers seed build +[ 2.7s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers run 1/3 — already recorded, skipping +[ 2.7s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers run 2/3 — already recorded, skipping +[ 2.7s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/headers 1.26s (min 1.24 / max 1.29, n=3) +[ 2.7s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules configure +[ 2.7s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules seed build +[ 20.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules run 1/3 — already recorded, skipping +[ 20.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules run 2/3 — already recorded, skipping +[ 20.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules 18.51s (min 17.92 / max 18.70, n=3) +[ 20.6s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules configure +[ 20.6s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules seed build +[ 20.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules run 1/3 — already recorded, skipping +[ 20.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules run 2/3 — already recorded, skipping +[ 20.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules 0.18s (min 0.16 / max 0.18, n=3) +[ 20.8s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules configure +[ 20.8s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules seed build +[ 21.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules run 1/3 — already recorded, skipping +[ 21.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules run 2/3 — already recorded, skipping +[ 21.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules 0.93s (min 0.91 / max 0.94, n=3) +[ 21.0s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules configure +[ 21.0s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules seed build +[ 21.1s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules run 1/3 — already recorded, skipping +[ 21.1s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules run 2/3 — already recorded, skipping +[ 21.1s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules 1.40s (min 1.39 / max 1.41, n=3) +[ 21.2s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules configure +[ 21.2s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules seed build +[ 21.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules run 1/3 — already recorded, skipping +[ 21.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules run 2/3 — already recorded, skipping +[ 21.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules 0.94s (min 0.92 / max 0.96, n=3) +[ 21.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules configure +[ 21.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules seed build +[ 22.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules run 1/3 — already recorded, skipping +[ 22.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules run 2/3 — already recorded, skipping +[ 22.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules 0.92s (min 0.92 / max 0.92, n=3) +[ 22.3s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl configure +[ 22.3s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl seed build +[ 27.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 27.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 27.6s] mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/cold/synth-20x3/modules-impl 5.35s (min 5.30 / max 5.36, n=3) +[ 27.6s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl configure +[ 27.6s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl seed build +[ 27.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 27.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 27.8s] mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/noop/synth-20x3/modules-impl 0.18s (min 0.16 / max 0.18, n=3) +[ 27.8s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl configure +[ 27.8s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl seed build +[ 28.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 28.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 28.0s] mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-hub/synth-20x3/modules-impl 0.48s (min 0.48 / max 0.50, n=3) +[ 28.0s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl configure +[ 28.0s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl seed build +[ 28.2s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 28.2s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 28.2s] mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/touch-leaf/synth-20x3/modules-impl 0.48s (min 0.46 / max 0.48, n=3) +[ 28.2s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl configure +[ 28.2s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl seed build +[ 28.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 28.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 28.3s] mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-body/synth-20x3/modules-impl 0.78s (min 0.76 / max 0.80, n=3) +[ 28.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl configure +[ 28.3s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl seed build +[ 29.1s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl run 1/3 — already recorded, skipping +[ 29.1s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl run 2/3 — already recorded, skipping +[ 29.1s] mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl run 3/3 — already recorded, skipping +mcpp@2026.8.13.1/gcc/release/edit-comment/synth-20x3/modules-impl 0.48s (min 0.48 / max 0.52, n=3) +[ 29.1s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers configure +[ 29.1s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers seed build +[ 30.5s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers run 1/3 — already recorded, skipping +[ 30.5s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers run 2/3 — already recorded, skipping +[ 30.5s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/headers 1.39s (min 1.34 / max 1.49, n=3) +[ 30.5s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers configure +[ 30.5s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers seed build +[ 30.7s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers run 1/3 — already recorded, skipping +[ 30.7s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers run 2/3 — already recorded, skipping +[ 30.7s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/headers 0.16s (min 0.16 / max 0.16, n=3) +[ 30.7s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers configure +[ 30.7s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers seed build +[ 30.9s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers run 1/3 — already recorded, skipping +[ 30.9s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers run 2/3 — already recorded, skipping +[ 30.9s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/headers 1.43s (min 1.37 / max 1.51, n=3) +[ 30.9s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers configure +[ 30.9s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers seed build +[ 31.1s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers run 1/3 — already recorded, skipping +[ 31.1s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers run 2/3 — already recorded, skipping +[ 31.2s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/headers 0.86s (min 0.84 / max 0.90, n=3) +[ 31.2s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers configure +[ 31.2s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers seed build +[ 31.4s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers run 1/3 — already recorded, skipping +[ 31.4s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers run 2/3 — already recorded, skipping +[ 31.4s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/headers 0.86s (min 0.86 / max 0.86, n=3) +[ 31.4s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers configure +[ 31.4s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers seed build +[ 32.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers run 1/3 — already recorded, skipping +[ 32.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers run 2/3 — already recorded, skipping +[ 32.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers run 3/3 — already recorded, skipping +mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/headers 1.35s (min 1.31 / max 1.39, n=3) +[ 32.3s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules configure +[ 32.3s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules seed build +[ 50.2s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules run 1/3 — already recorded, skipping +[ 50.2s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules run 2/3 +[ 68.5s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules 18.11s (min 17.96 / max 18.31, n=3) +[ 86.7s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules configure +[ 86.7s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules seed build +[ 86.8s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules run 1/3 +[ 87.0s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules run 2/3 +[ 87.2s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules 0.16s (min 0.16 / max 0.18, n=3) +[ 87.4s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules configure +[ 87.4s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules seed build +[ 87.5s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules run 1/3 +[ 105.8s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules run 2/3 +[ 123.9s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules 18.11s (min 18.07 / max 18.24, n=3) +[ 141.9s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules configure +[ 141.9s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules seed build +[ 142.1s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules run 1/3 +[ 143.6s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules run 2/3 +[ 145.1s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules 1.47s (min 1.47 / max 1.47, n=3) +[ 146.5s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules configure +[ 146.5s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules seed build +[ 146.7s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules run 1/3 +[ 164.9s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules run 2/3 +[ 183.0s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules 18.15s (min 18.08 / max 18.16, n=3) +[ 201.1s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules configure +[ 201.1s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules seed build +[ 219.2s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules run 1/3 +[ 237.4s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules run 2/3 +[ 255.6s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules 18.19s (min 18.19 / max 18.29, n=3) +[ 273.8s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl configure +[ 273.8s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl seed build +[ 279.3s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl run 1/3 +[ 284.8s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl run 2/3 +[ 290.2s] mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/cold/synth-20x3/modules-impl 5.47s (min 5.46 / max 5.63, n=3) +[ 295.9s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl configure +[ 295.9s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl seed build +[ 296.0s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl run 1/3 +[ 296.2s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl run 2/3 +[ 296.4s] mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/noop/synth-20x3/modules-impl 0.18s (min 0.18 / max 0.20, n=3) +[ 296.6s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl configure +[ 296.6s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl seed build +[ 296.8s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 302.3s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 307.8s] mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-hub/synth-20x3/modules-impl 5.49s (min 5.47 / max 5.50, n=3) +[ 313.3s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl configure +[ 313.3s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl seed build +[ 313.5s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 314.6s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 315.6s] mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-leaf/synth-20x3/modules-impl 1.08s (min 1.08 / max 1.10, n=3) +[ 316.8s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl configure +[ 316.8s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl seed build +[ 316.9s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 317.8s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 318.7s] mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-body/synth-20x3/modules-impl 0.88s (min 0.86 / max 0.88, n=3) +[ 319.6s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl configure +[ 319.6s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl seed build +[ 320.4s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 326.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 333.0s] mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-comment/synth-20x3/modules-impl 6.10s (min 5.88 / max 6.71, n=3) +[ 339.8s] cmake/gcc/release/cold/synth-20x3/headers configure +[ 341.5s] cmake/gcc/release/cold/synth-20x3/headers seed build +[ 343.6s] cmake/gcc/release/cold/synth-20x3/headers run 1/3 +[ 347.7s] cmake/gcc/release/cold/synth-20x3/headers run 2/3 +[ 351.7s] cmake/gcc/release/cold/synth-20x3/headers run 3/3 +cmake/gcc/release/cold/synth-20x3/headers 3.62s (min 3.61 / max 3.75, n=3) +[ 356.0s] cmake/gcc/release/noop/synth-20x3/headers configure +[ 356.6s] cmake/gcc/release/noop/synth-20x3/headers seed build +[ 356.9s] cmake/gcc/release/noop/synth-20x3/headers run 1/3 +[ 357.6s] cmake/gcc/release/noop/synth-20x3/headers run 2/3 +[ 358.3s] cmake/gcc/release/noop/synth-20x3/headers run 3/3 +cmake/gcc/release/noop/synth-20x3/headers 0.36s (min 0.36 / max 0.36, n=3) +[ 359.4s] cmake/gcc/release/touch-hub/synth-20x3/headers configure +[ 359.9s] cmake/gcc/release/touch-hub/synth-20x3/headers seed build +[ 360.3s] cmake/gcc/release/touch-hub/synth-20x3/headers run 1/3 +[ 362.5s] cmake/gcc/release/touch-hub/synth-20x3/headers run 2/3 +[ 364.8s] cmake/gcc/release/touch-hub/synth-20x3/headers run 3/3 +cmake/gcc/release/touch-hub/synth-20x3/headers 1.93s (min 1.93 / max 2.07, n=3) +[ 367.6s] cmake/gcc/release/touch-leaf/synth-20x3/headers configure +[ 368.1s] cmake/gcc/release/touch-leaf/synth-20x3/headers seed build +[ 368.5s] cmake/gcc/release/touch-leaf/synth-20x3/headers run 1/3 +[ 370.0s] cmake/gcc/release/touch-leaf/synth-20x3/headers run 2/3 +[ 371.5s] cmake/gcc/release/touch-leaf/synth-20x3/headers run 3/3 +cmake/gcc/release/touch-leaf/synth-20x3/headers 1.18s (min 1.16 / max 1.20, n=3) +[ 373.4s] cmake/gcc/release/edit-body/synth-20x3/headers configure +[ 373.9s] cmake/gcc/release/edit-body/synth-20x3/headers seed build +[ 374.3s] cmake/gcc/release/edit-body/synth-20x3/headers run 1/3 +[ 375.8s] cmake/gcc/release/edit-body/synth-20x3/headers run 2/3 +[ 377.3s] cmake/gcc/release/edit-body/synth-20x3/headers run 3/3 +cmake/gcc/release/edit-body/synth-20x3/headers 1.18s (min 1.16 / max 1.20, n=3) +[ 379.2s] cmake/gcc/release/edit-comment/synth-20x3/headers configure +[ 379.8s] cmake/gcc/release/edit-comment/synth-20x3/headers seed build +[ 380.9s] cmake/gcc/release/edit-comment/synth-20x3/headers run 1/3 +[ 383.2s] cmake/gcc/release/edit-comment/synth-20x3/headers run 2/3 +[ 385.4s] cmake/gcc/release/edit-comment/synth-20x3/headers run 3/3 +cmake/gcc/release/edit-comment/synth-20x3/headers 1.87s (min 1.87 / max 1.89, n=3) +[ 388.5s] cmake/gcc/release/cold/synth-20x3/modules configure +[ 390.2s] cmake/gcc/release/cold/synth-20x3/modules seed build +[ 412.6s] cmake/gcc/release/cold/synth-20x3/modules run 1/3 +[ 437.2s] cmake/gcc/release/cold/synth-20x3/modules run 2/3 +[ 461.9s] cmake/gcc/release/cold/synth-20x3/modules run 3/3 +cmake/gcc/release/cold/synth-20x3/modules 24.26s (min 24.22 / max 25.19, n=3) +[ 487.8s] cmake/gcc/release/noop/synth-20x3/modules configure +[ 488.3s] cmake/gcc/release/noop/synth-20x3/modules seed build +[ 488.7s] cmake/gcc/release/noop/synth-20x3/modules run 1/3 +[ 489.6s] cmake/gcc/release/noop/synth-20x3/modules run 2/3 +[ 490.3s] cmake/gcc/release/noop/synth-20x3/modules run 3/3 +cmake/gcc/release/noop/synth-20x3/modules 0.38s (min 0.38 / max 0.38, n=3) +[ 491.4s] cmake/gcc/release/touch-hub/synth-20x3/modules configure +[ 492.0s] cmake/gcc/release/touch-hub/synth-20x3/modules seed build +[ 492.4s] cmake/gcc/release/touch-hub/synth-20x3/modules run 1/3 +[ 514.9s] cmake/gcc/release/touch-hub/synth-20x3/modules run 2/3 +[ 537.5s] cmake/gcc/release/touch-hub/synth-20x3/modules run 3/3 +cmake/gcc/release/touch-hub/synth-20x3/modules 22.24s (min 22.20 / max 22.39, n=3) +[ 560.6s] cmake/gcc/release/touch-leaf/synth-20x3/modules configure +[ 561.2s] cmake/gcc/release/touch-leaf/synth-20x3/modules seed build +[ 561.5s] cmake/gcc/release/touch-leaf/synth-20x3/modules run 1/3 +[ 563.7s] cmake/gcc/release/touch-leaf/synth-20x3/modules run 2/3 +[ 565.9s] cmake/gcc/release/touch-leaf/synth-20x3/modules run 3/3 +cmake/gcc/release/touch-leaf/synth-20x3/modules 1.81s (min 1.79 / max 1.85, n=3) +[ 568.4s] cmake/gcc/release/edit-body/synth-20x3/modules configure +[ 568.9s] cmake/gcc/release/edit-body/synth-20x3/modules seed build +[ 569.3s] cmake/gcc/release/edit-body/synth-20x3/modules run 1/3 +[ 591.8s] cmake/gcc/release/edit-body/synth-20x3/modules run 2/3 +[ 614.4s] cmake/gcc/release/edit-body/synth-20x3/modules run 3/3 +cmake/gcc/release/edit-body/synth-20x3/modules 22.26s (min 22.21 / max 22.36, n=3) +[ 637.5s] cmake/gcc/release/edit-comment/synth-20x3/modules configure +[ 638.0s] cmake/gcc/release/edit-comment/synth-20x3/modules seed build +[ 660.4s] cmake/gcc/release/edit-comment/synth-20x3/modules run 1/3 +[ 683.1s] cmake/gcc/release/edit-comment/synth-20x3/modules run 2/3 +[ 705.8s] cmake/gcc/release/edit-comment/synth-20x3/modules run 3/3 +cmake/gcc/release/edit-comment/synth-20x3/modules 22.35s (min 22.22 / max 22.36, n=3) +[ 729.0s] cmake/gcc/release/cold/synth-20x3/modules-impl configure +[ 730.7s] cmake/gcc/release/cold/synth-20x3/modules-impl seed build +[ 736.7s] cmake/gcc/release/cold/synth-20x3/modules-impl run 1/3 +[ 744.7s] cmake/gcc/release/cold/synth-20x3/modules-impl run 2/3 +[ 752.7s] cmake/gcc/release/cold/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/cold/synth-20x3/modules-impl 7.71s (min 7.65 / max 7.73, n=3) +[ 761.2s] cmake/gcc/release/noop/synth-20x3/modules-impl configure +[ 761.7s] cmake/gcc/release/noop/synth-20x3/modules-impl seed build +[ 762.1s] cmake/gcc/release/noop/synth-20x3/modules-impl run 1/3 +[ 762.8s] cmake/gcc/release/noop/synth-20x3/modules-impl run 2/3 +[ 763.5s] cmake/gcc/release/noop/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/noop/synth-20x3/modules-impl 0.36s (min 0.36 / max 0.36, n=3) +[ 764.6s] cmake/gcc/release/touch-hub/synth-20x3/modules-impl configure +[ 765.1s] cmake/gcc/release/touch-hub/synth-20x3/modules-impl seed build +[ 765.5s] cmake/gcc/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 771.8s] cmake/gcc/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 778.1s] cmake/gcc/release/touch-hub/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/touch-hub/synth-20x3/modules-impl 5.97s (min 5.95 / max 6.02, n=3) +[ 784.8s] cmake/gcc/release/touch-leaf/synth-20x3/modules-impl configure +[ 785.3s] cmake/gcc/release/touch-leaf/synth-20x3/modules-impl seed build +[ 785.7s] cmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 787.4s] cmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 789.2s] cmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/touch-leaf/synth-20x3/modules-impl 1.40s (min 1.38 / max 1.41, n=3) +[ 791.2s] cmake/gcc/release/edit-body/synth-20x3/modules-impl configure +[ 791.8s] cmake/gcc/release/edit-body/synth-20x3/modules-impl seed build +[ 792.1s] cmake/gcc/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 793.7s] cmake/gcc/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 795.2s] cmake/gcc/release/edit-body/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/edit-body/synth-20x3/modules-impl 1.17s (min 1.16 / max 1.20, n=3) +[ 797.1s] cmake/gcc/release/edit-comment/synth-20x3/modules-impl configure +[ 797.7s] cmake/gcc/release/edit-comment/synth-20x3/modules-impl seed build +[ 798.8s] cmake/gcc/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 805.2s] cmake/gcc/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 811.5s] cmake/gcc/release/edit-comment/synth-20x3/modules-impl run 3/3 +cmake/gcc/release/edit-comment/synth-20x3/modules-impl 6.02s (min 5.98 / max 6.02, n=3) +[ 818.3s] xmake/gcc/release/cold/synth-20x3/headers configure +[ 818.6s] xmake/gcc/release/cold/synth-20x3/headers seed build +[ 822.9s] xmake/gcc/release/cold/synth-20x3/headers run 1/3 +[ 825.7s] xmake/gcc/release/cold/synth-20x3/headers run 2/3 +[ 828.5s] xmake/gcc/release/cold/synth-20x3/headers run 3/3 +xmake/gcc/release/cold/synth-20x3/headers 2.61s (min 2.55 / max 2.65, n=3) +[ 831.5s] xmake/gcc/release/noop/synth-20x3/headers configure +[ 831.8s] xmake/gcc/release/noop/synth-20x3/headers seed build +[ 832.1s] xmake/gcc/release/noop/synth-20x3/headers run 1/3 +[ 832.6s] xmake/gcc/release/noop/synth-20x3/headers run 2/3 +[ 833.1s] xmake/gcc/release/noop/synth-20x3/headers run 3/3 +xmake/gcc/release/noop/synth-20x3/headers 0.30s (min 0.30 / max 0.30, n=3) +[ 833.8s] xmake/gcc/release/touch-hub/synth-20x3/headers configure +[ 834.1s] xmake/gcc/release/touch-hub/synth-20x3/headers seed build +[ 834.4s] xmake/gcc/release/touch-hub/synth-20x3/headers run 1/3 +[ 836.8s] xmake/gcc/release/touch-hub/synth-20x3/headers run 2/3 +[ 839.0s] xmake/gcc/release/touch-hub/synth-20x3/headers run 3/3 +xmake/gcc/release/touch-hub/synth-20x3/headers 2.17s (min 1.99 / max 2.25, n=3) +[ 841.6s] xmake/gcc/release/touch-leaf/synth-20x3/headers configure +[ 841.9s] xmake/gcc/release/touch-leaf/synth-20x3/headers seed build +[ 842.2s] xmake/gcc/release/touch-leaf/synth-20x3/headers run 1/3 +[ 843.9s] xmake/gcc/release/touch-leaf/synth-20x3/headers run 2/3 +[ 844.4s] xmake/gcc/release/touch-leaf/synth-20x3/headers run 3/3 +xmake/gcc/release/touch-leaf/synth-20x3/headers 1.43s (min 0.30 / max 1.45, n=3) +[ 846.2s] xmake/gcc/release/edit-body/synth-20x3/headers configure +[ 846.5s] xmake/gcc/release/edit-body/synth-20x3/headers seed build +[ 846.8s] xmake/gcc/release/edit-body/synth-20x3/headers run 1/3 +[ 848.4s] xmake/gcc/release/edit-body/synth-20x3/headers run 2/3 +[ 850.0s] xmake/gcc/release/edit-body/synth-20x3/headers run 3/3 +xmake/gcc/release/edit-body/synth-20x3/headers 1.40s (min 0.30 / max 1.43, n=3) +[ 850.8s] xmake/gcc/release/edit-comment/synth-20x3/headers configure +[ 851.0s] xmake/gcc/release/edit-comment/synth-20x3/headers seed build +[ 852.5s] xmake/gcc/release/edit-comment/synth-20x3/headers run 1/3 +[ 855.1s] xmake/gcc/release/edit-comment/synth-20x3/headers run 2/3 +[ 857.2s] xmake/gcc/release/edit-comment/synth-20x3/headers run 3/3 +xmake/gcc/release/edit-comment/synth-20x3/headers 2.29s (min 1.93 / max 2.43, n=3) +[ 860.1s] xmake/gcc/release/cold/synth-20x3/modules configure +[ 860.5s] xmake/gcc/release/cold/synth-20x3/modules seed build +[ 884.9s] xmake/gcc/release/cold/synth-20x3/modules run 1/3 +[ 908.5s] xmake/gcc/release/cold/synth-20x3/modules run 2/3 +[ 932.1s] xmake/gcc/release/cold/synth-20x3/modules run 3/3 +xmake/gcc/release/cold/synth-20x3/modules 23.30s (min 23.28 / max 23.43, n=3) +[ 955.7s] xmake/gcc/release/noop/synth-20x3/modules configure +[ 956.0s] xmake/gcc/release/noop/synth-20x3/modules seed build +[ 956.3s] xmake/gcc/release/noop/synth-20x3/modules run 1/3 +[ 956.9s] xmake/gcc/release/noop/synth-20x3/modules run 2/3 +[ 957.4s] xmake/gcc/release/noop/synth-20x3/modules run 3/3 +xmake/gcc/release/noop/synth-20x3/modules 0.32s (min 0.32 / max 0.32, n=3) +[ 958.2s] xmake/gcc/release/touch-hub/synth-20x3/modules configure +[ 958.4s] xmake/gcc/release/touch-hub/synth-20x3/modules seed build +[ 958.8s] xmake/gcc/release/touch-hub/synth-20x3/modules run 1/3 +[ 981.8s] xmake/gcc/release/touch-hub/synth-20x3/modules run 2/3 +[ 1004.7s] xmake/gcc/release/touch-hub/synth-20x3/modules run 3/3 +xmake/gcc/release/touch-hub/synth-20x3/modules 22.75s (min 22.74 / max 22.80, n=3) +[ 1027.9s] xmake/gcc/release/touch-leaf/synth-20x3/modules configure +[ 1028.2s] xmake/gcc/release/touch-leaf/synth-20x3/modules seed build +[ 1028.5s] xmake/gcc/release/touch-leaf/synth-20x3/modules run 1/3 +[ 1030.7s] xmake/gcc/release/touch-leaf/synth-20x3/modules run 2/3 +[ 1032.8s] xmake/gcc/release/touch-leaf/synth-20x3/modules run 3/3 +xmake/gcc/release/touch-leaf/synth-20x3/modules 1.93s (min 1.93 / max 2.01, n=3) +[ 1035.2s] xmake/gcc/release/edit-body/synth-20x3/modules configure +[ 1035.4s] xmake/gcc/release/edit-body/synth-20x3/modules seed build +[ 1035.8s] xmake/gcc/release/edit-body/synth-20x3/modules run 1/3 +[ 1058.8s] xmake/gcc/release/edit-body/synth-20x3/modules run 2/3 +[ 1081.7s] xmake/gcc/release/edit-body/synth-20x3/modules run 3/3 +xmake/gcc/release/edit-body/synth-20x3/modules 22.77s (min 22.71 / max 22.80, n=3) +[ 1104.9s] xmake/gcc/release/edit-comment/synth-20x3/modules configure +[ 1105.2s] xmake/gcc/release/edit-comment/synth-20x3/modules seed build +[ 1127.7s] xmake/gcc/release/edit-comment/synth-20x3/modules run 1/3 +[ 1150.5s] xmake/gcc/release/edit-comment/synth-20x3/modules run 2/3 +[ 1173.2s] xmake/gcc/release/edit-comment/synth-20x3/modules run 3/3 +xmake/gcc/release/edit-comment/synth-20x3/modules 22.57s (min 22.44 / max 22.68, n=3) +[ 1196.5s] xmake/gcc/release/cold/synth-20x3/modules-impl configure +[ 1196.8s] xmake/gcc/release/cold/synth-20x3/modules-impl seed build +[ 1205.6s] xmake/gcc/release/cold/synth-20x3/modules-impl run 1/3 +[ 1213.0s] xmake/gcc/release/cold/synth-20x3/modules-impl run 2/3 +[ 1220.3s] xmake/gcc/release/cold/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/cold/synth-20x3/modules-impl 7.15s (min 7.11 / max 7.17, n=3) +[ 1227.9s] xmake/gcc/release/noop/synth-20x3/modules-impl configure +[ 1228.1s] xmake/gcc/release/noop/synth-20x3/modules-impl seed build +[ 1228.5s] xmake/gcc/release/noop/synth-20x3/modules-impl run 1/3 +[ 1229.0s] xmake/gcc/release/noop/synth-20x3/modules-impl run 2/3 +[ 1229.6s] xmake/gcc/release/noop/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/noop/synth-20x3/modules-impl 0.34s (min 0.32 / max 0.34, n=3) +[ 1230.3s] xmake/gcc/release/touch-hub/synth-20x3/modules-impl configure +[ 1230.6s] xmake/gcc/release/touch-hub/synth-20x3/modules-impl seed build +[ 1230.9s] xmake/gcc/release/touch-hub/synth-20x3/modules-impl run 1/3 +[ 1237.8s] xmake/gcc/release/touch-hub/synth-20x3/modules-impl run 2/3 +[ 1244.8s] xmake/gcc/release/touch-hub/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/touch-hub/synth-20x3/modules-impl 6.72s (min 6.70 / max 6.74, n=3) +[ 1251.9s] xmake/gcc/release/touch-leaf/synth-20x3/modules-impl configure +[ 1252.2s] xmake/gcc/release/touch-leaf/synth-20x3/modules-impl seed build +[ 1252.5s] xmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 1/3 +[ 1254.3s] xmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 2/3 +[ 1256.1s] xmake/gcc/release/touch-leaf/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/touch-leaf/synth-20x3/modules-impl 1.57s (min 1.57 / max 1.59, n=3) +[ 1258.0s] xmake/gcc/release/edit-body/synth-20x3/modules-impl configure +[ 1258.3s] xmake/gcc/release/edit-body/synth-20x3/modules-impl seed build +[ 1258.7s] xmake/gcc/release/edit-body/synth-20x3/modules-impl run 1/3 +[ 1260.2s] xmake/gcc/release/edit-body/synth-20x3/modules-impl run 2/3 +[ 1260.8s] xmake/gcc/release/edit-body/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/edit-body/synth-20x3/modules-impl 1.27s (min 0.38 / max 1.33, n=3) +[ 1262.4s] xmake/gcc/release/edit-comment/synth-20x3/modules-impl configure +[ 1262.7s] xmake/gcc/release/edit-comment/synth-20x3/modules-impl seed build +[ 1264.1s] xmake/gcc/release/edit-comment/synth-20x3/modules-impl run 1/3 +[ 1271.0s] xmake/gcc/release/edit-comment/synth-20x3/modules-impl run 2/3 +[ 1278.0s] xmake/gcc/release/edit-comment/synth-20x3/modules-impl run 3/3 +xmake/gcc/release/edit-comment/synth-20x3/modules-impl 6.71s (min 6.64 / max 6.84, n=3) + +=== relative to cmake (>1.00 = slower than the baseline) === + +-- headers / cold -- + mcpp@2026.8.13.1 1.31s 0.36x + mcpp@2026.8.11.3 1.39s 0.38x + cmake 3.62s 1.00x <- baseline + xmake 2.61s 0.72x + +-- headers / noop -- + mcpp@2026.8.13.1 0.16s 0.45x + mcpp@2026.8.11.3 0.16s 0.45x + cmake 0.36s 1.00x <- baseline + xmake 0.30s 0.83x + +-- headers / touch-hub -- + mcpp@2026.8.13.1 1.30s 0.68x + mcpp@2026.8.11.3 1.43s 0.74x + cmake 1.93s 1.00x <- baseline + xmake 2.17s 1.12x + +-- headers / touch-leaf -- + mcpp@2026.8.13.1 0.78s 0.66x + mcpp@2026.8.11.3 0.86s 0.73x + cmake 1.18s 1.00x <- baseline + xmake 1.43s 1.20x + +-- headers / edit-body -- + mcpp@2026.8.13.1 0.78s 0.66x + mcpp@2026.8.11.3 0.86s 0.73x + cmake 1.18s 1.00x <- baseline + xmake 1.40s 1.19x + +-- headers / edit-comment -- + mcpp@2026.8.13.1 1.26s 0.68x + mcpp@2026.8.11.3 1.35s 0.72x + cmake 1.87s 1.00x <- baseline + xmake 2.29s 1.23x + +-- modules / cold -- + mcpp@2026.8.13.1 18.51s 0.76x + mcpp@2026.8.11.3 18.11s 0.75x + cmake 24.26s 1.00x <- baseline + xmake 23.30s 0.96x + +-- modules / noop -- + mcpp@2026.8.13.1 0.18s 0.47x + mcpp@2026.8.11.3 0.16s 0.42x + cmake 0.38s 1.00x <- baseline + xmake 0.32s 0.84x + +-- modules / touch-hub -- + mcpp@2026.8.13.1 0.93s 0.04x + mcpp@2026.8.11.3 18.11s 0.81x + cmake 22.24s 1.00x <- baseline + xmake 22.75s 1.02x + +-- modules / touch-leaf -- + mcpp@2026.8.13.1 1.40s 0.78x + mcpp@2026.8.11.3 1.47s 0.81x + cmake 1.81s 1.00x <- baseline + xmake 1.93s 1.07x + +-- modules / edit-body -- + mcpp@2026.8.13.1 0.94s 0.04x + mcpp@2026.8.11.3 18.15s 0.82x + cmake 22.26s 1.00x <- baseline + xmake 22.77s 1.02x + +-- modules / edit-comment -- + mcpp@2026.8.13.1 0.92s 0.04x + mcpp@2026.8.11.3 18.19s 0.81x + cmake 22.35s 1.00x <- baseline + xmake 22.57s 1.01x + +-- modules-impl / cold -- + mcpp@2026.8.13.1 5.35s 0.69x + mcpp@2026.8.11.3 5.47s 0.71x + cmake 7.71s 1.00x <- baseline + xmake 7.15s 0.93x + +-- modules-impl / noop -- + mcpp@2026.8.13.1 0.18s 0.50x + mcpp@2026.8.11.3 0.18s 0.50x + cmake 0.36s 1.00x <- baseline + xmake 0.34s 0.94x + +-- modules-impl / touch-hub -- + mcpp@2026.8.13.1 0.48s 0.08x + mcpp@2026.8.11.3 5.49s 0.92x + cmake 5.97s 1.00x <- baseline + xmake 6.72s 1.13x + +-- modules-impl / touch-leaf -- + mcpp@2026.8.13.1 0.48s 0.34x + mcpp@2026.8.11.3 1.08s 0.77x + cmake 1.40s 1.00x <- baseline + xmake 1.57s 1.12x + +-- modules-impl / edit-body -- + mcpp@2026.8.13.1 0.78s 0.67x + mcpp@2026.8.11.3 0.88s 0.76x + cmake 1.17s 1.00x <- baseline + xmake 1.27s 1.09x + +-- modules-impl / edit-comment -- + mcpp@2026.8.13.1 0.48s 0.08x + mcpp@2026.8.11.3 6.10s 1.01x + cmake 6.02s 1.00x <- baseline + xmake 6.71s 1.11x + +report : /home/speak/workspace/github/mcpp-community/mcpp/bench/results/standard-20260814-linux-x86_64/gcc-fixture.json +cells : 72 ok, 0 failed, 0 not applicable diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.json b/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.json new file mode 100644 index 00000000..20a43a15 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.json @@ -0,0 +1,391 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-14T23:40:37Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 86.691, + "min_s": 84.787, + "max_s": 87.918, + "samples": [87.918, 86.691, 84.787] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.161, + "min_s": 0.161, + "max_s": 0.161, + "samples": [0.161, 0.161, 0.161] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.421, + "min_s": 0.381, + "max_s": 0.422, + "samples": [0.422, 0.421, 0.381] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 80.868, + "min_s": 80.728, + "max_s": 81.352, + "samples": [81.352, 80.728, 80.868] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 0.402, + "min_s": 0.381, + "max_s": 0.402, + "samples": [0.402, 0.381, 0.402] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 86.747, + "min_s": 85.995, + "max_s": 86.831, + "samples": [85.995, 86.831, 86.747] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 0.241, + "min_s": 0.241, + "max_s": 0.261, + "samples": [0.241, 0.261, 0.241] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 81.715, + "min_s": 80.240, + "max_s": 82.340, + "samples": [80.240, 82.340, 81.715] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 81.192, + "min_s": 81.000, + "max_s": 81.947, + "samples": [81.192, 81.947, 81.000] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: end-of-file", + "runs": 3, + "median_s": 79.113, + "min_s": 75.104, + "max_s": 79.326, + "samples": [79.113, 79.326, 75.104] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 35.731, + "min_s": 35.199, + "max_s": 35.943, + "samples": [35.943, 35.731, 35.199] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.181, + "min_s": 0.161, + "max_s": 0.181, + "samples": [0.161, 0.181, 0.181] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.422, + "min_s": 0.402, + "max_s": 0.443, + "samples": [0.443, 0.402, 0.422] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 29.827, + "min_s": 29.368, + "max_s": 29.953, + "samples": [29.953, 29.827, 29.368] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: end-of-file", + "runs": 3, + "median_s": 0.402, + "min_s": 0.402, + "max_s": 0.403, + "samples": [0.402, 0.403, 0.402] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 91.744, + "min_s": 91.590, + "max_s": 92.340, + "samples": [92.340, 91.744, 91.590] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.321, + "min_s": 0.302, + "max_s": 0.322, + "samples": [0.322, 0.321, 0.302] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 83.213, + "min_s": 82.881, + "max_s": 84.002, + "samples": [83.213, 82.881, 84.002] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 85.298, + "min_s": 84.846, + "max_s": 85.538, + "samples": [84.846, 85.298, 85.538] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: end-of-file", + "runs": 3, + "median_s": 83.210, + "min_s": 82.370, + "max_s": 83.406, + "samples": [83.406, 83.210, 82.370] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 90.540, + "min_s": 89.722, + "max_s": 90.840, + "samples": [90.840, 90.540, 89.722] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.382, + "min_s": 0.382, + "max_s": 0.382, + "samples": [0.382, 0.382, 0.382] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 82.481, + "min_s": 82.283, + "max_s": 83.071, + "samples": [83.071, 82.481, 82.283] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 84.325, + "min_s": 84.285, + "max_s": 84.601, + "samples": [84.285, 84.325, 84.601] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "mcpp-2026.8.11.3", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: end-of-file", + "runs": 3, + "median_s": 82.151, + "min_s": 82.135, + "max_s": 82.212, + "samples": [82.212, 82.135, 82.151] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.log b/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.log new file mode 100644 index 00000000..65e103b2 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.log @@ -0,0 +1,195 @@ +payload: payload:gcc → /home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++ +run id : 2c93b9dc (fresh) +host : linux x86_64 · 13th Gen Intel(R) Core(TM) i9-13900K · 32 logical / 24 physical (heterogeneous) +project: /home/speak/workspace/github/mcpp-community/mcpp/bench/projects/mcpp/mcpp-2026.8.11.3 (measured in place) + +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules configure +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules seed build +[ 86.2s] mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 174.2s] mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 260.9s] mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/cold/mcpp-2026.8.11.3/modules 86.69s (min 84.79 / max 87.92, n=3) +[ 345.7s] mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules configure +[ 345.7s] mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules seed build +[ 345.8s] mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 346.0s] mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 346.2s] mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/noop/mcpp-2026.8.11.3/modules 0.16s (min 0.16 / max 0.16, n=3) +[ 346.3s] mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 346.3s] mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 346.5s] mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 346.9s] mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 347.3s] mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/touch-hub/mcpp-2026.8.11.3/modules 0.42s (min 0.38 / max 0.42, n=3) +[ 347.7s] mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 347.7s] mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 347.9s] mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 429.2s] mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 510.0s] mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/edit-body/mcpp-2026.8.11.3/modules 80.87s (min 80.73 / max 81.35, n=3) +[ 590.8s] mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 590.8s] mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 671.8s] mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 672.2s] mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 672.6s] mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/edit-comment/mcpp-2026.8.11.3/modules 0.40s (min 0.38 / max 0.40, n=3) +[ 673.0s] mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules configure +[ 673.0s] mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules seed build +[ 759.7s] mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 845.8s] mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 932.6s] mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/cold/mcpp-2026.8.11.3/modules 86.75s (min 86.00 / max 86.83, n=3) +[ 1019.4s] mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules configure +[ 1019.4s] mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules seed build +[ 1019.7s] mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 1019.9s] mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 1020.2s] mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/noop/mcpp-2026.8.11.3/modules 0.24s (min 0.24 / max 0.26, n=3) +[ 1020.4s] mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 1020.4s] mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 1020.6s] mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 1100.9s] mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 1183.2s] mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-hub/mcpp-2026.8.11.3/modules 81.72s (min 80.24 / max 82.34, n=3) +[ 1265.0s] mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 1265.0s] mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 1265.2s] mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 1346.4s] mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 1428.3s] mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-body/mcpp-2026.8.11.3/modules 81.19s (min 81.00 / max 81.95, n=3) +[ 1509.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 1509.3s] mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 1591.5s] mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 1670.6s] mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 1749.9s] mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-comment/mcpp-2026.8.11.3/modules 79.11s (min 75.10 / max 79.33, n=3) +[ 1825.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules configure +[ 1825.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules seed build +[ 1860.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 1896.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 1932.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/cold/mcpp-2026.8.11.3/modules 35.73s (min 35.20 / max 35.94, n=3) +[ 1967.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules configure +[ 1967.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules seed build +[ 1967.4s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 1967.6s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 1967.8s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/noop/mcpp-2026.8.11.3/modules 0.18s (min 0.16 / max 0.18, n=3) +[ 1968.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 1968.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 1968.1s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 1968.6s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 1969.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/mcpp-2026.8.11.3/modules 0.42s (min 0.40 / max 0.44, n=3) +[ 1969.4s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 1969.4s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 1969.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 1999.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 2029.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/mcpp-2026.8.11.3/modules 29.83s (min 29.37 / max 29.95, n=3) +[ 2058.7s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 2058.7s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 2088.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 2088.6s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 2089.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/mcpp-2026.8.11.3/modules 0.40s (min 0.40 / max 0.40, n=3) +[ 2090.3s] cmake/gcc/release/cold/mcpp-2026.8.11.3/modules configure +[ 2091.6s] cmake/gcc/release/cold/mcpp-2026.8.11.3/modules seed build +[ 2182.5s] cmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 2275.2s] cmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 2367.4s] cmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 3/3 +cmake/gcc/release/cold/mcpp-2026.8.11.3/modules 91.74s (min 91.59 / max 92.34, n=3) +[ 2459.7s] cmake/gcc/release/noop/mcpp-2026.8.11.3/modules configure +[ 2460.2s] cmake/gcc/release/noop/mcpp-2026.8.11.3/modules seed build +[ 2460.6s] cmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 2461.2s] cmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 2461.9s] cmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 3/3 +cmake/gcc/release/noop/mcpp-2026.8.11.3/modules 0.32s (min 0.30 / max 0.32, n=3) +[ 2462.9s] cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 2463.4s] cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 2463.8s] cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 2547.3s] cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 2630.6s] cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +cmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules 83.21s (min 82.88 / max 84.00, n=3) +[ 2715.2s] cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 2715.8s] cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 2716.1s] cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 2801.3s] cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 2887.0s] cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +cmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules 85.30s (min 84.85 / max 85.54, n=3) +[ 2973.2s] cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 2973.7s] cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 3058.8s] cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 3142.5s] cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 3226.1s] cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +cmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules 83.21s (min 82.37 / max 83.41, n=3) +[ 3309.2s] xmake/gcc/release/cold/mcpp-2026.8.11.3/modules configure +[ 3309.6s] xmake/gcc/release/cold/mcpp-2026.8.11.3/modules seed build +[ 3400.7s] xmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 1/3 +[ 3491.8s] xmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 2/3 +[ 3582.6s] xmake/gcc/release/cold/mcpp-2026.8.11.3/modules run 3/3 +xmake/gcc/release/cold/mcpp-2026.8.11.3/modules 90.54s (min 89.72 / max 90.84, n=3) +[ 3672.7s] xmake/gcc/release/noop/mcpp-2026.8.11.3/modules configure +[ 3673.0s] xmake/gcc/release/noop/mcpp-2026.8.11.3/modules seed build +[ 3673.4s] xmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 1/3 +[ 3674.0s] xmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 2/3 +[ 3674.5s] xmake/gcc/release/noop/mcpp-2026.8.11.3/modules run 3/3 +xmake/gcc/release/noop/mcpp-2026.8.11.3/modules 0.38s (min 0.38 / max 0.38, n=3) +[ 3675.3s] xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules configure +[ 3675.6s] xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules seed build +[ 3676.0s] xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 1/3 +[ 3759.3s] xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 2/3 +[ 3842.0s] xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules run 3/3 +xmake/gcc/release/touch-hub/mcpp-2026.8.11.3/modules 82.48s (min 82.28 / max 83.07, n=3) +[ 3924.7s] xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules configure +[ 3925.0s] xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules seed build +[ 3925.4s] xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 1/3 +[ 4009.9s] xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 2/3 +[ 4094.4s] xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules run 3/3 +xmake/gcc/release/edit-body/mcpp-2026.8.11.3/modules 84.33s (min 84.28 / max 84.60, n=3) +[ 4179.4s] xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules configure +[ 4179.7s] xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules seed build +[ 4263.9s] xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 1/3 +[ 4346.4s] xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 2/3 +[ 4428.7s] xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules run 3/3 +xmake/gcc/release/edit-comment/mcpp-2026.8.11.3/modules 82.15s (min 82.14 / max 82.21, n=3) + +=== relative to 2026.8.11.3 (>1.00 = slower than the baseline) === + +-- modules / cold -- + mcpp@2026.8.13.1 86.69s 1.00x + mcpp@2026.8.11.3 86.75s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 35.73s 0.41x + cmake 91.74s 1.06x + xmake 90.54s 1.04x + +-- modules / noop -- + mcpp@2026.8.13.1 0.16s 0.67x + mcpp@2026.8.11.3 0.24s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 0.18s 0.75x + cmake 0.32s 1.33x + xmake 0.38s 1.58x + +-- modules / touch-hub -- + mcpp@2026.8.13.1 0.42s 0.01x + mcpp@2026.8.11.3 81.72s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 0.42s 0.01x + cmake 83.21s 1.02x + xmake 82.48s 1.01x + +-- modules / edit-body -- + mcpp@2026.8.13.1 80.87s 1.00x + mcpp@2026.8.11.3 81.19s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 29.83s 0.37x + cmake 85.30s 1.05x + xmake 84.33s 1.04x + +-- modules / edit-comment -- + mcpp@2026.8.13.1 0.40s 0.01x + mcpp@2026.8.11.3 79.11s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 0.40s 0.01x + cmake 83.21s 1.05x + xmake 82.15s 1.04x + +report : /home/speak/workspace/github/mcpp-community/mcpp/bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.json +cells : 25 ok, 0 failed, 0 not applicable diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.json b/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.json new file mode 100644 index 00000000..096d146b --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.json @@ -0,0 +1,391 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-15T01:30:01Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 91.462, + "min_s": 91.052, + "max_s": 97.009, + "samples": [91.052, 97.009, 91.462] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.725, + "min_s": 0.724, + "max_s": 0.726, + "samples": [0.724, 0.725, 0.726] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.789, + "min_s": 1.769, + "max_s": 1.791, + "samples": [1.791, 1.769, 1.789] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 87.695, + "min_s": 87.485, + "max_s": 94.874, + "samples": [87.485, 87.695, 94.874] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 93.585, + "min_s": 88.092, + "max_s": 95.752, + "samples": [95.752, 88.092, 93.585] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 93.945, + "min_s": 92.009, + "max_s": 100.921, + "samples": [93.945, 100.921, 92.009] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 1.709, + "min_s": 1.708, + "max_s": 1.709, + "samples": [1.708, 1.709, 1.709] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3", + "runs": 3, + "median_s": 90.088, + "min_s": 89.427, + "max_s": 97.756, + "samples": [90.088, 89.427, 97.756] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 89.361, + "min_s": 89.303, + "max_s": 96.832, + "samples": [89.303, 96.832, 89.361] + }, + { + "engine": "mcpp@2026.8.11.3", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.11.3 · perturbation: in-body", + "runs": 3, + "median_s": 89.376, + "min_s": 89.270, + "max_s": 96.101, + "samples": [89.270, 96.101, 89.376] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 33.267, + "min_s": 32.996, + "max_s": 33.466, + "samples": [33.267, 32.996, 33.466] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 0.724, + "min_s": 0.723, + "max_s": 0.724, + "samples": [0.724, 0.723, 0.724] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 3, + "median_s": 1.787, + "min_s": 1.770, + "max_s": 20.824, + "samples": [1.770, 20.824, 1.787] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 29.481, + "min_s": 28.777, + "max_s": 29.685, + "samples": [28.777, 29.481, 29.685] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 3, + "median_s": 29.900, + "min_s": 29.860, + "max_s": 35.321, + "samples": [35.321, 29.900, 29.860] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 113.884, + "min_s": 112.861, + "max_s": 114.146, + "samples": [113.884, 112.861, 114.146] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 0.361, + "min_s": 0.361, + "max_s": 0.382, + "samples": [0.382, 0.361, 0.361] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja", + "runs": 3, + "median_s": 97.120, + "min_s": 96.278, + "max_s": 97.691, + "samples": [96.278, 97.120, 97.691] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 98.228, + "min_s": 97.746, + "max_s": 98.538, + "samples": [98.228, 97.746, 98.538] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.4.2 + ninja · perturbation: in-body", + "runs": 3, + "median_s": 98.972, + "min_s": 98.847, + "max_s": 100.109, + "samples": [98.972, 98.847, 100.109] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 104.745, + "min_s": 104.421, + "max_s": 105.796, + "samples": [104.745, 104.421, 105.796] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 0.402, + "min_s": 0.402, + "max_s": 0.402, + "samples": [0.402, 0.402, 0.402] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 3, + "median_s": 100.116, + "min_s": 98.323, + "max_s": 101.395, + "samples": [98.323, 100.116, 101.395] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 99.039, + "min_s": 98.367, + "max_s": 100.584, + "samples": [100.584, 99.039, 98.367] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 3, + "median_s": 98.325, + "min_s": 97.908, + "max_s": 98.334, + "samples": [98.334, 98.325, 97.908] + } + ] +} diff --git a/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.log b/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.log new file mode 100644 index 00000000..c0bae192 --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.log @@ -0,0 +1,195 @@ +payload: payload:gcc → /home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++ +run id : 14a47a36 (fresh) +host : linux x86_64 · 13th Gen Intel(R) Core(TM) i9-13900K · 32 logical / 24 physical (heterogeneous) +project: /home/speak/workspace/github/mcpp-community/mcpp/bench/projects/xlings/xlings-2026.8.11.2 (measured in place) + +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules configure +[ 0.0s] mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules seed build +[ 95.9s] mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules run 1/3 +[ 187.0s] mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules run 2/3 +[ 284.1s] mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/cold/xlings-2026.8.11.2/modules 91.46s (min 91.05 / max 97.01, n=3) +[ 375.6s] mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules configure +[ 375.6s] mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules seed build +[ 384.2s] mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules run 1/3 +[ 384.9s] mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules run 2/3 +[ 385.6s] mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/noop/xlings-2026.8.11.2/modules 0.72s (min 0.72 / max 0.73, n=3) +[ 386.4s] mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules configure +[ 386.4s] mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules seed build +[ 387.1s] mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 1/3 +[ 388.9s] mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 2/3 +[ 390.7s] mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/touch-hub/xlings-2026.8.11.2/modules 1.79s (min 1.77 / max 1.79, n=3) +[ 392.4s] mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules configure +[ 392.4s] mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules seed build +[ 393.2s] mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules run 1/3 +[ 480.7s] mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules run 2/3 +[ 568.4s] mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/edit-body/xlings-2026.8.11.2/modules 87.69s (min 87.49 / max 94.87, n=3) +[ 663.2s] mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules configure +[ 663.2s] mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules seed build +[ 751.1s] mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 1/3 +[ 846.8s] mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 2/3 +[ 934.9s] mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1/gcc/release/edit-comment/xlings-2026.8.11.2/modules 93.58s (min 88.09 / max 95.75, n=3) +[ 1028.5s] mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules configure +[ 1028.5s] mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules seed build +[ 1128.9s] mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules run 1/3 +[ 1222.9s] mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules run 2/3 +[ 1323.9s] mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/cold/xlings-2026.8.11.2/modules 93.94s (min 92.01 / max 100.92, n=3) +[ 1415.9s] mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules configure +[ 1415.9s] mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules seed build +[ 1427.7s] mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules run 1/3 +[ 1429.4s] mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules run 2/3 +[ 1431.1s] mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/noop/xlings-2026.8.11.2/modules 1.71s (min 1.71 / max 1.71, n=3) +[ 1432.8s] mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules configure +[ 1432.8s] mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules seed build +[ 1434.6s] mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 1/3 +[ 1524.6s] mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 2/3 +[ 1614.1s] mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/touch-hub/xlings-2026.8.11.2/modules 90.09s (min 89.43 / max 97.76, n=3) +[ 1711.8s] mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules configure +[ 1711.8s] mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules seed build +[ 1713.5s] mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules run 1/3 +[ 1802.8s] mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules run 2/3 +[ 1899.7s] mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-body/xlings-2026.8.11.2/modules 89.36s (min 89.30 / max 96.83, n=3) +[ 1989.1s] mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules configure +[ 1989.1s] mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules seed build +[ 2085.5s] mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 1/3 +[ 2174.7s] mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 2/3 +[ 2270.8s] mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.11.3/gcc/release/edit-comment/xlings-2026.8.11.2/modules 89.38s (min 89.27 / max 96.10, n=3) +[ 2360.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules configure +[ 2360.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules seed build +[ 2398.6s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules run 1/3 +[ 2432.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules run 2/3 +[ 2465.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/cold/xlings-2026.8.11.2/modules 33.27s (min 33.00 / max 33.47, n=3) +[ 2498.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules configure +[ 2498.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules seed build +[ 2504.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules run 1/3 +[ 2504.7s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules run 2/3 +[ 2505.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/noop/xlings-2026.8.11.2/modules 0.72s (min 0.72 / max 0.72, n=3) +[ 2506.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules configure +[ 2506.2s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules seed build +[ 2506.9s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 1/3 +[ 2508.7s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 2/3 +[ 2529.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/touch-hub/xlings-2026.8.11.2/modules 1.79s (min 1.77 / max 20.82, n=3) +[ 2531.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules configure +[ 2531.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules seed build +[ 2532.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules run 1/3 +[ 2560.8s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules run 2/3 +[ 2590.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/edit-body/xlings-2026.8.11.2/modules 29.48s (min 28.78 / max 29.69, n=3) +[ 2620.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules configure +[ 2620.0s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules seed build +[ 2650.3s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 1/3 +[ 2685.6s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 2/3 +[ 2715.5s] mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 3/3 +mcpp@2026.8.13.1+schedule=on/gcc/release/edit-comment/xlings-2026.8.11.2/modules 29.90s (min 29.86 / max 35.32, n=3) +[ 2746.2s] cmake/gcc/release/cold/xlings-2026.8.11.2/modules configure +[ 2748.8s] cmake/gcc/release/cold/xlings-2026.8.11.2/modules seed build +[ 2858.8s] cmake/gcc/release/cold/xlings-2026.8.11.2/modules run 1/3 +[ 2973.2s] cmake/gcc/release/cold/xlings-2026.8.11.2/modules run 2/3 +[ 3086.4s] cmake/gcc/release/cold/xlings-2026.8.11.2/modules run 3/3 +cmake/gcc/release/cold/xlings-2026.8.11.2/modules 113.88s (min 112.86 / max 114.15, n=3) +[ 3201.3s] cmake/gcc/release/noop/xlings-2026.8.11.2/modules configure +[ 3201.9s] cmake/gcc/release/noop/xlings-2026.8.11.2/modules seed build +[ 3202.3s] cmake/gcc/release/noop/xlings-2026.8.11.2/modules run 1/3 +[ 3203.0s] cmake/gcc/release/noop/xlings-2026.8.11.2/modules run 2/3 +[ 3203.7s] cmake/gcc/release/noop/xlings-2026.8.11.2/modules run 3/3 +cmake/gcc/release/noop/xlings-2026.8.11.2/modules 0.36s (min 0.36 / max 0.38, n=3) +[ 3204.7s] cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules configure +[ 3205.3s] cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules seed build +[ 3205.7s] cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 1/3 +[ 3302.3s] cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 2/3 +[ 3399.8s] cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 3/3 +cmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules 97.12s (min 96.28 / max 97.69, n=3) +[ 3498.2s] cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules configure +[ 3498.8s] cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules seed build +[ 3499.1s] cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 1/3 +[ 3597.7s] cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 2/3 +[ 3695.8s] cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 3/3 +cmake/gcc/release/edit-body/xlings-2026.8.11.2/modules 98.23s (min 97.75 / max 98.54, n=3) +[ 3795.0s] cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules configure +[ 3795.6s] cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules seed build +[ 3893.3s] cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 1/3 +[ 3992.6s] cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 2/3 +[ 4091.8s] cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 3/3 +cmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules 98.97s (min 98.85 / max 100.11, n=3) +[ 4192.7s] xmake/gcc/release/cold/xlings-2026.8.11.2/modules configure +[ 4196.8s] xmake/gcc/release/cold/xlings-2026.8.11.2/modules seed build +[ 4302.9s] xmake/gcc/release/cold/xlings-2026.8.11.2/modules run 1/3 +[ 4408.0s] xmake/gcc/release/cold/xlings-2026.8.11.2/modules run 2/3 +[ 4512.6s] xmake/gcc/release/cold/xlings-2026.8.11.2/modules run 3/3 +xmake/gcc/release/cold/xlings-2026.8.11.2/modules 104.74s (min 104.42 / max 105.80, n=3) +[ 4618.8s] xmake/gcc/release/noop/xlings-2026.8.11.2/modules configure +[ 4619.1s] xmake/gcc/release/noop/xlings-2026.8.11.2/modules seed build +[ 4619.5s] xmake/gcc/release/noop/xlings-2026.8.11.2/modules run 1/3 +[ 4620.2s] xmake/gcc/release/noop/xlings-2026.8.11.2/modules run 2/3 +[ 4620.8s] xmake/gcc/release/noop/xlings-2026.8.11.2/modules run 3/3 +xmake/gcc/release/noop/xlings-2026.8.11.2/modules 0.40s (min 0.40 / max 0.40, n=3) +[ 4621.6s] xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules configure +[ 4621.9s] xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules seed build +[ 4622.2s] xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 1/3 +[ 4720.8s] xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 2/3 +[ 4821.1s] xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules run 3/3 +xmake/gcc/release/touch-hub/xlings-2026.8.11.2/modules 100.12s (min 98.32 / max 101.39, n=3) +[ 4922.9s] xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules configure +[ 4923.2s] xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules seed build +[ 4923.6s] xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 1/3 +[ 5024.4s] xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 2/3 +[ 5123.6s] xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules run 3/3 +xmake/gcc/release/edit-body/xlings-2026.8.11.2/modules 99.04s (min 98.37 / max 100.58, n=3) +[ 5222.4s] xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules configure +[ 5222.7s] xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules seed build +[ 5322.0s] xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 1/3 +[ 5420.6s] xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 2/3 +[ 5519.1s] xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules run 3/3 +xmake/gcc/release/edit-comment/xlings-2026.8.11.2/modules 98.33s (min 97.91 / max 98.33, n=3) + +=== relative to 2026.8.11.3 (>1.00 = slower than the baseline) === + +-- modules / cold -- + mcpp@2026.8.13.1 91.46s 0.97x + mcpp@2026.8.11.3 93.94s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 33.27s 0.35x + cmake 113.88s 1.21x + xmake 104.74s 1.11x + +-- modules / noop -- + mcpp@2026.8.13.1 0.72s 0.42x + mcpp@2026.8.11.3 1.71s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 0.72s 0.42x + cmake 0.36s 0.21x + xmake 0.40s 0.24x + +-- modules / touch-hub -- + mcpp@2026.8.13.1 1.79s 0.02x + mcpp@2026.8.11.3 90.09s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 1.79s 0.02x + cmake 97.12s 1.08x + xmake 100.12s 1.11x + +-- modules / edit-body -- + mcpp@2026.8.13.1 87.69s 0.98x + mcpp@2026.8.11.3 89.36s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 29.48s 0.33x + cmake 98.23s 1.10x + xmake 99.04s 1.11x + +-- modules / edit-comment -- + mcpp@2026.8.13.1 93.58s 1.05x + mcpp@2026.8.11.3 89.38s 1.00x <- baseline + mcpp@2026.8.13.1+schedule=on 29.90s 0.33x + cmake 98.97s 1.11x + xmake 98.33s 1.10x + +report : /home/speak/workspace/github/mcpp-community/mcpp/bench/results/standard-20260814-linux-x86_64/gcc-xlings-2026.8.11.2.json +cells : 25 ok, 0 failed, 0 not applicable diff --git a/bench/results/standard-20260814-linux-x86_64/probe-touch-hub-outlier-8-samples.json b/bench/results/standard-20260814-linux-x86_64/probe-touch-hub-outlier-8-samples.json new file mode 100644 index 00000000..17dd518b --- /dev/null +++ b/bench/results/standard-20260814-linux-x86_64/probe-touch-hub-outlier-8-samples.json @@ -0,0 +1,31 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-15T03:08:35Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 8, + "median_s": 1.787, + "min_s": 1.785, + "max_s": 1.809, + "samples": [1.790, 1.787, 1.787, 1.786, 1.785, 1.787, 1.809, 1.788] + } + ] +} diff --git a/bench/results/xlings-3way-20260814/xlings-combined-3way.json b/bench/results/xlings-3way-20260814/xlings-combined-3way.json new file mode 100644 index 00000000..3c016740 --- /dev/null +++ b/bench/results/xlings-3way-20260814/xlings-combined-3way.json @@ -0,0 +1,241 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T17:43:09Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 92.493, + "min_s": 92.493, + "max_s": 92.493, + "samples": [92.493] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.744, + "min_s": 0.744, + "max_s": 0.744, + "samples": [0.744] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 1.786, + "min_s": 1.786, + "max_s": 1.786, + "samples": [1.786] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 88.379, + "min_s": 88.379, + "max_s": 88.379, + "samples": [88.379] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 93.813, + "min_s": 93.813, + "max_s": 93.813, + "samples": [93.813] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 119.458, + "min_s": 119.458, + "max_s": 119.458, + "samples": [119.458] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 0.361, + "min_s": 0.361, + "max_s": 0.361, + "samples": [0.361] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 98.161, + "min_s": 98.161, + "max_s": 98.161, + "samples": [98.161] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja · perturbation: in-body", + "runs": 1, + "median_s": 98.434, + "min_s": 98.434, + "max_s": 98.434, + "samples": [98.434] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "cmake version 4.0.2 + ninja · perturbation: in-body", + "runs": 1, + "median_s": 97.979, + "min_s": 97.979, + "max_s": 97.979, + "samples": [97.979] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 105.024, + "min_s": 105.024, + "max_s": 105.024, + "samples": [105.024] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 0.401, + "min_s": 0.401, + "max_s": 0.401, + "samples": [0.401] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 98.157, + "min_s": 98.157, + "max_s": 98.157, + "samples": [98.157] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 1, + "median_s": 97.999, + "min_s": 97.999, + "max_s": 97.999, + "samples": [97.999] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 1, + "median_s": 97.974, + "min_s": 97.974, + "max_s": 97.974, + "samples": [97.974] + } + ] +} diff --git a/bench/results/xlings-3way-20260814/xlings-schedule.json b/bench/results/xlings-3way-20260814/xlings-schedule.json new file mode 100644 index 00000000..a93a6ca1 --- /dev/null +++ b/bench/results/xlings-3way-20260814/xlings-schedule.json @@ -0,0 +1,166 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T22:29:14Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 91.608, + "min_s": 91.608, + "max_s": 91.608, + "samples": [91.608] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.723, + "min_s": 0.723, + "max_s": 0.723, + "samples": [0.723] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 1.786, + "min_s": 1.786, + "max_s": 1.786, + "samples": [1.786] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 89.457, + "min_s": 89.457, + "max_s": 89.457, + "samples": [89.457] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 93.843, + "min_s": 93.843, + "max_s": 93.843, + "samples": [93.843] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 37.561, + "min_s": 37.561, + "max_s": 37.561, + "samples": [37.561] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.723, + "min_s": 0.723, + "max_s": 0.723, + "samples": [0.723] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 1.045, + "min_s": 1.045, + "max_s": 1.045, + "samples": [1.045] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 29.646, + "min_s": 29.646, + "max_s": 29.646, + "samples": [29.646] + }, + { + "engine": "mcpp@2026.8.13.1+schedule=on", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.11.2", + "variant": "modules", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 30.444, + "min_s": 30.444, + "max_s": 30.444, + "samples": [30.444] + } + ] +} diff --git a/bench/results/xlings-3way-20260814/xlings-split-3way.json b/bench/results/xlings-3way-20260814/xlings-split-3way.json new file mode 100644 index 00000000..5ee0c5f8 --- /dev/null +++ b/bench/results/xlings-3way-20260814/xlings-split-3way.json @@ -0,0 +1,241 @@ +{ + "protocol_version": 1, + "started_at": "2026-08-13T18:19:51Z", + "host": { + "os": "linux", + "arch": "x86_64", + "cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "logical_cores": 32, + "physical_cores": 24, + "heterogeneous": true, + "ram_bytes": 67147722752, + "toolchain": "/home/speak/.mcpp/registry/data/xpkgs/xim-x-gcc/16.1.0/bin/g++" + }, + "cells": [ + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 27.588, + "min_s": 27.588, + "max_s": 27.588, + "samples": [27.588] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 0.785, + "min_s": 0.785, + "max_s": 0.785, + "samples": [0.785] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1", + "runs": 1, + "median_s": 1.325, + "min_s": 1.325, + "max_s": 1.325, + "samples": [1.325] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 1.790, + "min_s": 1.790, + "max_s": 1.790, + "samples": [1.790] + }, + { + "engine": "mcpp@2026.8.13.1", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "mcpp 2026.8.13.1 · perturbation: in-body", + "runs": 1, + "median_s": 24.168, + "min_s": 24.168, + "max_s": 24.168, + "samples": [24.168] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 50.129, + "min_s": 50.129, + "max_s": 50.129, + "samples": [50.129] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 0.342, + "min_s": 0.342, + "max_s": 0.342, + "samples": [0.342] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja", + "runs": 1, + "median_s": 26.600, + "min_s": 26.600, + "max_s": 26.600, + "samples": [26.600] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja · perturbation: in-body", + "runs": 1, + "median_s": 1.349, + "min_s": 1.349, + "max_s": 1.349, + "samples": [1.349] + }, + { + "engine": "cmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "cmake version 4.0.2 + ninja · perturbation: in-body", + "runs": 1, + "median_s": 26.349, + "min_s": 26.349, + "max_s": 26.349, + "samples": [26.349] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "cold", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 41.904, + "min_s": 41.904, + "max_s": 41.904, + "samples": [41.904] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "noop", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 0.502, + "min_s": 0.502, + "max_s": 0.502, + "samples": [0.502] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "touch-hub", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua", + "runs": 1, + "median_s": 31.676, + "min_s": 31.676, + "max_s": 31.676, + "samples": [31.676] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-body", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 1, + "median_s": 1.486, + "min_s": 1.486, + "max_s": 1.486, + "samples": [1.486] + }, + { + "engine": "xmake", + "compiler": "gcc", + "profile": "release", + "scenario": "edit-comment", + "fixture": "xlings-2026.8.13.1", + "variant": "modules-impl", + "status": "ok", + "note": "xmake v3.1.0+HEAD.96ad28e, A cross-platform build utility based on Lua · perturbation: in-body", + "runs": 1, + "median_s": 31.281, + "min_s": 31.281, + "max_s": 31.281, + "samples": [31.281] + } + ] +} diff --git a/bench/run-standard.sh b/bench/run-standard.sh new file mode 100755 index 00000000..cb19864d --- /dev/null +++ b/bench/run-standard.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash +# bench/run-standard.sh — produce the STANDARD DATA SET on this machine. +# +# One command, no arguments. It replaces the CI matrix that used to live in +# .github/workflows/bench.yml, and it exists because that matrix was measuring +# far less than it appeared to: +# +# 10 cells, 32 foreign-engine arms, 12 of them (37%) waived by +# `allow_failed` — xmake had MORE arms waived (6) than measured (4) +# +# A job that goes green while a third of the comparison never ran is the exact +# failure this suite was built to remove, so it may not be how the suite runs. +# Two structural reasons on top of that: a shared runner measures the RUNNER +# (the same tree is 243s there and 79s on a developer box), and the gaps being +# waived are other people's tools, not mcpp. +# +# ── THE STANDARD SET IS DERIVED, NOT A SECOND LIST ───────────────────────── +# +# bench/matrix.json stays the single place the matrix is written down. This +# script does not repeat it; it selects the cells for THIS os and runs every +# engine they list. +# +# ⚠️ `allow_failed` IS DELIBERATELY IGNORED HERE, and the first version of this +# script got that wrong. Those waivers were recorded against failures on the CI +# RUNNER — cmake's `__CMAKE::CXX23` on the mcpp tree, cmake's `manifest has no +# sources` on the xlings tree — and both of those arms configure and generate +# perfectly well on a developer machine; they were verified doing so. Filtering +# by them would have carried a runner's limitation into local data and quietly +# published a smaller comparison than this machine can actually make. +# +# So nothing is pre-excluded and no `--allow-failed` is passed. An arm that +# cannot run here FAILS here, loudly, and that is data: a gap is only real once +# it has been reproduced on the machine making the claim. +# +# Usage: +# bash bench/run-standard.sh # everything for this OS +# bash bench/run-standard.sh --dry-run # print the plan and stop +# bash bench/run-standard.sh --runs 1 # faster, NOT publishable +# bash bench/run-standard.sh --resume # continue an interrupted run +# +# ── RESUMING ─────────────────────────────────────────────────────────────── +# +# mbench records every measured unit to `.mbench//journal.jsonl` as +# it goes, so an interrupted run costs at most the unit in flight. `--resume` +# re-runs this script against that cache: units already recorded are replayed +# from it, the rest are measured. +# +# It is a FLAG rather than the default because the two behaviours protect +# against opposite mistakes. Without it a second invocation must never write +# beside the first one's reports (see the supersede block below — two runs did +# end up spliced in one directory). With it, that is precisely what is wanted. +# The fingerprint is what makes it safe: it covers the whole configuration, so a +# resume that is not actually the same run lands in a different cache and starts +# from zero on its own. +# +# ⚠️ SEED BUILDS ARE NOT UNITS AND ARE REDONE. An incremental scenario needs a +# tree that is already up to date, and that state was built by a seed build that +# no journal can hold. A resumed cell therefore pays its seed again before it +# can skip anything — resume is cheap, not free. +set -uo pipefail + +# BENCH_ROOT lets this be run from a COPY. Editing a bash script while it is +# executing corrupts the running instance — bash reads the file incrementally, +# so an edit shifts byte offsets under it, and one run ended in +# line 193: ather: command not found +# line 199: tc: unbound variable +# That happened twice. Copy the script somewhere, point BENCH_ROOT at the +# repository, and the original can be edited freely while it runs. +ROOT="${BENCH_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +MATRIX="$ROOT/bench/matrix.json" +RUNS=3 +DRY=0 +RESUME=0 +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY=1 ;; + --resume) RESUME=1 ;; + --runs) RUNS="${2:?--runs needs a number}"; shift ;; + -h|--help) sed -n '2,55p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "unknown option: $1" >&2; exit 2 ;; + esac + shift +done + +[ -f "$MATRIX" ] || { echo "no bench/matrix.json at $MATRIX" >&2; exit 1; } + +case "$(uname -s)" in + Linux) OS=linux ;; + Darwin) OS=macos ;; + MINGW*|MSYS*|CYGWIN*) OS=windows ;; + *) echo "unsupported OS: $(uname -s)" >&2; exit 1 ;; +esac +ARCH="$(uname -m)" + +# ── Preconditions, each named rather than discovered three steps later ────── +# +# Every one of these has cost a full run before: a missing submodule makes the +# `hub` file absent and every perturbation scenario skip; an unpinned tool makes +# the numbers describe a different program; a dependency that was served from +# cache without being unpacked makes the cmake arm compile three units short. +fail=0 +say_missing() { echo " MISSING: $*"; fail=1; } + +BENCH="$(ls -t "$ROOT"/bench/target/*/*/bin/mbench 2>/dev/null | head -1)" +[ -n "$BENCH" ] || say_missing "the harness — run: (cd bench && mcpp build --release)" + +MCPP_BIN="$(ls -t "$ROOT"/target/*/*/bin/mcpp 2>/dev/null | head -1)" +[ -n "$MCPP_BIN" ] || say_missing "the mcpp under test — run: mcpp build --release" + +# ⚠️ THE ENGINE MUST BE A BINARY, NEVER THE BARE NAME `mcpp`. +# +# A bare `mcpp` resolves through PATH to the xlings shim, and that shim RE-PICKS +# its version from the working directory — which, for a `--project` run, is the +# measured tree. Every pinned workload carries its own pin, so the whole first +# run of this script measured mcpp@2026.8.11.3 in every cell: the released +# binary, not the branch, and with no old-vs-new column at all. The tell was in +# the report the entire time — `mcpp@2026.8.11.3` on rows that were supposed to +# be the build under test — and `touch-hub 76.55s` on mcpp's own tree, which is +# the old binary's byte-comparison behaviour, not this branch's. +# +# So `mcpp` in a cell's engine list is expanded here into explicit paths: the +# build under test, plus the released reference when it can be resolved AND +# asserts its own version. xlings unpacks each version to +# data/runtimedir/mcpp---/mcpp. +REFERENCE_MCPP="$(python3 -c "import json;print(json.load(open('$MATRIX',encoding='utf-8')).get('reference_mcpp',''))" 2>/dev/null)" +REF_BIN="" +if [ -n "$REFERENCE_MCPP" ]; then + for c in "$HOME"/.xlings/data/runtimedir/mcpp-"$REFERENCE_MCPP"-*/mcpp; do + [ -x "$c" ] || continue + got="$("$c" --version 2>/dev/null | grep -oE '[0-9]+(\.[0-9]+){2,3}' | head -1)" + if [ "$got" = "$REFERENCE_MCPP" ]; then REF_BIN="$c"; break; fi + echo " note: $c reports '$got', not '$REFERENCE_MCPP'; ignoring it" + done + [ -n "$REF_BIN" ] || echo " note: no mcpp@$REFERENCE_MCPP binary found; the old-vs-new column will be missing" +fi + +for t in cmake xmake bazel; do + want="$(python3 -c "import json;print(json.load(open('$MATRIX',encoding='utf-8'))['tools'].get('$t',''))" 2>/dev/null)" + have="$(command -v "$t" >/dev/null 2>&1 && "$t" --version 2>/dev/null | head -1 || true)" + if [ -z "$have" ]; then + echo " note: $t is not installed; its arms will report unavailable (pinned: $want)" + elif [ -n "$want" ] && ! printf '%s' "$have" | grep -q "$want"; then + echo " ⚠ $t is $have but matrix.json pins $want — the numbers would describe another tool" + fail=1 + fi +done + +# The pinned workloads. A submodule that is present-but-empty is the shape that +# reads as "checked out" to a path test and produces an empty measurement. +while read -r proj; do + [ -n "$proj" ] || continue + d="$ROOT/bench/projects" + case "$proj" in + mcpp-*) d="$d/mcpp/$proj" ;; + xlings-*) d="$d/xlings/$proj" ;; + *) continue ;; + esac + if [ ! -d "$d" ] || [ -z "$(ls -A "$d" 2>/dev/null)" ]; then + say_missing "workload $proj at $d — run: git submodule update --init" + fi +done < <(python3 -c " +import json +d = json.load(open('$MATRIX', encoding='utf-8')) +print('\n'.join(sorted({c['project'] for c in d['cells'] if c['project'] != 'fixture'}))) +" 2>/dev/null) + +[ "$fail" = 0 ] || { echo; echo "preconditions not met; nothing was run." >&2; exit 1; } + +# ── The plan: this OS's cells, minus every waived arm ─────────────────────── +PLAN="$(python3 - "$MATRIX" "$OS" <<'PY' +import json, sys +m = json.load(open(sys.argv[1], encoding="utf-8")) +os_want = sys.argv[2] +for c in m["cells"]: + if c.get("os") != os_want: + continue + # Every engine the cell lists. `allow_failed` is a CI-era record of what + # broke on a shared runner and says nothing about this machine. + engines = [e.strip() for e in c["engines"].split(",") if e.strip()] + if not engines: + continue + # ⚠️ NOT A TAB. `IFS=$'\t' read` treats tab as IFS WHITESPACE, so consecutive + # tabs COLLAPSE — a cell with no `leaf` shifted every later field one place + # left and cmake was handed the BASELINE as its source directory: + # CMake Error: The source directory ".../bench/projects/2026.8.11.3" + # does not exist. + # Five cmake cells failed that way, and the message names neither the field + # that was wrong nor the empty one before it. \x1f is not whitespace, so an + # empty field survives as an empty field. + print("\x1f".join([ + c["toolchain"], c["project"], ",".join(engines), c["variants"], c["scenarios"], + c.get("hub", ""), c.get("body", ""), c.get("leaf", ""), + c.get("buildfiles", c["project"]), c.get("baseline", m["baseline"]), + c.get("preset", "standard"), + ])) +PY +)" + +[ -n "$PLAN" ] || { echo "no cells for os=$OS in matrix.json" >&2; exit 1; } + +STAMP="$(date -u +%Y%m%d)" +OUT="$ROOT/bench/results/standard-$STAMP-$OS-$ARCH" +# ⚠️ SCRATCH DOES NOT LIVE IN results/. Putting the work tree under $OUT +# meant engine scratch — including JSON files whose top level is an ARRAY — +# landed in the directory the README guard scans for published medians, and +# it died with `AttributeError: 'list' object has no attribute 'get'`. +# It would also have been committed along with the data. +WORK="${TMPDIR:-/tmp}/bench-standard-$STAMP-$$" +# ⚠️ REFUSE TO MIX RUNS. A previous invocation's reports must not sit beside +# this one's under the same names. +# +# It happened: an earlier run was stopped with SIGTERM, did not die immediately, +# finished the cell it was on and wrote its report AFTER the `rm -rf` that was +# meant to clear the directory — so a 90-cell file from one run sat next to a +# 72-cell file from another, and the only way to tell was to read `started_at` +# out of the JSON. A table generated from that directory would have spliced two +# runs silently, which is the failure this suite exists to remove. +# +# Same shape as the timeout leak the harness itself had: a process outliving the +# command that was supposed to end it. +# One cache root for the whole set, pinned to the repository rather than to the +# working directory: resume must not depend on where the script was invoked from. +CACHE="$ROOT/.mbench" + +# WHICH BUILD OF mcpp THIS IS. Every engine labels itself from `--version`, and +# mcpp's version is a DATE: every commit on this branch reports `2026.8.13.1`, +# so the report could say which release it measured but never which build. The +# numbers here move with single commits, so that is not a detail. +# +# `-dirty` is not cosmetic either: a benchmark taken against uncommitted work +# describes a tree nobody else can check out. +UNDER_TEST="$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +if ! git -C "$ROOT" diff --quiet HEAD -- src/ 2>/dev/null; then + UNDER_TEST="$UNDER_TEST-dirty" + echo " ⚠ src/ has uncommitted changes; recording the build as $UNDER_TEST" +fi + +if [ "$RESUME" = 1 ]; then + if [ -d "$CACHE" ]; then + echo "resuming: $(find "$CACHE" -name journal.jsonl -exec cat {} + 2>/dev/null | wc -l)"\ + "unit(s) already recorded under ${CACHE#"$ROOT"/}" + else + echo "note: --resume given but ${CACHE#"$ROOT"/} does not exist; this run starts from zero." + fi + echo +elif [ -d "$OUT" ] && [ -n "$(ls -A "$OUT" 2>/dev/null)" ]; then + stamped="$OUT.superseded-$(date -u +%H%M%S)" + mv "$OUT" "$stamped" + echo "note: $(basename "$OUT") already held files from an earlier run;" + echo " moved to $(basename "$stamped") rather than writing beside them." + echo +fi + +echo "standard set: $(printf '%s\n' "$PLAN" | wc -l) cells, ${RUNS} run(s) each, os=$OS arch=$ARCH" +echo "output : ${OUT#"$ROOT"/}" +echo + +# ⚠️ NOT `printf ... | while`. A pipeline runs its right-hand side in a SUBSHELL, +# so a failure counter incremented inside the loop does not survive it and the +# script exits 0 no matter what happened. Verified with a stub engine that exits +# 7 for every cell: the old shape printed seven failures and then exited 0 — and +# went on to print "how to tell whether this data is publishable: 1. every cell +# exited 0", handing the reader a question it had the answer to. +# +# That is the exact defect this suite exists to remove, in the script written to +# replace a CI that had it. Process substitution keeps the loop in THIS shell. +FAILED=0 +while IFS=$'\x1f' read -r tc proj engines variants scenarios hub body leaf buildfiles baseline preset; do + echo " $tc/$proj [$engines]" + [ "$DRY" = 1 ] && continue + + mkdir -p "$OUT" + # Expand the bare `mcpp` into explicit binaries — see REFERENCE_MCPP above. + spec="" + for e in ${engines//,/ }; do + case "$e" in + # Bare `mcpp` is the old-vs-new pair: the build under test, plus the + # released reference when one could be resolved. + mcpp) spec="${spec:+$spec,}mcpp=$MCPP_BIN" + [ -n "$REF_BIN" ] && spec="$spec,mcpp=$REF_BIN" ;; + # `mcpp[...]` is an OPT-IN ARM of the build under test, and only of + # it. Running the reference with `schedule=on` would answer a + # question nobody asked — the released binary predates the fix in + # §8b, so the arm would measure a known-broken scheduler and the + # column would read as a regression in the feature. + "mcpp["*"]") spec="${spec:+$spec,}${e}=$MCPP_BIN" ;; + *) spec="${spec:+$spec,}$e" ;; + esac + done + + args=(--engines "$spec" --variants "$variants" --scenarios "$scenarios" + --baseline "$baseline" --profile release --runs "$RUNS" --timeout 1800 + --work "$WORK" --out "$OUT/$tc-$proj.json" --cache-root "$CACHE" + --under-test "$UNDER_TEST") + + # `payload:gcc` / `payload:clang` resolve to the hermetic driver every engine + # is handed — never `command -v g++`, which inside an xlings workspace is a + # shim whose include path moves with the workspace. + case "$tc" in + gcc) args+=(--compiler payload:gcc) ;; + clang) args+=(--compiler payload:clang) ;; + msvc) ;; # a label, not a path + esac + + if [ "$proj" = "fixture" ]; then + args+=(--preset "$preset") + else + case "$proj" in + mcpp-*) pdir="$ROOT/bench/projects/mcpp/$proj" ;; + xlings-*) pdir="$ROOT/bench/projects/xlings/$proj" ;; + esac + args+=(--project "$pdir" --buildfiles "$ROOT/bench/projects/$buildfiles") + [ -n "$hub" ] && args+=(--hub "$hub") + [ -n "$body" ] && args+=(--body "$body") + [ -n "$leaf" ] && args+=(--leaf "$leaf") + fi + + # `mcpp` is also passed as an explicit engine binary so the report labels it + # with the version that binary reports, rather than whatever a PATH shim + # resolves to from the measured tree's directory. + if [ "${BENCH_PRINT_ARGV:-0}" = 1 ]; then + printf ' argv:'; printf ' %q' "${args[@]}"; printf '\n' + continue + fi + "$BENCH" "${args[@]}" 2>&1 | tee "$OUT/$tc-$proj.log" | sed 's/^/ /' + rc=${PIPESTATUS[0]} + if [ "$rc" != 0 ]; then + echo " ^ cell exited $rc — see ${OUT#"$ROOT"/}/$tc-$proj.log" + FAILED=$((FAILED + 1)) + fi +done < <(printf '%s\n' "$PLAN") + +[ "$DRY" = 1 ] && exit 0 + +echo +if [ "$FAILED" != 0 ]; then + echo "── NOT PUBLISHABLE ─────────────────────────────────────────────────────" + echo " $FAILED cell(s) failed. Nothing here is pre-excluded, so each one is a" + echo " real failure on THIS machine: reproduce it by hand before calling it a" + echo " gap, and do not publish a table that quietly omits it." + echo + echo "reports: ${OUT#"$ROOT"/}" + exit 1 +fi + +echo "── the remaining checks, which this script cannot make for you ──────────" +echo " 2. no cell reports \`failed\` — nothing is pre-excluded, so a failure is real" +echo " (if one is, reproduce it by hand before calling it a gap)" +echo " 3. no \`cold\` tripped an invariant (vs its own noop, vs its peers)" +echo " 4. min/max within about ±20% of the median; wider means this machine was noisy" +echo +echo "reports: ${OUT#"$ROOT"/}" diff --git a/bench/src/analysis/graph.cpp b/bench/src/analysis/graph.cpp new file mode 100644 index 00000000..b557a5f0 --- /dev/null +++ b/bench/src/analysis/graph.cpp @@ -0,0 +1,62 @@ +// bench.analysis.graph — implementation. +// +// `module bench.analysis.graph;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. The ninja-file parser lives here. +module bench.analysis.graph; + +import std; +import bench.analysis.ninjalog; + +namespace bench::analysis { + +namespace detail { + +std::vector split_ws(std::string_view s) { + std::vector out; + std::size_t i = 0; + while (i < s.size()) { + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + auto b = i; + while (i < s.size() && s[i] != ' ' && s[i] != '\t') ++i; + if (i > b) out.emplace_back(s.substr(b, i - b)); + } + return out; +} + +std::optional parse_build(std::string_view line) { + constexpr std::string_view kw = "build "; + if (!line.starts_with(kw)) return std::nullopt; + auto body = line.substr(kw.size()); + auto colon = body.find(':'); + if (colon == std::string_view::npos) return std::nullopt; + + Stmt st; + for (auto& t : split_ws(body.substr(0, colon))) + if (t != "|") st.outs.push_back(t); + + auto toks = split_ws(body.substr(colon + 1)); + if (toks.empty()) return std::nullopt; + st.rule = toks.front(); + // Order-only deps still gate scheduling, so they are kept as real edges. + for (std::size_t i = 1; i < toks.size(); ++i) + if (toks[i] != "|" && toks[i] != "||") st.ins.push_back(toks[i]); + if (st.outs.empty()) return std::nullopt; + return st; +} + +std::string read_unfolded(const std::filesystem::path& p) { + std::ifstream in(p); + if (!in) return {}; + std::string all((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::string out; + out.reserve(all.size()); + for (std::size_t i = 0; i < all.size(); ++i) { + if (all[i] == '$' && i + 1 < all.size() && all[i + 1] == '\n') { out += ' '; ++i; } + else out += all[i]; + } + return out; +} + +} // namespace detail + +} // namespace bench::analysis diff --git a/bench/src/analysis/graph.cppm b/bench/src/analysis/graph.cppm new file mode 100644 index 00000000..1a9c016c --- /dev/null +++ b/bench/src/analysis/graph.cppm @@ -0,0 +1,108 @@ +// Reconstruct the dependency graph ninja actually executed. +// +// Two sources, and BOTH are required: +// 1. build.ninja — static edges, rule names +// 2. obj/**/*.ddi.dd — the dyndep files, which is where every real +// `import` edge lives for a C++ modules build +// +// Reading only build.ninja makes mcpp's critical path measure 22 s instead of +// 79 s: the module graph is invisible until dyndep is folded in. +// +// A second subtlety costs another 3x: dyndep attaches its deps to `obj/X.m.o`, +// but importers depend on the OTHER output of that same edge, +// `gcm.cache/X.gcm`. Unless all outputs of an edge are one graph node, the +// longest-path walk terminates after a couple of hops. +export module bench.analysis.graph; + +import std; +import bench.analysis.ninjalog; + +export namespace bench::analysis { + +struct Graph { + // node identity == the ninja Edge identity (its first sorted output) + std::unordered_map> deps; + std::unordered_map rule_of; + std::unordered_map node_of; // output -> node + + [[nodiscard]] std::string resolve(const std::string& output) const { + auto it = node_of.find(output); + return it == node_of.end() ? std::string{} : it->second; + } +}; + +namespace detail { + +std::vector split_ws(std::string_view s); + +// A ninja `build` statement: `build OUT... [| IMPLICIT_OUT...] : RULE IN... [| IMP] [|| ORDER]` +struct Stmt { + std::vector outs; + std::string rule; + std::vector ins; +}; + +std::optional parse_build(std::string_view line); + +// ninja continues a logical line with a trailing `$`. +std::string read_unfolded(const std::filesystem::path& p); + +} // namespace detail + +Graph build_graph(const std::filesystem::path& build_dir, const Log& log) { + Graph g; + + // Every output of a timed edge collapses onto that edge's identity. + for (const auto& e : log.edges) + for (const auto& o : e.outputs) g.node_of[o] = e.id(); + + auto ingest = [&](const std::string& text, bool dyndep_only) { + std::size_t pos = 0; + while (pos <= text.size()) { + auto nl = text.find('\n', pos); + auto line = std::string_view(text).substr( + pos, nl == std::string::npos ? std::string::npos : nl - pos); + pos = (nl == std::string::npos) ? text.size() + 1 : nl + 1; + + auto st = detail::parse_build(line); + if (!st) continue; + if (dyndep_only && st->rule != "dyndep") continue; + + // node identity: prefer the timed-edge identity, else invent one + std::string node; + for (const auto& o : st->outs) + if (auto it = g.node_of.find(o); it != g.node_of.end()) { node = it->second; break; } + if (node.empty()) node = *std::ranges::min_element(st->outs); + for (const auto& o : st->outs) g.node_of.emplace(o, node); + if (!dyndep_only) g.rule_of[node] = st->rule; + + for (const auto& in : st->ins) g.deps[node].insert(in); + } + }; + + ingest(detail::read_unfolded(build_dir / "build.ninja"), false); + + std::error_code ec; + auto objdir = build_dir / "obj"; + if (std::filesystem::exists(objdir, ec)) { + for (auto const& de : std::filesystem::recursive_directory_iterator(objdir, ec)) { + if (de.is_regular_file(ec) && de.path().extension() == ".dd") + ingest(detail::read_unfolded(de.path()), true); + } + } + + // Re-map every dependency name onto its node identity; drop self-edges and + // leaves (source files produced by nothing). + std::unordered_map> mapped; + for (auto& [node, ins] : g.deps) { + auto& set = mapped[node]; + for (const auto& in : ins) { + auto it = g.node_of.find(in); + if (it != g.node_of.end() && it->second != node) set.insert(it->second); + } + } + g.deps = std::move(mapped); + return g; +} + +} // namespace bench::analysis diff --git a/bench/src/analysis/ninjalog.cppm b/bench/src/analysis/ninjalog.cppm new file mode 100644 index 00000000..a72c019d --- /dev/null +++ b/bench/src/analysis/ninjalog.cppm @@ -0,0 +1,111 @@ +// Parse ninja's `.ninja_log` into per-EDGE timings. +// +// Format (v5/v6): start_ms \t end_ms \t mtime \t output \t command_hash +// +// The trap: an edge with several outputs (`build a.o | a.gcm : cxx_module ...`) +// writes ONE LINE PER OUTPUT, all sharing start/end/hash. Summing the lines +// double-counts the compile phase — for mcpp's own build that inflates module +// compile time from 302 s to 604 s. Edges are therefore keyed by +// (start, end, hash) and every output of an edge collapses onto one identity. +export module bench.analysis.ninjalog; + +import std; + +export namespace bench::analysis { + +struct Edge { + std::int64_t start_ms{}; + std::int64_t end_ms{}; + std::string hash; + std::vector outputs; // sorted; outputs[0] is the identity + + [[nodiscard]] std::int64_t duration_ms() const { return end_ms - start_ms; } + [[nodiscard]] const std::string& id() const { return outputs.front(); } +}; + +struct Log { + std::vector edges; + // `import std;` exports std::size_t but no global ::size_t — unqualified + // spellings that a headers build would accept do not compile here. + std::unordered_map edge_of_output; // output -> index + + [[nodiscard]] const Edge* find(const std::string& output) const { + auto it = edge_of_output.find(output); + return it == edge_of_output.end() ? nullptr : &edges[it->second]; + } + [[nodiscard]] std::int64_t makespan_ms() const { + if (edges.empty()) return 0; + auto lo = std::numeric_limits::max(); + auto hi = std::numeric_limits::min(); + for (const auto& e : edges) { lo = std::min(lo, e.start_ms); hi = std::max(hi, e.end_ms); } + return hi - lo; + } + [[nodiscard]] std::int64_t work_ms() const { + std::int64_t t = 0; + for (const auto& e : edges) t += e.duration_ms(); + return t; + } + [[nodiscard]] std::int64_t t0_ms() const { + auto lo = std::numeric_limits::max(); + for (const auto& e : edges) lo = std::min(lo, e.start_ms); + return edges.empty() ? 0 : lo; + } +}; + +// ninja APPENDS to .ninja_log and restarts its clock at 0 on every invocation, so +// a log touched by several builds holds overlapping time ranges. Mixing them +// yields a makespan SHORTER than the critical path (>100%) — that ratio is the +// tell that this filtering was skipped. Entries are written on completion, so +// `end` is non-decreasing within one run; a decrease starts a newer run. +std::expected parse_ninja_log(const std::filesystem::path& file, + bool last_run_only = true) { + std::ifstream in(file); + if (!in) return std::unexpected("cannot open " + file.string()); + + struct Row { std::int64_t s, e; std::string out, hash; }; + std::vector rows; + + std::string line; + while (std::getline(in, line)) { + if (line.empty() || line.front() == '#') continue; + std::vector f; + for (std::size_t p = 0; p <= line.size();) { + auto tab = line.find('\t', p); + if (tab == std::string::npos) { f.emplace_back(line.substr(p)); break; } + f.emplace_back(line.substr(p, tab - p)); + p = tab + 1; + } + if (f.size() < 5) continue; + std::int64_t s{}, e{}; + auto [p1, ec1] = std::from_chars(f[0].data(), f[0].data() + f[0].size(), s); + auto [p2, ec2] = std::from_chars(f[1].data(), f[1].data() + f[1].size(), e); + if (ec1 != std::errc{} || ec2 != std::errc{}) continue; + rows.push_back({s, e, f[3], f[4]}); + } + + std::size_t begin = 0; + if (last_run_only) { + for (std::size_t i = 1; i < rows.size(); ++i) + if (rows[i].e < rows[i - 1].e) begin = i; + } + + std::map, + std::vector> grouped; + for (std::size_t i = begin; i < rows.size(); ++i) + grouped[{rows[i].s, rows[i].e, rows[i].hash}].push_back(rows[i].out); + + Log log; + log.edges.reserve(grouped.size()); + for (auto& [key, outs] : grouped) { + auto& [s, e, h] = key; + Edge edge{s, e, h, outs}; + std::ranges::sort(edge.outputs); + log.edges.push_back(std::move(edge)); + } + for (std::size_t i = 0; i < log.edges.size(); ++i) + for (const auto& o : log.edges[i].outputs) + log.edge_of_output[o] = i; + return log; +} + +} // namespace bench::analysis diff --git a/bench/src/analysis/report.cpp b/bench/src/analysis/report.cpp new file mode 100644 index 00000000..b909197b --- /dev/null +++ b/bench/src/analysis/report.cpp @@ -0,0 +1,83 @@ +// bench.analysis.report — implementation. +// +// `module bench.analysis.report;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. The critical-path walk lives here. +module bench.analysis.report; + +import std; +import bench.analysis.ninjalog; +import bench.analysis.graph; + +namespace bench::analysis { + +namespace detail { + +std::pair> +longest_path(const Graph& g, const Log& log, const std::string& sink) { + auto dur = [&](const std::string& n) -> std::int64_t { + const auto* e = log.find(n); + return e ? e->duration_ms() : 0; + }; + + std::unordered_map indeg; + std::unordered_map> succ; + std::unordered_set nodes; + + for (const auto& [n, ds] : g.deps) { + nodes.insert(n); + for (const auto& d : ds) nodes.insert(d); + } + nodes.insert(sink); + for (const auto& n : nodes) indeg.try_emplace(n, 0); + for (const auto& [n, ds] : g.deps) { + indeg[n] = ds.size(); + for (const auto& d : ds) succ[d].push_back(n); + } + + std::unordered_map best; + std::unordered_map from; + std::vector ready; + for (const auto& [n, k] : indeg) + if (k == 0) ready.push_back(n); + + std::size_t relaxed = 0; + while (!ready.empty()) { + auto n = ready.back(); + ready.pop_back(); + ++relaxed; + std::int64_t b = 0; + std::string pick; + if (auto it = g.deps.find(n); it != g.deps.end()) { + for (const auto& d : it->second) { + auto v = best.contains(d) ? best[d] : 0; + if (v > b) { b = v; pick = d; } + } + } + best[n] = b + dur(n); + from[n] = pick; + if (auto it = succ.find(n); it != succ.end()) + for (const auto& s : it->second) + if (--indeg[s] == 0) ready.push_back(s); + } + if (relaxed != nodes.size()) { + // A cycle would leave nodes unrelaxed; the graph should be acyclic, so + // say so rather than silently reporting a short path. + std::println(std::cerr, + "buildstat: warning — {} of {} nodes unrelaxed (cycle in the graph?); " + "critical path is a lower bound", + nodes.size() - relaxed, nodes.size()); + } + + std::vector chain; + for (auto n = sink; !n.empty();) { + chain.push_back(n); + auto it = from.find(n); + n = (it == from.end()) ? std::string{} : it->second; + } + std::ranges::reverse(chain); + return {best.contains(sink) ? best[sink] : 0, chain}; +} + +} // namespace detail + +} // namespace bench::analysis diff --git a/bench/src/analysis/report.cppm b/bench/src/analysis/report.cppm new file mode 100644 index 00000000..de4c6955 --- /dev/null +++ b/bench/src/analysis/report.cppm @@ -0,0 +1,156 @@ +// Turn a parsed ninja log + graph into the four numbers that actually explain a +// modular C++ build's wall clock: +// +// work sum of every edge's duration "how much CPU the build costs" +// makespan last end - first start "what the user waited" +// critical longest dependency-weighted path "what no amount of cores fixes" +// concurrency work/makespan over time "where the machine went idle" +// +// For mcpp's own build these read 309 s / 79 s / 79 s / 3.9x — critical path is +// 100% of makespan, so the build is latency-bound, not throughput-bound, and +// buying more cores buys nothing. +export module bench.analysis.report; + +import std; +import bench.analysis.ninjalog; +import bench.analysis.graph; + +export namespace bench::analysis { + +struct RuleStat { + std::string rule; + std::size_t count{}; + std::int64_t total_ms{}; + std::int64_t max_ms{}; +}; + +struct Analysis { + std::int64_t work_ms{}; + std::int64_t makespan_ms{}; + std::int64_t critical_ms{}; + std::vector rules; // descending by total_ms + std::vector critical_chain; // node ids, source -> sink + std::vector concurrency; // per time bucket +}; + +namespace detail { + +// Longest path by Kahn topological relaxation. +// +// A recursive/stack DFS is the obvious implementation and it is WRONG here in a +// way that is quiet: when a dependency is already on the traversal stack (pushed +// via a sibling branch) it must not be treated as resolved, but the natural +// "skip what is on the stack" cycle guard does exactly that and scores it 0. The +// walk then terminates early — on mcpp's own build it reported 33.9 s over 10 +// nodes where the true answer is 79.0 s over 24, i.e. it turned a 100%-critical +// -path build into a 44% one and inverted the whole diagnosis. +// +// Topological order sidesteps it: a node is relaxed only once EVERY dependency +// has a final value. +std::pair> +longest_path(const Graph& g, const Log& log, const std::string& sink); + +} // namespace detail + +Analysis analyze(const Log& log, const Graph& g, std::size_t buckets = 20) { + Analysis a; + a.work_ms = log.work_ms(); + a.makespan_ms = log.makespan_ms(); + + std::map byrule; + for (const auto& e : log.edges) { + auto it = g.rule_of.find(e.id()); + auto name = it == g.rule_of.end() ? std::string("unknown") : it->second; + auto& r = byrule[name]; + r.rule = name; + ++r.count; + r.total_ms += e.duration_ms(); + r.max_ms = std::max(r.max_ms, e.duration_ms()); + } + for (auto& [_, r] : byrule) a.rules.push_back(r); + std::ranges::sort(a.rules, [](auto& x, auto& y) { return x.total_ms > y.total_ms; }); + + // Sink = the link edge if there is one, else the latest-finishing edge. + std::string sink; + std::int64_t latest = std::numeric_limits::min(); + for (const auto& e : log.edges) { + auto it = g.rule_of.find(e.id()); + if (it != g.rule_of.end() && it->second.contains("link")) { sink = e.id(); break; } + if (e.end_ms > latest) { latest = e.end_ms; sink = e.id(); } + } + if (!sink.empty()) { + auto [cost, chain] = detail::longest_path(g, log, sink); + a.critical_ms = cost; + a.critical_chain = std::move(chain); + } + + // Concurrency: fraction of each time bucket covered by running edges. + a.concurrency.assign(buckets, 0.0); + if (a.makespan_ms > 0) { + const double t0 = static_cast(log.t0_ms()); + const double binw = static_cast(a.makespan_ms) / static_cast(buckets); + for (const auto& e : log.edges) { + double s = static_cast(e.start_ms) - t0; + double f = static_cast(e.end_ms) - t0; + for (std::size_t b = 0; b < buckets; ++b) { + double bs = static_cast(b) * binw, be = bs + binw; + double ov = std::min(f, be) - std::max(s, bs); + if (ov > 0) a.concurrency[b] += ov / binw; + } + } + } + return a; +} + +void print_report(const Analysis& a, const Log& log, const Graph& g, + std::string_view label, std::size_t cores) { + auto sec = [](std::int64_t ms) { return static_cast(ms) / 1000.0; }; + std::println("### {}", label); + std::println("edges : {}", log.edges.size()); + std::println("makespan : {:.2f} s", sec(a.makespan_ms)); + std::println("work (sum dur) : {:.2f} s", sec(a.work_ms)); + std::println("avg parallelism: {:.2f} x (of {} hw threads)", + a.makespan_ms ? double(a.work_ms) / double(a.makespan_ms) : 0.0, cores); + std::println("critical path : {:.2f} s = {:.0f}% of makespan", sec(a.critical_ms), + a.makespan_ms ? 100.0 * double(a.critical_ms) / double(a.makespan_ms) : 0.0); + std::println(""); + std::println("{:<16}{:>7}{:>10}{:>9}{:>9}{:>8}", "rule", "count", "total_s", "avg_ms", + "max_ms", "%work"); + for (const auto& r : a.rules) { + std::println("{:<16}{:>7}{:>10.2f}{:>9.1f}{:>9}{:>7.1f}%", r.rule, r.count, + sec(r.total_ms), double(r.total_ms) / double(r.count), r.max_ms, + a.work_ms ? 100.0 * double(r.total_ms) / double(a.work_ms) : 0.0); + } + + std::println(""); + std::println("critical chain ({} nodes, non-zero shown):", a.critical_chain.size()); + for (const auto& n : a.critical_chain) { + const auto* e = log.find(n); + if (!e || e->duration_ms() == 0) continue; + auto it = g.rule_of.find(n); + std::println(" {:>7} ms {:<12} {}", e->duration_ms(), + it == g.rule_of.end() ? "?" : it->second, n); + } + + std::println(""); + std::println("concurrency over time:"); + const double binw = double(a.makespan_ms) / double(a.concurrency.size()) / 1000.0; + for (std::size_t b = 0; b < a.concurrency.size(); ++b) { + auto bars = static_cast(std::llround(a.concurrency[b] * 60.0 / double(cores))); + std::println(" t={:>6.1f}s {:>5.1f}x |{}", double(b) * binw, a.concurrency[b], + std::string(static_cast(std::max(0, bars)), '#')); + } + + std::vector slow; + for (const auto& e : log.edges) slow.push_back(&e); + std::ranges::sort(slow, [](auto* x, auto* y) { return x->duration_ms() > y->duration_ms(); }); + std::println(""); + std::println("slowest edges:"); + for (std::size_t i = 0; i < std::min(12, slow.size()); ++i) { + auto it = g.rule_of.find(slow[i]->id()); + std::println(" {:>7} ms {:<12} {}", slow[i]->duration_ms(), + it == g.rule_of.end() ? "?" : it->second, slow[i]->id()); + } +} + +} // namespace bench::analysis diff --git a/bench/src/engines/bazel.cpp b/bench/src/engines/bazel.cpp new file mode 100644 index 00000000..6d8478ab --- /dev/null +++ b/bench/src/engines/bazel.cpp @@ -0,0 +1,146 @@ +// bench.engines.bazel — implementation. +// +// `module bench.engines.bazel;` with no `export`: an implementation unit. The engine class +// stays declared in the interface and its bodies live here, so changing HOW +// this engine drives its tool does not change the BMI, and nothing that +// imports it has to be recompiled. +module bench.engines.bazel; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; + +namespace bench::engines { + +std::string_view BazelEngine::name() const{ return "bazel"; } + +Availability BazelEngine::probe() const{ + auto a = probe_program("bazel", {"bazel", "--version"}); + // The note rides into every result cell, so the one asymmetry a reader + // must know about is stated there rather than only in this source file: + // bazel keeps a warm server and an action cache OUTSIDE the workspace, + // and `clean` here is deliberately not `--expunge` (which would also + // discard the toolchain and turn the measurement into provisioning). + if (a.present) + a.note = std::format("{} (cold excludes server start; clean is not --expunge)", a.note); + return a; + } + +bool BazelEngine::supports(Variant v, std::string_view compiler) const{ + if (v == Variant::Headers) return true; + return is_clang(compiler); + } + +std::string BazelEngine::unsupported_reason(Variant v, std::string_view compiler) const{ + if (v == Variant::Headers || is_clang(compiler)) return {}; + return "bazel 9.2 builds C++20 modules with clang, but its ddi aggregator " + "cannot parse GCC's P1689 output (aggregate-ddi: \"Invalid JSON string\"); " + "re-run with --compiler to measure this cell"; + } + +platform::RunResult BazelEngine::configure(const Job&) const{ + return {0.0, 0}; // MODULE.bazel/BUILD are the configuration + } + +std::string BazelEngine::unbuildable_reason(const Job& job) const{ + platform::RunResult r; + const auto out = platform::run_capture( + {"bazel", "query", "kind(rule, //...)", "--noshow_progress"}, + job.buildfile_dir, &r); + // A query that ERRORS is not "no targets" — a broken query is a real + // failure, and it belongs in the build where the log gets reported. + if (!out || !r.ok()) return {}; + // The capture is stdout+stderr combined, so presence is tested on the one + // token bazel never prints by accident: a target label at line start. + for (std::string_view rest = *out; !rest.empty();) { + const auto nl = rest.find('\n'); + const auto line = rest.substr(0, nl); + if (line.starts_with("//")) return {}; + if (nl == std::string_view::npos) break; + rest.remove_prefix(nl + 1); + } + return std::format( + "{}/BUILD.bazel declares no rules, so `bazel build //...` would exit 0 " + "having compiled nothing and report a ~0.2s 'build'. bazel cannot glob " + "sources from outside its workspace and has no spelling for `import std;`", + job.buildfile_dir.filename().string()); + } + +platform::RunResult BazelEngine::build(const Job& job) const{ + std::vector argv{"bazel", "build", "//..."}; + if (job.jobs > 0) argv.push_back(std::format("--jobs={}", job.jobs)); + + // Applied to EVERY variant, not just the module ones, so bazel's own + // headers-vs-modules rows stay comparable to each other. + // + // It is also load-bearing for modules: cc_binary registers the ddi + // aggregation action for both the PIC and the non-PIC object sets, but + // names its output `.CXXModules.json` without a pic suffix, so + // analysis dies before a single file is compiled: + // Attempted action contains artifacts not in previous action: + // _objs/fx/unit_0.pic.ddi ... Outputs: are equal + // Forcing one object flavour leaves one action. PIC (rather than + // --features=-supports_pic) is the one that matches the other engines: + // it yields a PIE executable, which is what gcc/clang produce by default + // for everyone else in the table. + // + // ⚠️ POSIX ONLY. Windows has no PIC — code there is relocatable by + // construction — so its toolchain does not enable `supports_pic`, and + // asking for it is not ignored, it is fatal at ANALYSIS: + // + // Error in fail: PIC compilation is requested but the toolchain does + // not support it (feature named 'supports_pic' is not enabled) + // ERROR: Analysis of target '//:fx' failed; build aborted + // + // Every bazel cell of the windows/clang fixture died there, before a + // single file was compiled. The duplicate-action problem this flag + // solves is a POSIX one to begin with: it comes from bazel producing + // both a pic and a non-pic flavour of each object, which Windows does + // not do. + if constexpr (platform::OS_NAME != "windows") + argv.push_back("--force_pic"); + if (job.variant != Variant::Headers) { + // Both are required and they fail differently: without the first, + // `attribute module_interfaces: requires --experimental_cpp_modules`; + // without the second, `the feature cpp_modules must be enabled`. + argv.push_back("--experimental_cpp_modules"); + argv.push_back("--features=cpp_modules"); + } + argv.push_back(std::format("--compilation_mode={}", + job.profile == "debug" ? "dbg" : "opt")); + + // Pin the driver like every other engine. This is also what makes bazel + // WORK inside an xlings workspace: bazel autoconfigures its C++ toolchain + // by probing `$CC -E -v` for builtin include dirs, and a workspace shim + // reports directories that move with the workspace. The result is a + // build that fails with "undeclared inclusion(s)" against perfectly real + // system headers. + // + // BOTH mechanisms are needed and they are not interchangeable: + // CC in the environment — read by the `local_config_cc` REPOSITORY + // RULE when it autoconfigures the toolchain, + // which is where the include dirs are decided + // --action_env=CC — only reaches action execution, far too late + // to affect that probe + // Passing only the flag leaves the broken autoconfiguration in place; + // that is exactly how this failed until the environment was set too. + if (const auto cxx = resolve_cxx(job.compiler); !cxx.empty()) { + argv.push_back(std::format("--action_env=CC={}", cxx)); + platform::ScopedEnv pin("CC", cxx); + return platform::run(argv, job.project_dir, job.log_path, job.timeout_s); + } + return platform::run(argv, job.project_dir, job.log_path, job.timeout_s); + } + +void BazelEngine::clean(const Job& job) const{ + // Deliberately NOT --expunge: that would drop the downloaded toolchain + // and turn a build measurement into a provisioning measurement. + platform::run({"bazel", "clean"}, job.project_dir, job.log_path, job.timeout_s); + platform::remove_tree(job.build_dir); + } + +std::unique_ptr make_bazel() { return std::make_unique(); } + +} // namespace bench::engines diff --git a/bench/src/engines/bazel.cppm b/bench/src/engines/bazel.cppm new file mode 100644 index 00000000..37ef0725 --- /dev/null +++ b/bench/src/engines/bazel.cppm @@ -0,0 +1,58 @@ +// bench.engines.bazel — Bazel. +// +// Bazel DOES build C++20 named modules — a claim worth stating precisely, because +// the obvious guess is wrong in both directions. Measured on bazel 9.2.0 with +// rules_cc 0.2.22: +// +// * the `module_interfaces` attribute exists on cc_binary/cc_library, and +// * it needs BOTH `--experimental_cpp_modules` and `--features=cpp_modules` +// (each flag's absence produces a different, explicit error), and +// * with clang it builds and runs; with GCC it dies in bazel's own scanner: +// aggregate-ddi failed: ... what(): Invalid JSON string +// i.e. bazel's ddi aggregator cannot parse GCC's P1689 output. +// +// So module support here is CONDITIONAL ON THE COMPILER, which is why supports() +// takes one. +// +// Bazel is also the one engine whose "cold" is genuinely ambiguous. It keeps a +// persistent server and a large action cache outside the workspace, so +// `bazel clean` and `bazel clean --expunge` measure two very different things. +// This adapter uses the non-expunging form and says so in the result note, +// because expunging would also discard the downloaded toolchain — provisioning, +// not building. +export module bench.engines.bazel; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; + +namespace bench::engines { + +class BazelEngine : public Engine { +public: + std::string_view name() const override; + + Availability probe() const override; + + bool supports(Variant v, std::string_view compiler) const override; + + std::string unsupported_reason(Variant v, std::string_view compiler) const override; + + platform::RunResult configure(const Job&) const override; + + // `bazel build //...` over a package with no rules is a SUCCESS that compiles + // nothing, in about 0.2s. Ask bazel itself what it is about to build rather + // than reading the BUILD file here — a hand-rolled rule detector would be one + // more parser to keep in step with the file it parses. + std::string unbuildable_reason(const Job& job) const override; + + platform::RunResult build(const Job& job) const override; + + void clean(const Job& job) const override; +}; + +export std::unique_ptr make_bazel(); + +} // namespace bench::engines diff --git a/bench/src/engines/cmake.cpp b/bench/src/engines/cmake.cpp new file mode 100644 index 00000000..234459e6 --- /dev/null +++ b/bench/src/engines/cmake.cpp @@ -0,0 +1,127 @@ +// bench.engines.cmake — implementation. +// +// `module bench.engines.cmake;` with no `export`: an implementation unit. The engine class +// stays declared in the interface and its bodies live here, so changing HOW +// this engine drives its tool does not change the BMI, and nothing that +// imports it has to be recompiled. +module bench.engines.cmake; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; + +namespace bench::engines { + +std::string_view CMakeEngine::name() const{ return "cmake"; } + +Availability CMakeEngine::probe() const{ + auto a = probe_program("cmake", {"cmake", "--version"}); + if (!a.present) return a; + // Ninja is not optional here: the Makefile generator cannot express + // dyndep, so modules simply do not build. Reporting the real reason + // beats a confusing configure failure later. + if (!platform::have_program({"ninja", "--version"})) + return {false, "cmake present but ninja is not; the Makefile generator cannot build C++20 modules"}; + return {true, std::format("{} + ninja", a.note)}; + } + +bool CMakeEngine::supports(Variant v, std::string_view) const{ + if (v == Variant::Headers) return true; + const auto ver = version(); + return !(ver.major && ver.major < 4); + } + +std::string CMakeEngine::unsupported_reason(Variant v, std::string_view) const{ + if (v == Variant::Headers) return {}; + const auto ver = version(); + return std::format( + "cmake {}.{} is too old for `import std;` — the experimental gate key " + "changes with the version and these descriptions carry the 4.0 one " + "(bench/matrix.json pins 4.0.2)", ver.major, ver.minor); + } + +platform::RunResult CMakeEngine::configure(const Job& job) const{ + std::vector argv{ + "cmake", "-S", job.buildfile_dir.string(), "-B", job.build_dir.string(), + "-G", "Ninja", + std::format("-DCMAKE_BUILD_TYPE={}", job.profile == "debug" ? "Debug" : "Release"), + }; + if (const auto cxx = resolve_cxx(job.compiler); !cxx.empty()) + argv.push_back(std::format("-DCMAKE_CXX_COMPILER={}", cxx)); + auto r = platform::run(argv, {}, job.log_path, job.timeout_s); + + // ── When configure fails, append CMake's OWN detection log ────────── + // + // CMake's user-facing errors about the standard library are summaries: + // + // The "CXX_MODULE_STD" property ... requires that the + // "__CMAKE::CXX23" target exist, but it was not provided by the + // toolchain. Reason: Only `libstdc++` is supported + // + // names neither what it looked for nor what it found. Everything + // checkable from OUTSIDE has been checked for the linux/gcc arm and it + // all agrees with a machine where the same cmake and the same gcc + // succeed: the manifest is present on the runner, both sources it names + // are present, and the runner's exact flag shape reproduces and works + // locally. What is left is inside CMake's own probe, and CMake writes + // that down — in CMakeConfigureLog.yaml, which nobody ever reads + // because it lives in a build directory that CI deletes with the job. + // + // Appended to the cell's log ONLY on failure, so a green run costs + // nothing and a red one carries its own evidence. + if (!r.ok()) { + const auto detail = job.build_dir / "CMakeFiles" / "CMakeConfigureLog.yaml"; + // GREP, then tail. The first version took the last 120 lines and got + // the ABI/linker-id probe, because that is what CMake happens to + // write last — the std-module detection this was added for sits + // EARLIER in the file and was cut off. Same lesson as the build + // logs: a tail answers "what happened at the end", not "why did it + // fail". + std::ofstream log(job.log_path, std::ios::app); + if (log) { + if (const auto hits = platform::log_grep( + detail, + {"CXX_MODULE_STD", "IMPORT_STD", "import std", "CXX23", + "modules.json", "libstdc++", "std module"}, + 40); + !hits.empty()) + log << "\n--- CMakeConfigureLog.yaml (std-module entries) ---\n" + << hits; + if (const auto tail = platform::tail_of(detail, 60); !tail.empty()) + log << "\n--- CMakeConfigureLog.yaml (last 60 lines) ---\n" + << tail; + } + } + return r; + } + +platform::RunResult CMakeEngine::build(const Job& job) const{ + std::vector argv{"cmake", "--build", job.build_dir.string()}; + if (job.jobs > 0) { argv.push_back("-j"); argv.push_back(std::to_string(job.jobs)); } + return platform::run(argv, {}, job.log_path, job.timeout_s); + } + +CMakeEngine::Version CMakeEngine::version() const{ + if (!version_) { + Version v; + const auto a = probe_program("cmake", {"cmake", "--version"}); + if (a.present) { + // "cmake version X.Y.Z" — scan to the first digit rather than + // splitting on spaces, since the banner is localised on some + // builds and a missing version must read as 0, not as "new". + const auto pos = a.note.find_first_of("0123456789"); + if (pos != std::string::npos) + std::sscanf(a.note.c_str() + pos, "%d.%d", &v.major, &v.minor); + } + version_ = v; + } + return *version_; + } + +void CMakeEngine::clean(const Job& job) const{ platform::remove_tree(job.build_dir); } + +std::unique_ptr make_cmake() { return std::make_unique(); } + +} // namespace bench::engines diff --git a/bench/src/engines/cmake.cppm b/bench/src/engines/cmake.cppm new file mode 100644 index 00000000..6d943dab --- /dev/null +++ b/bench/src/engines/cmake.cppm @@ -0,0 +1,65 @@ +// bench.engines.cmake — CMake + Ninja. +// +// CMake has supported C++20 named modules since 3.28 (with Ninja >= 1.11), so it +// is the reference point for "the mainstream way to build modules today". +module; +// std::sscanf for the version banner; is not reachable through +// `import std;` for the C-library names in the global namespace. +#include + +export module bench.engines.cmake; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; + +namespace bench::engines { + +class CMakeEngine : public Engine { +public: + std::string_view name() const override; + + Availability probe() const override; + + // `import std;` sits behind an experimental gate whose KEY CHANGES WITH THE + // CMAKE VERSION, and the descriptions in projects/ carry the CMake 4.0 one. + // An older cmake does not reject the key, it simply does not recognise it — + // so the gate stays shut and configure dies on the first `import std;` with + // an error about the standard library rather than about the version. + // + // Reporting that as `failed` is what the runner images actually produced: + // cmake 3.31.6 ships in every GitHub image, and EVERY module cell in the + // matrix was recorded as a real finding against cmake. It is not one — it is + // "this engine, at this version, cannot express this cell", which is exactly + // what `unavailable` plus a reason is for. + // + // Only the module forms need it. `headers` builds fine on 3.28, which is why + // those six cells were the only ones in the whole matrix that ever passed. + bool supports(Variant v, std::string_view) const override; + std::string unsupported_reason(Variant v, std::string_view) const override; + + platform::RunResult configure(const Job& job) const override; + + platform::RunResult build(const Job& job) const override; + + // Parsed out of the probe banner ("cmake version 4.0.2"), and cached: the + // support question is asked once per cell and spawning cmake each time would + // add a process launch to every row of the matrix. + struct Version { int major{}; int minor{}; }; + Version version() const; + + // Artifacts only — the configure result lives in the same directory, so a + // "cold" build here re-runs configure. That is declared in the bench README + // rather than papered over: cmake genuinely cannot separate the two without + // keeping a second cache. + void clean(const Job& job) const override; + +private: + mutable std::optional version_; +}; + +export std::unique_ptr make_cmake(); + +} // namespace bench::engines diff --git a/bench/src/engines/engine.cpp b/bench/src/engines/engine.cpp new file mode 100644 index 00000000..cb532482 --- /dev/null +++ b/bench/src/engines/engine.cpp @@ -0,0 +1,81 @@ +// bench.engines.engine — implementation. +// +// `module bench.engines.engine;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. The interface is the adapter contract every +// engine implements; the shared probe helpers below are not part of it. +module bench.engines.engine; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; + +namespace bench::engines { + +std::string resolve_cxx(std::string_view compiler) { + if (compiler.empty() || compiler == "default") return {}; + if (compiler.find('/') != std::string_view::npos || + compiler.find('\\') != std::string_view::npos) + return std::string(compiler); + if (compiler == "gcc") return "g++"; + if (compiler == "clang") return "clang++"; + return std::string(compiler); +} + +std::string first_line(std::string_view text) { + std::string out; + for (std::size_t i = 0; i < text.size(); ++i) { + if (text[i] == '\n' || text[i] == '\r') break; + if (text[i] == '\x1b') { + // CSI = ESC '[' , parameter bytes 0x30-0x3F, intermediate bytes + // 0x20-0x2F, then ONE final byte 0x40-0x7E. Scanning straight for a + // byte in @-~ stops on the '[' itself, which leaves "0m" behind in + // every colour reset — the exact residue this used to produce. + ++i; + if (i < text.size() && text[i] == '[') ++i; + while (i < text.size() && text[i] >= '\x20' && text[i] <= '\x3f') ++i; + // land on the final byte; the loop's own ++i steps past it + continue; + } + out += text[i]; + } + while (!out.empty() && (out.back() == ' ' || out.back() == '\t')) out.pop_back(); + return out; +} + +bool looks_uninstalled(std::string_view banner) { + return banner.find("is not installed") != std::string_view::npos; +} + +std::string uninstalled_reason(std::string_view program, std::string_view banner) { + return std::format("{} resolves to a shim that reports it is not installed: {}", + program, banner); +} + +Availability probe_program(std::string_view program, + const std::vector& version_argv) { + platform::RunResult r; + const auto captured = platform::run_capture(version_argv, {}, &r); + if (!captured) + return {false, std::format("{} not found on PATH", program)}; + if (r.exit_code != 0) + return {false, std::format("{} present but `{}` exited {}", program, + version_argv.size() > 1 ? version_argv[1] : "--version", + r.exit_code)}; + auto banner = first_line(*captured); + // ⚠️ A SHIM THAT ANSWERS FOR A PROGRAM IT DOES NOT HAVE. xlings installs + // `bazel`, `mcpp` and friends as shims on PATH; ask one for its version + // when the package is not installed and it prints + // + // [error] xlings: 'bazel' is not installed + // + // and exits ZERO. Taken at face value that is "present, version = + // ", so every cell for that engine ran, failed, and was + // recorded as a FINDING against the engine rather than as "not installed + // here" — 18 cells per macOS job. + if (looks_uninstalled(banner)) + return {false, uninstalled_reason(program, banner)}; + return {true, banner.empty() ? std::string(program) : banner}; +} + +} // namespace bench::engines diff --git a/bench/src/engines/engine.cppm b/bench/src/engines/engine.cppm new file mode 100644 index 00000000..cdc3e414 --- /dev/null +++ b/bench/src/engines/engine.cppm @@ -0,0 +1,125 @@ +// bench.engines.engine — the adapter contract every build engine implements. +// +// Adding an engine is: one new module implementing this interface, plus one line +// in bench.registry. The runner, the protocol, the scenarios and the CI matrix +// all stay untouched. That property is the whole reason this interface exists. +// +// Two of the methods look optional and are not: +// +// probe() — "not installed here" and "ran and failed" are OPPOSITE +// conclusions. Without probe, a missing bazel would be recorded +// as a slow or broken bazel. Protocol invariant 2. +// supports() — not every engine can build every source form WITH EVERY +// COMPILER, and forcing a number out of one that cannot is +// worse than reporting that it cannot play. Measured examples: +// bazel 9.2 + rules_cc 0.2.22 builds C++20 modules with clang +// but not with gcc; meson 1.10.2 builds them with neither. +// Both are reported as `unavailable` WITH the measurement, +// never as a slow number. +export module bench.engines.engine; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; + +export namespace bench::engines { + +struct Availability { + bool present{}; + std::string note; // version when present; why not when absent +}; + +class Engine { +public: + virtual ~Engine() = default; + + virtual std::string_view name() const = 0; + + // Is this engine runnable on this machine right now? + virtual Availability probe() const = 0; + + // Can it build this source form, WITH THIS COMPILER? The compiler is part + // of the question: bazel builds C++20 modules with clang and fails with gcc + // (its ddi aggregator cannot parse GCC's P1689 output), so "does bazel + // support modules" has no answer that is independent of the run. + // A `false` becomes `unavailable` with a reason, never a timing. + virtual bool supports(Variant v, std::string_view compiler) const = 0; + + // Reason shown when supports() says no. Required, so the result file + // explains itself without a reader consulting this source. + virtual std::string unsupported_reason(Variant v, std::string_view compiler) const = 0; + + // "This engine cannot build THIS PROJECT" — the question `supports()` cannot + // ask, because it only sees the variant and the compiler. + // + // The gap was not theoretical. bench/projects/mcpp/BUILD.bazel declares no + // targets at all (bazel will not glob sources from outside its workspace, and + // `import std;` has no bazel spelling), so `bazel build //...` succeeded + // having built nothing, and the cell was published as + // bazel/clang/release/cold/mcpp-2026.8.11.3 0.43s + // next to mcpp's 12s and cmake's 94s. Every layer behaved correctly on its + // own: bazel exited 0, the runner timed it, the report printed it. + // + // Returning a non-empty reason marks the cell `unavailable` — a documented + // gap — instead of `ok` with a number that is off by two orders of magnitude. + // Empty (the default) means "nothing project-specific stops me". + virtual std::string unbuildable_reason(const Job&) const { return {}; } + + // Does `compiler` resolve to a clang driver? Several engines' module + // support is clang-only today. + static bool is_clang(std::string_view compiler) { + return compiler.find("clang") != std::string_view::npos; + } + + // One-time project setup (cmake/meson configure, xmake f, ...). Engines + // with no configure step return success without doing anything. + virtual platform::RunResult configure(const Job& job) const = 0; + + // The measured operation. Everything else exists to make this line fair. + virtual platform::RunResult build(const Job& job) const = 0; + + // Remove build artifacts ONLY — never the toolchain or package caches. + // A "cold build" is meant to measure building, not provisioning. + virtual void clean(const Job& job) const = 0; +}; + +// Resolves `Job::compiler` to a concrete C++ driver. +// +// FAIRNESS: every engine that can be told which compiler to use MUST be told the +// same one, or the comparison measures compilers instead of build engines. The +// previous round of this benchmark pinned xmake to mcpp's hermetic g++ by hand +// for exactly this reason; here it is the harness's job. +// +// A value containing a separator is taken as a path and passed through, so a +// caller can pin a hermetic payload (`--compiler /path/to/g++`) rather than +// whatever `g++` happens to mean on this host — which, inside an xlings +// workspace, is a shim whose include search list moves with the workspace. +std::string resolve_cxx(std::string_view compiler); + +// Trims to the first line and strips trailing whitespace and ANSI colour — some +// tools (xmake) colour their version banner, and a control sequence in a JSON +// result file is noise a reader has to decode. +std::string first_line(std::string_view text); + +// Does this version banner actually say the program is missing? +// +// EVERY engine's probe needs this, not just the ones going through +// `probe_program`: mcpp and bazel have their own probes, and putting the test in +// only one of them left 36 cells per job still reported as engine FAILURES on +// macOS. One spelling, three callers. +bool looks_uninstalled(std::string_view banner); + +std::string uninstalled_reason(std::string_view program, std::string_view banner); + +// Shared helper: probe by running ` --version` and keeping the reported +// VERSION as the note. Engines with a different version flag override probe(). +// +// The version, not just the name: a result file whose note reads "cmake" cannot +// answer "which cmake produced this?", and the answer moves the numbers a lot — +// cmake 4.0's module cold build is a different measurement from 3.28's. This is +// the same reason the report records host facts. +Availability probe_program(std::string_view program, + const std::vector& version_argv); + +} // namespace bench::engines diff --git a/bench/src/engines/mcpp.cpp b/bench/src/engines/mcpp.cpp new file mode 100644 index 00000000..44fbc8f5 --- /dev/null +++ b/bench/src/engines/mcpp.cpp @@ -0,0 +1,130 @@ +// bench.engines.mcpp — implementation. +// +// `module bench.engines.mcpp;` with no `export`: an implementation unit. The engine class +// stays declared in the interface and its bodies live here, so changing HOW +// this engine drives its tool does not change the BMI, and nothing that +// imports it has to be recompiled. +module bench.engines.mcpp; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.toolchain; +import bench.engines.engine; + +namespace bench::engines { + +McppEngine::McppEngine(std::string program, std::string label, std::map env) : program_(std::move(program)), label_(std::move(label)), env_(std::move(env)) {} + +std::string_view McppEngine::name() const { + if (label_.empty()) label_ = discover_label(); + return label_; + } + +Availability McppEngine::probe() const { + const auto v = version_string(); + if (v.empty()) + return {false, std::format("{} not runnable", program_)}; + // An xlings shim answers `--version` for a package it does not have, + // prints "is not installed", and exits ZERO — see looks_uninstalled. + if (looks_uninstalled(v)) return {false, uninstalled_reason(program_, v)}; + return {true, v}; + } + +bool McppEngine::supports(Variant, std::string_view) const { return true; } + +std::string McppEngine::unsupported_reason(Variant, std::string_view) const { return {}; } + +platform::RunResult McppEngine::configure(const Job&) const { + return {0.0, 0}; // no separate configure step by design + } + +platform::RunResult McppEngine::build(const Job& job) const { + const std::vector argv{ + program_, "build", job.profile == "debug" ? "--dev" : "--release"}; + + // Scoped, so a setting reaches THIS engine's child and is restored + // before the next engine runs. A run that leaked one would silently + // measure every later arm with the option on. + std::vector> scoped; + scoped.reserve(env_.size() + 1); + + // ── THE COMPILER, and the one place mcpp used to escape the axis ───── + // + // mcpp resolves its own toolchain from the MEASURED PROJECT's manifest + // and ignores the `--compiler` every other engine is handed. For the + // generated fixture that is fine, because the harness writes that + // manifest. For a REAL project it is not: the workloads are pinned + // submodules whose `[toolchain]` says `gcc@16.1.0`, so on a clang cell + // cmake and xmake were measured with clang while mcpp quietly used gcc — + // a compiler-vs-compiler comparison wearing an engine-vs-engine label, + // which is precisely what `resolve_cxx`'s fairness rule exists to stop. + // + // `--toolchain` is plumbed through MCPP_TOOLCHAIN, so the same mechanism + // the bracket options use covers this too. An explicit engine option + // wins, since that is the caller being specific on purpose. + if (!env_.contains("MCPP_TOOLCHAIN")) { + if (auto tc = toolchain_for(job.compiler); !tc.empty()) + scoped.push_back(std::make_unique("MCPP_TOOLCHAIN", tc)); + } + for (const auto& [k, v] : env_) + scoped.push_back(std::make_unique(k, v)); + return platform::run(argv, job.project_dir, job.log_path, job.timeout_s); + } + +void McppEngine::clean(const Job& job) const { + // Artifacts only. ~/.mcpp holds the toolchain and the dependency cache; + // removing those would measure provisioning, which is a different + // question, and would make "cold" mean something else for this engine + // than for the others. + platform::remove_tree(job.project_dir / "target"); + } + +std::string McppEngine::toolchain_for(std::string_view compiler) { + if (compiler.empty() || compiler == "default" || compiler == "msvc") return {}; + if (toolchain::is_clang_request(compiler)) + return std::format("llvm@{}", toolchain::on_windows() ? toolchain::kLlvmWindows + : toolchain::kLlvm); + // A path that is neither clang nor gcc-shaped is the caller pinning + // something the harness does not model; leave mcpp alone rather than + // guess a family for it. + if (compiler.find("gcc") != std::string_view::npos || + compiler.find("g++") != std::string_view::npos) + return std::format("gcc@{}", toolchain::kGcc); + return {}; + } + +std::string McppEngine::version_string() const { + const auto out = platform::run_capture({program_, "--version"}); + if (!out) return {}; + auto line = *out; + if (const auto nl = line.find('\n'); nl != std::string::npos) line.resize(nl); + while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) line.pop_back(); + return line; + } + +std::string McppEngine::discover_label() const { + const auto v = version_string(); // e.g. "mcpp 2026.8.12.1" + if (v.empty()) return "mcpp"; + const auto sp = v.rfind(' '); + if (sp == std::string::npos) return "mcpp"; + // "mcpp@2026.8.13.1" — distinct per version, so two binaries never + // collapse into one row of the result table. + // + // The env suffix is part of the identity for the same reason: the SAME + // binary with `schedule=on` is a different engine to measure, and two + // rows called `mcpp@2026.8.13.1` would be unreadable. + std::string out = std::format("mcpp@{}", v.substr(sp + 1)); + for (const auto& [k, val] : env_) { + std::string key = k; + // `MCPP_BMI_SCHEDULE` -> `schedule`: the label is read by people. + if (key.starts_with("MCPP_")) key.erase(0, 5); + if (key.ends_with("_SCHEDULE") || key == "BMI_SCHEDULE") key = "schedule"; + for (char& c : key) c = static_cast(std::tolower(c)); + out += std::format("+{}={}", key, val); + } + return out; + } + +} // namespace bench::engines diff --git a/bench/src/engines/mcpp.cppm b/bench/src/engines/mcpp.cppm new file mode 100644 index 00000000..e37b51c4 --- /dev/null +++ b/bench/src/engines/mcpp.cppm @@ -0,0 +1,90 @@ +// bench.engines.mcpp — mcpp as a measured engine. +// +// PARAMETERISED BY BINARY, not by a simulated flag. `--engines mcpp=/path/to/A, +// mcpp=/path/to/B` registers two engines that differ only in which mcpp runs, and +// each labels itself with the version it reports. That is how "did this release +// get faster?" is answered: by running both releases, not by approximating one +// of them. +// +// An earlier revision had an `mcpp-opt` engine that set SOURCE_DATE_EPOCH around +// the build to emulate an optimisation. It was removed: emulating a change in +// the harness measures the harness's idea of the change, and it silently stops +// tracking the real implementation the moment the two diverge. Optimisations +// belong in mcpp; the bench measures binaries. +export module bench.engines.mcpp; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.toolchain; +import bench.engines.engine; + +namespace bench::engines { + +class McppEngine : public Engine { +public: + // `program` may be a bare name resolved through PATH or an absolute path to + // a specific build. `label` is what appears in results; empty means "ask the + // binary", which is what makes a two-version comparison self-describing. + // + // `env` is how an OPT-IN BEHAVIOUR becomes a measurable engine. + // + // mcpp's split build schedule is `[build] bmi_schedule = "on"` in the measured + // project's manifest, and it is opt-in until it has been verified on every + // platform. That put the benchmark in an impossible position: the manifests + // belong to the pinned workloads (one of them is someone else's project), so + // the suite could not reach the single largest cold-build optimisation in + // the release it was supposed to be measuring — and the table read "no + // improvement on cold builds" for a change worth 2.29x. + // + // mcpp already exposes it as `MCPP_BMI_SCHEDULE`, so no flag had to be + // invented; the harness only had to set it. Setting it per ENGINE rather + // than per run is the point: both arms appear in the same report, against + // the same baseline, on the same machine, in the same minute. + explicit McppEngine(std::string program = "mcpp", std::string label = {}, std::map env = {}); + + std::string_view name() const override; + + Availability probe() const override; + + // mcpp compiles plain .cpp as readily as modules, and a Native project is + // whatever it already is, so every variant is in scope. + bool supports(Variant, std::string_view) const override; + std::string unsupported_reason(Variant, std::string_view) const override; + + platform::RunResult configure(const Job&) const override; + + platform::RunResult build(const Job& job) const override; + + void clean(const Job& job) const override; + +private: + std::string program_; + mutable std::string label_; + std::map env_; + + // `--compiler` -> the mcpp toolchain spec that names the SAME payload every + // other engine was handed. Empty for "default", where the project's own + // manifest is the right answer and nothing should override it. + // + // The versions come from bench.toolchain, which is also where `payload:gcc` + // is resolved — so the driver cmake is given and the toolchain mcpp is told + // to use cannot name different versions. + static std::string toolchain_for(std::string_view compiler); + + // `mcpp --version` prints "mcpp ". Empty means the binary could not + // be run at all — which probe() reports as unavailable rather than failed. + std::string version_string() const; + + std::string discover_label() const; +}; + +export std::unique_ptr make_mcpp(std::string program = "mcpp", + std::string label = {}, + std::map env = {}) { + return std::make_unique(std::move(program), std::move(label), + std::move(env)); +} + +} // namespace bench::engines diff --git a/bench/src/engines/xmake.cpp b/bench/src/engines/xmake.cpp new file mode 100644 index 00000000..06146bd0 --- /dev/null +++ b/bench/src/engines/xmake.cpp @@ -0,0 +1,205 @@ +// bench.engines.xmake — implementation. +// +// `module bench.engines.xmake;` with no `export`: an implementation unit. The engine class +// stays declared in the interface and its bodies live here, so changing HOW +// this engine drives its tool does not change the BMI, and nothing that +// imports it has to be recompiled. +module bench.engines.xmake; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.toolchain; +import bench.engines.engine; + +namespace bench::engines { + +std::string_view XmakeEngine::name() const { return "xmake"; } + +Availability XmakeEngine::probe() const { + return probe_program("xmake", {"xmake", "--version"}); + } + +bool XmakeEngine::supports(Variant, std::string_view) const { return true; } + +std::string XmakeEngine::unsupported_reason(Variant, std::string_view) const { return {}; } + +platform::RunResult XmakeEngine::configure(const Job& job) const { + std::vector argv{ + "xmake", "f", "-y", + // -P names the directory holding xmake.lua. For a fixture that is + // the tree itself; for a real project the description lives beside + // the bench and reaches back into the tree. + "-P", job.buildfile_dir.string(), + "-m", job.profile == "debug" ? "debug" : "release", + "-o", job.build_dir.string(), + // ⚠️ xmake's compiler cache is ON BY DEFAULT (`--ccache=y`) and it + // lives OUTSIDE the build directory, so `clean()` cannot reach it. + // A `cold` build then restores every object from it: + // seed build 105.059s + // timed run 0.106s <- "cold" + // which the report published as `cold 0.70s` against cmake's 103s. + // Both invariants caught it (cold vs its own noop, and cold vs the + // other engines on the same sources), which is the only reason this + // is a comment rather than a number in the README. + // + // Disabled rather than declared as an asymmetry: neither mcpp nor + // cmake has a compiler cache in this suite, so leaving it on would + // not be "xmake is faster", it would be "xmake did not compile". + // bazel's action cache is the one that IS declared instead — there + // it cannot be turned off without also discarding the toolchain. + "--ccache=n", + }; + // ── Tell xmake where libc++ lives, or `import std;` has no provider ── + // + // The payload SHIPS the std module (share/libc++/v1/std.cppm), but xmake + // looks for it under its own notion of an LLVM SDK and otherwise says + // warning: std and std.compat modules not found! + // maybe try to add --sdk= or install libc++ + // error: missing std dependency for module mcpp.build.provisions + // — a message naming a module of the project under test, so it reads as + // "this project is broken" rather than "the engine was not told where + // its standard library is". Every scenario in the windows/clang cell + // failed that way. + // + // Derived from the resolved driver (…/bin/clang++), which is the same + // path the toolchain pin already produced, so there is nothing to keep + // in step by hand. + if (const auto sdk = payload_sdk_root(job.compiler); !sdk.empty()) + argv.push_back("--sdk=" + sdk); + + // ── How the payload driver is pinned, and why it is not always CXX ── + // + // A real project's description (bench/projects/*/xmake.lua) DEFINES the + // payload as an xmake toolchain — compiler and its `-B` / + // `--sysroot` flags together — in ../common/xmake/payload.lua. Naming + // that toolchain is the only way to get both halves. + // + // Setting CXX instead hands xrepo a compiler WITHOUT those flags, and + // xmake then builds every dependency package with it. They fail: + // => install cmdline 0.0.2 .. failed + // => install mbedtls v3.6.7 .. failed + // => install ftxui v6.1.9 .. failed + // — while the identical `xmake f --toolchain=mcpp-gcc` run by hand + // succeeds, because there the toolchain carries the flags. + // + // The generated fixture has no such definition (bench.fixture.buildfiles + // writes the payload flags inline), so it still takes CXX. The two cases + // are told apart by whether the description lives beside the tree. + // BOTH mechanisms, because they cover different builds: + // + // --toolchain=mcpp-* the PROJECT's targets. Carries the payload + // compiler together with its -B/--sysroot. + // CC / CXX the DEPENDENCY packages xrepo builds. xmake + // does not apply the project toolchain to those, + // so without this they compile with whatever + // `cc` is first on PATH. + // + // In this repository that `cc` is the workspace xlings shim, whose + // include path lacks the kernel UAPI headers its own glibc needs, so + // every package build dies in a header three levels down: + // .../xim-x-glibc/2.39/include/bits/local_lim.h:38:10: + // fatal error: linux/limits.h: No such file or directory + // > in src/lua.c + // The same command run by hand looked fine only because the packages + // were already in xrepo's cache and never rebuilt. + // ── clang: xmake's BUILT-IN llvm toolchain, plus a RUNTIME ────────── + // + // The custom `mcpp-clang` toolchain below carries the payload's `-B` and + // include chain, which gcc needs. For clang it is actively wrong, and + // the reason is not obvious enough to leave undocumented. + // + // xmake picks the std module by C++ LIBRARY, and reads the library from + // the target's RUNTIME (rules/c++/modules/support.lua): + // + // has_runtime("c++_shared","c++_static") -> "c++" (libc++) + // ... no runtime given: fall back on the platform + // is_plat("linux", ...) -> "stdc++" (libstdc++) + // + // So a clang build with no runtime declared is treated as libstdc++: + // xmake looks for GCC's modules.json inside the LLVM payload, finds + // nothing, and prints + // + // warning: std and std.compat modules not found! + // maybe try to add --sdk= or install libc++ + // + // ⚠️ THAT SUGGESTION IS A DEAD END, and following it cost three rounds. + // `--sdk` is only read on the `c++` branch, which was never reached. The + // payload has carried `lib//libc++.modules.json` — precisely + // what that branch looks for — the entire time. + // + // Declaring the runtime on the TARGET does not fix it either: with the + // custom standalone toolchain the branch still is not taken. What works, + // verified end to end, is the built-in toolchain plus both flags — + // measured: the warning disappears and xmake starts emitting module + // BMIs. + const bool payload_clang = !payload_sdk_root(job.compiler).empty(); + if (payload_clang) { + argv.push_back("--toolchain=llvm"); + argv.push_back("--runtimes=c++_static"); // mcpp.toml: static_stdlib + if (const auto cxx = resolve_cxx(job.compiler); !cxx.empty()) { + auto cc = cxx; + if (const auto at = cc.rfind("clang++"); at != std::string::npos) + cc.replace(at, 7, "clang"); + platform::ScopedEnv pin_cxx("CXX", cxx); + platform::ScopedEnv pin_cc("CC", cc); + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + + const bool own_description = !job.buildfile_dir.empty() && + job.buildfile_dir != job.project_dir; + if (own_description) { + if (const auto tc = payload_toolchain(job.compiler); !tc.empty()) { + argv.push_back("--toolchain=" + tc); + const auto cxx = resolve_cxx(job.compiler); + if (!cxx.empty()) { + auto cc = cxx; + for (const auto& [from, to] : {std::pair{"clang++", "clang"}, + std::pair{"g++", "gcc"}}) { + if (const auto at = cc.rfind(from); at != std::string::npos) { + cc.replace(at, std::string_view(from).size(), to); + break; + } + } + platform::ScopedEnv pin_cxx("CXX", cxx); + platform::ScopedEnv pin_cc("CC", cc); + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + } + // Decided from the RESOLVED DRIVER, not from the literal string "clang" + // — main.cpp rewrites `--compiler payload:clang` into an absolute path + // before any engine sees it, so `job.compiler == "clang"` is false in + // exactly the cells that need this. xmake then fell back to g++ while + // the description carried clang's flags: + // g++: error: unrecognized command-line option '--no-default-config' + // Six fixture cells in the linux/clang job. Same rewrite, same mistake + // as `payload_toolchain` above — which I fixed without checking whether + // anything else tested the same string. + if (job.compiler.find("clang") != std::string::npos) + argv.push_back("--toolchain=llvm"); + // The driver is pinned through CXX so every engine compiles with the + // SAME binary; without it xmake resolves whatever `g++` means on this + // host, and the comparison silently becomes compiler-vs-compiler. + if (const auto cxx = resolve_cxx(job.compiler); !cxx.empty()) { + platform::ScopedEnv pin("CXX", cxx); + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + +platform::RunResult XmakeEngine::build(const Job& job) const { + std::vector argv{"xmake", "build", "-P", job.buildfile_dir.string()}; + if (job.jobs > 0) argv.push_back(std::format("-j{}", job.jobs)); + return platform::run(argv, job.buildfile_dir, job.log_path, job.timeout_s); + } + +void XmakeEngine::clean(const Job& job) const { platform::remove_tree(job.build_dir); } + +std::unique_ptr make_xmake() { return std::make_unique(); } + +} // namespace bench::engines diff --git a/bench/src/engines/xmake.cppm b/bench/src/engines/xmake.cppm new file mode 100644 index 00000000..13e4e08c --- /dev/null +++ b/bench/src/engines/xmake.cppm @@ -0,0 +1,83 @@ +// bench.engines.xmake — xmake, which drives its own scheduler rather than ninja. +export module bench.engines.xmake; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; +import bench.toolchain; + +namespace bench::engines { + +// The toolchain name ../common/xmake/payload.lua defines for a payload driver. +// Empty when the request is not a payload one (a bare `gcc`/`clang`/a path), in +// which case there is nothing pinned to name. +// Decided from the RESOLVED DRIVER PATH, not from the `payload:` request. +// main.cpp rewrites `--compiler payload:gcc` into an absolute path long before +// an engine sees it, so testing `starts_with("payload:")` here is always false — +// which is how this silently stopped passing `--toolchain` at all while looking +// correct. The test that survives that rewrite is where the binary lives. +inline std::string payload_toolchain(std::string_view compiler) { + if (compiler.find("/registry/data/xpkgs/") == std::string_view::npos && + compiler.find("\\registry\\data\\xpkgs\\") == std::string_view::npos) + return {}; + const bool clang = compiler.find("clang") != std::string_view::npos; + return clang ? "mcpp-clang" : "mcpp-gcc"; +} + +// The LLVM root a payload driver lives under: `…/xim-x-llvm//bin/clang++` +// → `…/xim-x-llvm/`. Empty for anything that is not a payload clang. +inline std::string payload_sdk_root(std::string_view compiler) { + if (compiler.find("/registry/data/xpkgs/") == std::string_view::npos && + compiler.find("\\registry\\data\\xpkgs\\") == std::string_view::npos) + return {}; + if (compiler.find("clang") == std::string_view::npos) return {}; + const auto slash = compiler.find_last_of("/\\"); + if (slash == std::string_view::npos) return {}; + const auto bin = compiler.substr(0, slash); // …//bin + const auto up = bin.find_last_of("/\\"); + if (up == std::string_view::npos) return {}; + return std::string(bin.substr(0, up)); // …/ +} + +class XmakeEngine : public Engine { +public: + std::string_view name() const override; + + Availability probe() const override; + + bool supports(Variant, std::string_view) const override; + std::string unsupported_reason(Variant, std::string_view) const override; + + platform::RunResult configure(const Job& job) const override; + + platform::RunResult build(const Job& job) const override; + + // ⚠️ EVERY COMMAND RUNS FROM `buildfile_dir`, i.e. the `-P` directory, and + // that is load-bearing rather than tidiness. + // + // xmake normalises `--builddir` (`-o`) to a path RELATIVE TO THE PROJECT + // DIRECTORY, then resolves that relative path against the process's cwd when + // it builds. Run it from anywhere other than `-P` and the two disagree. With + // `-P bench/projects/mcpp` and `-o /build`, running from + // `` put the artifacts in `/mcpp-2026.8.11.3/build` — + // the workload path DOUBLED — while clean() went on removing + // `/build`, which nothing ever wrote to. + // + // The symptom was a passing cell: `cold 0.60s` beside `touch-hub 82.79s`, + // status `ok`, samples present. Every xmake real-project cold number the + // suite produced was measuring an already-up-to-date tree. (The generated + // fixture was unaffected — there `buildfile_dir == project_dir`, so the two + // agreed by accident.) main.cpp now asserts cold > 2x that engine's own + // noop, which is what turns this class of defect into a red run. + // + // `.xmake/` holds the resolved configuration — the counterpart of a cmake + // cache or mcpp's resolution.json. Removing it would measure toolchain + // detection rather than the build, so only the artifact dir goes. + void clean(const Job& job) const override; +}; + +export std::unique_ptr make_xmake(); + +} // namespace bench::engines diff --git a/bench/src/fixture/buildfiles.cpp b/bench/src/fixture/buildfiles.cpp new file mode 100644 index 00000000..b48542fc --- /dev/null +++ b/bench/src/fixture/buildfiles.cpp @@ -0,0 +1,227 @@ +// bench.fixture.buildfiles — implementation. +// +// `module bench.fixture.buildfiles;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. Every engine's build description is emitted +// here; the interface is one function per engine plus `emit_all`. +module bench.fixture.buildfiles; + +import std; +import bench.protocol; +import bench.toolchain; +import bench.fixture.generate; + +namespace bench::fixture { + +namespace detail { + +void write(const std::filesystem::path& p, const std::string& text) { + std::ofstream out(p, std::ios::binary | std::ios::trunc); + out << text; +} + +std::string join(const std::vector& v, std::string_view sep, + std::string_view prefix, std::string_view suffix) { + std::string out; + for (std::size_t i = 0; i < v.size(); ++i) { + if (i) out += sep; + out += std::format("{}{}{}", prefix, v[i], suffix); + } + return out; +} + +std::string trim_copy(std::string_view s) { + while (!s.empty() && s.front() == ' ') s.remove_prefix(1); + while (!s.empty() && s.back() == ' ') s.remove_suffix(1); + return std::string(s); +} + +} // namespace detail + +SourceSet source_set(Variant variant, const Shape& s) { + SourceSet set; + for (int k = 0; k < s.units; ++k) { + const auto name = std::format("unit_{}", k); + if (variant == Variant::Headers) { + set.plain_sources.push_back(std::format("src/{}.cpp", name)); + } else { + set.module_interfaces.push_back(std::format("src/{}.cppm", name)); + if (variant == Variant::ModulesImpl) + set.plain_sources.push_back(std::format("src/{}_impl.cpp", name)); + } + } + set.plain_sources.push_back("src/main.cpp"); + return set; +} + +void emit_mcpp(const std::filesystem::path& root, Variant variant, const Shape&, + std::string_view compiler) { + // mcpp infers the source glob and the binary target from src/main.cpp, so + // the manifest only has to state what cannot be inferred. + std::string toml = + "[package]\n" + "name = \"fx\"\n" + "version = \"0.1.0\"\n" + "description = \"bench fixture\"\n" + "\n" + "[build]\n" + "default-profile = \"release\"\n"; + toml += variant == Variant::Headers ? "include_dirs = [\"include\"]\n" + : "include_dirs = [\"src\"]\n"; + // The toolchain is PINNED, matching every other mcpp project in this repo. + // Relying on the machine's global default makes the fixture build depend on + // ambient state — it works on a developer box that has one and fails on a + // fresh CI sandbox that does not, which is exactly how this surfaced: green + // locally, "seed build exited 1" on the runner. + // + // It also FOLLOWS `--compiler`. mcpp resolves its own toolchain and ignores + // the flag every other engine honours, so pinning gcc here while the harness + // hands clang to cmake/xmake/bazel would turn the table into a compiler + // comparison without saying so. + // + // The VERSION comes from bench.toolchain, which is also where the harness + // looks the payload driver up. Spelling it here as well is how the two + // drifted the first time: the manifest said gcc@16.1.0 and CI handed every + // other engine the runner's gcc 13. + toml += "\n[toolchain]\n"; + toml += std::format("default = \"{}\"\n", toolchain::mcpp_pin(compiler)); + detail::write(root / "mcpp.toml", toml); +} + +void emit_cmake(const std::filesystem::path& root, Variant variant, const Shape& s, + std::string_view compiler) { + const auto set = source_set(variant, s); + // The payload flags go in BEFORE project(), because cmake's "can the + // compiler build a trivial program" probe runs during project() — and a + // bare registry gcc fails that probe at the LINK with `cannot find crt1.o`, + // reported as a configure error that never mentions a sysroot. + const auto pf = toolchain::payload_flags(compiler); + std::string cm = + "# Generated by bench.fixture.buildfiles — do not edit.\n" + "cmake_minimum_required(VERSION 3.28)\n"; + if (!pf.compile.empty() || !pf.link.empty()) { + cm += std::format("set(CMAKE_CXX_FLAGS \"${{CMAKE_CXX_FLAGS}}{}\")\n" + "set(CMAKE_C_FLAGS \"${{CMAKE_C_FLAGS}}{}\")\n" + "set(CMAKE_EXE_LINKER_FLAGS \"${{CMAKE_EXE_LINKER_FLAGS}}{}\")\n", + pf.compile, pf.compile, pf.link); + } + cm += "project(fx CXX)\n" + "set(CMAKE_CXX_STANDARD 23)\n" + "set(CMAKE_CXX_STANDARD_REQUIRED ON)\n" + "set(CMAKE_CXX_EXTENSIONS OFF)\n" + "\n"; + cm += std::format("add_executable(fx\n {}\n)\n", + detail::join(set.plain_sources, "\n ")); + if (!set.module_interfaces.empty()) { + // FILE_SET CXX_MODULES is the only way CMake learns that these are + // interface units; listing them as plain sources compiles them as + // ordinary TUs and the link fails with missing module symbols. + cm += std::format("target_sources(fx\n PRIVATE\n FILE_SET CXX_MODULES FILES\n {}\n)\n", + detail::join(set.module_interfaces, "\n ")); + } + cm += std::format("target_include_directories(fx PRIVATE {})\n", + variant == Variant::Headers ? "include" : "src"); + detail::write(root / "CMakeLists.txt", cm); +} + +void emit_xmake(const std::filesystem::path& root, Variant variant, const Shape&, + std::string_view compiler) { + const auto pf = toolchain::payload_flags(compiler); + std::string lua = + "-- Generated by bench.fixture.buildfiles — do not edit.\n" + "set_project(\"fx\")\n" + "set_languages(\"c++23\")\n" + "add_rules(\"mode.debug\", \"mode.release\")\n" + "\n" + "target(\"fx\")\n" + " set_kind(\"binary\")\n"; + if (variant == Variant::Headers) { + lua += " add_files(\"src/*.cpp\")\n" + " add_includedirs(\"include\")\n"; + } else { + lua += " add_files(\"src/*.cppm\")\n" + " add_files(\"src/*.cpp\")\n" + " add_includedirs(\"src\")\n" + " set_policy(\"build.c++.modules\", true)\n"; + // The fixture never says `import std;`, so xmake must not spend time + // building the std module for it either — otherwise this engine pays a + // cost none of the others do. + lua += " set_policy(\"build.c++.modules.std\", false)\n"; + } + // ⚠️ PIE MUST BE EXPLICIT ON BOTH SIDES. The payload gcc defaults to a PIE + // link on the CI runners and to a non-PIE link on some developer boxes, so + // compiling without `-fPIE` produced objects the linker then refused: + // relocation R_X86_64_32 against `.rodata.str1.1' can not be used + // when making a PIE object; recompile with -fPIE + // ld: failed to set dynamic section sizes: bad value + // Twelve fixture cells red on CI, green locally — the difference was the + // driver's default, not the engine. mcpp and cmake produce PIE here (the + // bazel adapter passes --force_pic for the same reason), so saying it out + // loud keeps all four arms producing the same kind of executable. + lua += " add_cxflags(\"-fPIE\", {force = true})\n"; + lua += " add_ldflags(\"-pie\", {force = true})\n"; + + // Same payload flags as the cmake arm: xmake is handed the driver through + // CXX, and a registry gcc without -B/--sysroot cannot link. + if (!pf.compile.empty()) + lua += std::format(" add_cxflags(\"{}\", {{force = true}})\n", + detail::trim_copy(pf.compile)); + if (!pf.link.empty()) + lua += std::format(" add_ldflags(\"{}\", {{force = true}})\n", + detail::trim_copy(pf.link)); + detail::write(root / "xmake.lua", lua); +} + +void emit_bazel(const std::filesystem::path& root, Variant variant, const Shape& s) { + const auto set = source_set(variant, s); + + // bzlmod. `rules_cc` is NOT optional: bazel 9 removed the built-in cc_binary, + // so a BUILD file without the load() fails with "This rule has been removed + // from Bazel". The version matters too — `module_interfaces` only exists in + // recent rules_cc (0.1.x does not have it, which is what made bazel look like + // it could not build modules at all). + detail::write(root / "MODULE.bazel", + "module(name = \"fx\", version = \"0.1.0\")\n" + "bazel_dep(name = \"rules_cc\", version = \"0.2.22\")\n"); + + std::string bd = + "# Generated by bench.fixture.buildfiles — do not edit.\n" + "load(\"@rules_cc//cc:defs.bzl\", \"cc_binary\")\n" + "\n" + "cc_binary(\n" + " name = \"fx\",\n"; + + // Headers are listed in srcs, not hdrs: cc_binary has no hdrs attribute, and + // an undeclared header is a hard error under bazel's sandbox rather than the + // silent include it would be elsewhere. + std::vector srcs = set.plain_sources; + if (variant == Variant::Headers) { + for (int k = 0; k < s.units; ++k) srcs.push_back(std::format("include/unit_{}.hpp", k)); + srcs.push_back("include/fixture_support.hpp"); + } else { + srcs.push_back("src/fixture_support.hpp"); + } + bd += std::format(" srcs = [{}],\n", detail::join(srcs, ", ", "\"", "\"")); + + if (!set.module_interfaces.empty()) { + // The attribute that makes these interface units rather than ordinary + // TUs. Requires --experimental_cpp_modules AND --features=cpp_modules at + // build time; the adapter passes both. + bd += std::format(" module_interfaces = [{}],\n", + detail::join(set.module_interfaces, ", ", "\"", "\"")); + } + bd += std::format(" includes = [\"{}\"],\n", + variant == Variant::Headers ? "include" : "src"); + bd += " copts = [\"-std=c++23\"],\n"; + bd += ")\n"; + detail::write(root / "BUILD.bazel", bd); +} + +void emit_all(const std::filesystem::path& root, Variant variant, const Shape& s, + std::string_view compiler) { + emit_mcpp(root, variant, s, compiler); + emit_cmake(root, variant, s, compiler); + emit_xmake(root, variant, s, compiler); + emit_bazel(root, variant, s); +} + +} // namespace bench::fixture diff --git a/bench/src/fixture/buildfiles.cppm b/bench/src/fixture/buildfiles.cppm new file mode 100644 index 00000000..35ad88a2 --- /dev/null +++ b/bench/src/fixture/buildfiles.cppm @@ -0,0 +1,73 @@ +// bench.fixture.buildfiles — one project description per engine, for one variant. +// +// These files are what makes the comparison fair or meaningless, so each emitter +// pins the same four things: C++23, the same source set, the same optimisation +// level, and one executable. Anything an engine adds beyond that is noted in the +// emitted file itself, so a reader of the fixture can see the asymmetry without +// reading this module. +// +// `import std;` is deliberately ABSENT from every generated project. Engines +// differ wildly in how (and whether) they can build the std module — CMake needs +// a per-version experimental UUID, bazel needs libc++'s std.cppm listed by hand — +// and that difference would dominate the measurement. The fixture reaches the standard +// library through the global module fragment instead, which every engine handles +// identically. The suite measures MODULE MACHINERY, not std-module support. +export module bench.fixture.buildfiles; + +import std; +import bench.protocol; +import bench.toolchain; +import bench.fixture.generate; + +export namespace bench::fixture { + +// Collects the source lists a build description needs, derived from the variant +// rather than by globbing — a generator that guesses its own output is one +// rename away from silently building less than it claims. +struct SourceSet { + std::vector module_interfaces; // .cppm + std::vector plain_sources; // .cpp (incl. main + impl units) +}; + +SourceSet source_set(Variant variant, const Shape& s); + +namespace detail { + +void write(const std::filesystem::path& p, const std::string& text); + +std::string join(const std::vector& v, std::string_view sep, + std::string_view prefix = "", std::string_view suffix = ""); + + +// Leading space is convenient when concatenating flag strings and wrong inside +// a quoted xmake argument. +std::string trim_copy(std::string_view s); + +} // namespace detail + +// --- mcpp ----------------------------------------------------------------- + +void emit_mcpp(const std::filesystem::path& root, Variant variant, const Shape&, + std::string_view compiler = {}); + +// --- cmake ---------------------------------------------------------------- + +void emit_cmake(const std::filesystem::path& root, Variant variant, const Shape& s, + std::string_view compiler = {}); + +// --- xmake ---------------------------------------------------------------- + +void emit_xmake(const std::filesystem::path& root, Variant variant, const Shape&, + std::string_view compiler = {}); + +// --- bazel ---------------------------------------------------------------- + +void emit_bazel(const std::filesystem::path& root, Variant variant, const Shape& s); + +// Emits every build description a fixture instance can need. Engines that do +// not support the variant simply get no file, and their adapter reports +// `unavailable` with a reason rather than failing to find one. +void emit_all(const std::filesystem::path& root, Variant variant, const Shape& s, + std::string_view compiler = {}); + +} // namespace bench::fixture diff --git a/bench/src/fixture/generate.cpp b/bench/src/fixture/generate.cpp new file mode 100644 index 00000000..12f4ff53 --- /dev/null +++ b/bench/src/fixture/generate.cpp @@ -0,0 +1,169 @@ +// bench.fixture.generate — implementation. +// +// `module bench.fixture.generate;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. The interface keeps `Shape` and `Targets` — the +// vocabulary the runner and the buildfile emitters share — and nothing else. +module bench.fixture.generate; + +import std; +import bench.protocol; + +namespace bench::fixture { + +namespace detail { + +std::string unit_name(int k) { return std::format("unit_{}", k); } + +std::vector deps_of(int k, const Shape& s) { + std::vector d; + for (int j = std::max(0, k - s.fanin); j < k; ++j) d.push_back(j); + return d; +} + +std::string function_body(int k, const Shape& s) { + std::string b; + b += " long long acc = " + std::to_string(k) + ";\n"; + for (int w = 0; w < s.weight; ++w) + b += std::format(" acc += ::bench_fixture::work<{}>({});\n", w, k + w); + for (int d : deps_of(k, s)) + b += std::format(" acc += {}_value();\n", unit_name(d)); + b += " return static_cast(acc & 0x7fffffff);\n"; + return b; +} + +std::string support_header() { + return R"(#pragma once +#include +#include +#include +#include +#include + +namespace bench_fixture { + +// The tag makes every `work` a distinct instantiation of map, vector, string +// and sort. Without it the compiler instantiates one set and every later block +// is free — which is precisely why the previous knob did nothing. +template +struct Key { + int v; + friend bool operator<(const Key& a, const Key& b) { return a.v < b.v; } +}; + +template +long long work(int seed) { + std::map, std::vector> m; + for (int i = 0; i < 4; ++i) + m[Key{seed + i}].push_back(std::to_string(seed * i)); + std::vector v; + v.reserve(8); + for (const auto& [k, strs] : m) v.push_back(static_cast(k.v + strs.size())); + std::sort(v.begin(), v.end()); + return std::accumulate(v.begin(), v.end(), 0LL); +} + +} // namespace bench_fixture +)"; +} + +} // namespace detail + +Targets emit_sources(const std::filesystem::path& root, Variant variant, + const Shape& s) { + namespace fs = std::filesystem; + using detail::unit_name; + using detail::deps_of; + + fs::create_directories(root / "src"); + if (variant == Variant::Headers) fs::create_directories(root / "include"); + + auto write = [](const fs::path& p, const std::string& text) { + std::ofstream out(p, std::ios::binary | std::ios::trunc); + out << text; + }; + + // The support template lives in a header for every variant. In the module + // variants it is pulled in through the global module fragment, which is + // exactly how real module code reaches legacy headers — keeping it means + // the fixture exercises that path instead of pretending it does not exist. + write(root / (variant == Variant::Headers ? "include/fixture_support.hpp" + : "src/fixture_support.hpp"), + detail::support_header()); + + for (int k = 0; k < s.units; ++k) { + const auto name = unit_name(k); + const auto deps = deps_of(k, s); + + if (variant == Variant::Headers) { + std::string hpp = "#pragma once\n"; + for (int d : deps) hpp += std::format("#include \"{}.hpp\"\n", unit_name(d)); + hpp += std::format("\nint {}_value();\n", name); + write(root / "include" / (name + ".hpp"), hpp); + + std::string cpp = std::format("#include \"{}.hpp\"\n", name); + cpp += "#include \"fixture_support.hpp\"\n\n"; + cpp += std::format("int {}_value() {{\n{}}}\n", name, detail::function_body(k, s)); + write(root / "src" / (name + ".cpp"), cpp); + + } else { + std::string ixx = "module;\n#include \"fixture_support.hpp\"\n\n"; + ixx += std::format("export module fx.{};\n\n", name); + for (int d : deps) ixx += std::format("import fx.{};\n", unit_name(d)); + ixx += "\n"; + + if (variant == Variant::Modules) { + // Body IN the interface unit — the shape most module code takes, + // and the one whose BMI churns on every edit. + ixx += std::format("export int {}_value() {{\n{}}}\n", name, + detail::function_body(k, s)); + } else { + // Declaration only; the definition goes to an implementation + // unit, which produces no BMI at all. + ixx += std::format("export int {}_value();\n", name); + std::string impl = "module;\n#include \"fixture_support.hpp\"\n\n"; + impl += std::format("module fx.{};\n\n", name); + impl += std::format("int {}_value() {{\n{}}}\n", name, + detail::function_body(k, s)); + write(root / "src" / (name + "_impl.cpp"), impl); + } + write(root / "src" / (name + ".cppm"), ixx); + } + } + + // main pulls the last unit, which transitively reaches everything. + const int last = s.units - 1; + std::string main_cpp; + if (variant == Variant::Headers) { + main_cpp = std::format("#include \"{}.hpp\"\n#include \n\n" + "int main() {{ std::printf(\"%d\\n\", {}_value()); return 0; }}\n", + unit_name(last), unit_name(last)); + } else { + main_cpp = std::format("#include \nimport fx.{};\n\n" + "int main() {{ std::printf(\"%d\\n\", {}_value()); return 0; }}\n", + unit_name(last), unit_name(last)); + } + write(root / "src" / "main.cpp", main_cpp); + + // hub = unit 0: every other unit reaches it transitively, so an interface + // change there is the worst case for cascades. + // leaf = the last unit: only main depends on it. + // body = the hub too, because "edit a body in the most-depended-on unit" is + // the scenario that separates the three variants most sharply. + Targets t; + if (variant == Variant::Headers) { + t.hub = root / "include" / (unit_name(0) + ".hpp"); + t.leaf = root / "src" / (unit_name(last) + ".cpp"); + t.body = root / "src" / (unit_name(0) + ".cpp"); + } else if (variant == Variant::Modules) { + t.hub = root / "src" / (unit_name(0) + ".cppm"); + t.leaf = root / "src" / (unit_name(last) + ".cppm"); + t.body = root / "src" / (unit_name(0) + ".cppm"); + } else { + t.hub = root / "src" / (unit_name(0) + ".cppm"); + t.leaf = root / "src" / (unit_name(last) + ".cppm"); + t.body = root / "src" / (unit_name(0) + "_impl.cpp"); + } + return t; +} + +} // namespace bench::fixture diff --git a/bench/src/fixture/generate.cppm b/bench/src/fixture/generate.cppm new file mode 100644 index 00000000..818f2257 --- /dev/null +++ b/bench/src/fixture/generate.cppm @@ -0,0 +1,106 @@ +// bench.fixture.generate — the same logical project, emitted in three source +// forms. +// +// GENERATED, NOT CHECKED IN, and that is the point. Two hand-written "equivalent" +// projects are almost certainly inequivalent somewhere, and the difference lands +// exactly on the axis being measured. One generator means one definition of what +// the project IS; the variants differ only in how it is spelled. +// +// The three forms: +// headers unit_k.hpp declares, unit_k.cpp defines (the status quo) +// modules unit_k.cppm declares AND defines (what most module +// code looks like) +// modules-impl unit_k.cppm declares, unit_k_impl.cpp defines (interface and +// implementation split) +// +// modules-impl exists to give the "move bodies out of interface units" advice a +// number. What that number IS turns out to depend on the compiler, and an +// earlier version of this comment asserted the opposite of the measurement: +// +// * GCC 16.1 does NOT put the body of an exported non-template function into +// the BMI. Editing such a body changes the object file and leaves the BMI +// byte-identical apart from its embedded timestamps, so an engine that +// compares BMI CONTENT correctly rebuilds one unit and stops. +// * An engine that decides from the BMI's mtime cascades anyway, which is why +// cmake and xmake pay ~10 s for that edit where mcpp pays 0.3 s. +// +// So this variant measures the difference between the two decision rules, not a +// compiler limitation. Templates and inline functions in an interface unit are a +// different story and DO change the BMI — the advice survives, its justification +// is narrower than it was written to be. +export module bench.fixture.generate; + +import std; +import bench.protocol; + +export namespace bench::fixture { + +struct Shape { + // These defaults ARE the `standard` preset, deliberately: if "no flags" and + // "--preset standard" produced different fixtures, two people comparing + // results would have no way to tell which they each ran. + int units{20}; // how many translation units + int fanin{3}; // how many earlier units each one depends on → graph depth + // Distinct template-instantiation blocks per unit. Calibrated, not guessed: + // each block costs ~0.066 s on top of a 0.38 s floor, so 4 puts a unit at + // ~0.64 s — the same order as a real project's units (mcpp's are 0.57 s). + // See detail::support_header() for the measurements behind those numbers. + int weight{4}; +}; + +// Which files a scenario should perturb. The generator knows the shape, so it +// names them rather than leaving the runner to guess. +struct Targets { + std::filesystem::path hub; // depended on by many + std::filesystem::path leaf; // depended on by nobody + std::filesystem::path body; // holds a function body that can be edited +}; + +namespace detail { + +std::string unit_name(int k); + +std::vector deps_of(int k, const Shape& s); + +// Body shared by all three variants, so the WORK is identical and only the +// packaging differs. `weight` blocks, each a DISTINCT template instantiation — +// see support_header() for why distinctness is the whole point, and for what +// one block costs. +std::string function_body(int k, const Shape& s); + +// The template the bodies instantiate. Included by EVERY generated unit — in the +// global module fragment for the module variants, directly for the header +// variant — so all three pay the same per-translation-unit cost and differ only +// in how they share declarations. +// +// CALIBRATION, and why this is not the workload it started as. The first version +// measured almost no compilation: a unit cost 0.23 s of which 0.17 s was the +// compiler starting up — 74% process startup — and the `weight` knob barely +// moved it, because it emitted O(weight^2) instantiations of a single trivial +// constexpr recursion (a few hundred at weight 40, which a compiler does in +// microseconds). Measured on gcc 16.1.0, x86_64: +// +// empty module ................................. 0.17 s +// old fixture unit, weight 6 ................... 0.23 s +// one unit with a realistic global module fragment 0.97 s +// mcpp's own units (57k lines / 139 units) ..... 0.57 s +// +// So the workload is now built from what actually costs time in real C++: +// standard library headers, plus instantiation over DISTINCT types so the +// instantiations cannot be shared between blocks. Cost is 0.38 s + 0.066 s per +// weight unit, which puts the default weight at the same order as a real +// project's units instead of two orders below it. +std::string support_header(); + +} // namespace detail + +// Emits one variant of the project into `root`. Returns the perturbation +// targets for that layout. +Targets emit_sources(const std::filesystem::path& root, Variant variant, const Shape& s); + +// --------------------------------------------------------------------------- + +Targets emit_sources(const std::filesystem::path& root, Variant variant, + const Shape& s); + +} // namespace bench::fixture diff --git a/bench/src/journal.cpp b/bench/src/journal.cpp new file mode 100644 index 00000000..5d60a721 --- /dev/null +++ b/bench/src/journal.cpp @@ -0,0 +1,100 @@ +// bench.journal — implementation. +// +// `module bench.journal;` with no `export`: this is an implementation unit, so +// nothing here reaches the BMI. See journal.cppm for why that matters. +module bench.journal; + +import std; +import bench.protocol; + +namespace bench { +namespace { + +// Minimal JSON string escape, private to this unit. +std::string esc(std::string_view v) { + std::string out; + for (const char c : v) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; + } + } + return out; +} + +} // namespace + +Journal::Journal(std::filesystem::path path) : path_(std::move(path)) {} + +const std::filesystem::path& Journal::path() const { return path_; } + +std::string Journal::unit_id(std::string_view project, std::string_view variant, + std::string_view scenario, std::string_view engine, int run) { + return std::format("{}|{}|{}|{}|{}", project, variant, scenario, engine, run); +} + +void Journal::append(const JournalEntry& e) const { + std::ofstream out(path_, std::ios::app); + if (!out) return; + out << std::format( + R"({{"id":"{}","ver":"{}","project":"{}","variant":"{}","scenario":"{}",)" + R"("engine":"{}","run":{},"wall_s":{:.6f},"exit":{}}})", + esc(e.id), esc(e.engine_version), esc(e.project), esc(e.variant), esc(e.scenario), + esc(e.engine), e.run, e.wall_s, e.exit_code) + << '\n'; + out.flush(); +} + +Journal::Loaded Journal::load(std::string_view want_id) const { + Loaded out; + std::ifstream in(path_); + if (!in) return out; + std::string line; + while (std::getline(in, line)) { + // A line that is not a whole JSON object is skipped, not fatal: the last + // line of a killed run is expected to be half-written. + if (line.empty() || line.front() != '{' || line.back() != '}') { + if (!line.empty()) ++out.skipped_unparsable; + continue; + } + const auto get = [&](std::string_view k) -> std::string { + const auto at = line.find(std::format("\"{}\":", k)); + if (at == std::string::npos) return {}; + const auto v = line.find_first_not_of(' ', at + k.size() + 3); + if (v == std::string::npos) return {}; + if (line[v] == '"') { + const auto e = line.find('"', v + 1); + return e == std::string::npos ? std::string{} : line.substr(v + 1, e - v - 1); + } + const auto e = line.find_first_of(",}", v); + return line.substr(v, e - v); + }; + JournalEntry e; + e.id = get("id"); + if (e.id != want_id) { + ++out.skipped_other_id; + if (out.other_id.empty()) out.other_id = e.id; + continue; + } + e.engine_version = get("ver"); + e.project = get("project"); + e.variant = get("variant"); + e.scenario = get("scenario"); + e.engine = get("engine"); + const auto run_s = get("run"); + const auto wall_s = get("wall_s"); + const auto exit_s = get("exit"); + if (run_s.empty() || wall_s.empty()) { ++out.skipped_unparsable; continue; } + e.run = std::atoi(run_s.c_str()); + e.wall_s = std::atof(wall_s.c_str()); + e.exit_code = exit_s.empty() ? 0 : std::atoi(exit_s.c_str()); + out.units[unit_id(e.project, e.variant, e.scenario, e.engine, e.run)] = e; + } + return out; +} + +} // namespace bench diff --git a/bench/src/journal.cppm b/bench/src/journal.cppm new file mode 100644 index 00000000..23c60626 --- /dev/null +++ b/bench/src/journal.cppm @@ -0,0 +1,58 @@ +// bench.journal — the append-only record of measured units, and resume. +// +// INTERFACE ONLY. Definitions live in journal.cpp, the same split the xlings +// tree this suite measures uses for every module. +// +// ⚠️ THAT SPLIT IS NOT STYLE HERE, IT IS THE FIX. This started as one file with +// the class defined inline in the interface, and the whole suite then failed to +// compile with +// +// bench.registry: error: failed to read compiled module cluster 89: +// Bad file data +// fatal error: failed to load binding 'bench::make_engine@bench.registry' +// +// naming a module that has nothing to do with journals. Under GCC 16.1 an +// export whose INTERFACE carries std types (here a std::map, a +// std::filesystem::path and a nested struct) can poison the BMIs of everything +// downstream, and the error points somewhere unrelated. Declarations only keeps +// the BMI thin and the poisoning does not arise. +export module bench.journal; + +import std; +import bench.protocol; + +export namespace bench { + +// One line of the journal, parsed. +class Journal { +public: + explicit Journal(std::filesystem::path path); + + // Appends one measured unit and flushes. const because it writes a FILE and + // does not change the object. + void append(const JournalEntry& e) const; + + struct Loaded { + std::map units; // unit_id -> entry + std::size_t skipped_other_id{}; + std::size_t skipped_unparsable{}; + std::string other_id; + }; + + // Every entry carrying `want_id`, keyed by unit coordinate. A line that does + // not parse is SKIPPED: the last line of a killed run is expected to be + // half-written, and refusing the file because of it would discard everything + // that run did accomplish. + [[nodiscard]] Loaded load(std::string_view want_id) const; + + [[nodiscard]] static std::string unit_id(std::string_view project, std::string_view variant, + std::string_view scenario, std::string_view engine, + int run); + + [[nodiscard]] const std::filesystem::path& path() const; + +private: + std::filesystem::path path_; +}; + +} // namespace bench diff --git a/bench/src/main.cpp b/bench/src/main.cpp new file mode 100644 index 00000000..dd796aa9 --- /dev/null +++ b/bench/src/main.cpp @@ -0,0 +1,847 @@ +// bench — build-engine benchmark harness. +// +// bench [--engines a,b] [--variants headers,modules,modules-impl] +// [--scenarios cold,noop,...] [--profile release|debug] +// [--compiler default|gcc|clang] [--preset NAME] [--units N] [--fanin N] [--weight N] +// [--jobs N] [--runs N] [--work DIR] [--out FILE] [--list] +// +// Writes a protocol-versioned JSON report to --out (default bench-report.json) +// and a human summary to stdout. The two are separate on purpose: the JSON is +// what merges across machines, the summary is what a person reads. +import std; +import bench.protocol; +import bench.journal; +import bench.spec; +import bench.platform; +import bench.toolchain; +import bench.registry; +import bench.runner; +import bench.engines.engine; +import bench.fixture.generate; +import bench.analysis.ninjalog; +import bench.analysis.graph; +import bench.analysis.report; + +namespace { + +struct Options { + std::vector engines; + std::vector variants; + std::vector scenarios; + std::string profile{"release"}; + std::string compiler{"default"}; + bench::fixture::Shape shape{}; + int jobs{0}; + int runs{0}; + std::filesystem::path work{"bench-work"}; + std::filesystem::path out{"bench-report.json"}; + // Names this run, and therefore its journal. Empty → derived from + // os-toolchain-project, which is how the reports are already named. + std::string run_id; + // What is under test, in the caller's words (mcpp's commit). Recorded, never + // fingerprinted: folding it in would restart every resume on every rebuild. + std::string under_test; + // Where fingerprinted run caches live, like a build directory. + std::filesystem::path cache_root{".mbench"}; + std::filesystem::path analyze; // profile an existing ninja build dir instead + std::filesystem::path project; // measure an existing tree instead of a fixture + std::filesystem::path buildfiles;// foreign build descriptions for that tree + std::filesystem::path hub, leaf, body; // what the scenarios perturb there + // The engine every ratio is expressed against. cmake is the default and not + // an arbitrary one: it is the reference implementation of C++ module support + // (P1689 scanning + dyndep are its design), it is present on every machine + // this suite runs on, and it is what a reader already has a feel for. An + // absolute second count means nothing without knowing the runner; "1.8x + // cmake" survives being read on a different machine. + // + // Defaulting it rather than leaving it empty is deliberate: a run that + // forgot the flag produced a table of bare seconds, which is the one form + // of this data that cannot be compared to anything. + std::string baseline{"cmake"}; + // One configure or build may take this long before the child is killed. + // NOT unlimited by default: a hung engine used to consume the whole 120 + // minute CI budget and report nothing, because a cell only prints once it + // is over. 30 minutes is well clear of a cold cmake build of mcpp on a + // 4-core runner (~16 min measured) and still leaves room in the job. + double timeout_s{1800.0}; + // Engines whose `failed` must not fail the RUN. Empty by default: a failure + // is "the engine ran and did not produce the artifact", which is a finding, + // and a suite that reports findings with exit 0 is the one that let 48 of 72 + // cells fail unnoticed. A genuine known gap goes here WITH its reason in + // bench/matrix.json, so it stays visible instead of becoming invisible. + std::vector allow_failed; + bool list{false}; +}; + +// Does `engine` (a label like "mcpp@2026.8.13.1") match one of the names the +// caller marked as allowed-to-fail? Substring, like --baseline, so a versioned +// mcpp label is reachable by the bare name. +bool listed(const std::vector& names, std::string_view engine) { + return std::ranges::any_of(names, [&](const std::string& n) { + return !n.empty() && engine.find(n) != std::string_view::npos; + }); +} + +// Splits a comma-separated list, IGNORING commas inside `[...]`. +// +// An engine spec may carry bracketed options (`mcpp[schedule=on]=/path`), and a +// second option would be separated by a comma — which this function would +// otherwise cut in half, handing `make_engine` the fragments `mcpp[a=1` and +// `b=2]=/path` and reporting "unknown engine" for a perfectly valid spec. The +// list separator and the option separator are the same character, so the list +// splitter is the one that has to know about the brackets. +// +// Harmless for every other caller: `--variants`, `--scenarios` and +// `--allow-failed` contain no brackets, so the depth counter never leaves zero. +std::vector split(std::string_view s, char sep = ',') { + std::vector parts; + std::size_t start = 0, depth = 0; + for (std::size_t i = 0; i <= s.size(); ++i) { + if (i < s.size()) { + if (s[i] == '[') { ++depth; continue; } + if (s[i] == ']') { if (depth) --depth; continue; } + if (s[i] != sep || depth) continue; + } + if (i > start) parts.emplace_back(s.substr(start, i - start)); + start = i + 1; + } + return parts; +} + +void usage() { + std::println("bench — build-engine benchmark harness"); + std::println(""); + std::println(" --engines LIST mcpp,cmake,xmake,bazel (default: all)"); + std::println(" --variants LIST headers,modules,modules-impl (default: all)"); + std::println(" --scenarios LIST cold,noop,touch-hub,touch-leaf,edit-body,edit-comment"); + std::println(" --profile NAME release | debug (default: release)"); + std::println(" --compiler NAME default | gcc | clang | /path/to/g++ (default: default)"); + std::println(" payload:gcc / payload:clang — the driver out of MCPP'S OWN"); + std::println(" registry, i.e. the one mcpp itself builds with. Use this to"); + std::println(" compare engines rather than compilers; a host g++ that cannot"); + std::println(" build modules makes every other engine look broken."); + std::println(" --preset NAME smoke | standard | large — a NAMED size, so two runs on"); + std::println(" two machines compare. standard is the default shape."); + std::println(" smoke 4 units / fan-in 2 / weight 1 (~2s, CI)"); + std::println(" standard 20 units / fan-in 3 / weight 4 (~18s cold, mcpp)"); + std::println(" large 60 units / fan-in 3 / weight 6"); + std::println(" --units N fixture translation units (default: 20)"); + std::println(" --fanin N dependencies per unit (default: 3)"); + std::println(" --weight N distinct template blocks per unit (default: 4)"); + std::println(" --jobs N parallelism handed to each engine (default: engine's)"); + std::println(" --runs N repetitions per cell (default: per scenario)"); + std::println(" --work DIR scratch directory (default: bench-work)"); + std::println(" --id NAME one more field in the run's fingerprint — use it to keep"); + std::println(" two otherwise-identical runs apart"); + std::println(" --cache-root DIR where fingerprinted runs are cached (default: .mbench)"); + std::println(" --under-test TEXT what is being measured, in your words — for mcpp the"); + std::println(" commit, because its version is a DATE and every commit"); + std::println(" on a branch reports the same one. Recorded in the report"); + std::println(" and compared on resume; never part of the fingerprint."); + std::println(""); + std::println(" A run is fingerprinted over its WHOLE configuration (engines, variants,"); + std::println(" scenarios, runs, compiler, profile, project, shape, --id) and cached in"); + std::println(" //. Re-running the same configuration RESUMES from"); + std::println(" what is already recorded there, one measured unit at a time; changing any of"); + std::println(" it lands in a different directory instead of overwriting. Delete the"); + std::println(" directory to start that configuration over."); + std::println(" --out FILE JSON report path (default: bench-report.json)"); + std::println(" --baseline NAME normalise the summary against this engine (default: cmake)"); + std::println(" Substring match on the label; \"\" disables the column."); + std::println(" --timeout SEC kill one configure/build after this long (default: 1800, 0 = never)"); + std::println(" --allow-failed L engines whose failure must not fail the run (default: none)"); + std::println(""); + std::println("EXIT STATUS: 0 only when something was measured and nothing failed. A `failed`"); + std::println("cell means the engine ran and produced no artifact — that is a finding, not a"); + std::println("gap, so it is reported with a non-zero status unless --allow-failed names it."); + std::println("`unavailable` and `skipped` are gaps and never fail the run."); + std::println(" --list print engines and their availability, then exit"); + std::println(" --analyze DIR profile an existing ninja build dir (work, makespan,"); + std::println(" critical path, concurrency) instead of measuring"); + std::println(""); + std::println("Measuring a REAL project instead of a generated fixture:"); + std::println(" --project DIR build this tree as-is (e.g. mcpp itself)"); + std::println(" --buildfiles DIR where cmake/xmake read their description from, when the"); + std::println(" project does not carry one (bench/projects//)"); + std::println(" --hub FILE file with many dependents (touch-hub)"); + std::println(" --leaf FILE file with no dependents (touch-leaf)"); + std::println(" --body FILE file whose body gets edited (edit-body)"); + std::println(""); + std::println("Comparing two mcpp builds is an engine spec, not a flag:"); + std::println(" --engines mcpp=/usr/bin/mcpp,mcpp=./target/.../bin/mcpp"); + std::println(" each labels itself from the version it reports, so rows stay distinct"); +} + +std::expected parse(int argc, char** argv) { + Options o; + for (int i = 1; i < argc; ++i) { + const std::string_view a = argv[i]; + auto value = [&](std::string_view name) -> std::expected { + if (i + 1 >= argc) return std::unexpected(std::format("{} needs a value", name)); + return std::string(argv[++i]); + }; + auto take_int = [&](std::string_view name, int& dst) -> std::optional { + auto v = value(name); + if (!v) return v.error(); + dst = std::atoi(v->c_str()); + return std::nullopt; + }; + + if (a == "--engines") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.engines = split(*v); } + else if (a == "--profile") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.profile = *v; } + else if (a == "--compiler") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.compiler = *v; } + else if (a == "--work") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.work = *v; } + else if (a == "--cache-root") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.cache_root = *v; } + else if (a == "--id") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.run_id = *v; } + else if (a == "--under-test") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.under_test = *v; } + else if (a == "--out") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.out = *v; } + // Presets come FIRST so an explicit --units/--weight after one still + // wins. A benchmark whose size is a free-form pair of numbers cannot be + // compared between two people who ran it; a named size can. + else if (a == "--preset") { + if (i + 1 >= argc) return std::unexpected(std::string("--preset needs a value")); + const std::string_view v = argv[++i]; + if (v == "smoke") o.shape = {4, 2, 1}; + else if (v == "standard") o.shape = {20, 3, 4}; + else if (v == "large") o.shape = {60, 3, 6}; + else return std::unexpected(std::format( + "unknown preset '{}' (smoke | standard | large)", v)); + } + else if (a == "--units") { if (auto e = take_int(a, o.shape.units)) return std::unexpected(*e); } + else if (a == "--fanin") { if (auto e = take_int(a, o.shape.fanin)) return std::unexpected(*e); } + else if (a == "--weight") { if (auto e = take_int(a, o.shape.weight)) return std::unexpected(*e); } + else if (a == "--jobs") { if (auto e = take_int(a, o.jobs)) return std::unexpected(*e); } + else if (a == "--runs") { if (auto e = take_int(a, o.runs)) return std::unexpected(*e); } + else if (a == "--list") { o.list = true; } + else if (a == "--analyze") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.analyze = *v; } + // ABSOLUTE, for the same reason as --buildfiles below: `build_dir` is + // derived from this (`/build`) and handed to xmake as `-o`, + // which resolves a relative path against ITS cwd — the buildfile dir — + // while clean() resolves the same string against the harness's cwd. + // The two then name different directories: xmake wrote into + // bench/projects/xlings/bench/projects/xlings//build + // and clean() removed a path nothing had written to, so every `cold` + // measured an up-to-date tree (0.76s against cmake's 103s). The + // cold-vs-noop and cold-vs-peers invariants are what caught it. + else if (a == "--project") { + auto v = value(a); if (!v) return std::unexpected(v.error()); + std::error_code ec; + auto abs = std::filesystem::absolute(*v, ec); + o.project = ec ? std::filesystem::path(*v) : abs; + } + // ABSOLUTE, resolved against the cwd the harness was STARTED in. + // + // Every engine is spawned with its cwd set to this directory, and xmake + // is then handed it again as `-P`. A relative path therefore resolves + // twice: `--buildfiles bench/projects/xlings` became + // `bench/projects/xlings/bench/projects/xlings` and the whole arm failed + // with `error: project not found!` — while the identical command run by + // hand from the repository root worked, because there the doubling had + // nothing to double against. Same shape as the `--buildir` doubling this + // suite already fixed once; the fix belongs here, once, rather than in + // each adapter. + else if (a == "--buildfiles"){ + auto v = value(a); if (!v) return std::unexpected(v.error()); + std::error_code ec; + auto abs = std::filesystem::absolute(*v, ec); + o.buildfiles = ec ? std::filesystem::path(*v) : abs; + } + else if (a == "--hub") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.hub = *v; } + else if (a == "--leaf") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.leaf = *v; } + else if (a == "--body") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.body = *v; } + else if (a == "--baseline") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.baseline = *v; } + else if (a == "--allow-failed") { auto v = value(a); if (!v) return std::unexpected(v.error()); o.allow_failed = split(*v); } + else if (a == "--timeout") { + auto v = value(a); if (!v) return std::unexpected(v.error()); + o.timeout_s = std::atof(v->c_str()); + if (o.timeout_s < 0.0) return std::unexpected(std::string("--timeout must not be negative")); + } + else if (a == "-h" || a == "--help") { return std::unexpected("help"); } + else if (a == "--variants") { + auto v = value(a); if (!v) return std::unexpected(v.error()); + for (const auto& name : split(*v)) { + const auto parsed = bench::variant_from(name); + if (!parsed) return std::unexpected(std::format("unknown variant '{}'", name)); + o.variants.push_back(*parsed); + } + } else if (a == "--scenarios") { + auto v = value(a); if (!v) return std::unexpected(v.error()); + for (const auto& name : split(*v)) { + const auto parsed = bench::scenario_from(name); + if (!parsed) return std::unexpected(std::format("unknown scenario '{}'", name)); + o.scenarios.push_back(*parsed); + } + } else { + return std::unexpected(std::format("unknown argument '{}'", a)); + } + } + if (o.variants.empty()) { + // A real project has exactly one form — its own. Offering it the + // headers/modules axis would generate over the tree being measured. + o.variants = o.project.empty() + ? std::vector{bench::Variant::Headers, bench::Variant::Modules, + bench::Variant::ModulesImpl} + : std::vector{bench::Variant::Native}; + } + if (o.scenarios.empty()) + // ALL of them. A scenario that is defined, documented and advertised in + // --help but left out of this list runs only when someone names it + // explicitly, which in practice is never — `touch-leaf` sat unmeasured + // in every default run and every CI matrix cell for exactly that reason. + o.scenarios = {bench::Scenario::Cold, bench::Scenario::Noop, + bench::Scenario::TouchHub, bench::Scenario::TouchLeaf, + bench::Scenario::EditBody, bench::Scenario::EditComment}; + return o; +} + +} // namespace + +int main(int argc, char** argv) { + auto opts = parse(argc, argv); + if (!opts) { + if (opts.error() == "help") { usage(); return 0; } + std::println(std::cerr, "bench: {}", opts.error()); + usage(); + return 2; + } + + // Analysis mode short-circuits everything else: it reads a build that has + // already happened rather than causing one. + if (!opts->analyze.empty()) { + auto log = bench::analysis::parse_ninja_log(opts->analyze / ".ninja_log"); + if (!log) { + std::println(std::cerr, "bench: {}", log.error()); + return 1; + } + if (log->edges.empty()) { + std::println(std::cerr, "bench: .ninja_log has no edges — was anything built?"); + return 1; + } + const auto graph = bench::analysis::build_graph(opts->analyze, *log); + const auto a = bench::analysis::analyze(*log, graph); + const auto cores = bench::platform::host_facts().logical_cores; + bench::analysis::print_report(a, *log, graph, opts->analyze.string(), + static_cast(cores)); + return 0; + } + + // `payload:gcc` / `payload:clang` become a concrete path BEFORE anything + // else looks at the value, so the engines, the generated manifest, the cell + // key and the report all describe the same compiler. Resolving it later, or + // per engine, is how they came apart the first time. + if (opts->compiler.starts_with("payload:")) { + const auto want = opts->compiler.substr(std::string_view("payload:").size()); + const auto r = bench::toolchain::payload_cxx(want); + if (r.driver.empty()) { + // Hard error, not a fallback. Falling back to the host compiler is + // precisely what produced a matrix in which 48 of 72 cells failed + // while the job reported success. + std::println(std::cerr, "bench: --compiler {} could not be resolved: {}", + opts->compiler, r.why); + return 2; + } + std::println("payload: {} → {}", opts->compiler, r.driver.string()); + opts->compiler = r.driver.string(); + } + + const auto specs = opts->engines.empty() ? bench::default_engine_specs() : opts->engines; + std::vector> engines; + for (const auto& spec : specs) { + auto e = bench::make_engine(spec); + if (!e) { + std::println(std::cerr, "bench: unknown engine '{}'", spec); + return 2; + } + engines.push_back(std::move(e)); + } + + // Two binaries that report the SAME version produce the same label, and two + // rows with one name is a table nobody can read — the old-vs-new comparison + // silently stops being one the moment a branch forgets to bump its version. + // Say so rather than printing it twice. + { + std::vector seen; + for (std::size_t i = 0; i < engines.size(); ++i) { + const auto n = engines[i]->name(); + if (std::ranges::find(seen, n) != seen.end()) + std::println(std::cerr, + "bench: WARNING — engine #{} also calls itself '{}'. Two " + "binaries reporting one version cannot be told apart in " + "the report; bump one, or pass distinct labels.", + i + 1, n); + seen.push_back(n); + } + } + + if (opts->list) { + std::println("{:<10} {:<12} {}", "engine", "available", "note"); + for (const auto& e : engines) { + const auto a = e->probe(); + std::println("{:<10} {:<12} {}", e->name(), a.present ? "yes" : "no", a.note); + } + return 0; + } + + // A path pins the compiler; a label keeps the result readable. `--compiler + // /long/path/to/g++` would otherwise put that path in every cell key. + const std::string compiler_label = [&] { + const auto& c = opts->compiler; + if (c.empty() || c == "default") return std::string("default"); + if (c.find('/') == std::string::npos && c.find('\\') == std::string::npos) return c; + const auto stem = std::filesystem::path(c).filename().string(); + if (stem.starts_with("g++") || stem.starts_with("gcc")) return std::string("gcc"); + if (stem.starts_with("clang")) return std::string("clang"); + return stem; + }(); + + // A foreign build description for a REAL project cannot derive the tree from + // its own location — it lives in bench/projects// and the tree is + // elsewhere — so the harness has to tell it. Exported for the whole run + // because it is constant for the whole run. + // + // Without this the xlings arm could never run at all: its CMakeLists starts + // with a FATAL_ERROR demanding XLINGS_ROOT, nothing set it, and every cell + // was recorded as `configure exited 1`. Twelve cells per job, three jobs, + // all green. + std::optional project_root_env; + if (!opts->project.empty()) { + std::error_code ec; + auto abs = std::filesystem::absolute(opts->project, ec); + project_root_env.emplace("BENCH_PROJECT_ROOT", + (ec ? opts->project : abs).string()); + } + + const auto facts = bench::platform::host_facts(); + bench::Report report; + report.host = bench::HostInfo{facts.os, facts.arch, facts.cpu_model, + facts.logical_cores, facts.physical_cores, + facts.heterogeneous, facts.ram_bytes, opts->compiler}; + report.started_at = bench::platform::iso_now(); + report.under_test = opts->under_test; + + // Live progress. It goes to STDERR so that stdout stays the report and can + // still be redirected on its own, and it is flushed on every line because + // the whole point is to be readable WHILE the run is happening — a buffered + // progress line arrives with the summary, which is exactly too late. + const auto t0 = std::chrono::steady_clock::now(); + std::string current_cell; + const auto elapsed = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + }; + + bench::RunOptions ro; + ro.work_root = opts->work; + ro.shape = opts->shape; + ro.jobs = opts->jobs; + ro.runs_override = opts->runs; + ro.compiler = opts->compiler; + ro.buildfiles = opts->buildfiles; + ro.project = opts->project; + // --hub/--leaf/--body are PROJECT-RELATIVE, and have to be resolved here. + // + // They used to be taken as given, which made them relative to the harness's + // working directory instead. That is the same directory only when you are + // benchmarking the tree you happen to be standing in — true for mcpp + // measuring itself, false for every other project — and the failure is + // silent: `exists()` says no, the cell reports `skipped`, and the run still + // exits 0. An absolute path is left alone, so `--hub /tmp/x.cppm` still works. + const auto in_project = [&](const std::filesystem::path& p) { + if (p.empty() || p.is_absolute()) return p; + return opts->project.empty() ? p : opts->project / p; + }; + ro.project_targets = bench::fixture::Targets{in_project(opts->hub), + in_project(opts->leaf), + in_project(opts->body)}; + ro.timeout_s = opts->timeout_s; + ro.on_progress = [&](std::string_view what) { + bool first = true; + for (const auto part : std::views::split(what, '\n')) { + const std::string_view line(part.begin(), part.end()); + if (line.empty() && !first) continue; + if (first) std::print(std::cerr, "[{:>7.1f}s] {} {}\n", elapsed(), current_cell, line); + else std::print(std::cerr, " | {}\n", line); + first = false; + } + std::cerr.flush(); + }; + // Defined here rather than after the runner: the journal keys on it. + const auto fixture_name = opts->project.empty() + ? std::format("synth-{}x{}", opts->shape.units, opts->shape.fanin) + : opts->project.filename().string(); + const std::string fixture_name_for_key = fixture_name; + std::size_t units_done = 0; + + // ── The journal, and what makes its records usable ──────────────────── + // + // Beside the report, so a run's data and its record of how far it got stay + // together. Every measured unit is appended the moment it exists. + // ── The run's identity, and where its cache lives ───────────────────── + // + // One fingerprint over the whole configuration, the same way `mcpp build` + // works: same configuration → same directory → resume; different + // configuration → a different directory, instead of overwriting. + // `--id` is just one more field in the hash. + bench::RunId id; + { + std::string cfg = std::format( + "id={};profile={};runs={};compiler={};project={};buildfiles={};" + "units={};fanin={};weight={}", + opts->run_id, opts->profile, opts->runs, opts->compiler, + opts->project.string(), opts->buildfiles.string(), + opts->shape.units, opts->shape.fanin, opts->shape.weight); + for (const auto& e : engines) { cfg += ";e="; cfg += e->name(); } + for (const auto v : opts->variants) { cfg += ";v="; cfg += to_string(v); } + for (const auto sc : opts->scenarios) { cfg += ";s="; cfg += to_string(sc); } + id = bench::RunId::of(std::move(cfg)); + } + + // `.mbench//` beside the working directory, like a build cache. + const auto cache_dir = opts->cache_root / id.fingerprint; + std::error_code cache_ec; + std::filesystem::create_directories(cache_dir, cache_ec); + { + // The configuration in full, beside the journal: a fingerprint nobody + // can decode is a fingerprint nobody trusts. + std::ofstream cfg_out(cache_dir / "config.txt", std::ios::trunc); + if (cfg_out) cfg_out << id.config << '\n'; + } + + const bench::Journal journal(cache_dir / "journal.jsonl"); + auto loaded = journal.load(id.str()); + + // ⚠️ A RESUME MUST NOT SILENTLY SPAN TWO BUILDS OF WHAT IS UNDER TEST. + // + // The fingerprint deliberately excludes the binary, so that rebuilding does + // not throw the cache away — and that is exactly what makes this possible: + // resume, rebuild, resume again, and the report carries samples from two + // different binaries under one heading. `engine_version` catches a version + // change, but mcpp's version is a DATE, so every commit on a branch reports + // the same one and that check sees nothing. + // + // Recorded here rather than fingerprinted, and REPORTED on mismatch: the + // caller is entitled to resume across a rebuild (that is the normal case + // while developing), but not to be unaware that they did. + { + const auto stamp = cache_dir / "under-test.txt"; + std::string previous; + if (std::ifstream in(stamp); in) std::getline(in, previous); + if (!previous.empty() && !opts->under_test.empty() && previous != opts->under_test) + std::println(std::cerr, + "bench: ⚠️ this cache holds samples measured with '{}' but this run is " + "'{}'. The report will mix them. Use --id to fork a fresh cache, or " + "delete {}.", + previous, opts->under_test, cache_dir.string()); + if (!opts->under_test.empty() && previous != opts->under_test) { + std::ofstream out(stamp, std::ios::trunc); + if (out) out << opts->under_test << '\n'; + } + } + + if (loaded.skipped_other_id) + std::println(std::cerr, + "bench: {} journal entries carry a different fingerprint and are ignored " + "(this run {}, theirs {}).", + loaded.skipped_other_id, id.str(), loaded.other_id); + if (loaded.skipped_unparsable) + std::println(std::cerr, + "bench: {} journal line(s) unreadable and skipped — the last line of a " + "killed run is expected to be half-written.", loaded.skipped_unparsable); + std::println("run id : {} ({})", id.fingerprint, + loaded.units.empty() ? "fresh" + : std::format("resuming, {} unit(s) recorded", + loaded.units.size())); + + ro.already_done = [&](std::string_view sc, std::string_view en, + std::string_view va, int run) { + return loaded.units.contains( + bench::Journal::unit_id(fixture_name_for_key, va, sc, en, run)); + }; + ro.recorded_sample = [&](std::string_view sc, std::string_view en, + std::string_view va, int run) { + const auto it = loaded.units.find( + bench::Journal::unit_id(fixture_name_for_key, va, sc, en, run)); + if (it == loaded.units.end()) return std::pair{0.0, 0}; + // The binary is NOT part of the fingerprint — a rebuild must not throw + // the cache away — so adopting a record measured with a different one is + // reported instead. This is the one way a resume can mix two binaries. + for (const auto& e : engines) { + if (e->name() != en) continue; + const auto now = e->probe().note; + if (!it->second.engine_version.empty() && it->second.engine_version != now) + std::println(std::cerr, + "bench: adopting a {} record measured with '{}', but it now " + "reports '{}' — both are in this table.", + en, it->second.engine_version, now); + break; + } + return std::pair{it->second.wall_s, it->second.exit_code}; + }; + ro.record = [&](std::string_view sc, std::string_view en, std::string_view va, + int run, double wall_s, int exit_code) { + std::string ver; + for (const auto& e : engines) if (e->name() == en) { ver = e->probe().note; break; } + journal.append(bench::JournalEntry{id.str(), ver, fixture_name_for_key, + std::string(va), std::string(sc), + std::string(en), run, wall_s, exit_code}); + }; + + const bench::Runner runner(ro); + + + std::println("host : {} {} · {} · {} logical / {} physical{}", + facts.os, facts.arch, facts.cpu_model, facts.logical_cores, + facts.physical_cores, facts.heterogeneous ? " (heterogeneous)" : ""); + if (opts->project.empty()) + std::println("fixture: {} units, fanin {}, weight {}", + opts->shape.units, opts->shape.fanin, opts->shape.weight); + else + std::println("project: {} (measured in place)", opts->project.string()); + std::println(""); + + for (const auto& engine : engines) { + for (const auto variant : opts->variants) { + // Materialise once per (engine, variant): the scenarios of a pair + // share a tree on purpose, since generation time belongs to none of + // them. Cells that will not run skip the cost entirely. + const bool will_run = engine->probe().present && engine->supports(variant, opts->compiler); + std::optional inst; + if (will_run) inst = runner.materialise(engine->name(), variant); + + for (const auto scenario : opts->scenarios) { + bench::CellResult cell; + if (will_run) { + current_cell = bench::CellKey{ + std::string(engine->name()), compiler_label, opts->profile, + std::string(to_string(scenario)), fixture_name, + std::string(to_string(variant))}.str(); + cell = runner.measure(*engine, *inst, variant, scenario, opts->profile, + opts->compiler, compiler_label, fixture_name); + } else { + cell.key = bench::CellKey{std::string(engine->name()), compiler_label, + opts->profile, std::string(to_string(scenario)), + fixture_name, std::string(to_string(variant))}; + const auto a = engine->probe(); + cell.status = bench::Status::Unavailable; + cell.note = a.present ? engine->unsupported_reason(variant, opts->compiler) : a.note; + } + + if (cell.status == bench::Status::Ok) { + std::println("{:<38} {:>8.2f}s (min {:.2f} / max {:.2f}, n={})", + cell.key.str(), cell.median_s(), cell.min_s(), + cell.max_s(), cell.samples.size()); + } else { + std::println("{:<38} {:>9} {}", cell.key.str(), + bench::to_string(cell.status), cell.note); + } + report.cells.push_back(std::move(cell)); + } + } + } + + // Normalised summary. A column of raw seconds answers "how long"; a column of + // ratios answers "compared to what", and the second question is the one a + // build-engine comparison is actually asking. + if (!opts->baseline.empty()) { + std::println(""); + std::println("=== relative to {} (>1.00 = slower than the baseline) ===", opts->baseline); + + // Group by (variant, scenario): a ratio only means something against the + // SAME source form and the SAME perturbation. + std::vector> groups; + for (const auto& c : report.cells) { + auto key = std::pair{c.key.variant, c.key.scenario}; + if (std::ranges::find(groups, key) == groups.end()) groups.push_back(key); + } + + for (const auto& [variant, scenario] : groups) { + const bench::CellResult* base = nullptr; + for (const auto& c : report.cells) + if (c.key.variant == variant && c.key.scenario == scenario + && c.key.engine.find(opts->baseline) != std::string::npos + && c.status == bench::Status::Ok) + base = &c; + + std::println(""); + std::println("-- {} / {} --", variant, scenario); + if (!base) { + // Saying "no baseline" beats printing ratios against nothing, and + // beats silently omitting the group. + std::println(" (no successful '{}' cell here; ratios omitted)", opts->baseline); + } + for (const auto& c : report.cells) { + if (c.key.variant != variant || c.key.scenario != scenario) continue; + if (c.status != bench::Status::Ok) { + std::println(" {:<22} {:>9} {}", c.key.engine, + bench::to_string(c.status), c.note); + continue; + } + if (base && base->median_s() > 0.0) + std::println(" {:<22} {:>8.2f}s {:>6.2f}x{}", c.key.engine, + c.median_s(), c.median_s() / base->median_s(), + (&c == base) ? " <- baseline" : ""); + else + std::println(" {:<22} {:>8.2f}s", c.key.engine, c.median_s()); + } + } + } + + std::ofstream out(opts->out, std::ios::binary | std::ios::trunc); + out << bench::to_json(report); + std::println(""); + std::println("report : {}", opts->out.string()); + + // --- exit status ------------------------------------------------------- + // + // A benchmark that cannot assert on TIMINGS (shared runners, changing CPU + // models) can still assert that it MEASURED SOMETHING. Not doing so cost + // this suite weeks: a matrix job in which 48 of 72 cells failed, 18 were + // unavailable and the only 6 that ran were one engine on one variant, + // reported success — as did an xlings job whose every single cell was + // skipped because --hub named a file that no longer existed. + // + // `failed` is the finding: the engine ran and produced no artifact. + // `unavailable` and `skipped` are gaps, are documented in the note, and + // never fail the run. + // --- internal consistency: a `cold` build must out-work its own `noop` --- + // + // NOT a performance threshold. The suite deliberately has none, because a + // shared runner's variance would turn into red crosses people mute. This is + // an INVARIANT: `cold` removes the build directory and rebuilds everything, + // `noop` does nothing, so a `cold` in the same league as its own engine's + // `noop` did not rebuild — the engine's clean() missed where that engine + // actually keeps its artifacts. + // + // It is worth a check because the failure is invisible: the cell is `ok`, + // it carries samples, and it reports a spectacular number. xmake on the + // pinned mcpp workload produced `cold 0.60s` next to `touch-hub 82.79s`, + // and nothing in the report said the first of those was not a build. + // + // 2x is the same floor README §4a R1 uses for "this is measuring process + // startup". For every other scenario that is a caveat a reader applies; for + // `cold` it is a defect. + std::size_t suspect = 0; + for (const auto& c : report.cells) { + if (c.status != bench::Status::Ok || c.key.scenario != "cold") continue; + const bench::CellResult* noop = nullptr; + for (const auto& n : report.cells) + if (n.status == bench::Status::Ok && n.key.scenario == "noop" + && n.key.engine == c.key.engine && n.key.variant == c.key.variant) + noop = &n; + if (!noop || noop->median_s() <= 0.0) continue; + if (c.median_s() < noop->median_s() * 2.0) { + ++suspect; + std::println(std::cerr, + "bench: {} reports cold={:.2f}s against its own noop={:.2f}s — a cold " + "build cannot be that cheap, so clean() did not remove this engine's " + "artifacts and the cell measured an up-to-date tree.", + c.key.str(), c.median_s(), noop->median_s()); + } + } + + // --- cross-engine consistency: a cold build cannot be 20x cheaper than every + // other engine building the same sources --- + // + // The check above is RELATIVE TO ONE ENGINE, so it is blind to the case that + // actually shipped: an engine that builds NOTHING has a cheap cold AND a + // cheap noop, and their ratio looks healthy. bazel on the pinned mcpp tree + // reported cold=0.43s / noop=0.22s — a ratio of 1.95, missing the 2x trip + // wire by one hundredth of a second — while compiling zero of 137 units. + // + // Peers are the honest yardstick here, and they are already in the report: + // engines in the same cell compile the same sources on the same machine, so + // a 20x gap is not a fast engine, it is a different workload. The factor is + // deliberately far past any real result (mcpp's best measured win over cmake + // is 3.1x) so this fires on phantoms and never on a good number. + for (const auto& c : report.cells) { + if (c.status != bench::Status::Ok || c.key.scenario != "cold") continue; + std::vector peers; + for (const auto& p : report.cells) + if (p.status == bench::Status::Ok && p.key.scenario == "cold" + && p.key.fixture == c.key.fixture && p.key.variant == c.key.variant + && p.key.engine != c.key.engine && p.median_s() > 0.0) + peers.push_back(p.median_s()); + if (peers.empty() || c.median_s() <= 0.0) continue; + std::ranges::sort(peers); + const double peer_median = peers[peers.size() / 2]; + if (c.median_s() * 20.0 < peer_median) { + ++suspect; + std::println(std::cerr, + "bench: {} reports cold={:.2f}s while other engines building the same " + "sources take {:.2f}s — {:.0f}x is not a faster engine, it is a smaller " + "workload; check that this engine's description actually names the sources.", + c.key.str(), c.median_s(), peer_median, peer_median / c.median_s()); + } + } + + std::size_t ok = 0, failed = 0, waived = 0; + for (const auto& c : report.cells) { + if (c.status == bench::Status::Ok) { ++ok; continue; } + if (c.status != bench::Status::Failed) continue; + if (listed(opts->allow_failed, c.key.engine)) ++waived; else ++failed; + } + std::println("cells : {} ok, {} failed{}, {} not applicable", ok, failed, + waived ? std::format(" ({} waived by --allow-failed)", waived) : "", + report.cells.size() - ok - failed - waived); + + // ── A WAIVER MAY HIDE A FAILURE. IT MUST NOT HIDE THE COMPARISON. ─────── + // + // `--allow-failed` exists so ONE arm with a documented gap does not turn a + // whole matrix red. It was never meant to cover every foreign engine at + // once — but nothing stopped it, and the result is a job that exits 0 while + // its log reads + // + // => install ftxui v6.1.9 .. failed + // => install mcpplibs-tinyhttps 0.2.9 .. failed + // error: missing std dependency for module ... + // cells : 10 ok, 0 failed (10 waived by --allow-failed) + // + // A cell that calls itself a three-engine comparison, measures mcpp against + // mcpp, and reports green. That is the exact shape this suite exists to + // remove, so it may not be the suite's own. + // + // Named per ENGINE rather than counted: "10 waived" is a number, "cmake and + // xmake produced nothing here" is the fact a reader needs. + { + std::map> per_engine; // ok, waived + for (const auto& c : report.cells) { + auto& e = per_engine[c.key.engine]; + if (c.status == bench::Status::Ok) ++e.first; + else if (c.status == bench::Status::Failed && + listed(opts->allow_failed, c.key.engine)) ++e.second; + } + std::vector silenced; + for (const auto& [engine, counts] : per_engine) + if (counts.first == 0 && counts.second > 0) silenced.push_back(engine); + if (!silenced.empty()) { + std::string list; + for (const auto& e : silenced) { if (!list.empty()) list += ", "; list += e; } + std::println(std::cerr, + "bench: WAIVED AWAY ENTIRELY: {} — every cell of {} failed and was " + "waived, so this run contains no measurement of {} at all. The run " + "is green by policy, not because those engines worked; whatever this " + "report is compared against, it is not them.", + list, silenced.size() == 1 ? "it" : "them", + silenced.size() == 1 ? "it" : "them"); + } + } + + if (failed) { + std::println(std::cerr, + "bench: {} cell(s) FAILED — the engine ran and produced no artifact. " + "Each one's reason and log tail are above.", failed); + return 1; + } + if (suspect) { + std::println(std::cerr, + "bench: {} `cold` cell(s) did not actually rebuild (see above). Those " + "numbers are not measurements of a cold build.", suspect); + return 1; + } + if (ok == 0) { + std::println(std::cerr, + "bench: nothing was measured. Every cell was unavailable or skipped, " + "so this run contains no data; see each cell's note for why."); + return 1; + } + return 0; +} diff --git a/bench/src/platform.cpp b/bench/src/platform.cpp new file mode 100644 index 00000000..e82dcd1d --- /dev/null +++ b/bench/src/platform.cpp @@ -0,0 +1,178 @@ +// bench.platform — implementation. +// +// `module bench.platform;` with no `export`: an implementation unit. The +// interface declares; the bodies live here, so nothing below reaches an +// importer's BMI. Everything the suite does to the OS still passes through this +// one module, and the `#if defined(...)` blocks the partitions cannot own — the +// architecture probe, and MSVC's getenv deprecation — stay confined to this +// file rather than spreading into runner or the engines. +// +// A module implementation unit implicitly imports its primary interface, so the +// `platform_impl::` names the :posix / :windows partitions export are visible. +module bench.platform; + +import std; + +namespace bench::platform { + +ScopedEnv::ScopedEnv(std::string key, const std::string& value) : key_(std::move(key)) { + // MSVC's CRT deprecates getenv in favour of _dupenv_s. Reading it is + // safe here (single-threaded setup, value copied immediately) and the + // portable spelling keeps this out of the platform partitions. +#if defined(_MSC_VER) +#pragma warning(suppress : 4996) +#endif + if (const char* prev = std::getenv(key_.c_str())) { + had_previous_ = true; + previous_ = prev; + } + set_env(key_, value); + } + ScopedEnv::~ScopedEnv() { + if (had_previous_) set_env(key_, previous_); + else unset_env(key_); + } + +bool RunResult::ok() const { return exit_code == 0; } + +bool RunResult::started() const { return !launch_failed; } + +RunResult run(const std::vector& argv, + const std::filesystem::path& cwd, + const std::filesystem::path& log, + double timeout_s) { + double wall = 0.0; + bool hung = false; + std::string why; + bool failed_to_launch = false; + const int rc = run_process(argv, cwd, log, &wall, timeout_s, &hung, &why, + &failed_to_launch); + return RunResult{wall, rc, hung, std::move(why), failed_to_launch}; +} + +bool log_mentions(const std::filesystem::path& p, + std::initializer_list markers) { + std::ifstream in(p, std::ios::binary); + if (!in) return false; + std::string line; + while (std::getline(in, line)) + for (const auto m : markers) + if (line.find(m) != std::string::npos) return true; + return false; +} + +std::string log_grep(const std::filesystem::path& p, + std::initializer_list markers, + std::size_t max) { + std::ifstream in(p, std::ios::binary); + if (!in) return {}; + std::string out, line; + std::size_t kept = 0; + // ⚠️ THE MESSAGE IS OFTEN ON THE NEXT LINE. cmake writes + // CMake Error at xpkg_source_library.cmake:231 (message): + // bench: 's manifest has no `sources` list + // and a grep that returns only matching lines keeps the location and throws + // away WHAT WENT WRONG. That happened three separate times today, each time + // costing a round trip to CI for a sentence that was already in the file. + // `after` carries the following lines of a hit through. + std::size_t after = 0; + while (kept < max && std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + bool hit = false; + for (const auto m : markers) + if (line.find(m) != std::string::npos) { hit = true; break; } + if (!hit) { + if (after == 0 || line.empty()) continue; + --after; // trailing context of the previous hit + } else { + after = 2; + } + // Long lines here are usually a whole compiler command line; the cause + // is at the front of them. + if (line.size() > 400) { line.resize(400); line += " …"; } + out += " "; out += line; out += '\n'; + ++kept; + } + return out; +} + +std::string tail_of(const std::filesystem::path& p, std::size_t lines) { + std::ifstream in(p, std::ios::binary); + if (!in) return {}; + std::deque keep; + std::string line; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + keep.push_back(std::move(line)); + if (keep.size() > lines) keep.pop_front(); + } + std::string out; + for (const auto& l : keep) { out += l; out += '\n'; } + return out; +} + +bool have_program(const std::vector& version_argv) { + return run(version_argv).started(); +} + +std::optional run_capture(const std::vector& argv, + const std::filesystem::path& cwd, + RunResult* result) { + std::error_code ec; + auto tmp = std::filesystem::temp_directory_path(ec); + if (ec) return std::nullopt; + tmp /= std::format("bench-capture-{}.txt", + std::chrono::steady_clock::now().time_since_epoch().count()); + + const auto r = run(argv, cwd, tmp); + if (result) *result = r; + if (!r.started()) { std::filesystem::remove(tmp, ec); return std::nullopt; } + + std::ifstream in(tmp, std::ios::binary); + std::string text; + if (in) text.assign((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + in.close(); + std::filesystem::remove(tmp, ec); + return text; +} + +HostFacts host_facts() { + HostFacts f; + f.os = std::string(OS_NAME); + f.cpu_model = platform_impl::cpu_model(); + f.logical_cores = platform_impl::cpu_logical(); + f.physical_cores = platform_impl::cpu_physical(); + f.heterogeneous = platform_impl::heterogeneous_cpu(); + f.ram_bytes = platform_impl::ram_bytes(); + // Architecture, unlike the OS, is not a partition concern: both partitions + // would carry an identical copy of this. It is a property of the build, so + // it is detected once, here. +#if defined(__aarch64__) || defined(_M_ARM64) + f.arch = "aarch64"; +#elif defined(__x86_64__) || defined(_M_X64) + f.arch = "x86_64"; +#else + f.arch = "unknown"; +#endif + return f; +} + +void remove_tree(const std::filesystem::path& p) { + std::error_code ec; + std::filesystem::remove_all(p, ec); // absent is success, not failure +} + +bool touch(const std::filesystem::path& p) { + std::error_code ec; + if (!std::filesystem::exists(p, ec)) return false; + std::filesystem::last_write_time(p, std::filesystem::file_time_type::clock::now(), ec); + return !ec; +} + +std::string iso_now() { + return std::format("{:%FT%TZ}", + std::chrono::floor( + std::chrono::system_clock::now())); +} + +} // namespace bench::platform diff --git a/bench/src/platform.cppm b/bench/src/platform.cppm new file mode 100644 index 00000000..412bc2ca --- /dev/null +++ b/bench/src/platform.cppm @@ -0,0 +1,164 @@ +// bench.platform — the suite's single door to the operating system. +// +// The per-platform partitions each guard their whole body with one macro and +// export the SAME names, so exactly one definition of each survives in any +// build. This module re-exports them and adds the parts that need no platform +// knowledge. Consequence: `#if defined(_WIN32)` appears in the two partitions +// and nowhere else — runner, engines, fixtures and analysis contain no platform +// conditionals at all. +export module bench.platform; + +import std; + +export import :posix; +export import :windows; + +export namespace bench::platform { + +// Platform-selected primitives, lifted out of the partitions. +using platform_impl::OS_NAME; +using platform_impl::cpu_logical; +using platform_impl::cpu_physical; +using platform_impl::cpu_model; +using platform_impl::ram_bytes; +using platform_impl::heterogeneous_cpu; +using platform_impl::run_process; +using platform_impl::set_env; +using platform_impl::unset_env; + +// Sets an environment variable for the lifetime of the guard and restores the +// previous state — including "was not set at all", which is distinct from "was +// empty" to a child process. Engines use this to toggle a build flag for one +// measured cell without leaking it into the next. +class ScopedEnv { +public: + ScopedEnv(std::string key, const std::string& value); + ~ScopedEnv(); + ScopedEnv(const ScopedEnv&) = delete; + ScopedEnv& operator=(const ScopedEnv&) = delete; + +private: + std::string key_; + std::string previous_; + bool had_previous_{}; +}; + +struct RunResult { + double wall_s{}; + int exit_code{}; + // Set when the child was killed for exceeding its deadline. A separate flag + // rather than a reserved exit code: 124 is the `timeout(1)` convention and + // is a perfectly legal thing for a build tool to exit with on its own, so + // "hung" and "exited 124" must stay distinguishable. + bool timed_out{}; + // WHY the child could not be launched, in the OS's own words. Empty unless + // `started()` is false. + // + // "could not start the process" is true and useless: on Windows it covers a + // program that is not on PATH, a cwd that does not exist, a bad handle and + // an ACL, and those are four different fixes. A windows/clang cell reported + // exactly that message while `xmake --version` had just succeeded in the + // same job — the only difference between the two calls being a cwd and a + // log path — and there was no way to tell which from CI. One + // GetLastError/errno turns that into a sentence. + std::string start_error; + [[nodiscard]] bool ok() const; + // Distinguishes "could not start" from "started and failed" — the whole + // basis for reporting an engine as unavailable rather than broken. + // + // ⚠️ NOT `exit_code >= 0`. That was the test, and on Windows it is wrong in + // the one case that matters: a child that CRASHES exits with a status like + // 0xC0000135 (a DLL it needs is missing) or 0xC0000005 (access violation), + // and `GetExitCodeProcess` hands back a DWORD that becomes a NEGATIVE int. + // The harness then reported + // + // could not start the process (no log written) — + // check the engine's program path + // + // for an xmake that `xlings install` had just installed successfully and + // that the harness's own probe had just run. The advice was wrong, the + // diagnosis was wrong, and it cost two matrix cycles chasing a PATH that was + // never the problem. + // + // Launch failure is now its own fact, reported by the platform layer, + // instead of being inferred from the shape of a number that has a different + // meaning on each OS. + [[nodiscard]] bool started() const; + bool launch_failed{}; +}; + +// Run argv, discarding the child's output unless a log path is given. The +// harness never lets build noise reach its own stdout: the report IS the +// output, and a mixed stream cannot be parsed. +// +// `timeout_s` <= 0 waits forever, which is right for a version probe and wrong +// for a build — see run_process. +RunResult run(const std::vector& argv, + const std::filesystem::path& cwd = {}, + const std::filesystem::path& log = {}, + double timeout_s = 0.0); + +// The last `lines` lines of a file, for showing WHY a cell failed. +// +// Without this a red benchmark is as uninformative as the green one it +// replaces: the harness records `see .../logs/cmake-cold.log`, and on a CI +// runner that file is deleted with the machine. Every module cell in the matrix +// failed for weeks behind exactly that sentence. +// Does the log contain any of these markers? Used to decide how much of it is +// worth showing — a crash needs far more context than a compile error. +bool log_mentions(const std::filesystem::path& p, + std::initializer_list markers); + +// The lines anywhere in `p` that look like a cause, not a progress report. +// +// A tail cannot answer "why did this fail" for a tool that prints a line per +// translation unit: the error scrolled past hundreds of lines ago and the last +// 20 are all `[ 2%]: generating.module.deps ...`. That is exactly how an +// `xmake exited 255` cell reached CI with nothing to diagnose it by. +// +// Deliberately a keyword sieve rather than per-engine parsing: every engine +// here is a different program with a different diagnostic format, and one that +// is merely APPROXIMATELY right on all of them beats four that are exactly +// right until a tool changes its wording. False positives cost a line of noise; +// a false negative costs a matrix cycle. +std::string log_grep(const std::filesystem::path& p, + std::initializer_list markers, + std::size_t max = 12); + +std::string tail_of(const std::filesystem::path& p, std::size_t lines = 20); + +bool have_program(const std::vector& version_argv); + +// Run argv and return its combined stdout+stderr, or nullopt if it could not be +// started. Goes through a temp file rather than a pipe: a pipe needs +// platform-specific plumbing on both sides, and the outputs captured here are +// version banners — a few dozen bytes, once per engine. +// `result`, when given, receives the child's RunResult so a caller needing both +// the output and the exit status does not have to run the command twice. +std::optional run_capture(const std::vector& argv, + const std::filesystem::path& cwd = {}, + RunResult* result = nullptr); + +struct HostFacts { + std::string os; + std::string arch; + std::string cpu_model; + int logical_cores{}; + int physical_cores{}; + bool heterogeneous{}; + std::uint64_t ram_bytes{}; +}; + +HostFacts host_facts(); + +// --- portable helpers: std::filesystem needs no per-platform split --------- + +void remove_tree(const std::filesystem::path& p); + +// mtime bump with no content change — the `touch-*` scenarios turn on exactly +// this distinction, so it must not rewrite the file. +bool touch(const std::filesystem::path& p); + +std::string iso_now(); + +} // namespace bench::platform diff --git a/bench/src/platform/posix.cppm b/bench/src/platform/posix.cppm new file mode 100644 index 00000000..965bf91b --- /dev/null +++ b/bench/src/platform/posix.cppm @@ -0,0 +1,310 @@ +// bench.platform:posix — process launch, wall-clock timing and host facts on +// POSIX (Linux + macOS). +// +// SHAPE: the ENTIRE body is inside `#if !defined(_WIN32)`. On Windows this file +// still compiles — it just declares the partition and exports nothing. The peer +// partition exports the same names, so exactly one definition of each exists in +// any build and the compiler selects the platform for us. No stubs, no dead +// branches, no `if constexpr` dispatch at the call sites. +// +// This is the convention used by xlings' src/platform/*.cppm; bench follows it +// so the two codebases read the same way. +module; + +#if !defined(_WIN32) +#include +#include +#include // kill/SIGKILL for the run_process timeout +#include +#include +#include // setenv / unsetenv — needed on Darwin too, where they + // live in <_stdlib.h> and are NOT reachable through the + // other POSIX headers this file pulls in +#include +#include +#if defined(__APPLE__) +#include +#include +#else +#include +#include +#endif +extern "C" char** environ; +#endif + +export module bench.platform:posix; + +import std; + +#if !defined(_WIN32) + +namespace bench::platform_impl { + +export constexpr std::string_view OS_NAME = +#if defined(__APPLE__) + "macos"; +#else + "linux"; +#endif + +static unsigned long long now_ns() { + struct timespec ts{}; + ::clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000ULL + + static_cast(ts.tv_nsec); +} + +// Runs argv in `cwd`, child stdout+stderr to `log` (empty → discarded). +// Returns the exit status, or -1 if the child could not be started; the +// distinction matters because "could not start" is what tells probe() an engine +// is absent rather than broken. +// +// `timeout_s` <= 0 means wait forever. A build engine CAN hang — bazel fetching +// a module from a registry that never answers is the one seen here — and a +// benchmark that hangs with it is worse than one that fails: the CI job burns +// its whole budget and the log says nothing, because the harness only prints a +// cell once the cell is over. Two jobs sat 25 minutes inside one child that way, +// on a cell whose sibling finished in four. +export int run_process(const std::vector& argv, + const std::filesystem::path& cwd, + const std::filesystem::path& log, + double* out_wall_s, + double timeout_s = 0.0, + bool* out_timeout = nullptr, + std::string* out_error = nullptr, + bool* out_launch_failed = nullptr) { + if (out_wall_s) *out_wall_s = 0.0; + if (out_timeout) *out_timeout = false; + if (argv.empty()) { + if (out_error) *out_error = "empty argv"; + if (out_launch_failed) *out_launch_failed = true; + return -1; + } + + std::vector raw; + raw.reserve(argv.size() + 1); + for (const auto& a : argv) raw.push_back(const_cast(a.c_str())); + raw.push_back(nullptr); + + posix_spawn_file_actions_t actions; + if (::posix_spawn_file_actions_init(&actions) != 0) return -1; + + // chdir must happen in the CHILD. A process-wide chdir here would race with + // everything else the harness does and would leave the wrong cwd behind on + // any early return. + if (!cwd.empty()) { + const std::string cwd_s = cwd.string(); + if (::posix_spawn_file_actions_addchdir_np(&actions, cwd_s.c_str()) != 0) { + ::posix_spawn_file_actions_destroy(&actions); + return -1; + } + } + + const std::string log_s = log.empty() ? std::string("/dev/null") : log.string(); + // APPEND, not truncate. A cell runs configure, then a seed build, then N + // timed builds — all into one log path. Truncating meant the build's output + // erased the configure's, so when a cold cell came back at 0.60s the log + // held one line ("build ok, spent 0.111s") and nothing about the configure + // that had just been asked to set the output directory. The runner clears + // the file once per cell (see Runner::measure), which is the right grain. + const int flags = log.empty() ? O_WRONLY : (O_WRONLY | O_CREAT | O_APPEND); + ::posix_spawn_file_actions_addopen(&actions, 1, log_s.c_str(), flags, 0644); + ::posix_spawn_file_actions_adddup2(&actions, 1, 2); + + // The child leads its OWN process group, so a timeout can kill everything it + // started rather than just the tool itself. + // + // A build engine is a process tree: ninja and a pool of compilers, bazel and + // a server, xmake and its own children. Killing only the direct child leaves + // that pool running — reparented, invisible, and still holding the CPU while + // the NEXT cells are being timed. A single timeout would then inflate every + // measurement after it, with nothing in the report to show why. That is the + // expensive shape: not a failure, a quietly wrong number. + // + // It must be a new group and not the harness's: `kill(-pid)` on a shared + // group reaches the harness too. + posix_spawnattr_t attr; + bool own_group = false; + if (::posix_spawnattr_init(&attr) == 0) { + if (::posix_spawnattr_setpgroup(&attr, 0) == 0 && + ::posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP) == 0) + own_group = true; + } + + const unsigned long long t0 = now_ns(); + ::pid_t pid = 0; + const int rc = ::posix_spawnp(&pid, raw[0], &actions, own_group ? &attr : nullptr, + raw.data(), ::environ); + ::posix_spawn_file_actions_destroy(&actions); + ::posix_spawnattr_destroy(&attr); + if (rc != 0) { + // posix_spawnp reports through its RETURN VALUE, not errno. + if (out_error) + *out_error = std::format("posix_spawnp('{}'): {}", raw[0], std::strerror(rc)); + if (out_launch_failed) *out_launch_failed = true; + return -1; + } + + int status = 0; + if (timeout_s <= 0.0) { + while (::waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) return -1; + } + } else { + // Poll rather than alarm/sigtimedwait: the harness must not install a + // signal handler, because the child inherits the disposition and a + // compiler that ignores SIGALRM is a compiler that behaves differently + // under measurement than in real use. + // + // 20ms is well under the noise floor of anything timed here (the + // fastest measured cell is a ~10ms noop) and costs ~50 wakeups a second + // on a machine already running a compiler. + const auto deadline_ns = t0 + static_cast(timeout_s * 1e9); + for (;;) { + const ::pid_t r = ::waitpid(pid, &status, WNOHANG); + if (r == pid) break; + if (r < 0) { if (errno == EINTR) continue; return -1; } + if (now_ns() >= deadline_ns) { + // SIGKILL, not SIGTERM: the thing being killed is a build tool + // that may have spawned a job server and a pool of compilers, + // and a polite signal it chooses to handle leaves the harness + // waiting on exactly the hang it is trying to escape. + // + // THE GROUP, not just the child — that is what the spawn above + // set up. `kill(-pid)` reaches the compilers the tool started; + // killing the tool alone leaves them running and stealing CPU + // from every cell measured afterwards. Falls back to the single + // process when the group could not be set (the flag is POSIX, + // but this must not depend on it succeeding). + if (own_group) ::kill(-pid, SIGKILL); + ::kill(pid, SIGKILL); + while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) {} + if (out_wall_s) *out_wall_s = static_cast(now_ns() - t0) / 1e9; + if (out_timeout) *out_timeout = true; + // 124 is what `timeout(1)` reports, so the number is already + // familiar; `out_timeout` is what callers actually branch on, + // since a build tool may legitimately exit 124 on its own. + return 124; + } + struct timespec nap{0, 20 * 1000 * 1000}; + ::nanosleep(&nap, nullptr); + } + } + if (out_wall_s) *out_wall_s = static_cast(now_ns() - t0) / 1e9; + + if (WIFEXITED(status)) return WEXITSTATUS(status); + if (WIFSIGNALED(status)) return 128 + WTERMSIG(status); + return -1; +} + +// Environment mutation is a platform concern (setenv here, _putenv_s on +// Windows), so it lives with the other platform primitives rather than being +// #if'd at a call site. +export void set_env(const std::string& key, const std::string& value) { + ::setenv(key.c_str(), value.c_str(), /*overwrite*/ 1); +} + +export void unset_env(const std::string& key) { ::unsetenv(key.c_str()); } + +export int cpu_logical() { + const long n = ::sysconf(_SC_NPROCESSORS_ONLN); + return n > 0 ? static_cast(n) : 1; +} + +#if defined(__APPLE__) + +export int cpu_physical() { + int value = 0; + std::size_t len = sizeof(value); + if (::sysctlbyname("hw.physicalcpu", &value, &len, nullptr, 0) == 0 && value > 0) + return value; + return cpu_logical(); +} + +export std::string cpu_model() { + char buf[256] = {}; + std::size_t len = sizeof(buf); + if (::sysctlbyname("machdep.cpu.brand_string", buf, &len, nullptr, 0) != 0) return {}; + return std::string(buf); +} + +export std::uint64_t ram_bytes() { + std::uint64_t value = 0; + std::size_t len = sizeof(value); + if (::sysctlbyname("hw.memsize", &value, &len, nullptr, 0) == 0) return value; + return 0; +} + +// Apple Silicon is performance/efficiency by construction. `hw.nperflevels` +// states it directly; on Intel Macs it is absent and the answer is no. +export bool heterogeneous_cpu() { + int value = 0; + std::size_t len = sizeof(value); + if (::sysctlbyname("hw.nperflevels", &value, &len, nullptr, 0) == 0) return value > 1; + return false; +} + +#else // Linux + +static std::string read_cpuinfo_field(std::string_view key) { + std::ifstream in("/proc/cpuinfo"); + if (!in) return {}; + std::string line; + while (std::getline(in, line)) { + if (!std::string_view(line).starts_with(key)) continue; + const auto colon = line.find(':'); + if (colon == std::string::npos) continue; + auto value = std::string_view(line).substr(colon + 1); + while (!value.empty() && (value.front() == ' ' || value.front() == '\t')) + value.remove_prefix(1); + return std::string(value); + } + return {}; +} + +export int cpu_physical() { + // "cpu cores" is per-socket. Multi-socket would need the full topology walk; + // over-counting would be worse than falling back to the logical count, which + // is never wrong, only imprecise. + const auto s = read_cpuinfo_field("cpu cores"); + if (!s.empty()) { + if (const int n = std::atoi(s.c_str()); n > 0) return n; + } + return cpu_logical(); +} + +export std::string cpu_model() { return read_cpuinfo_field("model name"); } + +export std::uint64_t ram_bytes() { + const long pages = ::sysconf(_SC_PHYS_PAGES); + const long size = ::sysconf(_SC_PAGE_SIZE); + if (pages > 0 && size > 0) + return static_cast(pages) * static_cast(size); + return 0; +} + +// Hybrid x86 (P-cores + E-cores) reports differing per-CPU max frequencies. +// Cheapest reliable signal short of CPUID: on a homogeneous part every +// cpuinfo_max_freq is identical. No cpufreq at all → cannot tell → say no, +// because a false "heterogeneous" would misread every parallelism figure. +export bool heterogeneous_cpu() { + const int n = cpu_logical(); + if (n <= 1) return false; + long first = -1; + for (int i = 0; i < n; ++i) { + std::ifstream in(std::format( + "/sys/devices/system/cpu/cpu{}/cpufreq/cpuinfo_max_freq", i)); + if (!in) return false; + long v = 0; + if (!(in >> v)) return false; + if (first < 0) first = v; + else if (v != first) return true; + } + return false; +} + +#endif // __APPLE__ + +} // namespace bench::platform_impl + +#endif // !_WIN32 diff --git a/bench/src/platform/windows.cppm b/bench/src/platform/windows.cppm new file mode 100644 index 00000000..96a2f8f9 --- /dev/null +++ b/bench/src/platform/windows.cppm @@ -0,0 +1,326 @@ +// bench.platform:windows — process launch, wall-clock timing and host facts on +// Windows. +// +// SHAPE: the ENTIRE body is inside `#if defined(_WIN32)`. On POSIX this file +// still compiles and exports nothing; the peer partition exports the same names. +// Exactly one definition of each exists in any build, so the platform is chosen +// at compile time with no stubs and no dispatch. Same convention as +// xlings' src/platform/windows.cppm. +module; + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +// RegOpenKeyExA / RegQueryValueExA live in advapi32, which lld-link does NOT +// pull in by default — the build fails at link with "undefined symbol: +// __declspec(dllimport) RegOpenKeyExA". Declaring the dependency in the source +// keeps this partition self-contained instead of pushing an ldflag into every +// consumer's manifest. +#pragma comment(lib, "advapi32.lib") +#endif + +export module bench.platform:windows; + +import std; + +#if defined(_WIN32) + +namespace bench::platform_impl { + +export constexpr std::string_view OS_NAME = "windows"; + +// CreateProcess takes ONE command line, not a vector, and parses it with the +// CRT rules: quote an argument containing space/tab/quote, backslash-escape +// embedded quotes, and DOUBLE any run of backslashes that immediately precedes +// a quote. That last rule is the one usually missed — it is why "works until a +// path ends in a backslash" is a classic Windows bug, and here it would change +// WHICH tree gets built rather than fail loudly. +static void append_quoted(std::string& out, const std::string& arg) { + const bool needs = arg.empty() + || arg.find_first_of(" \t\"") != std::string::npos; + if (!needs) { out += arg; return; } + + out += '"'; + for (std::size_t i = 0; i < arg.size(); ) { + std::size_t slashes = 0; + while (i < arg.size() && arg[i] == '\\') { ++slashes; ++i; } + if (i == arg.size()) { + out.append(slashes * 2, '\\'); + break; + } + if (arg[i] == '"') { + out.append(slashes * 2 + 1, '\\'); + out += '"'; + } else { + out.append(slashes, '\\'); + out += arg[i]; + } + ++i; + } + out += '"'; +} + +// `timeout_s` <= 0 means wait forever — see the peer partition for why a +// benchmark that can hang forever is worse than one that fails. +export int run_process(const std::vector& argv, + const std::filesystem::path& cwd, + const std::filesystem::path& log, + double* out_wall_s, + double timeout_s = 0.0, + bool* out_timeout = nullptr, + std::string* out_error = nullptr, + bool* out_launch_failed = nullptr) { + if (out_wall_s) *out_wall_s = 0.0; + if (out_timeout) *out_timeout = false; + if (argv.empty()) { + if (out_error) *out_error = "empty argv"; + if (out_launch_failed) *out_launch_failed = true; + return -1; + } + + std::string cmdline; + for (std::size_t i = 0; i < argv.size(); ++i) { + if (i) cmdline += ' '; + append_quoted(cmdline, argv[i]); + } + + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + + const std::string log_s = log.empty() ? std::string("NUL") : log.string(); + // FILE_APPEND_DATA + OPEN_ALWAYS, the peer partition's O_APPEND: a cell's + // configure, seed build and timed builds all share one log path, and + // truncating meant each step erased the previous one's output. The runner + // clears the file once per cell. + HANDLE sink = ::CreateFileA(log_s.c_str(), + log.empty() ? GENERIC_WRITE : FILE_APPEND_DATA, + FILE_SHARE_READ, &sa, + log.empty() ? OPEN_EXISTING : OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + + STARTUPINFOA si{}; + si.cb = sizeof(si); + if (sink != INVALID_HANDLE_VALUE) { + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = sink; + si.hStdError = sink; + si.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); + } + + LARGE_INTEGER freq{}, t0{}, t1{}; + ::QueryPerformanceFrequency(&freq); + ::QueryPerformanceCounter(&t0); + + const std::string cwd_s = cwd.string(); + PROCESS_INFORMATION pi{}; + + // A JOB OBJECT so a timeout can kill the whole process TREE. + // + // `TerminateProcess` reaches one process. A build engine is a tree — ninja + // and a pool of compilers, bazel and a server — so killing the tool alone + // leaves that pool running, holding the CPU while the REMAINING CELLS of + // this same run are being timed. One timeout then inflates every number + // after it, and nothing in the report says why. (The POSIX peer solves the + // same problem with a process group.) + // + // CREATE_SUSPENDED so the child cannot spawn anything before it is inside + // the job; KILL_ON_JOB_CLOSE so the tree also dies if the harness itself is + // killed, which is the case a timeout handler cannot cover. + // + // ⚠️ NOT VERIFIED ON WINDOWS — this repository's author has no Windows + // machine, and CI does not time out, so no job here exercises it. It is + // written to be non-regressive rather than to be trusted: every step is + // checked, and any failure falls through to exactly the previous behaviour + // (a plain CreateProcess and a TerminateProcess on the one handle). Wine is + // not evidence either — see the note in .agents/docs about Z: mapping. + HANDLE job = ::CreateJobObjectA(nullptr, nullptr); + if (job) { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION li{}; + li.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!::SetInformationJobObject(job, JobObjectExtendedLimitInformation, + &li, sizeof(li))) { + ::CloseHandle(job); + job = nullptr; + } + } + + const BOOL ok = ::CreateProcessA(nullptr, cmdline.data(), nullptr, nullptr, + /*bInheritHandles*/ TRUE, + job ? CREATE_SUSPENDED : 0, nullptr, + cwd.empty() ? nullptr : cwd_s.c_str(), &si, &pi); + if (!ok) { + // ⚠️ GetLastError FIRST — CloseHandle overwrites it, and this is the + // whole reason the message exists. "could not start the process" covers + // a program not on PATH (2), a cwd that does not exist (267), a bad + // handle and an ACL; those are four different fixes and the harness + // could not tell them apart from CI. + const DWORD err = ::GetLastError(); + if (out_error) { + char* text = nullptr; + const DWORD n = ::FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, err, 0, reinterpret_cast(&text), 0, nullptr); + std::string msg = (n && text) ? std::string(text, n) : std::string(); + if (text) ::LocalFree(text); + while (!msg.empty() && (msg.back() == '\n' || msg.back() == '\r' || + msg.back() == ' ' || msg.back() == '.')) + msg.pop_back(); + *out_error = std::format("CreateProcess failed ({}): {} [cwd={}]", + static_cast(err), + msg.empty() ? "unknown error" : msg, + cwd.empty() ? std::string("") : cwd_s); + } + if (out_launch_failed) *out_launch_failed = true; + if (sink != INVALID_HANDLE_VALUE) ::CloseHandle(sink); + if (job) ::CloseHandle(job); + return -1; + } + if (job) { + // If assignment fails the child is still suspended and must be resumed + // anyway — dropping the job is a lost optimisation, not a lost build. + if (!::AssignProcessToJobObject(job, pi.hProcess)) { + ::CloseHandle(job); + job = nullptr; + } + ::ResumeThread(pi.hThread); + } + + const DWORD wait_ms = timeout_s <= 0.0 + ? INFINITE + : static_cast(timeout_s * 1000.0); + if (::WaitForSingleObject(pi.hProcess, wait_ms) == WAIT_TIMEOUT) { + // THE JOB first, which reaches the compilers the tool started; then the + // process itself, which is all that was possible before the job object + // above and is still the fallback when it could not be created. + // + // TerminateProcess alone does not reach a child's children, and "they + // are reaped when the CI job ends" — the old justification here — only + // covers the harness's own exit. It does not cover the cells measured + // between the timeout and that exit, which are the numbers this suite + // exists to produce. + if (job) ::TerminateJobObject(job, 124); + ::TerminateProcess(pi.hProcess, 124); + ::WaitForSingleObject(pi.hProcess, 5000); + ::QueryPerformanceCounter(&t1); + if (out_wall_s && freq.QuadPart) + *out_wall_s = static_cast(t1.QuadPart - t0.QuadPart) + / static_cast(freq.QuadPart); + if (out_timeout) *out_timeout = true; + ::CloseHandle(pi.hThread); + ::CloseHandle(pi.hProcess); + // The job handle closes on EVERY path. One per spawned process, and a + // matrix spawns hundreds; KILL_ON_JOB_CLOSE also means an unclosed job + // keeps its tree alive rather than merely leaking a handle. Safe here + // because the wait above has already returned. + if (job) ::CloseHandle(job); + if (sink != INVALID_HANDLE_VALUE) ::CloseHandle(sink); + return 124; // the `timeout(1)` convention; callers branch on out_timeout + } + ::QueryPerformanceCounter(&t1); + + DWORD code = 0; + ::GetExitCodeProcess(pi.hProcess, &code); + ::CloseHandle(pi.hThread); + ::CloseHandle(pi.hProcess); + if (job) ::CloseHandle(job); // see the timeout path + if (sink != INVALID_HANDLE_VALUE) ::CloseHandle(sink); + + if (out_wall_s && freq.QuadPart > 0) + *out_wall_s = static_cast(t1.QuadPart - t0.QuadPart) + / static_cast(freq.QuadPart); + return static_cast(code); +} + +// Peer of the POSIX setenv/unsetenv. `SetEnvironmentVariableA(key, nullptr)` +// is the documented way to REMOVE a variable — passing "" would leave an empty +// one behind, which a child process sees as set. +export void set_env(const std::string& key, const std::string& value) { + ::SetEnvironmentVariableA(key.c_str(), value.c_str()); +} + +export void unset_env(const std::string& key) { + ::SetEnvironmentVariableA(key.c_str(), nullptr); +} + +export int cpu_logical() { + SYSTEM_INFO si{}; + ::GetSystemInfo(&si); + return si.dwNumberOfProcessors > 0 ? static_cast(si.dwNumberOfProcessors) : 1; +} + +// The relationship table needs the two-call pattern: its length is not knowable +// up front, so ask for the size, allocate, then ask again. +static std::vector processor_info(LOGICAL_PROCESSOR_RELATIONSHIP rel) { + DWORD bytes = 0; + ::GetLogicalProcessorInformationEx(rel, nullptr, &bytes); + if (bytes == 0) return {}; + std::vector buf(bytes); + if (!::GetLogicalProcessorInformationEx( + rel, reinterpret_cast(buf.data()), + &bytes)) + return {}; + buf.resize(bytes); + return buf; +} + +export int cpu_physical() { + auto buf = processor_info(RelationProcessorCore); + int count = 0; + for (DWORD off = 0; off < buf.size(); ) { + auto* info = reinterpret_cast(buf.data() + off); + if (info->Size == 0) break; + ++count; + off += info->Size; + } + return count > 0 ? count : cpu_logical(); +} + +export std::string cpu_model() { + HKEY key{}; + if (::RegOpenKeyExA(HKEY_LOCAL_MACHINE, + "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", + 0, KEY_READ, &key) != ERROR_SUCCESS) + return {}; + char buf[256] = {}; + DWORD size = sizeof(buf); + DWORD type = 0; + const LSTATUS st = ::RegQueryValueExA(key, "ProcessorNameString", nullptr, &type, + reinterpret_cast(buf), &size); + ::RegCloseKey(key); + if (st != ERROR_SUCCESS || type != REG_SZ) return {}; + return std::string(buf); +} + +export std::uint64_t ram_bytes() { + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + if (::GlobalMemoryStatusEx(&status)) return status.ullTotalPhys; + return 0; +} + +// Windows states efficiency class per core; more than one distinct class is +// exactly what "hybrid" means here — no frequency heuristics needed. +export bool heterogeneous_cpu() { + auto buf = processor_info(RelationProcessorCore); + int first = -1; + for (DWORD off = 0; off < buf.size(); ) { + auto* info = reinterpret_cast(buf.data() + off); + if (info->Size == 0) break; + const int cls = static_cast(info->Processor.EfficiencyClass); + if (first < 0) first = cls; + else if (cls != first) return true; + off += info->Size; + } + return false; +} + +} // namespace bench::platform_impl + +#endif // _WIN32 diff --git a/bench/src/protocol.cpp b/bench/src/protocol.cpp new file mode 100644 index 00000000..6af39992 --- /dev/null +++ b/bench/src/protocol.cpp @@ -0,0 +1,137 @@ +// bench.protocol — implementation. +// +// `module bench.protocol;` with no `export`: an implementation unit. Nothing +// written here lands in the BMI, which is the point — protocol is imported by +// every other module in the suite, so it has the largest blast radius of any +// interface here. Under GCC 16.1 an exported definition mentioning std types +// can corrupt the BMIs of importers and report the failure against an unrelated +// module (this suite lost an afternoon to exactly that; see journal.cppm). +// +// What stays in the interface: enums, aggregates, and the `constexpr` mappings +// over them — those must be usable at compile time, and none of them carries a +// std container. +module bench.protocol; + +import std; + +namespace bench { + +std::string CellKey::str() const { + return std::format("{}/{}/{}/{}/{}/{}", engine, compiler, profile, scenario, fixture, variant); +} + +bool CellResult::has_timing() const { return status == Status::Ok && !samples.empty(); } + +double CellResult::median_s() const { + if (!has_timing()) return 0.0; + std::vector v; + v.reserve(samples.size()); + for (const auto& s : samples) v.push_back(s.wall_s); + std::ranges::sort(v); + const auto n = v.size(); + return (n % 2) ? v[n / 2] : (v[n / 2 - 1] + v[n / 2]) / 2.0; +} + +double CellResult::min_s() const { + if (!has_timing()) return 0.0; + return std::ranges::min(samples, {}, &Sample::wall_s).wall_s; +} + +double CellResult::max_s() const { + if (!has_timing()) return 0.0; + return std::ranges::max(samples, {}, &Sample::wall_s).wall_s; +} + +std::string RunId::str() const { return fingerprint; } + +RunId RunId::of(std::string config_text) { + std::uint64_t h = 1469598103934665603ULL; + for (unsigned char c : config_text) { h ^= c; h *= 1099511628211ULL; } + return RunId{std::format("{:08x}", static_cast(h ^ (h >> 32))), + std::move(config_text)}; +} + +namespace detail { + +std::string escape(std::string_view s) { + std::string out; + out.reserve(s.size() + 8); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (static_cast(c) < 0x20) + out += std::format("\\u{:04x}", static_cast(c)); + else + out += c; + } + } + return out; +} + +std::string q(std::string_view s) { return std::format("\"{}\"", escape(s)); } + +// Fixed precision everywhere: a result file is diffed and merged across runs, +// and shortest-round-trip formatting makes those diffs noisy for no gain. +std::string num(double v) { return std::format("{:.3f}", v); } + +} // namespace detail + + +std::string to_json(const Report& r) { + using detail::q; + using detail::num; + std::string out; + out += "{\n"; + out += std::format(" \"protocol_version\": {},\n", kProtocolVersion); + out += std::format(" \"started_at\": {},\n", q(r.started_at)); + out += std::format(" \"under_test\": {},\n", q(r.under_test)); + out += " \"host\": {\n"; + out += std::format(" \"os\": {},\n", q(r.host.os)); + out += std::format(" \"arch\": {},\n", q(r.host.arch)); + out += std::format(" \"cpu_model\": {},\n", q(r.host.cpu_model)); + out += std::format(" \"logical_cores\": {},\n", r.host.logical_cores); + out += std::format(" \"physical_cores\": {},\n", r.host.physical_cores); + out += std::format(" \"heterogeneous\": {},\n", r.host.heterogeneous ? "true" : "false"); + out += std::format(" \"ram_bytes\": {},\n", r.host.ram_bytes); + out += std::format(" \"toolchain\": {}\n", q(r.host.toolchain)); + out += " },\n"; + out += " \"cells\": [\n"; + for (std::size_t i = 0; i < r.cells.size(); ++i) { + const auto& c = r.cells[i]; + out += " {\n"; + out += std::format(" \"engine\": {},\n", q(c.key.engine)); + out += std::format(" \"compiler\": {},\n", q(c.key.compiler)); + out += std::format(" \"profile\": {},\n", q(c.key.profile)); + out += std::format(" \"scenario\": {},\n", q(c.key.scenario)); + out += std::format(" \"fixture\": {},\n", q(c.key.fixture)); + out += std::format(" \"variant\": {},\n", q(c.key.variant)); + out += std::format(" \"status\": {},\n", q(to_string(c.status))); + out += std::format(" \"note\": {},\n", q(c.note)); + out += std::format(" \"runs\": {},\n", c.samples.size()); + if (c.has_timing()) { + out += std::format(" \"median_s\": {},\n", num(c.median_s())); + out += std::format(" \"min_s\": {},\n", num(c.min_s())); + out += std::format(" \"max_s\": {},\n", num(c.max_s())); + out += " \"samples\": ["; + for (std::size_t k = 0; k < c.samples.size(); ++k) + out += std::format("{}{}", k ? ", " : "", num(c.samples[k].wall_s)); + out += "]\n"; + } else { + // No timing keys at all rather than zeros: a reader that forgets to + // check `status` gets a missing key (loud) instead of a 0.0 (silent). + out += " \"samples\": []\n"; + } + out += (i + 1 == r.cells.size()) ? " }\n" : " },\n"; + } + out += " ]\n"; + out += "}\n"; + return out; +} + + +} // namespace bench diff --git a/bench/src/protocol.cppm b/bench/src/protocol.cppm new file mode 100644 index 00000000..05ca38fe --- /dev/null +++ b/bench/src/protocol.cppm @@ -0,0 +1,206 @@ +// The bench result protocol: what a measurement IS, independent of who produced +// it or what reads it. +// +// This module is the one piece of the suite that is expensive to change, so it +// is deliberately small and carries no logic beyond serialisation. Engines, +// scenarios and analysis all write toward these types; none of them may add a +// field without bumping kProtocolVersion. +// +// Three invariants are encoded here rather than left to convention, because each +// one was violated by the shell harness this suite replaces: +// +// 1. A FAILURE MUST NOT BE ABLE TO LOOK LIKE A MEASUREMENT. `status` and the +// timings are separate fields, and a non-ok status carries no median. The +// old harness formatted a failed cell as "0.000 s" and three of them went +// into a results file looking like the fastest builds ever recorded. +// 2. A SKIP MUST CARRY ITS REASON. "bazel is not installed here" and "bazel +// ran and failed" are opposite conclusions; `Unavailable` vs `Failed` plus +// a mandatory note keeps them apart. +// 3. RESULTS TRAVEL WITH THEIR HOST. A wall-clock number without the machine +// that produced it is not comparable to anything — least of all on a +// heterogeneous CPU, where "32 cores" is not 32 of the same thing. +export module bench.protocol; + +import std; + +export namespace bench { + +// Bump on ANY field addition/removal/semantic change. Readers compare against +// their own expectation and degrade explicitly rather than mis-parsing. +inline constexpr int kProtocolVersion = 2; // +Report::under_test + +// --------------------------------------------------------------------------- + +enum class Status { Ok, Failed, Skipped, Unavailable }; + +constexpr std::string_view to_string(Status s) { + switch (s) { + case Status::Ok: return "ok"; + case Status::Failed: return "failed"; + case Status::Skipped: return "skipped"; + case Status::Unavailable: return "unavailable"; + } + return "unknown"; +} + +// The source form the fixture is expressed in. This is the axis the whole suite +// exists to measure, so it is a first-class enum rather than a string tag. +enum class Variant { + Headers, // classic headers + separate .cpp implementation + Modules, // module interface units carrying their implementations + ModulesImpl, // module interface units + separate implementation units + // An existing project measured as-is. The variant axis does not apply: the + // project is whatever it already is, and generating over it would destroy + // the very thing being measured. + Native, +}; + +constexpr std::string_view to_string(Variant v) { + switch (v) { + case Variant::Headers: return "headers"; + case Variant::Modules: return "modules"; + case Variant::ModulesImpl: return "modules-impl"; + case Variant::Native: return "native"; + } + return "unknown"; +} + +constexpr std::optional variant_from(std::string_view s) { + if (s == "headers") return Variant::Headers; + if (s == "modules") return Variant::Modules; + if (s == "modules-impl") return Variant::ModulesImpl; + if (s == "native") return Variant::Native; + return std::nullopt; +} + +// --------------------------------------------------------------------------- + +struct HostInfo { + std::string os; + std::string arch; + std::string cpu_model; + int logical_cores{}; + int physical_cores{}; + // A 13900K is 8 P-cores + 16 E-cores. Reading its 32 threads as 32 equal + // cores makes every parallelism figure wrong, so the fact is recorded + // rather than inferred by whoever reads the numbers later. + bool heterogeneous{}; + std::uint64_t ram_bytes{}; + std::string toolchain; // e.g. "gcc 16.1.0" +}; + +// The full coordinate of one measurement. Every field is part of the identity; +// two cells differing in any of them are different measurements, never repeats. +struct CellKey { + std::string engine; + std::string compiler; + std::string profile; + std::string scenario; + std::string fixture; + std::string variant; + + [[nodiscard]] std::string str() const; +}; + +struct Sample { + double wall_s{}; + int exit_code{}; +}; + +class CellResult { +public: + CellKey key; + std::vector samples; + Status status{Status::Skipped}; + std::string note; // required whenever status != Ok + + // Timings are DERIVED, never set alongside a non-ok status — that is what + // makes invariant 1 structural instead of a review comment. + [[nodiscard]] bool has_timing() const; + [[nodiscard]] double median_s() const; + [[nodiscard]] double min_s() const; + [[nodiscard]] double max_s() const; +}; + +struct Report { + HostInfo host; + std::string started_at; // ISO-8601, filled by the caller + std::vector cells; + // Identifies WHICH RUN produced this. See RunId. + std::string run_id; + // WHAT WAS UNDER TEST, in the caller's own words — for mcpp, the commit. + // + // The engines label themselves from `--version`, and mcpp's version is a + // DATE: every commit on a branch reports the same `2026.8.13.1`. So a report + // could say which release it measured but never which build of it, and the + // numbers in this suite move with single commits. Supplied rather than + // derived because the harness measures a BINARY and cannot know what tree it + // came from. + std::string under_test; +}; + +// ── The identity of a run: ONE fingerprint over the whole configuration ──── +// +// Same shape as `mcpp build`: everything that decides WHAT is measured is +// hashed into one value, that value names a cache directory, a re-run with the +// same configuration hits it, and a different configuration lands somewhere +// else instead of overwriting. `--id` is not a separate concept — it is simply +// one more input to the hash, so passing it forks a fresh cache exactly the way +// changing the engine list does. +// +// Hashed: engines, variants, scenarios, run count, compiler, profile, fixture +// shape, project, buildfiles, and `--id`. +// +// ⚠️ NOT hashed: the mcpp binary, the installed cmake, anything that can change +// underneath while the command line stays the same. Folding those in would +// restart from zero on every rebuild — the normal case while developing, and +// exactly when resume is worth having. They are recorded per entry as observed +// facts, and adopting a record measured with a different one is REPORTED. +// +// ⚠️ WHY AN IDENTITY AT ALL: resuming means treating an old record as this +// run's result, so without one a resumable benchmark is a machine for silently +// splicing runs together. Not hypothetical — a killed run did not die on +// SIGTERM, finished its cell and wrote its report AFTER the `rm -rf` meant to +// clear the directory, leaving a 90-cell file beside a 72-cell file under names +// that gave no hint. The only field that told them apart was `started_at`, +// inside the JSON. +struct RunId { + std::string fingerprint; // 8 hex chars, like mcpp's build directories + std::string config; // the text it was taken over, for `--explain` + + [[nodiscard]] std::string str() const; + [[nodiscard]] bool operator==(const RunId&) const = default; + + // FNV-1a. Short and stable; a cache key, not a security boundary. + [[nodiscard]] static RunId of(std::string config_text); +}; + +// One measured unit: the smallest thing worth saving, and the granularity a +// resume works at. +// +// os · toolchain · project · variant · scenario · engine · run index +// +// Appended to the journal the moment it is measured, so a kill costs at most +// the unit in flight rather than the whole invocation. +struct JournalEntry { + std::string id; // RunId::str() + // The engine's own version banner AT THE TIME this was measured. Not part + // of the id — a rebuilt binary must not invalidate a resume — but recorded + // so that adopting a record measured with a different one is REPORTED + // rather than silent. + std::string engine_version; + std::string project; + std::string variant; + std::string scenario; + std::string engine; + int run{}; // 1-based + double wall_s{}; + int exit_code{}; +}; + +// Serialisation. Hand-rolled on purpose: the suite must build with nothing but +// `import std;` so it can be the FIRST thing that runs on a fresh machine. +// Defined in protocol.cpp — see the header comment for why. +std::string to_json(const Report& r); + +} // namespace bench diff --git a/bench/src/registry.cpp b/bench/src/registry.cpp new file mode 100644 index 00000000..ba5abf87 --- /dev/null +++ b/bench/src/registry.cpp @@ -0,0 +1,88 @@ +// bench.registry — implementation. +// +// `module bench.registry;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. This is the one module that knows every engine, +// so keeping its bodies out of the BMI stops a change to any single engine +// from rippling through the importer graph. +module bench.registry; + +import std; +import bench.engines.engine; +import bench.engines.mcpp; +import bench.engines.cmake; +import bench.engines.xmake; +import bench.engines.bazel; + +namespace bench { + +std::string anchor_program(std::string program) { + if (program.empty()) return program; + if (program.find('/') == std::string::npos && + program.find('\\') == std::string::npos) + return program; // bare name → PATH, cwd-independent + std::error_code ec; + auto abs = std::filesystem::absolute(program, ec); + if (ec) return program; // leave it; probe() will report it + // weakly_canonical also collapses `..`, which absolute() keeps. + auto canon = std::filesystem::weakly_canonical(abs, ec); + return ec ? abs.string() : canon.string(); +} + +std::optional> engine_option( + std::string_view engine, std::string_view key, std::string_view value) { + if (engine == "mcpp" && key == "schedule") + return std::pair{std::string("MCPP_BMI_SCHEDULE"), std::string(value)}; + return std::nullopt; +} + +std::unique_ptr make_engine(std::string_view spec) { + std::string name(spec); + std::string program; + std::map env; + + // The BRACKETS are parsed first, then `=program`. Order matters: the option + // list contains `=` itself (`mcpp[schedule=on]=/path`), so splitting on the + // first `=` yields the name `mcpp[schedule`, and the whole spec is rejected + // as an unknown engine. + std::string opts; + if (const auto lb = spec.find('['); lb != std::string_view::npos) { + const auto rb = spec.find(']', lb); + if (rb == std::string_view::npos) return nullptr; // unterminated: reject + name = std::string(spec.substr(0, lb)); + opts = std::string(spec.substr(lb + 1, rb - lb - 1)); + auto rest = spec.substr(rb + 1); + if (!rest.empty()) { + if (rest.front() != '=') return nullptr; // trailing junk: reject + program = anchor_program(std::string(rest.substr(1))); + } + } else if (const auto eq = spec.find('='); eq != std::string_view::npos) { + name = std::string(spec.substr(0, eq)); + program = anchor_program(std::string(spec.substr(eq + 1))); + } + + for (std::size_t at = 0; at <= opts.size();) { + const auto end = std::min(opts.find(',', at), opts.size()); + const auto item = std::string_view(opts).substr(at, end - at); + at = end + 1; + if (item.empty()) continue; + const auto sep = item.find('='); + if (sep == std::string_view::npos) return nullptr; + auto mapped = engine_option(name, item.substr(0, sep), item.substr(sep + 1)); + if (!mapped) return nullptr; // unknown: reject loudly + env.emplace(std::move(mapped->first), std::move(mapped->second)); + } + + if (name == "mcpp") + return engines::make_mcpp(program.empty() ? "mcpp" : program, {}, std::move(env)); + if (!env.empty()) return nullptr; // no other engine takes options yet + if (name == "cmake") return engines::make_cmake(); + if (name == "xmake") return engines::make_xmake(); + if (name == "bazel") return engines::make_bazel(); + return nullptr; +} + +std::vector default_engine_specs() { + return {"mcpp", "cmake", "xmake", "bazel"}; +} + +} // namespace bench diff --git a/bench/src/registry.cppm b/bench/src/registry.cppm new file mode 100644 index 00000000..232086de --- /dev/null +++ b/bench/src/registry.cppm @@ -0,0 +1,60 @@ +// bench.registry — turning `--engines` text into engine objects. +// +// Adding an engine is: write bench.engines., then add ONE line to +// `make_engine`. Nothing else in the suite — runner, protocol, scenarios, CI — +// changes. +// +// A spec is either a bare name (`cmake`) or `name=program` (`mcpp=/path/to/mcpp`). +// The second form is what makes "is the new release faster?" a normal query: +// +// --engines mcpp=/usr/bin/mcpp,mcpp=./target/x86_64-linux-gnu/*/bin/mcpp +// +// registers two mcpp engines that label themselves from the version each binary +// reports, so the two rows never collapse into one. +export module bench.registry; + +import std; +import bench.engines.engine; +import bench.engines.mcpp; +import bench.engines.cmake; +import bench.engines.xmake; +import bench.engines.bazel; + +export namespace bench { + +// Makes a program spec independent of the current directory. +// +// Every measured command runs with its cwd set to the project under test, so a +// relative `--engines mcpp=./mcpp-old` resolves against the FIXTURE rather than +// the shell the user typed it in. The spawn then fails with "could not start", +// which is reported per cell as `exited -1` — a whole matrix of failures whose +// cause is one missing `./`. Resolving here, once, at the only place a spec +// becomes an engine, removes the class of bug rather than documenting it. +// +// Bare names (`mcpp`, `cmake`) are left alone: those are PATH lookups, which the +// child performs itself and which cwd does not affect. +std::string anchor_program(std::string program); + +// A spec may carry ENGINE OPTIONS in brackets: `mcpp[schedule=on]=/path/to/mcpp`. +// +// This exists for opt-in behaviour. mcpp's split build schedule is a key in the +// MEASURED PROJECT's manifest, and the measured projects are pinned workloads — +// one of them belongs to someone else — so the suite had no way to reach the +// largest cold-build change in the release it was benchmarking, and reported +// "no improvement" for something worth 2.29x. +// +// Bracket options become environment variables for that engine's child only, so +// both arms sit in one report against one baseline on one machine. Unbracketed +// specs are untouched, and an unknown option is an error rather than a silently +// ignored word — a benchmark that quietly measures the default when you asked +// for the option is the exact failure this is meant to remove. +std::optional> engine_option( + std::string_view engine, std::string_view key, std::string_view value); + +std::unique_ptr make_engine(std::string_view spec); + +// The default set, used when --engines is omitted. Order is the reporting order, +// chosen for reading: mcpp first (the subject), then the others. +std::vector default_engine_specs(); + +} // namespace bench diff --git a/bench/src/runner.cpp b/bench/src/runner.cpp new file mode 100644 index 00000000..fd3af4c7 --- /dev/null +++ b/bench/src/runner.cpp @@ -0,0 +1,387 @@ +// bench.runner — implementation. +// +// `module bench.runner;` with no `export`: an implementation unit. The runner is +// where a measurement actually happens — seed build, perturbation, timed build, +// journal append — and none of that belongs in a BMI. Only `RunOptions`, +// `Runner` and `Instance` stay declared in the interface, because those are what +// main.cpp names. +module bench.runner; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; +import bench.fixture.generate; +import bench.fixture.buildfiles; + +namespace bench { + +namespace detail { + +std::optional insert_into_first_body( + const std::filesystem::path& file, int nonce, bool statement) { + std::ifstream in(file, std::ios::binary); + if (!in) return std::nullopt; + std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + in.close(); + + // Insert inside the first function body: immediately after the first '{' + // that follows a ')'. Anchoring on the brace rather than a name keeps this + // working for all three variants, whose function text differs. + // + // ⚠️ AFTER THE BRACE, not after the newline that follows it. Those are the + // same position only when the body spans several lines. Given a one-line + // body — `export int f() { return 1; }` — the newline is past the CLOSING + // brace, so the statement landed at namespace scope and the build died with + // + // error: expected unqualified-id + // volatile int bench_nonce_0 = 0; (void)bench_nonce_0; + // + // pointing at a file the harness had just written. Honest (the cell failed + // loudly) but wrong: the perturbation is supposed to be applicable to any + // function, and "your function is on one line" is not a real limitation. + // + // A file may legitimately have NO function body — the modules-impl variant's + // interface unit only declares — so a comment falls back to end-of-file + // rather than reporting the scenario as inapplicable. A statement has no + // such fallback: there is nowhere to put it that would mean the same thing. + const auto paren = text.find(") {"); + const auto brace = paren == std::string::npos ? std::string::npos : paren + 2; + std::string_view form = "in-body"; + if (brace == std::string::npos) { + if (statement) return std::nullopt; + form = "end-of-file"; + text += std::format("\n// bench: comment perturbation #{}\n", nonce); + } else { + // `volatile` so no optimiser can delete the edit and hand back the + // previous object file — that would quietly turn a semantic edit back + // into a no-op, i.e. straight back into the bug this split exists to fix. + // + // The name carries the nonce because perturbations ACCUMULATE across the + // repetitions of one cell: a fixed name redeclares itself on run 2 and + // the build fails, which is exactly what the first version did. + // The leading newline is what makes a one-line body work: the text + // opens its own line immediately after `{`, whatever followed it. + text.insert(brace + 1, + statement + ? std::format("\n volatile int bench_nonce_{0} = {0};" + " (void)bench_nonce_{0};", nonce) + : std::format("\n // bench: comment perturbation #{}", nonce)); + } + + std::ofstream out(file, std::ios::binary | std::ios::trunc); + out << text; + return form; +} + +} // namespace detail + +std::string Runner::failure_note(std::string_view what, const platform::RunResult& r, const std::filesystem::path& log) { + if (!r.started()) + // The OS's own reason when there is one. "check the engine's program + // path" was the only advice this could give, and it is wrong as + // often as it is right — a windows/clang cell reported it while + // `xmake --version` had just succeeded in the same job, so the path + // was demonstrably fine and something else (a cwd, a handle) was not. + return std::format("{}: could not start the process (no log written){}", + what, + r.start_error.empty() + ? std::string(" — check the engine's program path") + : std::format(" — {}", r.start_error)); + if (r.timed_out) + return std::format("{} TIMED OUT after {:.0f}s and was killed (see {})", + what, r.wall_s, log.string()); + return std::format("{} exited {} (see {})", what, r.exit_code, log.string()); + } + +Runner::Runner(RunOptions opt) : opt_(std::move(opt)) {} + +std::filesystem::path Runner::log_dir() const { + std::error_code ec; + auto d = opt_.work_root / "logs"; + std::filesystem::create_directories(d, ec); + return d; + } + +Runner::Instance Runner::materialise(std::string_view engine, Variant variant) const { + // PROJECT MODE. The tree already exists and belongs to someone; nothing + // here may create or delete it. In particular the remove_tree below must + // never run against it — deleting the user's repository is the one + // failure mode this whole function has to make impossible. + if (!opt_.project.empty()) { + Instance inst; + inst.project_dir = opt_.project; + inst.build_dir = opt_.project / "build"; // used by cmake/meson/xmake + inst.targets = opt_.project_targets; + return inst; + } + + // The engine LABEL can carry a version ("mcpp@2026.8.12.1"); the + // directory name must stay predictable and portable, so it is slugged. + std::string slug(engine); + for (char& c : slug) + if (c == '@' || c == '/' || c == '\\' || c == ':' || c == ' ') c = '-'; + const auto dir = opt_.work_root / std::format("{}-{}", slug, to_string(variant)); + platform::remove_tree(dir); + std::filesystem::create_directories(dir); + Instance inst; + inst.project_dir = dir; + inst.build_dir = dir / "build"; + inst.targets = fixture::emit_sources(dir, variant, opt_.shape); + fixture::emit_all(dir, variant, opt_.shape, opt_.compiler); + return inst; + } + +CellResult Runner::measure(engines::Engine& engine, const Instance& inst, Variant variant, Scenario scenario, std::string_view profile, std::string_view compiler, std::string_view compiler_label, std::string_view fixture_name) const { + CellResult cell; + cell.key = CellKey{std::string(engine.name()), std::string(compiler_label), + std::string(profile), std::string(to_string(scenario)), + std::string(fixture_name), std::string(to_string(variant))}; + + // Availability before anything else: "not installed" must never be + // reported as a slow or broken engine. + const auto avail = engine.probe(); + if (!avail.present) { + cell.status = Status::Unavailable; + cell.note = avail.note; + return cell; + } + if (!engine.supports(variant, compiler)) { + cell.status = Status::Unavailable; + cell.note = engine.unsupported_reason(variant, compiler); + return cell; + } + + // A scenario that needs a file nobody named cannot be run. Reporting it + // as `skipped` with the reason beats perturbing an arbitrary file, which + // would produce a number that looks valid and measures something else. + if (const auto missing = unmet_target(inst, scenario); !missing.empty()) { + cell.status = Status::Skipped; + cell.note = missing; + return cell; + } + + Job job; + job.project_dir = inst.project_dir; + // mcpp reads its own manifest from the tree and ignores this; cmake and + // xmake are told to look here for their description. + job.buildfile_dir = opt_.buildfiles.empty() ? inst.project_dir : opt_.buildfiles; + job.build_dir = inst.build_dir; + // The child log goes in the WORK directory, never inside the measured + // tree. In --project mode that tree is the user's repository, and a + // harness that drops files into it is one `git add -A` away from + // committing its own scratch (which is exactly what happened once). + job.log_path = log_dir() / std::format("{}-{}.log", engine.name(), + to_string(scenario)); + // Cleared ONCE per cell; every child then appends. The alternative — + // truncating per child — is what made a 0.60s "cold" build unexplainable: + // the timed build's one line of output had erased the configure that + // preceded it. + { std::ofstream clear(job.log_path, std::ios::binary | std::ios::trunc); } + job.variant = variant; + job.profile = std::string(profile); + job.compiler = std::string(compiler); + job.jobs = opt_.jobs; + job.timeout_s = opt_.timeout_s; + + // Turns a failure into something a reader can act on WITHOUT the log + // file, which on a CI runner is deleted with the machine. Every module + // cell in the matrix failed behind a bare "see .../cmake-cold.log" and + // the job stayed green; neither half of that was noticed for weeks. + const auto fail = [&](std::string_view what, const platform::RunResult& r) { + cell.status = Status::Failed; + cell.note = failure_note(what, r, job.log_path); + report(cell.note); + // 20 lines is right for an ordinary failure and useless for the one + // that most needs a log: a compiler CRASH. clang prints ~40 lines of + // stack dump after the line that names the file and the pass, so a + // 20-line tail shows frames #30..#36 and the bare + // `clang frontend command failed with exit code 139` — everything + // identifying WHAT it was compiling has already scrolled past. That + // is exactly what happened to the xlings/clang cell, and it is why + // that crash is still undiagnosed. + const auto crashed = platform::log_mentions( + job.log_path, {"PLEASE submit a bug report", "Stack dump"}); + + // ⚠️ A TAIL IS THE WRONG SHAPE WHEN THE TOOL IS CHATTY. Every build + // engine here prints a progress line per translation unit, so 20 + // lines of tail is 20 lines of `generating.module.deps ...` and the + // error that actually stopped it — printed once, hundreds of lines + // earlier — is gone. That is not hypothetical: `xmake/clang` failed + // with `seed build exited 255` and the captured tail contained + // nothing but progress, so the cell could not be diagnosed from CI + // at all and cost a full matrix cycle to learn nothing. + // + // So the lines that LOOK like an error are pulled out first, from + // anywhere in the file, and the tail follows as context. Cheap, and + // it is the difference between "exited 255" and a cause. + if (const auto why = platform::log_grep( + job.log_path, + // ⚠️ EVERY ENGINE SPELLS IT DIFFERENTLY, and the first + // version of this list only knew the compiler's spelling. + // cmake writes `CMake Error in CMakeLists.txt:` — no colon + // after "Error", capital E — so the very next failure it + // was meant to explain slipped straight through it. That is + // the sieve's own version of the bug it exists to catch. + {"error:", "error :", "ERROR:", " error ", "Error in", + "Error at", "Error:", "fatal", "not found", "No such file", + "cannot find", "undefined", "failed to", "step failed", + "requires that", "Assertion", "abort"}, + /*max=*/12); + !why.empty()) + report(std::format("--- error lines from {} ---\n{}", + job.log_path.filename().string(), why)); + + if (const auto tail = platform::tail_of(job.log_path, crashed ? 80 : 20); + !tail.empty()) + report(std::format("--- last lines of {} ---\n{}", + job.log_path.filename().string(), tail)); + }; + + // Asked once the Job exists, because the answer depends on the PROJECT — + // which is why it cannot live beside the `supports()` check above. + if (auto why = engine.unbuildable_reason(job); !why.empty()) { + cell.status = Status::Unavailable; + cell.note = std::move(why); + report(cell.note); + return cell; + } + + report("configure"); + if (const auto cfg = engine.configure(job); !cfg.ok()) { + fail("configure", cfg); + return cell; + } + + // One untimed seed build. An incremental scenario is only incremental + // against an up-to-date tree, and it warms the page cache so run 1 is + // not systematically slower than the rest. + report("seed build"); + if (const auto seed = engine.build(job); !seed.ok()) { + fail("seed build", seed); + return cell; + } + + // edit-body rewrites a source file. In project mode that file belongs to + // the user, so its exact bytes are captured first and restored no matter + // how this function exits — including on a failed build. + // Both editing scenarios rewrite a source file. In project mode that file + // belongs to the user, so the guard must cover each of them — an + // unguarded scenario silently leaves the perturbation behind. + const SourceGuard guard( + scenario == Scenario::EditBody ? inst.targets.body : + scenario == Scenario::EditComment ? inst.targets.hub : std::filesystem::path{}); + + std::string_view perturbForm; + const int runs = opt_.runs_override > 0 ? opt_.runs_override : default_runs(scenario); + for (int i = 0; i < runs; ++i) { + // Already in the journal under this run key: adopt it and move on. + // + // The PERTURBATION IS STILL SKIPPED ALONG WITH THE MEASUREMENT, and + // that is the subtle part. `edit-body` inserts text carrying the run + // index, so replaying a recorded run must not re-apply its edit — + // doing so would leave the tree carrying an edit that no timing in + // this run accounts for, and the NEXT run's build would inherit it. + if (opt_.already_done && + opt_.already_done(to_string(scenario), engine.name(), + to_string(variant), i + 1)) { + report(std::format("run {}/{} — already recorded, skipping", i + 1, runs)); + if (opt_.recorded_sample) { + const auto s = opt_.recorded_sample(to_string(scenario), engine.name(), + to_string(variant), i + 1); + cell.samples.push_back(Sample{s.first, s.second}); + } + continue; + } + report(std::format("run {}/{}", i + 1, runs)); + const auto form = perturb(engine, job, inst, scenario, i); + if (!form) { + cell.status = Status::Failed; + cell.note = std::format("could not apply scenario '{}'", to_string(scenario)); + report(cell.note); + return cell; + } + if (!form->empty()) perturbForm = *form; + + // COLD IS "from nothing to a binary", so it must include configure. + // Not a detail: cmake and meson keep their configure output INSIDE + // the build dir that clean() just removed, so building without + // re-configuring simply fails — which is how this was found. Timing + // configure separately would also be wrong: a user waiting for a + // clean build waits for both, and engines that fold configure into + // the build (mcpp, bazel) would otherwise get a discount for it. + double extra = 0.0; + if (scenario == Scenario::Cold) { + const auto cfg = engine.configure(job); + if (!cfg.ok()) { + fail(std::format("re-configure on run {}", i + 1), cfg); + return cell; + } + extra = cfg.wall_s; + } + + const auto r = engine.build(job); + if (!r.ok()) { + fail(std::format("build on run {}", i + 1), r); + return cell; + } + cell.samples.push_back(Sample{extra + r.wall_s, r.exit_code}); + if (opt_.record) + opt_.record(to_string(scenario), engine.name(), to_string(variant), + static_cast(i) + 1, extra + r.wall_s, r.exit_code); + } + cell.status = Status::Ok; + // The engine's version banner, plus HOW the source was perturbed when + // that choice was not fixed by the scenario name alone. + cell.note = perturbForm.empty() + ? avail.note + : std::format("{} · perturbation: {}", avail.note, perturbForm); + return cell; + } + +void Runner::report(std::string_view what) const { + if (opt_.on_progress) opt_.on_progress(what); + } + +std::string Runner::unmet_target(const Instance& inst, Scenario scenario) const { + const std::filesystem::path* want = nullptr; + std::string_view which; + switch (scenario) { + case Scenario::TouchHub: want = &inst.targets.hub; which = "--hub"; break; + case Scenario::TouchLeaf: want = &inst.targets.leaf; which = "--leaf"; break; + case Scenario::EditBody: want = &inst.targets.body; which = "--body"; break; + // Deliberately the HUB, not the body target: the question is whether + // a non-semantic change to a widely-imported INTERFACE cascades. In + // the modules-impl variant `body` is an implementation unit with no + // BMI at all, so asking it there would measure nothing. + case Scenario::EditComment: want = &inst.targets.hub; which = "--hub"; break; + default: return {}; + } + if (want->empty()) + return std::format("scenario '{}' needs a file to perturb; pass {} ", + to_string(scenario), which); + std::error_code ec; + if (!std::filesystem::exists(*want, ec)) + return std::format("{} points at a file that does not exist: {}", + which, want->string()); + return {}; + } + +std::optional Runner::perturb(engines::Engine& engine, const Job& job, const Instance& inst, Scenario scenario, int nonce) const { + const std::optional applied{std::string_view{}}; + const auto done = [&](bool ok) { return ok ? applied : std::nullopt; }; + switch (scenario) { + case Scenario::Cold: engine.clean(job); return applied; + case Scenario::Noop: return applied; + case Scenario::TouchHub: return done(platform::touch(inst.targets.hub)); + case Scenario::TouchLeaf: return done(platform::touch(inst.targets.leaf)); + case Scenario::EditBody: + return detail::insert_into_first_body(inst.targets.body, nonce, true); + case Scenario::EditComment: + return detail::insert_into_first_body(inst.targets.hub, nonce, false); + } + return std::nullopt; + } + +} // namespace bench diff --git a/bench/src/runner.cppm b/bench/src/runner.cppm new file mode 100644 index 00000000..e9749f9e --- /dev/null +++ b/bench/src/runner.cppm @@ -0,0 +1,199 @@ +// bench.runner — turns a cell coordinate into a measurement. +// +// The runner knows how to time and how to perturb; it knows nothing about any +// particular engine or source form. Everything engine-specific arrives through +// the Engine interface, everything project-specific through the fixture. +export module bench.runner; + +import std; +import bench.protocol; +import bench.spec; +import bench.platform; +import bench.engines.engine; +import bench.fixture.generate; +import bench.fixture.buildfiles; + +export namespace bench { + +struct RunOptions { + std::filesystem::path work_root{"bench-work"}; + fixture::Shape shape{}; + int jobs{0}; + int runs_override{0}; // 0 → per-scenario default + + // Project mode: measure an EXISTING tree instead of a generated fixture. + // This is how mcpp benchmarks itself, and how the suite is pointed at any + // real codebase — a synthetic graph cannot reproduce the shape of one. + // One directory holding the FOREIGN build descriptions for --project mode + // (CMakeLists.txt, xmake.lua, ...). Empty → each engine reads its + // description from the project tree, which is what a generated fixture does. + std::filesystem::path buildfiles; + std::filesystem::path project; // empty → generate a fixture + fixture::Targets project_targets{}; // which files the scenarios perturb + // The requested compiler, so the generated mcpp manifest can pin the same + // family the other engines are handed. + std::string compiler; + + // How long ONE configure or build may run. 0 = forever, which is the right + // default for a library and the wrong one for CI — main gives it a value. + double timeout_s{0.0}; + + // Live progress, and the ONLY thing that makes a long run legible while it + // is happening. A cell prints when it finishes, so a matrix cell that hangs + // in its third engine looks identical to one that hangs in its first: two + // CI jobs sat 25 minutes inside a child with a completely silent log. + // + // A callback rather than a print, because the runner must not own an output + // policy — the tests drive it with no sink at all. + std::function on_progress; + + // ── Resume, at the granularity of ONE measured unit ──────────────────── + // + // The unit is (project, variant, scenario, engine, run). `already_done` + // answers "is this one already in the journal for THIS run key"; `record` + // appends it the moment it is measured. + // + // Hooks rather than a Journal member so the runner keeps knowing nothing + // about files — the same reason `on_progress` is a callback. + // + // ⚠️ THE SEED BUILD IS NOT A UNIT and is redone on resume. An incremental + // scenario only means anything against an up-to-date tree, and that state + // is what the seed establishes; it lives on disk, not in the journal. So + // resuming a partially-measured cell costs one seed build. Stated because + // otherwise resume looks free and someone will be surprised by the clock. + std::function already_done; + std::function record; + // The recorded (wall_s, exit) for a unit `already_done` said yes to. + std::function(std::string_view scenario, std::string_view engine, + std::string_view variant, int run)> recorded_sample; +}; + +namespace detail { + +// EditBody must present content the previous build has never seen, on EVERY +// repetition. An idempotent edit is a real edit on run 1 and a bare `touch` on +// runs 2..N — which measures a different, much cheaper scenario and silently +// drags the median toward it. The counter is what keeps every run honest. +// Inserts text into the first function body of `file`. +// +// `statement` decides WHAT this measures, and the two are not interchangeable: +// +// true — a real statement. The function's object code changes, and for an +// inline body in an interface unit the BMI changes too, so a cascade +// is the CORRECT answer, not a defect. +// false — a comment. The bytes change but nothing observable does, so an +// engine that compares the produced BMI can stop the cascade while an +// mtime-only engine cannot. +// +// They used to be one function that inserted a comment and was called +// "edit_body". Every "engine X is N times faster on edits" number it produced +// was really a statement about comments. +// Returns WHICH FORM the perturbation took, or nullopt if it could not be +// applied. The form is not a detail — it changes what the cell measures: +// +// "in-body" the text lands inside the first function body, so every +// subsequent line in the file moves. GCC records source +// locations for inline bodies in the BMI, so the BMI genuinely +// changes and a cascade is CORRECT. +// "end-of-file" the unit had no function body to insert into, so a comment is +// appended instead. No existing line moves, the BMI is +// unchanged, and an engine that compares BMIs skips the whole +// cascade. +// +// Those are different questions, and the suite was answering both under one +// scenario name. Measured on the same engine, same compiler, same day: +// `edit-comment` on mcpp's hub (66 lines, no bodies → end-of-file) came out at +// 0.38s, and on xlings' hub (566 lines, 56 bodies → in-body) at 95.02s. Read +// side by side without knowing the form, that reads as "the optimisation works +// on one project and not the other", which is not what happened at all. +// +// So the form goes into the cell's note. Same rule as `status` carrying its +// reason: a number whose meaning depends on an invisible choice is not a +// measurement. +std::optional insert_into_first_body( + const std::filesystem::path& file, int nonce, bool statement); + +} // namespace detail + +class Runner { +public: + // Explains a non-zero result. `RunResult::started()` is false when the child + // never ran at all (bad path, not executable, missing loader) — in that case + // the log file exists but is EMPTY, and pointing a reader at it sends them + // looking for a compiler error that was never emitted. Say which of the two + // happened. + static std::string failure_note(std::string_view what, const platform::RunResult& r, const std::filesystem::path& log); + + explicit Runner(RunOptions opt); + + // Materialise one fixture instance. Kept separate from measure() so a single + // tree is reused across scenarios of the same (engine, variant) pair — the + // generation cost is real and belongs to neither measurement. + struct Instance { + std::filesystem::path project_dir; + std::filesystem::path build_dir; + fixture::Targets targets; + }; + + // Where child stdout/stderr is collected. Always under the work root, so it + // is disposable and never lands in the project being measured. + std::filesystem::path log_dir() const; + + Instance materialise(std::string_view engine, Variant variant) const; + + // `compiler` is what the engine is told to use (possibly an absolute path + // to a hermetic payload); `compiler_label` is the short name that goes into + // the result key. Keeping them apart matters: the path pins fairness, the + // label is what makes a result table readable and mergeable across machines + // where the same compiler lives at a different path. + CellResult measure(engines::Engine& engine, const Instance& inst, Variant variant, Scenario scenario, std::string_view profile, std::string_view compiler, std::string_view compiler_label, std::string_view fixture_name) const; + +private: + RunOptions opt_; + + void report(std::string_view what) const; + + // Restores a file's exact bytes on destruction. Not a convenience: without + // it a benchmark run leaves edit markers in the measured repository, and a + // failed cell leaves them silently. + class SourceGuard { + public: + explicit SourceGuard(std::filesystem::path file) : file_(std::move(file)) { + if (file_.empty()) return; + std::ifstream in(file_, std::ios::binary); + if (!in) { file_.clear(); return; } + saved_.assign((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + } + ~SourceGuard() { + if (file_.empty()) return; + std::ofstream out(file_, std::ios::binary | std::ios::trunc); + out << saved_; + } + SourceGuard(const SourceGuard&) = delete; + SourceGuard& operator=(const SourceGuard&) = delete; + private: + std::filesystem::path file_; + std::string saved_; + }; + + // Which target a scenario needs, and whether it is present and real. + std::string unmet_target(const Instance& inst, Scenario scenario) const; + + // nullopt = could not apply. A non-empty string_view describes the FORM, + // which the caller records in the cell note — see insert_into_first_body. + // nullopt — could not apply; the cell fails and says so. + // an EMPTY view — applied, and the scenario's name already says everything + // about what was done (`cold` cleans, `touch-*` bumps an + // mtime; there is only one way to do either). + // a NON-EMPTY view — applied in one of several forms, and the form changes + // what the cell measures, so it is recorded in the note. + // Only the editing scenarios have this; see + // insert_into_first_body. + std::optional perturb(engines::Engine& engine, const Job& job, const Instance& inst, Scenario scenario, int nonce) const; +}; + +} // namespace bench diff --git a/bench/src/spec.cppm b/bench/src/spec.cppm new file mode 100644 index 00000000..5803bdf3 --- /dev/null +++ b/bench/src/spec.cppm @@ -0,0 +1,102 @@ +// bench.spec — WHAT gets measured, expressed as data. +// +// Scenarios, jobs and the matrix live here so that adding a measurement never +// means editing the runner. The runner knows how to time a thing; this module +// knows which things are worth timing and how to perturb the tree first. +export module bench.spec; + +import std; +import bench.protocol; + +export namespace bench { + +// The perturbation applied immediately before a timed build. Every one of these +// answers a different question, and the names are the vocabulary the whole +// suite (CLI, CI inputs, result files) speaks. +enum class Scenario { + Cold, // no build dir at all: full graph construction + every compile + Noop, // nothing changed: how cheap is "already up to date" + TouchHub, // mtime bump on a widely-imported unit, CONTENT UNCHANGED — + // can the engine prove the interface did not change? + EditBody, // real SEMANTIC edit inside a function body — the everyday + // developer loop. For an inline body in an interface unit this + // legitimately changes the BMI and a cascade is CORRECT; that + // is the point of comparing it against `modules-impl`, where + // the same edit touches no interface at all. + EditComment, // the file's bytes change but its interface does not (a comment + // is inserted into a widely-imported unit). Distinct from + // TouchHub: mtime engines see a real content change here, so + // only an engine that compares the produced BMI can avoid the + // cascade. Keeping this separate from EditBody is what stops a + // "12x faster on edits" claim that is really about comments. + TouchLeaf, // mtime bump on a unit nobody imports: recompile 1 + link +}; + +constexpr std::string_view to_string(Scenario s) { + switch (s) { + case Scenario::Cold: return "cold"; + case Scenario::Noop: return "noop"; + case Scenario::TouchHub: return "touch-hub"; + case Scenario::EditBody: return "edit-body"; + case Scenario::EditComment: return "edit-comment"; + case Scenario::TouchLeaf: return "touch-leaf"; + } + return "unknown"; +} + +constexpr std::optional scenario_from(std::string_view s) { + if (s == "cold") return Scenario::Cold; + if (s == "noop") return Scenario::Noop; + if (s == "touch-hub") return Scenario::TouchHub; + if (s == "edit-body") return Scenario::EditBody; + if (s == "edit-comment") return Scenario::EditComment; + if (s == "touch-leaf") return Scenario::TouchLeaf; + return std::nullopt; +} + +// Everything an engine needs to act, and nothing about how it is timed. +struct Job { + std::filesystem::path project_dir; // the fixture instance (holds the sources) + // Where THIS engine's build description lives. Equal to project_dir for a + // generated fixture, where the harness emits one file per engine into the + // tree it just created. + // + // For a REAL project they separate. mcpp is built by mcpp, so a CMakeLists + // and an xmake.lua at its root are files every contributor has to learn to + // ignore; they live in bench/projects// instead and reach back into + // the tree. The alternative — copying them in for the duration of a run — + // writes into the user's repository, which this harness refuses to do. + std::filesystem::path buildfile_dir; + std::filesystem::path build_dir; // where this engine may write + std::filesystem::path log_path; // child stdout+stderr goes here + Variant variant{Variant::Modules}; + std::string profile{"release"}; // release | debug + std::string compiler{"default"}; // gcc | clang | default + int jobs{0}; // 0 = let the engine decide + // How long ONE configure or build may take before the child is killed and + // the cell reported as `failed` with "timed out". 0 = wait forever. + // + // It lives on the Job rather than inside the runner because every engine + // has to pass it to platform::run itself, and an engine that forgets is an + // engine that can still hang the whole matrix — which is what happened. + double timeout_s{0.0}; +}; + +// NOTE: which file each scenario perturbs is NOT declared here. Only the fixture +// knows its own shape — "hub" means something different in a 10-unit synthetic +// project than in mcpp's 137-module graph — so `fixture::Targets` owns it and +// this module stays free of any assumption about the project being measured. + +// Cold builds are expensive and their variance is low; incremental scenarios are +// cheap and noisier, so they would justify more repetitions. Encoded here rather +// than in the runner so the policy is visible next to the scenario it applies to. +// +// THREE, not five. On a real project an "incremental" scenario is not cheap — +// `edit-comment` on xlings rebuilds 45 importers, so five repetitions is five +// near-full rebuilds and a windows/clang cell spent over half an hour on one +// engine. The extra samples were not buying accuracy worth that: the spread +// across runs on the pinned workloads is under 2%, and every published table is +// a median. A matrix nobody waits for is one nobody reads. +constexpr int default_runs(Scenario) { return 3; } + +} // namespace bench diff --git a/bench/src/toolchain.cpp b/bench/src/toolchain.cpp new file mode 100644 index 00000000..d44643f6 --- /dev/null +++ b/bench/src/toolchain.cpp @@ -0,0 +1,167 @@ +// bench.toolchain — implementation. +// +// `module bench.toolchain;` with no `export`: an implementation unit, so nothing below +// reaches an importer's BMI. The pinned version constants stay in the interface — +// they are `constexpr` and callers use them at compile time. +module bench.toolchain; + +import std; +import bench.platform; + +namespace bench::toolchain { + +bool on_windows() { return platform::OS_NAME == "windows"; } + +bool is_clang_request(std::string_view compiler) { + return compiler.find("clang") != std::string_view::npos + || compiler.find("llvm") != std::string_view::npos; +} + +bool resolves_to_clang(std::string_view compiler) { + return is_clang_request(compiler) || platform::OS_NAME != "linux"; +} + +std::string mcpp_pin(std::string_view compiler) { + if (resolves_to_clang(compiler)) + return std::format("llvm@{}", on_windows() ? kLlvmWindows : kLlvm); + return std::format("gcc@{}", kGcc); +} + +std::filesystem::path registry_xpkgs() { +#if defined(_MSC_VER) +#pragma warning(suppress : 4996) +#endif + if (const char* home = std::getenv("MCPP_HOME")) + return std::filesystem::path(home) / "registry" / "data" / "xpkgs"; +#if defined(_MSC_VER) +#pragma warning(suppress : 4996) +#endif + const char* user = std::getenv(platform::OS_NAME == "windows" ? "USERPROFILE" : "HOME"); + if (!user) return {}; + return std::filesystem::path(user) / ".mcpp" / "registry" / "data" / "xpkgs"; +} + +Resolved payload_cxx(std::string_view compiler) { + const auto xpkgs = registry_xpkgs(); + if (xpkgs.empty()) + return {{}, "neither MCPP_HOME nor HOME/USERPROFILE is set"}; + + const bool clang = resolves_to_clang(compiler); + const std::string pkg = clang ? "xim-x-llvm" : "xim-x-gcc"; + const std::string ver{clang ? (on_windows() ? kLlvmWindows : kLlvm) : kGcc}; + const std::string exe = std::string(clang ? "clang++" : "g++") + + (on_windows() ? ".exe" : ""); + + const auto driver = xpkgs / pkg / ver / "bin" / exe; + std::error_code ec; + if (std::filesystem::exists(driver, ec)) return {driver, {}}; + + return {{}, std::format("{} is not unpacked (run `mcpp toolchain install {}@{}`)", + driver.string(), clang ? "llvm" : "gcc", ver)}; +} + +PayloadFlags payload_flags(std::string_view compiler) { + PayloadFlags f; + // Only a compiler FROM the registry gets these; a host compiler already + // knows where its own runtime is, and adding a registry sysroot to it + // produces a mixed build that fails somewhere unrelated. + if (compiler.find("xpkgs") == std::string_view::npos) return f; + + const auto xpkgs = registry_xpkgs(); + if (xpkgs.empty()) return f; + std::error_code ec; + + if (!is_clang_request(compiler)) { + // gcc: -B for `as`/`ld`, --sysroot for headers and startup files, and + // BOTH must reach compile and link — the driver spawns `as` at compile + // time and `ld` at link time, so one side alone silently falls back. + for (const auto& e : std::filesystem::directory_iterator( + xpkgs / "xim-x-binutils", ec)) { + f.compile += " -B" + (e.path() / "bin").string(); + f.link += " -B" + (e.path() / "bin").string(); + break; + } + auto sysroot = registry_xpkgs().parent_path().parent_path() + / "subos" / "default"; + if (std::filesystem::is_directory(sysroot, ec)) { + f.compile += " --sysroot=" + sysroot.string(); + f.link += " --sysroot=" + sysroot.string(); + } + return f; + } + + // ⚠️ macOS GETS NOTHING, AND THAT IS THE CORRECT ANSWER. + // + // Pointing `-L`/`-rpath` at the registry's lib directory puts a second + // libc++ where the platform toolchain can find it, and Apple's own linker + // links against libc++ — so `ld` itself was resolved against the payload's + // copy and died before it linked anything: + // + // dyld: Symbol not found: __ZdaPv + // Referenced from: /Applications/Xcode_*.app/.../usr/bin/ld + // Expected in: …/registry/…/lib/libc++.1.0.dylib + // + // cmake, bazel and the reference mcpp all failed identically while the mcpp + // UNDER TEST passed on the same runner — which is the proof that mcpp does + // not pass these flags on macOS either. The payload clang already knows + // where its own libc++ is, and cmake supplies `-isysroot` itself. + if (platform::OS_NAME == "macos") return f; + + // clang elsewhere: an explicit libc++ chain rather than --sysroot, which is + // what mcpp itself drives clang with. Handing clang gcc's sysroot is the + // mirror of the gcc branch's bug — one arm on the payload libc, the other + // on the host's. + const std::string ver{on_windows() ? kLlvmWindows : kLlvm}; + const auto root = xpkgs / "xim-x-llvm" / ver; + if (std::filesystem::is_directory(root / "include" / "c++" / "v1", ec)) { + f.compile += " --no-default-config -nostdinc++" + " -isystem" + (root / "include" / "c++" / "v1").string(); + // ⚠️ AND THE PER-TRIPLE DIRECTORY, which is where `__config_site` lives. + // libc++'s `__config` includes it, so without this every TU dies with + // + // __config:13:10: fatal error: '__config_site' file not found + // + // pointing inside the standard library rather than at a missing flag. + // hermetic_payload.cmake globs for it; this port dropped that line. + for (const auto& d : std::filesystem::directory_iterator(root / "include", ec)) { + const auto cand = d.path() / "c++" / "v1"; + if (std::filesystem::is_directory(cand, ec)) + f.compile += " -isystem" + cand.string(); + } + // ⚠️ AND THE PER-TRIPLE lib DIRECTORY. In this payload libc++ lives in + // `lib/x86_64-unknown-linux-gnu/`, not `lib/`. A clang DRIVER finds it + // by itself, which is why the cmake arm worked with `-L…/lib` alone — + // but an engine that links through a different driver does not, and + // xmake failed with + // + // ld: cannot find -lc++: No such file or directory + // + // Naming both directories makes the flags independent of who links. + // ⚠️ -L TELLS THE LINKER; -rpath TELLS THE LOADER. They are different + // questions and this payload needs both answered: libc++ lives inside + // the registry, nowhere the dynamic loader looks by default. With only + // -L, every engine on macOS produced a binary that linked cleanly and + // then died the moment it ran: + // + // dyld: Symbol not found: __ZdaPv (operator delete[]) + // + // cmake, mcpp and bazel all failed identically, which is the tell that + // it was the flags rather than any one engine. + const auto add_libdir = [&](const std::filesystem::path& d) { + f.link += " -L" + d.string() + " -Wl,-rpath," + d.string(); + }; + f.link += " -nostdlib++"; + add_libdir(root / "lib"); + for (const auto& d : std::filesystem::directory_iterator(root / "lib", ec)) { + if (!d.is_directory()) continue; + if (std::filesystem::exists(d.path() / "libc++.so", ec) || + std::filesystem::exists(d.path() / "libc++.dylib", ec) || + std::filesystem::exists(d.path() / "libc++.a", ec)) + add_libdir(d.path()); + } + f.link += " -lc++ -lc++abi"; + } + return f; +} + +} // namespace bench::toolchain diff --git a/bench/src/toolchain.cppm b/bench/src/toolchain.cppm new file mode 100644 index 00000000..6fa2289e --- /dev/null +++ b/bench/src/toolchain.cppm @@ -0,0 +1,107 @@ +// bench.toolchain — WHICH compiler every engine is handed, and where it lives. +// +// This module exists because the same decision was being made in two places. +// The fixture's generated `mcpp.toml` pinned `gcc@16.1.0`, and the CI workflow +// separately resolved `command -v g++` for cmake/xmake/bazel. Those two are not +// the same compiler, and nothing anywhere said so: +// +// * mcpp built the fixture with the registry's gcc 16.1.0 and passed; +// * cmake and xmake were handed the runner's gcc 13.3.0, which cannot build +// C++23 modules at all — cmake failed to configure, xmake crashed gcc with +// an internal compiler error, and both were recorded as `failed`. +// +// Forty-eight of the seventy-two cells in a "passing" matrix job failed that +// way. The suite's own fairness rule (see `resolve_cxx`) says every engine that +// can be told which compiler to use MUST be told the same one; this module is +// what makes that rule reachable, by naming ONE payload and handing it to +// everybody including mcpp. +// +// The versions are pinned rather than "whatever is newest" for the reason every +// other pin in this repository exists: a benchmark whose toolchain moves under +// it reports the toolchain's change as the engine's. +export module bench.toolchain; + +import std; +import bench.platform; + +export namespace bench::toolchain { + +// The payload every arm of the benchmark compiles against. +// +// Windows is on llvm 20.1.7 rather than 22.1.8 because that is the version +// mcpp's registry actually ships for the PE target; pinning a version that is +// not there does not produce a slower number, it produces `unavailable`. +inline constexpr std::string_view kGcc = "16.1.0"; +inline constexpr std::string_view kLlvm = "22.1.8"; +inline constexpr std::string_view kLlvmWindows = "20.1.7"; + +bool on_windows(); + +// Is this compiler request a clang one? The single spelling of that test, used +// by both the manifest emitter and the payload lookup. +bool is_clang_request(std::string_view compiler); + +// Which FAMILY a compiler request resolves to on this host — the single +// decision `mcpp_pin` and `payload_cxx` both read, so the toolchain mcpp is told +// to use and the driver every other engine is handed cannot disagree. +// +// ⚠️ THE HOST IS PART OF THE ANSWER. There is no gcc payload for macOS in mcpp's +// registry (bench/matrix.json excludes the macos/gcc cell for exactly that +// reason), and the Windows payload is llvm. A pin that reads `gcc@16.1.0` +// everywhere fails on those hosts with +// +// error: toolchain 'gcc@16.1.0': package 'xim:gcc@16.1.0' not found +// +// which is what happened the moment this replaced the old emitter's explicit +// `macos = "llvm@..."` / `windows = "llvm@..."` overrides with one `default`. +bool resolves_to_clang(std::string_view compiler); + +// What the fixture's `mcpp.toml` must say so that mcpp uses the same compiler +// every other engine was handed. +std::string mcpp_pin(std::string_view compiler); + +// Where mcpp keeps its packages. MCPP_HOME first, matching mcpp's own +// resolution order and the CMake helper in projects/common/. +std::filesystem::path registry_xpkgs(); + +// The C++ driver for `compiler` inside that payload, or nullopt with a reason. +// +// Returning the REASON rather than a bare nullopt matters: "the payload is not +// unpacked" and "this machine has no mcpp" lead to different fixes, and a +// benchmark that silently falls back to the host compiler when it cannot find +// the payload is the exact failure this module was written to end. +struct Resolved { + std::filesystem::path driver; + std::string why; // set when `driver` is empty +}; + +Resolved payload_cxx(std::string_view compiler); + +// The flags a FOREIGN engine needs so that a payload compiler can actually +// build and link — the generated fixture's counterpart of +// bench/projects/common/cmake/hermetic_payload.cmake. +// +// ⚠️ WHY THIS EXISTS AT ALL, given that file exists. The checked-in project +// descriptions `include()` it; the fixture is GENERATED into a scratch +// directory by a binary that may live anywhere, so it has no path to include. +// The two are the same decision in two places and must be kept in step — the +// alternative considered (emit an `include()` of an absolute path) makes every +// generated fixture depend on this checkout still being where it was. +// +// It is needed because `--compiler payload:gcc` hands cmake and xmake a +// compiler out of mcpp's registry, and a bare registry gcc has no idea where +// its assembler, linker or libc are: +// +// /usr/bin/ld: cannot find crt1.o: No such file or directory +// /usr/bin/ld: cannot find -lm: No such file or directory +// +// which cmake reports as "the C++ compiler is not able to compile a simple +// test program", i.e. as a configure failure with no mention of a sysroot. +struct PayloadFlags { + std::string compile; + std::string link; +}; + +PayloadFlags payload_flags(std::string_view compiler); + +} // namespace bench::toolchain diff --git a/bench/tests/harness.sh b/bench/tests/harness.sh new file mode 100755 index 00000000..8c2e4d38 --- /dev/null +++ b/bench/tests/harness.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +# requires: python3 +# bench/ harness: builds with mcpp, measures a fixture, and emits a valid report. +# +# This is an INTEGRATION test for the benchmark suite, not a benchmark: it uses +# the smallest fixture that still exercises the module graph, and asserts on the +# protocol rather than on any timing. Timings on CI are noise; the contract is not. +set -e + +# bench/tests -> two levels up is the repository root. +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +TMP=$(mktemp -d) +# `|| true`: on Windows a build killed by the --timeout test (§12) leaves the +# compilers ninja spawned still running for a moment, and they hold the child +# log open — `rm -rf` then fails with "Device or resource busy" and, because the +# trap runs on EXIT, ITS status becomes the script's. The suite printed +# "bench harness OK" and the job went red anyway. Failing to delete a temp +# directory is not a test result. +trap "rm -rf $TMP || true" EXIT + +# mcpp's e2e runner exports MCPP as the binary under test; a standalone run has +# nothing to inherit and used to die on `line 22: : command not found`, which +# names neither the variable nor the fix. +# +# NOT defaulted to PATH. `command -v mcpp` returns the xlings SHIM, and a shim +# re-resolves which mcpp to exec from the workspace it is invoked in — while +# bench deliberately runs every engine with its cwd inside the tree under test. +# The shim then picked a different mcpp than the one being tested and the cells +# failed as `mcpp@2026.8.11.2 | unknown command: build`, naming a version nobody +# asked for. This is the same defect as the reference-mcpp arm on CI; the rule +# from bench/README §4 is absolute: an engine is named by BINARY, never by PATH. +[ -n "${MCPP:-}" ] || { + echo "FAIL: MCPP is unset. Set it to a REAL mcpp binary, not a PATH name:" + echo " MCPP=\$(bash .github/tools/newest_artifact.sh . mcpp) bash bench/tests/harness.sh" + echo " (mcpp's e2e runner exports it; \`bash tests/e2e/230_bench_harness.sh\` works too.)" + exit 1 +} +# Absolutised against the INVOKING cwd, which is why this runs before the `cd` +# below: `MCPP=./target/.../mcpp` is what every natural way of producing the +# path yields, and a relative one silently stops resolving the moment the script +# changes directory — including the command this very message suggests. +case "$MCPP" in + /*|?:[/\\]*) ;; + *) MCPP="$PWD/$MCPP" ;; +esac +[ -x "$MCPP" ] || { echo "FAIL: MCPP=$MCPP is not an executable file"; exit 1; } + +cd "$REPO/bench" +echo "harness built with: $("$MCPP" --version 2>&1 | head -1) ($MCPP)" +"$MCPP" build > /dev/null + +# NEWEST, not `find | head -1`: target/ holds one directory per toolchain +# fingerprint, so a tree built more than once has several binaries with this +# name and `head -1` picks whichever the filesystem lists first — routinely a +# stale one. That is a test exercising code that has already been replaced, with +# no symptom at all. See .github/tools/newest_artifact.sh. +# +# ⚠️ THE NAME COMES FROM THE MANIFEST, NOT FROM THIS LINE. The harness binary was +# renamed `bench` -> `mbench` and this kept asking for `bench`, so the test died +# with `newest_artifact: no 'bench' under target/*/*/bin/` — a rename caught only +# by running it, because nothing ties the two together. Reading the name out of +# bench/mcpp.toml is what ties them. +BENCH_NAME="$(sed -n 's/^ *name *= *"\([^"]*\)".*/\1/p' "$REPO/bench/mcpp.toml" | head -1)" +[ -n "$BENCH_NAME" ] || { echo "FAIL: no package name in bench/mcpp.toml"; exit 1; } +BENCH="$REPO/bench/$(bash "$REPO/.github/tools/newest_artifact.sh" target "$BENCH_NAME")" + +# ⚠️ A TEST MUST MEASURE, NOT RESUME. +# +# mbench caches every measured sample under `--cache-root` (default `.mbench` in +# the CURRENT DIRECTORY) and replays it when the configuration matches. Run from +# the repository root that is the repository's own cache: this test both wrote +# into it and, on the second run, reported +# run 1/1 — already recorded, skipping +# i.e. it stopped exercising the thing it exists to exercise while still passing. +# Pinning the cache into $TMP makes every run of this test a fresh measurement +# and leaves the repository alone. +bench() { "$BENCH" --cache-root "$TMP/mbench-cache" "$@"; } + +# 1. Availability listing must classify mcpp itself as present. If this fails the +# probe path is broken, and every later cell would be reported `unavailable` +# for the wrong reason. +# Engines are named by BINARY, not by PATH lookup: `$MCPP` is the build under +# test, while a bare `mcpp` resolves to whatever the sandbox has — on CI that is +# an xlings shim reporting "'mcpp' is not installed", which failed every cell. +out=$(bench --list --engines "mcpp=$MCPP") +# The label carries the version it discovered ("mcpp@2026.8.12.1"), which is what +# makes a two-binary comparison legible; match the prefix, not the whole token. +echo "$out" | grep -qE '^mcpp(@[^ ]+)? +yes' || { echo "mcpp not reported available:"; echo "$out"; exit 1; } +# The note carries each engine's reported VERSION, so a result file can answer +# "which cmake produced this?". Some tools colour that banner, and the escape +# sequences must be stripped before they reach a JSON result — an ESC here means +# the CSI parser regressed (it once left the "0m" of every colour reset behind). +# Written as an explicit `if` rather than `grep -q ... && { ... }`: under +# `set -e` the exit status of an AND-OR list whose left side fails is the exact +# corner this suite has been bitten by before. +if printf '%s' "$out" | grep -q "$(printf '\033')"; then + echo "engine notes contain ANSI escapes:"; printf '%s' "$out" | cat -v; exit 1 +fi + +# 2. A real measurement over the modules variant. Tiny on purpose: 4 units still +# produce a module graph with depth, which is what the harness is for. +# On failure the child's build log is the only thing that explains why — and the +# trap deletes $TMP on exit, so a message that merely names the path is useless +# in CI. Dump it here instead of leaving a dangling reference. +dump_child_logs() { + echo "--- harness stdout ---"; cat "$TMP/stdout.txt" 2>/dev/null + for log in "$TMP"/work/logs/*.log; do + [ -f "$log" ] || continue + echo "--- $log ---"; tail -40 "$log" + done +} + +# --preset names the size instead of spelling it out, which is also the only +# place the preset code path gets exercised. +bench --engines "mcpp=$MCPP" --variants modules --scenarios cold,noop \ + --preset smoke --runs 1 \ + --work "$TMP/work" --out "$TMP/report.json" > "$TMP/stdout.txt" \ + || { echo "harness exited non-zero"; dump_child_logs; exit 1; } + +# 3. The report must be a protocol-shaped document, not merely non-empty. +# ⚠️ THE EXPECTED VERSION COMES FROM THE PROTOCOL, NOT FROM THIS LINE. +# +# This asserted `"protocol_version": 1` literally. The protocol says to bump on +# any field addition — which is exactly what adding `under_test` did — and the +# test then failed with +# report is missing protocol_version +# while printing a report whose very next line read `"protocol_version": 2`. The +# message named the wrong thing because the check was really "is it 1?". +# +# Reading kProtocolVersion out of the source keeps the two in step, and still +# fails loudly if the field disappears entirely. +WANT_PROTO="$(sed -n 's/.*kProtocolVersion *= *\([0-9][0-9]*\).*/\1/p' \ + "$REPO/bench/src/protocol.cppm" | head -1)" +[ -n "$WANT_PROTO" ] || { echo "FAIL: no kProtocolVersion in bench/src/protocol.cppm"; exit 1; } +grep -q "\"protocol_version\": $WANT_PROTO" "$TMP/report.json" \ + || { echo "report does not carry protocol_version $WANT_PROTO"; cat "$TMP/report.json"; exit 1; } +grep -q '"status": "ok"' "$TMP/report.json" \ + || { echo "no cell succeeded"; cat "$TMP/report.json"; dump_child_logs; exit 1; } + +# 4. INVARIANT 1: a non-ok cell must never carry a timing. Asserted from BOTH +# sides — checking only that ok cells have medians would pass a harness that +# emitted medians for everything, which is exactly the bug this protocol was +# designed to make impossible. +python3 - "$TMP/report.json" <<'PY' +import json, sys +cells = json.load(open(sys.argv[1]))["cells"] +assert cells, "report has no cells" +for c in cells: + if c["status"] == "ok": + assert "median_s" in c, f"ok cell without a median: {c}" + assert c["runs"] > 0, f"ok cell with zero runs: {c}" + else: + assert "median_s" not in c, f"non-ok cell carrying a timing: {c}" + assert c["note"], f"non-ok cell without a reason: {c}" +PY + +# 5. Host facts must be populated — a result without its host is not comparable +# to anything, so an empty one is a defect rather than a cosmetic gap. +python3 - "$TMP/report.json" <<'PY' +import json, sys +h = json.load(open(sys.argv[1]))["host"] +assert h["os"], "host.os is empty" +assert h["logical_cores"] >= 1, f"implausible core count: {h}" +assert h["arch"] != "unknown", f"arch not detected: {h}" +PY + +# 6. The three fixture variants must all generate and differ in SHAPE, not just +# in file names: modules-impl is the variant whose whole point is that bodies +# live outside the interface unit. +bench --engines "mcpp=$MCPP" --variants headers,modules,modules-impl --scenarios noop \ + --units 3 --fanin 1 --weight 1 --runs 1 \ + --work "$TMP/w2" --out "$TMP/r2.json" > /dev/null +# Directory names are slugged from the engine label, which carries a version, so +# resolve them by suffix instead of hard-coding the label. +hdr=$(echo "$TMP"/w2/*-headers); mods=$(echo "$TMP"/w2/*-modules) +impl=$(echo "$TMP"/w2/*-modules-impl) +[ -f "$hdr/include/unit_0.hpp" ] || { echo "headers variant missing its header"; exit 1; } +[ -f "$mods/src/unit_0.cppm" ] || { echo "modules variant missing its interface"; exit 1; } +[ -f "$impl/src/unit_0_impl.cpp" ] || { echo "modules-impl variant has no implementation unit"; exit 1; } +grep -q 'export int unit_0_value();' "$impl/src/unit_0.cppm" \ + || { echo "modules-impl interface should DECLARE, not define"; exit 1; } +grep -q 'export int unit_0_value() {' "$mods/src/unit_0.cppm" \ + || { echo "modules interface should DEFINE inline"; exit 1; } + +# 7. No fixture may say `import std;`. Engines differ wildly in std-module +# support and that difference would dominate every measurement — the suite +# measures module machinery, not std-module support. +if grep -rq 'import std;' "$TMP/w2"/*/src/ 2>/dev/null; then + echo "a generated fixture imports std, which breaks cross-engine comparability" + exit 1 +fi + +# 8. A RELATIVE engine program path must still resolve. Every measured command +# runs with its cwd set to the project under test, so `--engines mcpp=./bin` +# used to resolve against the fixture and fail to spawn — reported per cell as +# `exited -1` across the whole matrix, with an empty log to explain it. +# +# The run happens from the binary's OWN directory, with the fixture under +# $TMP: that is all the bug needs (cwd at launch != the tree the child is +# later run in) and it is expressible everywhere. Deriving a relative path +# between two arbitrary directories is not — on Windows `$MCPP` and `$TMP` +# routinely sit on different drives (`path is on mount 'D:', start on mount +# 'C:'`), and on macOS `mktemp -d` returns `/var/folders/...` while the +# process's real cwd is `/private/var/folders/...`, one level deeper. +# +# The binary is REFERENCED where it is, never copied: mcpp locates its +# payloads relative to its own installation, so a copy in a scratch dir would +# fail for a reason that has nothing to do with the path handling under test. +BINDIR=$(dirname "$MCPP") +BINNAME=$(basename "$MCPP") +( cd "$BINDIR" \ + && bench --engines "mcpp=./$BINNAME" --variants modules --scenarios cold \ + --units 3 --fanin 1 --weight 1 --runs 1 \ + --work "$TMP/w3" --out "$TMP/r3.json" > "$TMP/stdout3.txt" ) \ + || { echo "harness exited non-zero on a relative engine path"; cat "$TMP/stdout3.txt"; exit 1; } +python3 - "$TMP/r3.json" <<'PY' +import json, sys +cells = json.load(open(sys.argv[1]))["cells"] +assert cells, "no cells for a relative engine path" +bad = [c for c in cells if c["status"] != "ok"] +assert not bad, f"relative engine path did not resolve: {bad}" +PY + +# 9. And a program that cannot be run at all must be reported with a reason that +# stands on its own. Here the probe catches it first (`unavailable`), but the +# invariant is the same one `failure_note` enforces further in: never point a +# reader at a log the child never got far enough to write. +bench --engines "mcpp=$TMP/definitely-not-here" --variants modules --scenarios cold \ + --units 3 --fanin 1 --weight 1 --runs 1 \ + --work "$TMP/w4" --out "$TMP/r4.json" > /dev/null 2>&1 || true +python3 - "$TMP/r4.json" <<'PY' +import json, sys +cells = json.load(open(sys.argv[1]))["cells"] +assert cells, "no cells for a missing engine binary" +for c in cells: + assert c["status"] != "ok", f"a missing binary produced a timing: {c}" + assert c["note"], f"a missing binary produced no reason: {c}" + assert "see " not in c["note"], \ + f"reason points at a log that was never written: {c['note']}" +PY + +# 10. --hub/--leaf/--body are PROJECT-RELATIVE, and must resolve from anywhere. +# +# They used to be taken as given, i.e. relative to the harness's own working +# directory. That is the project directory only when you are benchmarking +# the tree you are standing in — true for mcpp measuring itself, false for +# every other project — and it fails SILENTLY: exists() says no, the cell +# reports `skipped --hub points at a file that does not exist`, and the run +# still exits 0. Three CI jobs reported success with zero measurements. +# +# Driven from a different cwd on purpose; that difference IS the bug. +mkdir -p "$TMP/proj/src" +cat > "$TMP/proj/mcpp.toml" <<'TOML' +[package] +name = "relhub" +version = "0.1.0" +TOML +printf 'export module hub;\nexport int hub_value() { return 1; }\n' > "$TMP/proj/src/hub.cppm" +printf 'import hub;\nint main() { return hub_value() - 1; }\n' > "$TMP/proj/src/main.cpp" + +( cd "$TMP" \ + && bench --engines "mcpp=$MCPP" --project "$TMP/proj" --variants native \ + --scenarios touch-hub,edit-body --runs 1 \ + --hub src/hub.cppm --body src/hub.cppm \ + --work "$TMP/w5" --out "$TMP/r5.json" > "$TMP/stdout5.txt" 2>&1 ) \ + || { echo "harness exited non-zero on project-relative targets"; cat "$TMP/stdout5.txt"; exit 1; } +python3 - "$TMP/r5.json" <<'PY' +import json, sys +cells = json.load(open(sys.argv[1]))["cells"] +assert cells, "no cells for project-relative targets" +missing = [c["note"] for c in cells if "does not exist" in c["note"]] +assert not missing, ("--hub/--body were resolved against the harness's cwd " + f"rather than the project: {missing}") +PY + +# 11. EXIT STATUS. A run that measured nothing must not report success — the +# whole matrix did exactly that for weeks (6 ok / 48 failed / 18 +# unavailable, and green), as did an xlings job whose every cell was +# skipped. Asserted from BOTH sides, because a harness that always exited +# non-zero would sail through a one-sided check: every successful run above +# is the other half. +if bench --engines "mcpp=$TMP/definitely-not-here" --variants modules \ + --scenarios cold --units 3 --fanin 1 --weight 1 --runs 1 \ + --work "$TMP/w6" --out "$TMP/r6.json" > /dev/null 2>&1; then + echo "a run in which nothing was measured exited 0"; exit 1 +fi + +# 12. --timeout must KILL a child rather than wait on it, and must say that is +# what happened. Without it a hung engine consumes the whole CI budget and +# the log stays empty, because a cell only prints once it is over: two jobs +# sat 25 minutes inside one child that way. One second is far below any real +# cold build, so the deadline is certain to fire. +bench --engines "mcpp=$MCPP" --variants modules --scenarios cold \ + --units 3 --fanin 1 --weight 1 --runs 1 --timeout 1 \ + --work "$TMP/w7" --out "$TMP/r7.json" > "$TMP/stdout7.txt" 2>&1 || true +grep -qi 'timed out' "$TMP/stdout7.txt" || { + echo "a 1-second deadline neither fired nor was reported:" + cat "$TMP/stdout7.txt"; exit 1; } + +echo "bench harness OK" diff --git a/bench/tools/report.py b/bench/tools/report.py new file mode 100755 index 00000000..23b930ee --- /dev/null +++ b/bench/tools/report.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Turn one or more bench report JSONs into the markdown table they describe. + +WHY THIS EXISTS. The published tables in bench/results/, bench/README.md and the +root README were transcribed by hand from harness output. Transcription is +exactly the failure this whole suite is built to remove — a benchmark whose +headline number was mistyped is indistinguishable from one that was measured, +and there is no test that can catch it. So the tables come from the JSON. + + bench/tools/report.py run.json [more.json ...] [--baseline NAME] + +Rows are scenarios, columns are engines, ordered as the report file lists them. +Each cell is `s · x`, the ratio against `--baseline` within the +same (variant, scenario) group — the same grouping the harness's own summary +uses, because a ratio across source forms or perturbations is not a ratio. + +A cell that has no measurement renders as `-`, and one that HAS a measurement +which did not succeed renders as its status in italics. Those two are opposite +claims and must never collapse into the same mark: + + - not measured — no data exists for this combination + _failed_ the engine ran and produced no artifact — a FINDING + _unavailable_ the engine is not installed, or cannot express this cell + +Never a blank, and never `0.00s`: protocol invariant 1 says a failure must not be +able to look like a measurement, and a table is where that invariant is most +easily lost — the shell harness this suite replaces formatted three failed cells +as `0.000 s` and they were published as the fastest builds ever recorded. + +The PERTURBATION FORM is carried through into a footnote when a group has more +than one, because `edit-comment` means two different things depending on whether +the target unit had a function body (see SPEC.md §4). +""" +import json +import sys +from collections import OrderedDict + + +def load_journal(path): + """Reduce a `.mbench//journal.jsonl` to report cells. + + ⚠️ WHY THIS EXISTS. The journal records one line per measured SAMPLE, but a + report JSON is only written when a whole CELL finishes — so an interrupted + run left its samples on disk with no way to look at them. 42 measured points + of an in-flight cell were invisible to every table in the repository while + sitting in a file. The design says the journal is the source of truth and the + report is derived from it; until this function, that was only half true. + + The reduction is deliberately the same one the harness does: group by + (project, variant, scenario, engine), median/min/max over the samples + present. A partial group is reported with the count it actually has, never + padded — `runs` in the output is how many samples exist, not how many were + planned. + """ + groups = OrderedDict() + for line in open(path, encoding="utf-8"): + line = line.strip() + if not line.startswith("{") or not line.endswith("}"): + continue # a half-written last line is expected after a kill + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + key = (e.get("project", ""), e.get("variant", ""), + e.get("scenario", ""), e.get("engine", "")) + groups.setdefault(key, []).append(e) + + cells = [] + for (project, variant, scenario, engine), samples in groups.items(): + walls = sorted(s["wall_s"] for s in samples) + n = len(walls) + failed = [s for s in samples if s.get("exit", 0) != 0] + cell = { + "engine": engine, "compiler": "", "profile": "", + "scenario": scenario, "fixture": project, "variant": variant, + "runs": n, + "note": "" if not failed else f"{len(failed)} of {n} samples exited non-zero", + "status": "ok" if not failed else "failed", + "samples": [{"wall_s": w} for w in walls], + } + if not failed: + cell["median_s"] = walls[n // 2] if n % 2 else (walls[n // 2 - 1] + walls[n // 2]) / 2 + cell["min_s"] = walls[0] + cell["max_s"] = walls[-1] + cells.append(cell) + return cells + + +def load(paths): + cells, hosts = [], [] + for p in paths: + # A journal and a report describe the same thing at different stages; + # accepting both means a table can be drawn at any moment, not only + # after a cell completes. + if p.endswith(".jsonl"): + cells += load_journal(p) + hosts.append({}) + continue + d = json.load(open(p, encoding="utf-8")) + cells += d["cells"] + hosts.append(d.get("host", {})) + return cells, hosts + + +def form_of(note): + marker = "perturbation: " + return note.split(marker, 1)[1].strip() if marker in note else "" + + +def render(cells, baseline): + engines = list(OrderedDict.fromkeys(c["engine"] for c in cells)) + groups = list(OrderedDict.fromkeys((c["variant"], c["scenario"]) for c in cells)) + # Group by variant so a table never mixes source forms. + variants = list(OrderedDict.fromkeys(v for v, _ in groups)) + out, notes = [], [] + + for variant in variants: + out.append(f"\n**`{variant}`**\n") + out.append("| scenario | " + " | ".join(f"`{e}`" for e in engines) + " |") + out.append("|---" * (len(engines) + 1) + "|") + for v, scenario in groups: + if v != variant: + continue + row = [f"`{scenario}`"] + here = {c["engine"]: c for c in cells + if c["variant"] == v and c["scenario"] == scenario} + base = next((c for e, c in here.items() + if baseline in e and c["status"] == "ok"), None) + forms = {form_of(c.get("note", "")) for c in here.values()} - {""} + if len(forms) > 1: + notes.append(f"`{variant}`/`{scenario}` mixes perturbation forms " + f"({', '.join(sorted(forms))}) — those are different " + f"questions; see SPEC.md §4") + for e in engines: + c = here.get(e) + if c is None: + row.append("-") # not measured — see the legend above + elif c["status"] != "ok": + row.append(f"_{c['status']}_") + elif base and base.get("median_s"): + ratio = c["median_s"] / base["median_s"] + mark = " ← baseline" if c is base else "" + row.append(f"{c['median_s']:.2f}s · {ratio:.2f}x{mark}") + else: + row.append(f"{c['median_s']:.2f}s") + out.append("| " + " | ".join(row) + " |") + for n in OrderedDict.fromkeys(notes): + out.append(f"\n> ⚠️ {n}") + return "\n".join(out) + + +# ── the root README's headline table ─────────────────────────────────────── +# +# That table was the last hand-transcribed thing in the repository, and it was +# also the WORST place for one: it is the first table a reader sees, and it was +# stitched from three separate runs because no single run measured every column. +# The standard set now does, so it can be generated — and once it is generated, +# `233_bench_matrix.sh` can check it against ONE file instead of a hard-coded +# list of three. +# +# ⚠️ The wording is deliberately not translated field-by-field. A benchmark table +# that says different things in two languages is two claims, and only one of them +# can be checked against the data. +ALLOW_SUSPECT = [False] +HEADLINE_SCENARIOS = ["cold", "noop", "touch-hub", "edit-body", "edit-comment"] +HEADLINE_WHAT = { + "en": { + "cold": "nothing built yet", + "noop": "nothing at all", + "touch-hub": "mtime only, content unchanged", + "edit-body": "a real edit inside a function body", + "edit-comment": "a comment added to a hub interface", + "scenario": "scenario", "what": "what changed", + }, + "zh": { + "cold": "还没编过", + "noop": "什么都没改", + "touch-hub": "只碰 mtime,内容不变", + "edit-body": "真的改了一个函数体", + "edit-comment": "在 hub 接口里加一行注释", + "scenario": "场景", "what": "改了什么", + }, +} + + +def engine_order(engines): + """mcpp arms first, newest version first, each default before its opt-in arm. + + Reading order matters more than it looks: the build under test and its + `+schedule=on` arm answer one question ("what does the key buy?") and the + released reference answers a different one ("did this get faster?"). Sorting + the raw strings interleaved them — reference, opt-in, default — so neither + pair sat together and every comparison was two columns apart. + """ + def version_key(e): + head = e.split("+", 1)[0] + ver = head.split("@", 1)[1] if "@" in head else "" + parts = [] + for piece in ver.split("."): + parts.append(int(piece) if piece.isdigit() else 0) + return parts + [0] * (4 - len(parts)) + + def key(e): + if not e.startswith("mcpp@"): + return (1, [], 0, e) # other engines after every mcpp arm + # negated version → newest first; base arm before its own option arm + return (0, [-p for p in version_key(e)], 1 if "+" in e else 0, e) + return sorted(engines, key=key) + + +# Short COLUMN names. The engine labels are the truth — `mcpp@2026.8.13.1`, +# `mcpp@2026.8.13.1+schedule=on`, `mcpp@2026.8.11.3` — but three of them side by +# side make the table wider than a README renders, so it arrives collapsed with a +# horizontal scrollbar and the reader sees two columns of a five-column +# comparison. The identity moves to the footnote, where it is read once. +# +# ⚠️ THE MAPPING IS EMITTED, NOT REPEATED. A short name in the table and a long +# one in the data is exactly the drift this tool exists to remove, so the header +# carries a machine-readable `` line and the guard reads the +# mapping from there instead of keeping a second copy. +def short_name(engine, newest, lang="en"): + """The column header. + + ⚠️ NOT `mcpp +schedule`. "schedule" is the name of the MECHANISM, and a + reader meeting this table for the first time has no idea whether a build + scheduler makes things faster, slower or merely different. The column is + there to say "this is the opt-in speed-up"; what it actually turns on is one + line further down, in the legend, where a name has room to be precise. + """ + if not engine.startswith("mcpp@"): + return engine + base, _, opt = engine.partition("+") + if opt: + return "mcpp +优化" if lang == "zh" else "mcpp +opt" + if base == newest: + return "mcpp" + return "mcpp (旧版)" if lang == "zh" else "mcpp (old)" + + +def columns_legend(short, engines, lang): + """The sentence that has to accompany short column names. + + Shortening a header is only safe if the identity it dropped is still stated + somewhere the reader will see. This is that somewhere, generated from the + same mapping the table uses so the two cannot disagree.""" + parts = [] + for e in engines: + name = short[e] + if not e.startswith("mcpp@"): + continue + # Keyed on the SHORT NAME this run produced, never on a literal spelling + # of it. Renaming the column used to leave this branch unmatched, and the + # opt-in arm then fell through to the `else` and was described as "the + # previously published release" — the wrong sentence, printed with total + # confidence, about the one column whose whole point is that it is the + # SAME binary as the first. + if name == "mcpp": + parts.append(f"`mcpp` = {e}, the build under test" + if lang == "en" else f"`mcpp` = {e},被测的这一版") + elif "+" in name: + parts.append(f"`{name}` = the SAME binary as `mcpp`, with the opt-in key " + "`[build] bmi_schedule = \"on\"` (off by default)" + if lang == "en" else + f"`{name}` = **和 `mcpp` 同一个二进制**,开了 opt-in 的 " + "`[build] bmi_schedule = \"on\"`(默认关闭)") + else: + parts.append(f"`{name}` = {e}, the previously published release" + if lang == "en" else f"`{name}` = {e},上一个已发布版") + return " · ".join(parts) + + +def headline(cells, baseline, lang): + w = HEADLINE_WHAT[lang] + # The real workload only: a headline table that silently mixed a generated + # fixture into it would be comparing two different questions. + # + # ⚠️ THE DISCRIMINATOR IS THE FIXTURE NAME, NOT THE VARIANT. `variant` is + # whatever the cell declared, and the same real project is labelled `modules` + # in one published run and `native` in another — so filtering on `native` + # silently dropped every cmake and xmake column and the table rendered with + # two columns and no ratios at all. + cells = [c for c in cells if not c["fixture"].startswith("synth-")] + projects = {c["fixture"] for c in cells} + if len(projects) > 1: + raise SystemExit(f"headline: {len(projects)} projects in these reports " + f"({', '.join(sorted(projects))}) — one table, one workload; " + f"pass the report for a single project") + if not cells: + raise SystemExit("headline: no real-workload cells in these reports " + "(only generated fixtures)") + engines = engine_order({c["engine"] for c in cells}) + newest = next((e.split("+", 1)[0] for e in engines if e.startswith("mcpp@")), "") + short = {e: short_name(e, newest, lang) for e in engines} + + # ⚠️ A COLD BUILD THAT IS NOT SLOWER THAN A NO-OP DID NOT BUILD ANYTHING. + # + # Not a hypothetical: rendering the previously published files produced + # xmake cold 0.60s · 153.1x + # next to cmake's 92s — xmake was resolving `--buildir` relative to `-P` and + # configuring into a directory that was already populated, so it exited + # having compiled nothing. The number was real, the measurement was not, and + # in a headline table it reads as xmake being 153x faster than cmake. + # + # This is the whole failure mode of the suite in one cell, so it BLOCKS + # rather than warns: a table nobody can publish is better than one that is + # wrong in the reader's favour. + suspect = [] + for e in engines: + cold = next((c for c in cells if c["engine"] == e and c["scenario"] == "cold" + and c["status"] == "ok"), None) + noop = next((c for c in cells if c["engine"] == e and c["scenario"] == "noop" + and c["status"] == "ok"), None) + if cold and noop and noop["median_s"] > 0 and cold["median_s"] < noop["median_s"] * 5: + suspect.append(f"{e}: cold {cold['median_s']:.2f}s vs noop " + f"{noop['median_s']:.2f}s — a cold build that is not at least " + f"5x its own no-op did not build the project") + if suspect and not ALLOW_SUSPECT[0]: + raise SystemExit("headline: refusing to render — " + + "; ".join(suspect) + + "\n(pass --allow-suspect to see it anyway)") + + rows = ["", + f"| {w['scenario']} | " + " | ".join(f"`{short[e]}`" for e in engines) + " |", + "|---" * (len(engines) + 1) + "|"] + for sc in HEADLINE_SCENARIOS: + here = {c["engine"]: c for c in cells if c["scenario"] == sc} + if not here: + continue + base = next((c for e, c in here.items() + if baseline in e and c["status"] == "ok"), None) + ok = [c for c in here.values() if c["status"] == "ok" and c.get("median_s") is not None] + best = min((c["median_s"] for c in ok), default=None) + cellsr = [] + for e in engines: + c = here.get(e) + if c is None: + cellsr.append("-") # not measured; see the legend above + elif c["status"] != "ok": + cellsr.append(f"_{c['status']}_") + else: + txt = f"{c['median_s']:.2f}s" + if base and base.get("median_s"): + txt += f" · {base['median_s'] / c['median_s']:.1f}x" + # NOT a code span. Wrapping `86.69s · 1.1x` was the worry, but + # a span renders MONOSPACE — wider than proportional text for the + # same characters — so it widened the very table it was meant to + # keep on screen. The fix for wrapping is fewer columns (below), + # not stiffer cells. + # Bold the fastest arm in the row, so the table reads without + # the reader dividing anything in their head. + cellsr.append(f"**{txt}**" if c["median_s"] == best else txt) + rows.append(f"| `{sc}` | " + " | ".join(cellsr) + " |") + return "\n".join(rows) + + +def main(argv): + baseline = "cmake" + mode = "table" + lang = "en" + paths = [] + it = iter(argv) + for a in it: + if a == "--baseline": + baseline = next(it, "cmake") + elif a == "--headline": + mode = "headline" + elif a == "--allow-suspect": + ALLOW_SUSPECT[0] = True + elif a == "--lang": + lang = next(it, "en") + else: + paths.append(a) + if not paths: + print(__doc__) + return 2 + if mode == "headline": + cells, hosts = load(paths) + h = hosts[0] + print(headline(cells, baseline, lang)) + print() + _engines = engine_order({c["engine"] for c in cells + if not c["fixture"].startswith("synth-")}) + _newest = next((e.split("+", 1)[0] for e in _engines if e.startswith("mcpp@")), "") + _scen = [c["scenario"] for c in cells if not c["fixture"].startswith("synth-")] + print("SCENARIOS: " + " · ".join( + f"`{s}` {HEADLINE_WHAT[lang][s]}" for s in HEADLINE_SCENARIOS if s in _scen)) + print("COLUMNS: " + columns_legend({e: short_name(e, _newest, lang) for e in _engines}, + _engines, lang)) + # The toolchain is recorded as the driver PATH; a footnote wants the + # compiler, not where this machine happens to keep it. + tc = str(h.get("toolchain", "")) + for part in tc.replace("\\", "/").split("/"): + if any(ch.isdigit() for ch in part) and "." in part: + tc = part + print(f"{h.get('os')} {h.get('arch')} · {h.get('cpu_model')} · " + f"{tc} · n={max((len(c.get('samples', [])) for c in cells), default=0)}" + f"") + return 0 + cells, hosts = load(paths) + if not cells: + print("no cells in those reports", file=sys.stderr) + return 1 + h = next((x for x in hosts if x), {}) + if h: + print(f"host: {h.get('os')} {h.get('arch')} · {h.get('cpu_model')} · " + f"{h.get('logical_cores')} logical / {h.get('physical_cores')} physical" + f"{' (heterogeneous)' if h.get('heterogeneous') else ''}") + else: + # A journal records samples, not the machine — that is in the report the + # run writes at the end. Say so rather than printing a row of `None`, + # which reads like the host detection failed. + print("host: not recorded in a journal (see the run's report JSON)") + print(f"baseline: {baseline}") + incomplete = sorted({c["runs"] for c in cells}) + if len(incomplete) > 1: + print(f"⚠️ sample counts differ across cells ({incomplete}) — this is a " + f"PARTIAL run; a cell with fewer samples has less dispersion, not less variance") + print(render(cells, baseline)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/docs/05-mcpp-toml.md b/docs/05-mcpp-toml.md index 2286f9ee..1a7f4b78 100644 --- a/docs/05-mcpp-toml.md +++ b/docs/05-mcpp-toml.md @@ -170,6 +170,8 @@ target = "x86_64-linux-musl" # Default build target when no --target is pa # (≙ cargo build.target; e.g. "ship fully-static") macos_deployment_target = "14.0" # Minimum supported OS version for macOS artifacts (macOS only) cache = "global" # Global dependency cache: global (default) | local | off (§2.10) +jobs = "auto" # Concurrent compiles: a positive number, or "auto" (§ below) +bmi_schedule = "auto" # Module-edge scheduling: auto (= off) | on | off (§ below) ``` `include_dirs_after` (#249) lists header directories that are searched **after** @@ -196,6 +198,55 @@ baseline, and 14.0 is the floor of LLVM's official static libraries themselves). This value enters the BMI fingerprint, so switching targets automatically rebuilds the module cache. +### Build concurrency (`jobs`) and module scheduling (`bmi_schedule`) + +```toml +[build] +jobs = "auto" # or a positive number; --jobs / MCPP_JOBS override it +bmi_schedule = "off" # auto (default, = off) | on | off +``` + +`jobs` is how many compiles run at once. `"auto"` is resolved **against the +machine doing the build**, never frozen into the manifest: it takes the physical +core count on a heterogeneous CPU (a 13900K is 8 P-cores + 16 E-cores, so its 32 +threads are not 32 equal workers) and clamps that by free memory, because a +single module interface compile peaks at 0.5–1.0 GB. Precedence is +`--jobs` / `MCPP_JOBS` > this key > the backend's own default. A malformed value +is **reported, never silently treated as the default** — a typo that quietly +restores the default is a build mysteriously slower than you asked for. + +`bmi_schedule` decides when importers are unblocked. + +| value | | +|---|---| +| `"auto"` | **the default, and it currently means OFF** | +| `"on"` | split the module edge: importers start when the BMI is published, not when the compiler exits | +| `"off"` | one edge per module | + +Only those three spellings are accepted. `"ON"`, `"true"` and `"yes"` are +**rejected with a diagnostic** rather than quietly meaning off — and they are not +harmless typos: the value enters the build fingerprint, so a rejected spelling +used to select a different build directory (a full rebuild) while changing +nothing about the schedule. + +**Why `auto` is off.** 86% of a module interface compile is code generation that +no importer reads, so publishing the BMI early is worth a lot — measured on mcpp +itself, `cold` 86.7s → 35.7s and `edit-body` 80.9s → 29.8s. But a scheduling +change that is wrong is wrong *silently*: a missed dependency does not fail the +build, it just stops rebuilding something. It stays opt-in until it has been +through CI on every platform. + +**What it does not help.** Where mcpp already skips the cascade — `touch-hub`, +`edit-comment` — there is no owed work to move off the critical path, and the +key buys nothing. See the [benchmark](../README.md#benchmark). + +**How it works** differs per compiler and is chosen for you: gcc publishes its +BMI with `rename()`, so code generation is detached and the edge returns at +publication; clang gets two ordinary edges instead, because it writes the BMI to +its final path with `O_TRUNC` and a reader could observe a half-written file. +MSVC is left alone — neither `/ifcOnly`'s cost nor `.ifc` atomicity has been +measured, and guessing either wrong is silent. + ### Module interface extensions (`module_extensions`) mcpp treats `.cppm` as a module interface unit. The C++ ecosystem has not @@ -360,8 +411,20 @@ toolchain's libraries first. `defines` takes **bare** macro names (no `-D`) and desugars each entry to `-D` on both the C and C++ compile channels. It reaches every TU in the package — module -interface units included — so it also reaches the P1689 module scan, which is what -makes a macro-guarded `import` resolvable. Assembly units pick it up too. It is a +interface units included — so it also reaches the compiler's own P1689 module scan. + +> ⚠️ **It does not make a macro-guarded `import` acceptable.** mcpp runs its own +> lexical pre-scan before the compiler ever sees the file, and that scanner +> rejects an `import` inside **any** `#if` / `#ifdef` block without evaluating the +> condition: +> +> ``` +> error: import statement inside conditional preprocessor block (forbidden in M1) +> ``` +> +> So a `#ifdef FOO` / `import bar;` pair fails even when `FOO` is in `defines`. +> Put the conditional around an `#include` in the global module fragment instead. +> Tracked as mcpp-community/mcpp#421. Assembly units pick it up too. It is a build input like any other, so `[target.'cfg(...)'.build]` can carry it: ```toml diff --git a/docs/zh/05-mcpp-toml.md b/docs/zh/05-mcpp-toml.md index b8c9c934..6ee1bc2d 100644 --- a/docs/zh/05-mcpp-toml.md +++ b/docs/zh/05-mcpp-toml.md @@ -161,6 +161,8 @@ defines = ["BIZ=1", "QUX"] # 作用于每个 TU 的预处理宏(脱糖为 cxx_runtime = "self-contained" # C++ 运行时契约(见下节);static_stdlib 是旧拼写 macos_deployment_target = "14.0" # macOS 产物的最低支持系统版本(仅 macOS 生效) cache = "global" # 依赖的全局构建缓存:global(默认)| local | off(见 §2.10) +jobs = "auto" # 并发编译数:正整数,或 "auto"(见下节) +bmi_schedule = "auto" # 模块边调度:auto(= 关)| on | off(见下节) ``` `include_dirs_after`(#249)列出**排在工具链系统目录之后**搜索的头文件目录 @@ -181,6 +183,47 @@ cargo/rustc、cc 等同样尊重该变量)> 本字段(项目默认,类似 SwiftP 14.0 即 LLVM 官方静态库自身的下限)。该值会进入 BMI 指纹——切换 target 会自动重建模块缓存。 +### 构建并发(`jobs`)与模块调度(`bmi_schedule`) + +```toml +[build] +jobs = "auto" # 或正整数;--jobs / MCPP_JOBS 覆盖它 +bmi_schedule = "off" # auto(默认,= 关)| on | off +``` + +`jobs` 是同时跑几个编译。`"auto"` **在构建这台机器上现算**,绝不冻进 manifest: +异构 CPU 上取物理核数(13900K 是 8 P-core + 16 E-core,它的 32 个线程不是 32 个 +等价的工人),再按可用内存夹一次 —— 单个模块接口编译峰值 0.5–1.0 GB。 +优先级:`--jobs` / `MCPP_JOBS` > 这个键 > 后端自己的默认值。写错的值会被 +**明确报出来,绝不静默当成默认值** —— 一个悄悄退回默认的拼写错误,表现是 +「构建莫名其妙比我要求的慢」。 + +`bmi_schedule` 决定**导入方什么时候被解锁**。 + +| 值 | | +|---|---| +| `"auto"` | **默认值,而它目前等于「关」** | +| `"on"` | 拆开模块边:BMI 一发布导入方就能开始,而不是等编译器退出 | +| `"off"` | 每个模块一条边 | + +只认这三种拼写。`"ON"`、`"true"`、`"yes"` 会被**拒绝并给出诊断**,而不是悄悄 +当成关 —— 而且它们不是无害的笔误:这个值会进构建指纹,所以一个被拒的拼写 +以前会选到**另一个构建目录**(即一次全量重建),同时对调度没有任何影响。 + +**`auto` 为何等于关闭。** 模块接口编译中约 86% 是任何导入方都不会读取的代码生成, +因此提前发布 BMI 收益显著 —— 在 mcpp 自身上实测:`cold` 86.7s → 35.7s、 +`edit-body` 80.9s → 29.8s。但调度错误的表现是静默失效:缺少一条依赖不会使构建 +报错,只会使某个目标不再重建。因此在所有平台完成 CI 验证前,该键保持 opt-in。 + +**该键无效的场景。** mcpp 本来就跳过级联的地方(`touch-hub`、`edit-comment`) +没有可以移出关键路径的必需工作,该键不产生收益。见 +[性能对比](../../README.zh-CN.md#性能对比)。 + +**实现方式**按编译器确定,无需用户选择:gcc 用 `rename()` 发布 BMI,所以代码 +生成被分离出去、边在发布时就返回;clang 换成两条普通边 —— 它把 BMI 直接 +`O_TRUNC` 写到最终路径,读的人可能看到写了一半的文件。MSVC 不动:`/ifcOnly` +的代价和 `.ifc` 是否原子发布都没测过,而这两件事猜错都是无声的。 + ### 模块接口扩展名(`module_extensions`) mcpp 把 `.cppm` 视为模块接口单元。C++ 生态并没有收敛到一种拼法 —— Clang 还认 @@ -271,7 +314,7 @@ cxx_runtime = "host-coupled" # 例如这次构建是为发行版打 | `host-coupled` | 驱动默认解析到的那份(通常是系统运行时) | 发行版打包;必须与宿主共用同一份运行时的 `dlopen` 插件 | **默认即自包含(portable by default)**:macOS 上这会静态链入 LLVM 自带的 -libc++/libc++abi —— 系统 libc++ 会把实际可运行版本钉死在构建机的 OS(老系统 +libc++/libc++abi —— 系统 libc++ 会把实际可运行版本固定在构建机的 OS(老系统 缺新符号,如 `std::print` 的支撑符号),只有静态化才能真正兑现 `macos_deployment_target` 的 floor。Linux/MinGW 上它是 `-static-libstdc++` (GCC)或整条链的 `-static`(MinGW);Linux 上的 clang/libc++ 工具链则显式链入 @@ -319,8 +362,22 @@ C++ 运行时的进程。 > 把流顶上去,你的代码不需要做任何事。详见 mcpp-community/mcpp#336。 `defines` 接受**裸**宏名(不带 `-D`),把每个条目脱糖为 `-D`,同时作用于 C 和 -C++ 编译通道。它覆盖包内每个 TU(含模块接口单元),因此也会进入 P1689 模块扫描 -—— 这正是被宏保护的 `import` 能被解析的前提。汇编单元同样能拿到。它是普通的构建 +C++ 编译通道。它覆盖包内每个 TU(含模块接口单元),因此也会进入**编译器自己的** +P1689 模块扫描。 + +> ⚠️ **但它不会让被宏保护的 `import` 变得可用。** mcpp 在编译器看到文件之前先跑 +> 自己的词法预扫描,而那个扫描器对**任何** `#if` / `#ifdef` 块内的 `import` +> 一律拒绝,不求值条件: +> +> ``` +> error: import statement inside conditional preprocessor block (forbidden in M1) +> ``` +> +> 所以即使 `FOO` 写在 `defines` 里,`#ifdef FOO` / `import bar;` 仍然会失败。 +> 替代写法是把条件放在全局模块片段的 `#include` 上。见 +> mcpp-community/mcpp#421。 + +汇编单元同样能拿到。它是普通的构建 输入,所以 `[target.'cfg(...)'.build]` 也能承载它: ```toml diff --git a/mcpp.toml b/mcpp.toml index ebdebc2f..88acf547 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,6 +1,6 @@ [package] name = "mcpp" -version = "2026.8.11.3" +version = "2026.8.15.1" description = "Modern C++ build & package management tool" license = "Apache-2.0" authors = ["mcpp-community"] @@ -15,6 +15,27 @@ default-profile = "release" # nlohmann/json.hpp lives in src/libs/json/; expose it to the global # module fragment `#include ` in src/libs/json.cppm. include_dirs = ["src/libs/json"] +# Split the module edge: importers start when the BMI is published rather than +# when the compiler exits. See docs/05-mcpp-toml.md. +# +# ON HERE, `auto` (= off) EVERYWHERE ELSE. The key stays opt-in until it has been +# through CI on every platform, and nothing was exercising it, so it could never +# accumulate the evidence needed to change that. mcpp's own build is where that +# evidence is cheapest: whatever this schedule produces has to pass 83 unit tests +# and the full e2e suite, on Linux (detach-codegen, gcc) and on macOS and Windows +# (two-phase, clang) — three platforms, every PR. +# +# ⚠️ THAT SAFETY NET IS NOT DECORATION — it has already caught one defect. The +# windows-host cross build to x86_64-linux-musl failed on the BMI edge with +# +# failed: gcm.cache/mcpp.libs.json.gcm +# src/libs/json.cppm:3: fatal error: json.hpp: No such file or directory +# +# because `cmd.exe /c` does not use CreateProcess argument quoting, so the +# compiler ran with `-I` missing. Fixed in detach_codegen.cppm; see #425. +# +# Revert is this one line if anything here is ever unexplained. +bmi_schedule = "on" [toolchain] default = "gcc@16.1.0" diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 68acae33..33cea412 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -29,6 +29,8 @@ import mcpp.platform.xlings.subos_info; import mcpp.platform.runtime_binding; import mcpp.log; import mcpp.platform; +import mcpp.platform.capacity; +import mcpp.build.schedule.policy; // resolve_jobs — one answer to "how many at once" import mcpp.fetcher.progress; import mcpp.project; import mcpp.ui; @@ -375,6 +377,12 @@ compute_subos_env(const mcpp::build::BuildPlan& plan) { // Compile a prepared BuildContext. Shared between `mcpp build` and `mcpp run` // so the latter doesn't call prepare_build twice (and re-print the toolchain // resolution banner). +// How many compiles to run at once. +// Concurrency and the module-edge schedule are resolved together in +// mcpp.build.schedule.policy and stamped onto the plan, so this reads one value +// instead of re-deriving it. `scheduleNinjaJobs` is NOT the compiler cap under +// detach-codegen: a detached compiler stops holding a ninja slot, so ninja is +// handed a larger number on purpose. export int run_build_plan(BuildContext& ctx, bool verbose, bool no_cache, std::string_view targetOverride = "") { // `--cache=off` means a cold build: no global cache, and target/ cleared — @@ -444,8 +452,33 @@ export int run_build_plan(BuildContext& ctx, bool verbose, bool no_cache, } } + // ⚠️ RECLAIM THE STALE CONCURRENCY TOKENS, HERE AND NOT IN prepare. + // + // detach-codegen bounds real compiler concurrency with a semaphore of + // directories under `/.mcpp-sched`, released by the supervisor + // holding each token. A supervisor that never runs its cleanup — Ctrl-C on + // the build, the OOM killer, a reboot — leaves its directory behind, and + // nothing else deletes one. Every such event permanently lowers the cap for + // that build directory; after `cap` of them the next build waits for a token + // that can never be released and hangs with no output at all. + // + // This was first placed in prepare, beside the schedule decision, and an + // e2e that plants a full set of stale tokens showed it never running: an + // incremental build takes the project-level fast path, which replays + // build.ninja without re-deriving the plan. The reclaim has to sit on the + // path EVERY build takes, which is the line below this one. + // + // Safe here because ninja has not been spawned yet, so no token in the + // directory can have a live owner. + if (ctx.plan.scheduleTag == "detach-codegen") { + std::error_code semEc; + std::filesystem::remove_all( + std::filesystem::path(ctx.plan.outputDir) / ".mcpp-sched", semEc); + } + mcpp::build::BuildOptions opts; opts.verbose = verbose; + opts.parallelJobs = static_cast(ctx.plan.scheduleNinjaJobs); auto r = be->build(ctx.plan, opts); if (!r) { std::fflush(stdout); @@ -1323,9 +1356,21 @@ export int run_tests(std::span passthrough, // 6. Phase B. First a single keep-going bulk build over every selected // test goal — ninja parallelizes across tests and a failing test does // not stop the rest (-k 0). The result is deliberately ignored: the - // per-test loop below re-drives each goal, where successes are cache - // hits (near no-ops) and failures re-fail fast, yielding cleanly - // attributed per-test diagnostics without sacrificing parallelism. + // per-test loop below re-drives each goal so a failure is attributed to + // exactly one test. + // + // ...but ONLY when this bulk build failed. A re-drive was assumed to be + // a near no-op, and it is not: a drive re-emits build.ninja, rewrites + // compile_commands.json, spawns ninja and re-validates the runtime + // closure. Measured on the 83-test suite AFTER the rule E fix, that is + // still ~39ms x 83 = 3.2s of a 5.3s hot run — spent re-asking a question + // the bulk build just answered for every test at once. + // + // `-k 0` means the bulk exit code is 0 IFF every selected goal built, so + // it carries exactly the information the loop was re-deriving. When it + // is non-zero the loop runs as before and each failure still names its + // own test. + bool bulkBuiltEverything = false; { mcpp::build::BuildOptions bulk; bulk.keepGoing = true; @@ -1335,7 +1380,7 @@ export int run_tests(std::span passthrough, bulk.ninjaTargets.push_back(lu.output.generic_string()); if (!bulk.ninjaTargets.empty()) { auto tBulk = std::chrono::steady_clock::now(); - (void)backend->build(ctx->plan, bulk); + bulkBuiltEverything = backend->build(ctx->plan, bulk).has_value(); summary.buildMs += std::chrono::duration_cast( std::chrono::steady_clock::now() - tBulk).count(); } @@ -1366,6 +1411,128 @@ export int run_tests(std::span passthrough, } } + // How many test binaries run at once. + // + // The tests themselves were never the slow part — MEASURED on the 83-test + // suite, the whole run phase is 1.8s against a 190s total — so this is the + // tail, not the fix. It is still worth having: after the build-side work + // (rule E, the per-test re-drive) the run phase is HALF of what is left. + // + // ONE test runs in the foreground, unbuffered. That is the debugging case: + // a single long test streaming its progress is worth more than the ~0ms + // concurrency would save on it, and capturing would hold that output back + // until the test ended — including when it hangs, which is exactly when a + // reader needs it. + const int runJobs = [&] { + int j = mcpp::build::schedule::resolve_jobs(ctx->manifest); + if (j <= 0) j = static_cast(std::thread::hardware_concurrency()); + return j > 0 ? j : 1; + }(); + + struct Runnable { + std::string name; + std::vector argv; + std::vector> env; + }; + std::vector runnable; + + // Executes `list`, appending to `results` and emitting the per-test line. + // + // Output is CAPTURED whenever more than one test runs, and printed as one + // contiguous block when that test finishes. Streaming N tests straight to + // the terminal interleaves them line by line, which does not just look + // untidy — it makes a failing assertion unattributable, and the whole + // reason the per-test loop exists is attribution. + auto run_tests_now = [&](std::vector& list) { + if (list.empty()) return; + const bool capture = json || list.size() > 1; + const auto deadline = std::chrono::milliseconds( + static_cast(testOpts.timeoutSecs) * 1000); + const int workers = capture + ? std::min(runJobs, static_cast(list.size())) : 1; + + auto tRunPhase = std::chrono::steady_clock::now(); + std::atomic next{0}; + std::mutex reportMutex; + + auto worker = [&] { + for (;;) { + std::size_t i = next.fetch_add(1); + if (i >= list.size()) return; + auto& r = list[i]; + + // Stamped HERE, immediately before the exec — not when the + // test was queued. + // + // Discovery, building and attribution all happen in a first + // pass that completes before any test runs, and the workers + // then take tests off a queue. A start time captured at queue + // time therefore includes the whole preparation phase plus + // however long this test waited for a worker, and `ok (2.30s)` + // for a test that ran in 30ms is not a slow test, it is a + // mislabelled one. The phase's own wall time is measured + // separately by `tRunPhase` below. + const auto tStart = std::chrono::steady_clock::now(); + bool timedOut = false; + int exitCode = 0; + std::string runOutput; + if (capture) { + auto rr = mcpp::platform::process::capture_exec_deadline( + r.argv, r.env, deadline, &timedOut); + exitCode = rr.exit_code; + runOutput = std::move(rr.output); + } else { + mcpp::ui::status("Running", std::format("bin/{}", r.name)); + exitCode = mcpp::platform::process::run_exec_deadline( + r.argv, r.env, deadline, &timedOut); + } + auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - tStart).count(); + + std::scoped_lock lock(reportMutex); + if (timedOut) { + if (!json) mcpp::ui::plain(std::format( + "{} ... FAIL (timeout after {}s)", r.name, testOpts.timeoutSecs)); + results.push_back({r.name, TestResult::St::RunFail, exitCode, {}, + runOutput, ms, true}); + } else if (exitCode == 0) { + if (!json) mcpp::ui::plain(std::format( + "{} ... ok ({:.2f}s)", r.name, static_cast(ms) / 1000.0)); + results.push_back({r.name, TestResult::St::Pass, 0, {}, + runOutput, ms}); + } else { + if (!json) mcpp::ui::plain(std::format( + "{} ... FAIL (exit {}, {:.2f}s)", r.name, exitCode, + static_cast(ms) / 1000.0)); + results.push_back({r.name, TestResult::St::RunFail, exitCode, {}, + runOutput, ms}); + } + // The captured output belongs directly under its own line, or + // it is attributable to nothing. + if (!json && capture && !runOutput.empty()) { + std::fputs(runOutput.c_str(), stdout); + if (runOutput.back() != '\n') std::fputc('\n', stdout); + } + std::fflush(stdout); + emit_json(results.back()); + } + }; + + if (workers <= 1) { + worker(); + } else { + std::vector pool; + pool.reserve(static_cast(workers)); + for (int w = 0; w < workers; ++w) pool.emplace_back(worker); + for (auto& t : pool) t.join(); + } + // WALL time of the phase, not the sum of the per-test durations: with + // N running at once that sum exceeds the elapsed time and the summary + // would report a run phase longer than the whole command. + summary.runMs += std::chrono::duration_cast( + std::chrono::steady_clock::now() - tRunPhase).count(); + }; + for (auto& lu : ctx->plan.linkUnits) { if (!filter_match(lu)) continue; @@ -1377,13 +1544,16 @@ export int run_tests(std::span passthrough, mcpp::ui::status("Compiling", std::format("{} (test)", lu.targetName)); - mcpp::build::BuildOptions bOpts; - bOpts.ninjaTargets = {lu.output.generic_string()}; - bOpts.buildTimeoutSecs = static_cast(testOpts.buildTimeoutSecs); - auto tBuild = std::chrono::steady_clock::now(); - auto b = backend->build(ctx->plan, bOpts); - summary.buildMs += std::chrono::duration_cast( - std::chrono::steady_clock::now() - tBuild).count(); + std::expected b{}; + if (!bulkBuiltEverything) { + mcpp::build::BuildOptions bOpts; + bOpts.ninjaTargets = {lu.output.generic_string()}; + bOpts.buildTimeoutSecs = static_cast(testOpts.buildTimeoutSecs); + auto tBuild = std::chrono::steady_clock::now(); + b = backend->build(ctx->plan, bOpts); + summary.buildMs += std::chrono::duration_cast( + std::chrono::steady_clock::now() - tBuild).count(); + } if (!b) { if (!json) { // The test's own diagnostics, right under its FAIL line — a @@ -1410,7 +1580,6 @@ export int run_tests(std::span passthrough, } auto exe = ctx->outputDir / lu.output; - mcpp::ui::status("Running", std::format("bin/{}", lu.targetName)); std::vector argv; argv.push_back(exe.string()); @@ -1440,45 +1609,12 @@ export int run_tests(std::span passthrough, } } - // JSON mode captures the test's combined stdout+stderr into the - // record; human mode streams it to the terminal as before. - auto deadline = std::chrono::milliseconds( - static_cast(testOpts.timeoutSecs) * 1000); - bool timedOut = false; - int exitCode; - std::string runOutput; - auto tRun = std::chrono::steady_clock::now(); - if (json) { - auto rr = mcpp::platform::process::capture_exec_deadline( - argv, childEnv, deadline, &timedOut); - exitCode = rr.exit_code; - runOutput = std::move(rr.output); - } else { - exitCode = mcpp::platform::process::run_exec_deadline( - argv, childEnv, deadline, &timedOut); - } - summary.runMs += std::chrono::duration_cast( - std::chrono::steady_clock::now() - tRun).count(); - - if (timedOut) { - if (!json) mcpp::ui::plain(std::format("{} ... FAIL (timeout after {}s)", - lu.targetName, testOpts.timeoutSecs)); - results.push_back({lu.targetName, TestResult::St::RunFail, exitCode, {}, - std::move(runOutput), test_ms(), true}); - } else if (exitCode == 0) { - if (!json) mcpp::ui::plain(std::format("{} ... ok ({:.2f}s)", lu.targetName, - static_cast(test_ms()) / 1000.0)); - results.push_back({lu.targetName, TestResult::St::Pass, 0, {}, - std::move(runOutput), test_ms()}); - } else { - if (!json) mcpp::ui::plain(std::format("{} ... FAIL (exit {}, {:.2f}s)", - lu.targetName, exitCode, - static_cast(test_ms()) / 1000.0)); - results.push_back({lu.targetName, TestResult::St::RunFail, exitCode, {}, - std::move(runOutput), test_ms()}); - } - emit_json(results.back()); + runnable.push_back({lu.targetName, std::move(argv), std::move(childEnv)}); } + + // Pass 2: run them. Concurrently unless there is exactly one — see + // `runJobs` for why the single-test case is deliberately different. + run_tests_now(runnable); summary.elapsedMs = member_ms(); // 7. Summary. diff --git a/src/build/flags.cppm b/src/build/flags.cppm index 3948dc16..61a650d5 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -585,7 +585,10 @@ CompileFlags compute_flags(const BuildPlan& plan) { std::string msvc_base; if (isMsvcDialect) { msvc_base = std::format(" {}", d.alwaysFlags); - msvc_base += (plan.manifest.buildConfig.linkage == "static") ? " /MT" : " /MD"; + // ONE derivation, shared with the std module build — see + // `msvc_crt_flag` in mcpp.toolchain.dialect and #422. + msvc_base += std::format(" {}", mcpp::toolchain::msvc_crt_flag( + d, plan.manifest.buildConfig.linkage == "static")); } // User link flags diff --git a/src/build/graph_shape.cppm b/src/build/graph_shape.cppm index 385f9ba6..a74f8594 100644 --- a/src/build/graph_shape.cppm +++ b/src/build/graph_shape.cppm @@ -47,8 +47,15 @@ std::string_view to_string(GraphShape shape) { // The marker line, without its newline. A ninja comment, so it costs nothing // and older ninja versions do not care. -std::string header_line(GraphShape shape) { - return std::format("# mcpp:graph={}", to_string(shape)); +// `scheduleTag` names the SHAPE OF THE MODULE EDGES (see +// mcpp.build.schedule.policy): "none", "two-phase", "detach-codegen". It rides +// the same line for the same reason the shape does — build.ninja is shared +// mutable state and the fast path replays it, so a graph built under one +// schedule must not be replayed under another. Flipping the switch has to +// invalidate the graph, and the only way that cannot be forgotten is if the +// graph says which schedule produced it. +std::string header_line(GraphShape shape, std::string_view scheduleTag) { + return std::format("# mcpp:graph={};schedule={}", to_string(shape), scheduleTag); } // Read the shape back. `nullopt` means "this file does not say" — a build.ninja @@ -68,6 +75,10 @@ std::optional read_shape(const std::filesystem::path& ninjaPath) { auto value = std::string_view(line).substr(prefix.size()); while (!value.empty() && (value.back() == '\r' || value.back() == ' ')) value.remove_suffix(1); + // `graph=[;schedule=]`. Split before comparing, so adding + // the schedule field does not turn every existing graph into "unknown". + if (const auto semi = value.find(';'); semi != std::string_view::npos) + value = value.substr(0, semi); if (value == "normal") return GraphShape::Normal; if (value == "test") return GraphShape::WithTests; // A shape this binary does not know is not `Normal`. An older mcpp @@ -77,7 +88,40 @@ std::optional read_shape(const std::filesystem::path& ninjaPath) { return std::nullopt; } +// The schedule tag this graph was written with. Empty means the file predates +// the field — which is NOT the same as "none": an unlabelled graph is exactly +// the case that must not be replayed blind, so callers compare and miss. +std::string read_schedule(const std::filesystem::path& ninjaPath) { + std::ifstream input(ninjaPath); + if (!input) return {}; + std::string line; + for (int i = 0; i < 8 && std::getline(input, line); ++i) { + constexpr std::string_view prefix = "# mcpp:graph="; + if (!line.starts_with(prefix)) continue; + auto value = std::string_view(line).substr(prefix.size()); + while (!value.empty() && (value.back() == '\r' || value.back() == ' ')) + value.remove_suffix(1); + const auto semi = value.find(';'); + if (semi == std::string_view::npos) return {}; + auto rest = value.substr(semi + 1); + constexpr std::string_view schedPrefix = "schedule="; + if (!rest.starts_with(schedPrefix)) return {}; + return std::string(rest.substr(schedPrefix.size())); + } + return {}; +} + // The one question every fast path asks. +// +// It deliberately does NOT compare the schedule tag. The fast paths run BEFORE +// a plan exists, so they have no toolchain to derive the expected schedule +// from — and passing one in would mean deriving the same decision a second +// time, in a place that cannot see the compiler. +// +// Instead the schedule SWITCH is part of the toolchain fingerprint, so flipping +// it lands in a different build directory: a graph written under one schedule +// is structurally unreachable from a build configured with another. The tag on +// the line is then for humans and for `mcpp explain`, not for invalidation. bool is_plain_build_graph(const std::filesystem::path& ninjaPath) { return read_shape(ninjaPath) == GraphShape::Normal; } diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 067f2311..8674883c 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -41,6 +41,7 @@ import mcpp.toolchain.registry; import mcpp.platform.xlings; import mcpp.platform; import mcpp.ui; +import mcpp.log; export namespace mcpp::build { @@ -203,7 +204,22 @@ std::string shared_soname_flag(const LinkUnit& lu) { #endif } +// Write only when the bytes would actually change. +// +// One of these files is a BUILD INPUT: `obj/mcpp_ios_init.c`, the generated +// initializer-ordering TU (#336, macOS static libc++ only). It was rewritten on +// every drive, so its mtime moved on every drive, so ninja recompiled it on +// every build — including no-op ones. That is one object, but `mcpp test` +// drives the backend once per test, and the symptom read as "the split module +// schedule is not incremental" on macOS while being invisible on Linux, where +// the shim does not exist. void write_file(const std::filesystem::path& p, std::string_view content) { + std::error_code ec; + if (std::filesystem::file_size(p, ec) == content.size() && !ec) { + std::ifstream is(p, std::ios::binary); + std::string prev((std::istreambuf_iterator(is)), {}); + if (prev == content) return; + } std::filesystem::create_directories(p.parent_path()); std::ofstream os(p); os << content; @@ -399,6 +415,26 @@ std::string emit_ninja_string(const BuildPlan& plan) { bool has_scanner = caps.has_builtin_p1689_scan || !plan.scanDepsPath.empty(); bool dyndep = dyndep_mode_enabled() && has_scanner; auto traits = mcpp::toolchain::bmi_traits(plan.toolchain); + // The module-edge shape, decided once in prepare (mcpp.build.schedule). + // dyndep is a precondition: without it nothing declares BMIs as outputs, so + // there is no BMI edge for importers to depend on. + const bool splitBmi = plan.scheduleTag == "detach-codegen" && dyndep; + // The other split shape. Clang publishes no BMI early — it writes the BMI + // at the end of the compile — so the GCC trick of releasing the file + // mid-compile has nothing to release. What clang has instead is a driver + // mode that stops once the BMI exists, so the split is two INDEPENDENT + // PROCESSES over the same source: one emits the BMI (fast; importers wait + // only on this), one emits the object (slow; nobody waits on it). + // + // The object edge recompiles the source rather than reading the BMI back. + // That is not a missed shortcut — see BmiTraits::bmiOnlyFlags: the BMI that + // CAN be read back is clang's *full* BMI, which is ~2x larger and makes + // clang 22.1.8 miscompile a downstream TU on mcpp's own graph. MEASURED on + // src/build/prepare.cppm: BMI edge 1.67s, object edge 7.31s, against 7.35s + // for the single edge — so ~22% more CPU buys a 4.4x shorter critical path, + // and both outputs are byte-identical to the single-edge build's. + const bool twoPhase = plan.scheduleTag == "two-phase" && dyndep + && !traits.bmiOnlyFlags.empty(); const auto& dial = mcpp::toolchain::dialect_for(plan.toolchain); std::string out; auto append = [&](std::string s) { out += std::move(s); }; @@ -407,7 +443,7 @@ std::string emit_ninja_string(const BuildPlan& plan) { // #407: the graph declares which mode produced it, because three modes // write this one file and the fast path has to know what it is about to // replay. Must stay within the first few lines — see read_shape. - append(mcpp::build::header_line(plan.graphShape) + "\n"); + append(mcpp::build::header_line(plan.graphShape, plan.scheduleTag) + "\n"); append("ninja_required_version = 1.11\n\n"); // All compile/link flags are computed once via flags.cppm. @@ -499,9 +535,14 @@ std::string emit_ninja_string(const BuildPlan& plan) { append(" restat = 1\n\n"); // P1: per-file dyndep rule. Converts one .ddi → .dd independently. + // + // `$bind` is per-edge, not per-graph: under two-phase only the units that + // are actually SPLIT bind their record to the BMI. An implementation unit + // or a plain .cpp still compiles in one edge whose output is the object, + // and a `--target-bmi` there would name an edge nobody declared. append(std::format( "rule cxx_dyndep\n" - " command = $mcpp dyndep --single --bmi-dir {} --bmi-ext {} $expect --output $out $in\n" + " command = $mcpp dyndep --single --bmi-dir {} --bmi-ext {} $bind $expect --output $out $in\n" " description = DYNDEP $out\n" " restat = 1\n\n", traits.bmiDir, traits.bmiExt)); @@ -690,8 +731,13 @@ std::string emit_ninja_string(const BuildPlan& plan) { // must precede `-c $in` (which compile_tail carries) or it // applies to nothing. "$cxx $local_includes $cxxflags $unit_cxxflags{}{} {}{}{} && " + // `$mcpp bmi-equal`, not `cmp -s`: GCC stamps a wall clock into + // the BMI content, so a byte compare NEVER reports "unchanged" + // and this whole fast path was dead. Measured: touching a module + // with 46 importers and no content change cost 73.0 s before, + // 0.22 s after. "if [ -n \"$bmi_out\" ] && [ -f \"$bmi_out.bak\" ] && " - "cmp -s \"$bmi_out\" \"$bmi_out.bak\"; then " + "$mcpp bmi-equal \"$bmi_out\" \"$bmi_out.bak\"; then " "mv \"$bmi_out.bak\" \"$bmi_out\"; " "else " "rm -f \"$bmi_out.bak\"; " @@ -704,6 +750,113 @@ std::string emit_ninja_string(const BuildPlan& plan) { append(" restat = 1\n"); append("\n"); + if (splitBmi) { + // Two edges driven by ONE compiler process. `$out` is the BMI here, so + // the object path travels as `$obj_out`. + // + // The compiler command goes through a response file rather than the + // command line: it is already joined and quoted for a shell, and + // splitting it back into argv would mean reimplementing the shell's + // rules — the assumption "one flag element == one argv token" has been + // wrong in this file before. + append("rule cxx_module_bmi\n"); + append(" command = $mcpp bmi-compile --bmi $bmi_out --slot $slot" + " --self $mcpp --sem $sched_sem --cap $sched_cap" + " --command-file $out.cmd --dep-from $scan_dep --dep-to $out.d\n"); + append(std::format( + " rspfile = $out.cmd\n" + " rspfile_content = $cxx $local_includes $cxxflags $unit_cxxflags{}{} {} $in {}$obj_out\n", + module_output_flag, module_src_flags, + dial.compileOnly, dial.outputObjPrefix)); + // ⚠️ THE SUPERVISOR OUTLIVES THIS EDGE, THE RSPFILE DOES NOT. + // + // ninja DELETES `$out.cmd` as soon as the edge finishes, and the edge + // finishes when `bmi-compile` returns — which is the moment the BMI is + // published, with the compiler still running under a detached + // supervisor that was handed this same path. + // + // It is safe because of an ordering worth writing down: the supervisor + // reads the file at start-up, and the BMI cannot appear until the + // compiler it launched has produced one, so every SUCCESSFUL exit from + // phase 1 is already downstream of that read. + // + // Phase 1 also has two exits that do NOT require the compiler to have + // started — the no-supervisor grace and the hard limit. Both fail the + // edge, so ninja does not proceed and a supervisor arriving afterwards + // to a deleted file changes nothing. If a phase-1 exit is ever added + // that reports SUCCESS without the compiler having started, this stops + // holding: pass the command by value at that point, not by path. + // The depfile is ADOPTED from the P1689 scan, not produced here. + // MEASURED: the compiler's own -MMD file lands at 16.39s of a 16.55s + // compile — AFTER the BMI is published at 2.36s — so this edge is over + // before it exists. The scan's is equivalent for headers (compared on + // real modules: the only prerequisites it lacks are `.gcm`, and those + // are ninja's through dyndep). Copied rather than pointed at, because + // ninja DELETES a depfile once it has folded it into .ninja_deps. + append(" depfile = $out.d\n"); + append(" description = BMI $out\n"); + append(" restat = 1\n\n"); + + append("rule cxx_module_obj\n"); + append(" command = $mcpp bmi-await --slot $slot --object $out\n"); + append(" description = OBJ $out\n"); + // The object is written by the detached compiler, so its mtime moves + // outside this edge. Without restat ninja treats every join as having + // changed its output and cascades into the link every time. + append(" restat = 1\n\n"); + } + + if (twoPhase) { + // Edge A — the BMI, and nothing else. `$out` is the BMI here. + append("rule cxx_precompile\n"); + if constexpr (mcpp::platform::is_windows) { + const std::string payload = " $local_includes"; + append(std::format( + " command = $cxx{} $cxxflags $unit_cxxflags{}{} $in {}$out\n", + rsp_ref(payload), traits.bmiOnlyFlags, module_src_flags, + dial.outputObjPrefix)); + append_rspfile(payload); + append_deps(); + } else { + // Same bak / bmi-equal / restore dance as cxx_module, and for the + // same reason: ninja's `restat` compares the output's MTIME, and a + // compiler that rewrites a byte-identical BMI still moves it. What + // suppresses the cascade is putting the old file back. + append(std::format( + " command = " + "if [ -f \"$out\" ]; then cp -p \"$out\" \"$out.bak\"; fi && " + "$cxx $local_includes $cxxflags $unit_cxxflags{}{} {}$in {}$out && " + "if [ -f \"$out.bak\" ] && $mcpp bmi-equal \"$out\" \"$out.bak\"; then " + "mv \"$out.bak\" \"$out\"; " + "else rm -f \"$out.bak\"; fi\n", + traits.bmiOnlyFlags, module_src_flags, mmd_flag, + dial.outputObjPrefix)); + append_cxx_deps(); + } + append(" description = BMI $out\n"); + append(" restat = 1\n\n"); + + // Edge B — the object, compiled from the SAME SOURCE, with no + // `-fmodule-output`: the BMI is edge A's output and two edges must not + // write one file. Identical to cxx_object except for the language flag + // that tells the driver this source is a module interface. + append("rule cxx_module_object\n"); + if constexpr (mcpp::platform::is_windows) { + const std::string payload = " $local_includes"; + append(std::format(" command = $cxx{} $cxxflags $unit_cxxflags{} {}\n", + rsp_ref(payload), module_src_flags, compile_tail)); + append_rspfile(payload); + append_deps(); + } else { + append(std::format( + " command = $cxx $local_includes $cxxflags $unit_cxxflags{} {}{}{}\n", + module_src_flags, mmd_flag, compile_tail, mmd_filter)); + append_cxx_deps(); + } + append(" description = OBJ $out\n"); + append(" restat = 1\n\n"); + } + append("rule cxx_object\n"); if constexpr (mcpp::platform::is_windows) { const std::string payload = " $local_includes"; @@ -925,7 +1078,7 @@ std::string emit_ninja_string(const BuildPlan& plan) { // GCC path: compiler-integrated P1689 scanning. append(std::format(" command = $cxx{} $cxxflags $unit_cxxflags -fmodules " "-fdeps-format=p1689r5 " - "-fdeps-file=$out -fdeps-target=$compile_target " + "-fdeps-file=$out -fdeps-target=$deps_target " "-M -MM -MF $out.dep $unit_lang -E $in -o $compile_target\n", rsp_ref(scanPayload))); } else { @@ -976,6 +1129,54 @@ std::string emit_ninja_string(const BuildPlan& plan) { } bool has_std_artifacts = !plan.stdBmiPath.empty() && !plan.stdObjectPath.empty(); + + // WHICH LINK UNITS ACTUALLY NEED `std.o` (#416). + // + // `has_std_artifacts` only says "this toolchain has a prebuilt std module". + // It was also the whole condition for appending `std.o` to every Binary, + // TestBinary and SharedLibrary — so a link unit with no `import std` + // anywhere in it still got the module's global initialiser linked in. + // + // ⚠️ THE TEST HAS TO BE TRANSITIVE. A unit that never writes `import std` + // itself still needs the initialiser when a module it imports does; asking + // only about the unit's own source is the same "the edge exists but nobody + // depends on it" mistake as #405. So: reachability over the module graph. + // + // Getting it wrong UNDER-includes, and that fails loudly — `std.o` defines + // exactly one symbol (`_ZGIW3std`, measured) and an importing TU references + // it, so a missed unit is an undefined symbol at link time rather than a + // silent miscompile. + std::unordered_map byModule; + for (auto& cu : plan.compileUnits) + if (cu.providesModule) byModule.emplace(*cu.providesModule, &cu); + + auto reaches_std = [&](const CompileUnit& start) { + std::vector stack{&start}; + std::unordered_set seen; + while (!stack.empty()) { + const CompileUnit* cu = stack.back(); + stack.pop_back(); + for (auto& imp : cu->imports) { + if (imp == "std" || imp == "std.compat") return true; + if (!seen.insert(imp).second) continue; + if (auto it = byModule.find(imp); it != byModule.end()) + stack.push_back(it->second); + } + } + return false; + }; + + std::unordered_map objectNeedsStd; + for (auto& cu : plan.compileUnits) + objectNeedsStd[cu.object.generic_string()] = reaches_std(cu); + + auto unit_needs_std = [&](const LinkUnit& lu) { + for (auto& o : lu.objects) { + auto it = objectNeedsStd.find(o.generic_string()); + if (it != objectNeedsStd.end() && it->second) return true; + } + return false; + }; if (has_std_artifacts) { append(std::format("build {} : stage_file {}\n", escape_ninja_path(std_bmi_dst), escape_ninja_path(plan.stdBmiPath))); @@ -1153,7 +1354,21 @@ std::string emit_ninja_string(const BuildPlan& plan) { ddi_paths.push_back(ddi); append(std::format("build {} : cxx_scan {}{}\n", escape_ninja_path(ddi), escape_ninja_path(cu.source), stagedOrderOnly)); + // `-o` and `-fdeps-target` are DIFFERENT under the split shape and + // must not share a variable. The scan writes a throwaway object to + // `-o`, but the dyndep file has to bind the BMI edge — that is the + // edge whose inputs are the imported BMIs. + // + // Pointing both at the BMI made the SCAN try to create + // `gcm.cache/.gcm` before anything had made that directory: + // cc1plus: fatal error: opening output file gcm.cache/fx.unit_19.gcm + // It survived on this repository only because gcm.cache already + // existed there from an earlier build — a fresh project failed. append(std::format(" compile_target = {}\n", escape_ninja_path(cu.object))); + append(std::format(" deps_target = {}\n", + splitBmi && cu.providesModule + ? bmi_path(*cu.providesModule) + : escape_ninja_path(cu.object))); if (auto includes = local_include_flags(cu, dial); !includes.empty()) append(std::format(" local_includes ={}\n", includes)); if (auto flags = join_flags(cu.packageCxxflags); !flags.empty()) @@ -1218,10 +1433,27 @@ std::string emit_ninja_string(const BuildPlan& plan) { if (exp.empty()) exp = "--expect-none"; ddi_expect[ddi] = std::move(exp); } + // Which units get the split shape. Computed ONCE and consulted from + // both loops below: the dyndep record and the edge it augments have to + // agree on the target, and when they disagree ninja blames the edge + // ("'…pcm' not mentioned in its dyndep file") rather than the record. + std::set two_phase_ddi; + if (twoPhase) { + for (auto& cu : plan.compileUnits) { + if (cu.servedFromCache) continue; + if (is_scan_exempt(cu)) continue; + if (!cu.providesModule) continue; + if (cu.kind != mcpp::SourceKind::ModuleInterface) continue; + two_phase_ddi.insert( + (cu.object.parent_path() / cu.source.filename()).string() + ".ddi"); + } + } for (auto& ddi : ddi_paths) { auto dd = ddi + ".dd"; // e.g. obj/cli.cppm.ddi.dd ddi_to_dd[ddi] = dd; append(std::format("build {} : cxx_dyndep {}\n", dd, ddi)); + if (two_phase_ddi.contains(ddi)) + append(" bind = --split-module\n"); if (auto it = ddi_expect.find(ddi); it != ddi_expect.end()) append(std::format(" expect = {}\n", it->second)); } @@ -1234,6 +1466,103 @@ std::string emit_ninja_string(const BuildPlan& plan) { if (cu.servedFromCache) continue; // a stage_file edge owns these outputs std::string rule = pick_rule(cu); + if (splitBmi && cu.providesModule && + cu.kind == mcpp::SourceKind::ModuleInterface) { + const auto bmi = bmi_path(*cu.providesModule); + const auto obj = escape_ninja_path(cu.object); + const auto slot = obj + ".sched"; + const auto ddi = (cu.object.parent_path() / cu.source.filename()) + .string() + ".ddi"; + auto it = ddi_to_dd.find(ddi); + if (it != ddi_to_dd.end()) { + std::string e = std::format("build {} : cxx_module_bmi {} | {}", + bmi, escape_ninja_path(cu.source), + it->second); + e += stagedOrderOnly; + e += "\n dyndep = " + it->second + "\n"; + e += " bmi_out = " + bmi + "\n"; + e += " obj_out = " + obj + "\n"; + e += " slot = " + slot + "\n"; + e += " scan_dep = " + escape_ninja_path(ddi) + ".dep\n"; + e += " sched_sem = .mcpp-sched\n"; + e += std::format(" sched_cap = {}\n", plan.scheduleCompilerCap); + if (auto inc = local_include_flags(cu, dial); !inc.empty()) + e += " local_includes =" + inc + "\n"; + if (auto fl = join_flags(cu.packageCxxflags); !fl.empty()) + e += " unit_cxxflags =" + fl + "\n"; + append(std::move(e)); + // The join. THE SOURCE IS AN INPUT, and the BMI is only an + // implicit one — the reverse of what this was. + // + // ⚠️ WITH THE BMI AS THE ONLY INPUT THIS EDGE GETS CLEANED + // BY THE VERY OPTIMISATION IT IS PART OF. The BMI edge sets + // `restat = 1`, and when the new BMI turns out equivalent + // `settle_bmi` puts the previous file back so its mtime does + // not advance — that is what stops the cascade to importers, + // and it is correct. But ninja's restat then cleans every + // edge whose only reason to be dirty was that output, and + // this edge was one: it was skipped, and the LINK with it. + // + // The unit's own object is a different question from the + // cascade. Editing a function body does not change a GCC + // BMI — bodies are not in it — so importers genuinely need + // no rebuild, while THIS unit's object genuinely does. + // + // Measured on a two-module repro (`export int leaf_value() + // { return 1; }` → `42`): the rebuild reported success in + // 0.02s having run 3 of 8 edges, and the binary still + // printed 1. No link error, no diagnostic — the detached + // compiler wrote the correct object a fifth of a second + // later, after ninja had already decided not to link it. + // On the generated fixture the same defect surfaces instead + // as `undefined reference to unit_19_value@fx.unit_19()`, + // which is the same skip in the case where the symbol did + // not exist beforehand. + // + // With the source as an input the edge has a reason to be + // dirty that restat cannot clean away, and the BMI stays as + // an implicit input so ninja still orders this after phase 1 + // — `bmi-await` must not run before a compiler was started. + append(std::format("build {} : cxx_module_obj {} | {}\n slot = {}\n", + obj, escape_ninja_path(cu.source), bmi, slot)); + continue; + } + // No dyndep file for this unit: fall through to the single-edge + // shape rather than emitting a BMI edge nothing can order. + } + + if (twoPhase && cu.providesModule && + cu.kind == mcpp::SourceKind::ModuleInterface) { + const auto bmi = bmi_path(*cu.providesModule); + const auto obj = escape_ninja_path(cu.object); + const auto ddi = (cu.object.parent_path() / cu.source.filename()) + .string() + ".ddi"; + auto it = ddi_to_dd.find(ddi); + if (it != ddi_to_dd.end()) { + // Both edges read the same source and so need the same + // imported BMIs; `--split-module` made the .dd carry a + // record for each. They are otherwise INDEPENDENT — the + // object edge does not wait for the BMI edge, which is what + // lets codegen drift behind the front of the graph. + auto edge = [&](std::string_view rule, const std::string& out) { + std::string e = std::format("build {} : {} {} | {}", + out, rule, + escape_ninja_path(cu.source), + it->second); + e += stagedOrderOnly; + e += "\n dyndep = " + it->second + "\n"; + if (auto inc = local_include_flags(cu, dial); !inc.empty()) + e += " local_includes =" + inc + "\n"; + if (auto fl = join_flags(cu.packageCxxflags); !fl.empty()) + e += " unit_cxxflags =" + fl + "\n"; + append(std::move(e)); + }; + edge("cxx_precompile", bmi); + edge("cxx_module_object", obj); + continue; + } + } + std::string out_line = "build " + escape_ninja_path(cu.object); if (cu.providesModule) { out_line += " | " + bmi_path(*cu.providesModule); @@ -1363,7 +1692,7 @@ std::string emit_ninja_string(const BuildPlan& plan) { switch (lu.kind) { case LinkUnit::Binary: case LinkUnit::TestBinary: - if (has_std_artifacts) + if (has_std_artifacts && unit_needs_std(lu)) ins += " " + escape_ninja_path(std_o_dst); if (has_std_compat) ins += " " + escape_ninja_path(compat_o_dst); @@ -1373,7 +1702,7 @@ std::string emit_ninja_string(const BuildPlan& plan) { rule = "cxx_archive"; break; case LinkUnit::SharedLibrary: - if (has_std_artifacts) + if (has_std_artifacts && unit_needs_std(lu)) ins += " " + escape_ninja_path(std_o_dst); if (has_std_compat) ins += " " + escape_ninja_path(compat_o_dst); @@ -1608,11 +1937,24 @@ std::optional check_inline_command_lengths(const std::string& manif std::expected NinjaBackend::build(const BuildPlan& plan, const BuildOptions& opts) { auto t0 = std::chrono::steady_clock::now(); + // Where a drive's wall clock went. `mcpp test` calls this once per test on + // an already-built tree, so anything here that is not proportional to the + // work done is paid N times — and that is invisible from the outside, + // because the only number reported is the total. + auto tStage = t0; + auto stage = [&](std::string_view what) { + if (!mcpp::log::is_verbose()) { tStage = std::chrono::steady_clock::now(); return; } + auto now = std::chrono::steady_clock::now(); + auto ms = std::chrono::duration_cast(now - tStage).count(); + tStage = now; + if (ms >= 1) mcpp::log::verbose("build/stage", std::format("{}: {}ms", what, ms)); + }; // Captured before ninja touches any link output. The post-build runtime // validator compares this snapshot, so a hot no-op performs zero ELF // parses and an output rebuilt behind an unchanged build.ninja is caught. auto runtimeBefore = mcpp::build::runtime_validation::snapshot_link_artifacts(plan); + stage("snapshot"); std::error_code ec; std::filesystem::create_directories(plan.outputDir, ec); @@ -1623,6 +1965,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan auto ninja_path = plan.outputDir / "build.ninja"; auto manifest = emit_ninja_string(plan); + stage("emit-ninja"); // Command-length backstop (see // .agents/docs/2026-08-06-command-length-architecture.md). The structural @@ -1635,10 +1978,13 @@ std::expected NinjaBackend::build(const BuildPlan& plan return std::unexpected(BuildError{*over, ninja_path}); auto goalArg = append_goal_phony(manifest, opts.ninjaTargets); write_file(ninja_path, manifest); + stage("write-ninja"); // compile_commands.json — via the dedicated module. auto flags = compute_flags(plan); + stage("compute-flags"); auto cdb = write_compile_commands(plan, flags); + stage("compile-commands"); if (!cdb) { if (opts.requireCompileDatabase) { return std::unexpected(BuildError{ @@ -1685,6 +2031,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan plan.manifest.buildConfig.allowHostLibs); !h) { return std::unexpected(BuildError{h.error(), {}}); } + stage("hermetic-check"); // When the toolchain comes from mcpp's private sandbox, use the // sandbox-local ninja absolute path (skip the system xlings ninja @@ -1771,6 +2118,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan nargv, nenv, std::chrono::milliseconds(static_cast(opts.buildTimeoutSecs) * 1000), &buildTimedOut); + stage("ninja"); std::string out = cap.output; bool ok = (cap.exit_code == 0) && !buildTimedOut; @@ -1794,6 +2142,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan auto runtimeReport = mcpp::build::runtime_validation::validate_changed_artifacts( plan, runtimeBefore); + stage("runtime-validate"); std::string runtimeFailure; std::filesystem::path runtimeFailureArtifact; for (auto const& checked : runtimeReport.artifacts) { @@ -1836,6 +2185,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan continue; mcpp::ui::warning(finding.explain()); } + stage("loader-tags"); if (opts.verbose && !out.empty()) std::fputs(out.c_str(), stdout); std::set want(opts.ninjaTargets.begin(), opts.ninjaTargets.end()); diff --git a/src/build/plan.cppm b/src/build/plan.cppm index 7090b742..e3a36e08 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -145,6 +145,18 @@ struct BuildPlan { // share an output directory and overwrite each other's graph; this is what // lets a fast path tell them apart (mcpp#407, mcpp.build.graph_shape). GraphShape graphShape = GraphShape::Normal; + // The module-edge schedule this plan will emit, resolved ONCE (see + // mcpp.build.schedule.policy). The backend writes the graph in this shape, + // the graph records the tag, and the fast path compares against it — three + // readers, one derivation. Deriving it separately in the backend and in the + // executor is how the BMI-equivalence check and the job count drifted into + // disagreeing about what a module edge is. + std::string scheduleTag = "none"; + // What to hand ninja. Under detach-codegen a compiler stops holding a slot + // when it publishes, so this must exceed the real compiler cap or the ready + // frontier starves — see the hazard note in schedule/detach_codegen. + int scheduleNinjaJobs = 0; + int scheduleCompilerCap = 0; // One immutable snapshot selected before workspace member substitution. // Build/run/test and cache fast paths consume this value; none may re-read // xlings active/current state. @@ -711,6 +723,36 @@ std::vector runtime_search_closure( const bool elfTarget = triple.empty() ? bool(mcpp::platform::is_linux) : (triple.os != "macos" && triple.os != "windows"); + + // THE ARTIFACT'S OWN DIRECTORY — `$ORIGIN` (#415). + // + // It is emitted onto the link line by `shared_library_link_flags`, on a + // per-unit channel that never came through here, so the record and the + // artifact's DT_RPATH were STRUCTURALLY not comparable: e2e 219 had to + // special-case `$ORIGIN` to compare them at all, and that exception was + // the shape of the gap rather than a detail of the test. + // + // Recorded, not emitted: the link flag still comes from the same place it + // did. This closes the record, not the producer — making the closure the + // sole rpath producer means teaching it per-unit scope, which is a much + // larger change for a much smaller gain. + // + // ELF only, matching the guard below: `$ORIGIN` is the ELF spelling, and a + // format that gets no DT_RPATH gets no entry either. + // + // ⚠️ AND ONLY WHEN SOMETHING ACTUALLY EMITS IT. `$ORIGIN` comes from + // `shared_library_link_flags`, i.e. per CONSUMER of a shared library — a + // project with no shared library never gets one. Recording it + // unconditionally would swap this issue's asymmetry for its mirror image: + // the record would carry an entry the artifact does not, and e2e 219 would + // still need an exception to compare them. The point is that no exception + // is needed in either direction. + const bool buildsSharedLib = std::ranges::any_of( + plan.linkUnits, [](const LinkUnit& lu) { + return lu.kind == LinkUnit::SharedLibrary; + }); + if (elfTarget && buildsSharedLib && !plan.outputDir.empty()) + closure.push_back({plan.outputDir / "bin", Origin::Artifact}); // The binding names its libc as `@`; the triple names it // as an ABI env (`gnu` ⇒ glibc). A MISMATCH must be PROVEN, not assumed: // an undeclared SubOS has no runtime identity at all, and refusing its own diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index 87cc66bf..17a2e7e7 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -9,6 +9,12 @@ module; export module mcpp.build.prepare; +// The cfg() predicate evaluator and the fingerprint canonicalisers moved out — +// see mcpp.build.prepare_inputs. Re-exported so every existing caller of +// `target_dir` / `canonical_compile_flags` keeps working: a split whose only +// visible effect is that other files stop compiling is not an improvement. +export import mcpp.build.prepare_inputs; + import std; import mcpp.diag; import mcpp.home; @@ -33,6 +39,10 @@ import mcpp.toolchain.post_install; import mcpp.toolchain.abi; import mcpp.toolchain.triple; import mcpp.build.plan; +import mcpp.build.schedule.policy; +import mcpp.build.flags; // compute_flags — the per-role contracts (#418) +import mcpp.build.distribution; // dist::Role / dist::Contract to_string +import mcpp.platform.capacity; // the host fallback handed to schedule::decide import mcpp.build.graph_shape; // #407: the graph says which mode wrote it import mcpp.build.runtime_validation; // declared artifact -> identity verdict import mcpp.build.cache_key; @@ -97,347 +107,6 @@ inline void warn_unknown_xpkg_keys(const mcpp::manifest::Manifest& dm, } } -// ── L1 platform-conditional config: cfg() predicate evaluation ────────────── -// Context = the RESOLVED target's coordinates. A `[target.'cfg(...)'.build]` -// predicate is evaluated against this (target triple for a cross build, host -// for a native build), so conditional flags follow what the binary will run on -// — not the build host. See the manifest design doc. -namespace cfgpred { - -struct Ctx { std::string os, arch, family, env; }; - -// Derive the cfg context from the resolved --target triple, falling back to -// the host for a native build. Parsing goes through triple.cppm — the single -// triple parser — so the cfg vocabulary IS the canonical triple vocabulary -// (os: linux|macos|windows, arch: GNU spellings, env: gnu|musl|msvc), and -// alias spellings ("x86_64-w64-mingw32") evaluate identically to canonical. -inline Ctx context_for(std::string_view targetTriple) { - namespace triple = mcpp::toolchain::triple; - Ctx c; - auto t = targetTriple.empty() - ? std::optional(triple::host_triple()) - : triple::parse(targetTriple); - if (t) { - c.os = t->os; - c.arch = t->arch; - c.env = t->env; - c.family = t->family(); - } else { - // Escape-hatch triple outside the language: only the leading arch - // segment is derivable; other dimensions stay empty (never match). - auto dash = targetTriple.find('-'); - c.arch = std::string(dash == std::string_view::npos ? targetTriple - : targetTriple.substr(0, dash)); - } - return c; -} - -// Recursive-descent evaluator over the inside of `cfg(...)`: -// expr := all(list) | any(list) | not(expr) | key="value" | bareword -// key ∈ {os, arch, family, env} bareword ∈ {windows, unix, linux, macos} -struct Parser { - std::string_view s; std::size_t i = 0; const Ctx& c; - void ws() { while (i < s.size() && std::isspace((unsigned char)s[i])) ++i; } - bool eat(char ch) { ws(); if (i < s.size() && s[i] == ch) { ++i; return true; } return false; } - std::string ident() { - ws(); std::size_t b = i; - while (i < s.size() && (std::isalnum((unsigned char)s[i]) || s[i] == '_')) ++i; - return std::string(s.substr(b, i - b)); - } - std::string str() { - ws(); if (i >= s.size() || s[i] != '"') return {}; - ++i; std::size_t b = i; while (i < s.size() && s[i] != '"') ++i; - auto v = std::string(s.substr(b, i - b)); if (i < s.size()) ++i; return v; - } - bool match_alias(const std::string& a) { - if (a == "windows") return c.os == "windows"; - if (a == "linux") return c.os == "linux"; - if (a == "macos") return c.os == "macos"; - if (a == "unix") return c.family == "unix"; - return false; // unknown bareword → no match - } - bool match_kv(const std::string& k, const std::string& v) { - if (k == "os") return c.os == v; - if (k == "arch") return c.arch == v; - if (k == "family") return c.family == v; - if (k == "env") return c.env == v; - return false; - } - bool expr() { - std::string id = ident(); - if (id == "all" || id == "any") { - eat('('); - bool acc = (id == "all"); - ws(); - if (!(i < s.size() && s[i] == ')')) { - do { bool r = expr(); acc = (id == "all") ? (acc && r) : (acc || r); } - while (eat(',')); - } - eat(')'); - return acc; - } - if (id == "not") { eat('('); bool r = expr(); eat(')'); return !r; } - ws(); - if (i < s.size() && s[i] == '=') { ++i; return match_kv(id, str()); } - return match_alias(id); - } -}; - -// Evaluate a `[target.]` key. Returns the cfg() result, or — for a -// non-cfg key (a bare triple) — an exact match against the resolved triple. -inline bool matches(const std::string& predicate, const Ctx& c, std::string_view triple) { - std::string_view k = predicate; - if (k.starts_with("cfg(") && k.ends_with(")")) { - Parser p{ k.substr(4, k.size() - 5), 0, c }; - return p.expr(); - } - // Bare OS/family alias sugar: `[target.linux]` ≡ `[target.'cfg(linux)']`. - // These aliases are never valid triples (no dash), so there is no ambiguity - // with the exact-triple namespace. Evaluated as the cfg bareword. - if (predicate == "windows" || predicate == "linux" || - predicate == "macos" || predicate == "unix") { - Parser p{ predicate, 0, c }; - return p.expr(); - } - // Bare-triple match, spelling-independent: a `[target.x86_64-w64-mingw32]` - // key matches a resolved `x86_64-windows-gnu` build (and vice versa) — - // both normalize through triple::parse. Unparseable keys (the explicit- - // section escape hatch) fall back to exact string comparison. - if (triple.empty()) return false; - if (auto p = mcpp::toolchain::triple::parse(predicate)) { - if (auto rt = mcpp::toolchain::triple::parse(triple)) - return p->str() == rt->str(); - } - return predicate == triple; -} - -} // namespace cfgpred - -export std::filesystem::path target_dir(const mcpp::toolchain::Toolchain& tc, - const mcpp::toolchain::Fingerprint& fp, - const std::filesystem::path& root) -{ - // Canonical triple names the output directory (D1: `target/ - // x86_64-windows-gnu/`, not the GNU spelling the compiler reports via - // -dumpmachine) — alias inputs land in the same directory. Triples - // outside the language keep their raw spelling. - auto triple = tc.targetTriple.empty() ? std::string{"unknown"} : tc.targetTriple; - if (auto t = mcpp::toolchain::triple::parse(triple)) triple = t->str(); - return root / "target" / triple / fp.hex; -} - - -// Compose a stable canonical compile-flags string for fingerprinting. -// Exported so the "every build-variant knob is in here" invariant is machine- -// checkable: the profile knobs were absent for a long time precisely because -// nothing could assert on this string. -export std::string canonical_compile_flags(const mcpp::manifest::Manifest& m) { - std::string s; - s += "-std="; s += m.package.standard; - s += " -fmodules"; - // macOS deployment target changes the effective compile triple - // (arm64-apple-macosxNN) — a std.pcm built for one target cannot be - // loaded by a TU compiled for another. Fold the resolved value - // (env override > [build] macos_deployment_target manifest default) - // into the fingerprint so switching targets rebuilds the BMI cache - // instead of dying with a module config mismatch. - // - // The built-in default floor (rustc-style) lives in the single - // resolver (platform::macos::deployment_target), so this rule, the - // flags and the std-module prebuild always agree — the 0.0.50-era - // attempt to inject a default here alone left the test build's - // std.pcm unstaged (import std failed wholesale on macos CI). - if constexpr (mcpp::platform::is_macos) { - auto dtv = mcpp::platform::macos::deployment_target( - m.buildConfig.macosDeploymentTarget); - if (!dtv.empty()) { - s += " macos_deployment_target="; - s += dtv; - } - } - if (!m.buildConfig.cStandard.empty()) { - s += " c_standard="; - s += m.buildConfig.cStandard; - } - for (auto const& flag : m.buildConfig.cflags) { - s += " cflag:"; - s += flag; - } - for (auto const& flag : m.buildConfig.cxxflags) { - s += " cxxflag:"; - s += flag; - } - // Explicit [build] dialect_cxxflags (auto-promoted ones are already in - // cxxflags above) — they change every BMI in the graph. - for (auto const& flag : m.buildConfig.dialectCxxflags) { - s += " dialect:"; - s += flag; - } - for (auto const& flag : m.buildConfig.ldflags) { - s += " ldflag:"; - s += flag; - } - // Per-glob flags (G4): full ordered serialization — glob + every list — - // so editing any entry (or reordering) re-fingerprints the output dir. - for (auto const& gf : m.buildConfig.globFlags) { - s += " globflags:"; s += gf.glob; - for (auto const& f : gf.cflags) { s += " gc:"; s += f; } - for (auto const& f : gf.cxxflags) { s += " gxx:"; s += f; } - for (auto const& f : gf.asmflags) { s += " gas:"; s += f; } - for (auto const& f : gf.defines) { s += " gd:"; s += f; } - } - // [build] module_extensions changes WHICH FILES ARE MODULE INTERFACES, - // i.e. the shape of the graph: which units emit a BMI, which objects link - // unconditionally, which ninja rule each unit gets. That is a build - // variant, so it belongs in the fingerprint — mcpp.toml's mtime alone only - // protects the fast path within one output dir, not the BMI cache. - // - // Contrast [build] build_program_timeout, which is deliberately absent: - // it changes no edge. See BuildConfig::buildProgramTimeoutSecs. - for (auto const& e : m.buildConfig.moduleExtensions) { - s += " modext:"; - s += e; - } - // The resolved [profile] knobs. These are NOT in cflags/cxxflags: the - // profile block (see the profile resolution below) lands them in - // buildConfig.optLevel/debug/lto/strip and flags.cppm turns them into - // -O/-g/-flto at command-construction time. Leaving them out made - // `--dev`, `--release` and `--profile dist` share ONE fingerprint, hence - // one target/// directory AND one global cache entry — so a - // release build could be served -O0 -g dependency objects. They are - // build-variant by definition; they belong here. - s += " opt="; s += m.buildConfig.optLevel; - s += " debug="; s += m.buildConfig.debug ? "1" : "0"; - s += " lto="; s += m.buildConfig.lto ? "1" : "0"; - s += " strip="; s += m.buildConfig.strip ? "1" : "0"; - return s; -} - -std::string canonical_package_build_metadata( - const std::vector& packages) -{ - std::string s; - for (auto const& pkg : packages) { - s += "\npackage:"; - s += pkg.manifest.package.namespace_; - s += "/"; - s += pkg.manifest.package.name; - s += "@"; - s += pkg.manifest.package.version; - s += " source="; - s += pkg.manifest.package.sourceProvenance; - auto const& runtime = pkg.manifest.runtimeConfig; - for (auto const& requirement : runtime.requirements) { - s += " runtime-need:"; - s += requirement.kind; - s += ':'; - s += requirement.value; - s += ':'; - s += requirement.phase; - s += requirement.required ? ":required" : ":optional"; - } - for (auto const& artifact : runtime.artifacts) { - s += " runtime-artifact:"; - s += artifact.role; - s += ':'; - s += artifact.path.generic_string(); - s += ':'; - s += artifact.provenance; - s += ':'; - s += artifact.abi; - s += ':'; - s += artifact.digest; - s += ':'; - s += artifact.hostFingerprint; - } - for (auto const& value : runtime.linkIntent.libraries) - s += " link-library:" + value; - for (auto const& value : runtime.linkIntent.linkLibraryDirs) - s += " link-dir:" + value.generic_string(); - for (auto const& value : runtime.linkIntent.transitiveNeededDirs) - s += " needed-dir:" + value.generic_string(); - for (auto const& value : runtime.linkIntent.runtimeSearchDirs) - s += " runtime-dir:" + value.generic_string(); - for (auto const& value : runtime.linkIntent.frameworks) - s += " framework:" + value; - for (auto const& value : runtime.linkIntent.deployFiles) - s += " deploy:" + value.generic_string(); - // Legacy fields remain fingerprinted while they are readable. - for (auto const& value : runtime.libraryDirs) - s += " legacy-runtime-dir:" + value.generic_string(); - for (auto const& value : runtime.dlopenLibs) - s += " legacy-soname:" + value; - for (auto const& value : runtime.capabilities) - s += " legacy-capability:" + value; - for (auto const& value : runtime.provides) - s += " legacy-provides:" + value; - for (auto const& [capability, provider] : runtime.providerOverrides) - s += " provider-override:" + capability + '=' + provider; - if (!pkg.manifest.buildConfig.cStandard.empty()) { - s += " c_standard="; - s += pkg.manifest.buildConfig.cStandard; - } - for (auto const& flag : pkg.manifest.buildConfig.cflags) { - s += " cflag:"; - s += flag; - } - for (auto const& flag : pkg.manifest.buildConfig.cxxflags) { - s += " cxxflag:"; - s += flag; - } - for (auto const& flag : pkg.manifest.buildConfig.ldflags) { - s += " ldflag:"; - s += flag; - } - // Per-glob flags — same full ordered serialization as the root-side - // block above. Until #253 dependency globFlags were unfingerprinted - // (held only by "descriptor frozen per version" + "feature toggles - // always change cflags via -DMCPP_FEATURE_*"); feature-folded entries - // make the vector build-variant, so fingerprint it directly. - // featureOrigin is diagnostic-only and deliberately NOT serialized - // (the active feature set is already in cflags above). - for (auto const& gf : pkg.manifest.buildConfig.globFlags) { - s += " globflags:"; s += gf.glob; - for (auto const& f : gf.cflags) { s += " gc:"; s += f; } - for (auto const& f : gf.cxxflags) { s += " gxx:"; s += f; } - for (auto const& f : gf.asmflags) { s += " gas:"; s += f; } - for (auto const& f : gf.defines) { s += " gd:"; s += f; } - } - // Same reason as the root block, and it cannot be skipped on the - // grounds that "a descriptor is frozen per version": path and git - // dependencies are not frozen, and this key changes their products. - for (auto const& e : pkg.manifest.buildConfig.moduleExtensions) { - s += " modext:"; - s += e; - } - if (pkg.usageResolved) { - for (auto const& dir : pkg.privateBuild.includeDirs) { - s += " private_include:"; - s += dir.generic_string(); - } - for (auto const& dir : pkg.publicUsage.includeDirs) { - s += " public_include:"; - s += dir.generic_string(); - } - for (auto const& dir : pkg.privateBuild.includeDirsAfter) { - s += " private_include_after:"; - s += dir.generic_string(); - } - for (auto const& dir : pkg.publicUsage.includeDirsAfter) { - s += " public_include_after:"; - s += dir.generic_string(); - } - } - for (auto const& [path, content] : pkg.manifest.buildConfig.generatedFiles) { - s += " genfile:"; - s += path.generic_string(); - s += "="; - s += content; - } - } - return s; -} - std::expected materialize_generated_files(const std::filesystem::path& root, const mcpp::manifest::Manifest& manifest) @@ -1362,6 +1031,21 @@ prepare_build(bool print_fingerprint, // silently overrule one the user wrote down. auto tcOrigin = tcSpec.has_value() ? TcOrigin::ManifestToolchain : TcOrigin::None; + // `--toolchain` (arriving as MCPP_TOOLCHAIN, the same side channel + // `--offline` and `--jobs` use) beats everything, including the manifest. + // + // This is the usable form of "which compiler". On this repository the + // choice is worth 2.48x — gcc@16.1.0 builds mcpp in 79.9s, llvm@22.1.8 in + // 32.2s — but CHANGING THE DEFAULT is an ecosystem decision, not a + // performance one: it invalidates every published package's fingerprint and + // the three platforms do not yet ship the same llvm. Selecting per build + // costs nobody anything and needs no coordination. + // + // It counts as user-explicit, so mcpp will not quietly revise it. + if (const char* tcEnv = std::getenv("MCPP_TOOLCHAIN"); tcEnv && *tcEnv) { + tcSpec = std::string(tcEnv); + tcOrigin = TcOrigin::ManifestToolchain; + } if (!tcSpec.has_value()) { auto cfg = get_cfg(); if (cfg && !(*cfg)->defaultToolchain.empty()) { @@ -5226,6 +4910,17 @@ prepare_build(bool print_fingerprint, fpi.cppStandard = m->package.standard; fpi.compileFlags = canonical_compile_flags(*m) + canonical_package_build_metadata(packages); + // The module-edge schedule changes the SHAPE of build.ninja, and the fast + // path replays that file without a plan to compare against. Folding the + // switch into the fingerprint puts a differently-scheduled build in a + // different directory, which makes replaying the wrong shape structurally + // impossible instead of merely guarded. Only appended when non-default, so + // existing build directories keep their identity. + if (const auto sched = mcpp::build::schedule::requested_switch(*m); + sched != "auto") { + fpi.compileFlags += " #schedule="; + fpi.compileFlags += sched; + } if (m->cppStandard.experimental) { // c++fly gate flags are derived (not manifest-declared): fold them in // so a cppfly table change across mcpp versions re-fingerprints. @@ -5249,10 +4944,18 @@ prepare_build(bool print_fingerprint, // a std BMI built without it structurally lacks std::meta). Both // pieces were already in the fingerprint; this fixes the COMMAND // construction the fingerprint promised (stdFlagAndDialect above). + // #422: the CRT model reaches the std module too. Derived from the + // SAME expression the project's TUs use (flags.cppm), through the one + // helper, so the two cannot drift. Non-MSVC dialects yield "" and the + // command is unchanged. + const auto& stdDialect = mcpp::toolchain::dialect_for(*tc); + const auto stdCrt = mcpp::toolchain::msvc_crt_flag( + stdDialect, m->buildConfig.linkage == "static"); auto sm = mcpp::toolchain::ensure_built( *tc, m->package.standard, stdFlagAndDialect, mcpp::platform::macos::deployment_target( - m->buildConfig.macosDeploymentTarget)); + m->buildConfig.macosDeploymentTarget), + mcpp::toolchain::default_cache_root(), stdCrt); if (!sm) return std::unexpected(sm.error().message); stdBmiPath = sm->bmiPath; stdObjectPath = sm->objectPath; @@ -5311,6 +5014,39 @@ prepare_build(bool print_fingerprint, ctx.plan.graphShape = (includeDevDeps || !extraTargets.empty()) ? mcpp::build::GraphShape::WithTests : mcpp::build::GraphShape::Normal; + // Resolve the module-edge schedule ONCE, here, where both the toolchain and + // the manifest are in hand. The backend writes the graph in this shape, the + // graph records the tag, and `mcpp build --verbose` prints the reason — all + // three read this, none of them re-derives it. + { + const auto decision = mcpp::build::schedule::decide( + ctx.plan.toolchain, + // Warned HERE and not at the fingerprint call above, which reads the + // same switch a few hundred lines earlier: both get the normalised + // value, only one of them says anything, so a typo produces exactly + // one warning rather than two identical ones. + mcpp::build::schedule::requested_switch(*m, [](std::string_view bad) { + mcpp::ui::warning(std::format( + "ignoring invalid bmi_schedule '{}' (expected \"auto\", \"on\" or \"off\")", bad)); + }), + mcpp::build::schedule::resolve_jobs(*m, [](std::string_view bad) { + mcpp::ui::warning(std::format( + "ignoring invalid job count '{}' (expected a positive number or 'auto')", bad)); + }), + // What this machine would pick if asked. Impure, so it is resolved + // here and handed to the pure `decide`. Only DetachCodegen uses it, + // and only when the user gave no job count — without it that + // strategy ships `sched_cap = 0`, which disables the semaphore that + // is its ONLY bound on how many compilers run at once. + mcpp::platform::capacity::recommended_jobs( + mcpp::platform::capacity::host_capacity())); + ctx.plan.scheduleTag = std::string(mcpp::build::schedule::to_string(decision.strategy)); + ctx.plan.scheduleNinjaJobs = decision.ninjaJobs; + ctx.plan.scheduleCompilerCap = decision.compilerCap; + mcpp::log::verbose("build", std::format("schedule: {} — {}", + ctx.plan.scheduleTag, decision.reason)); + + } ctx.plan.runtimeBinding = runtimeBindingSnapshot; mcpp::build::merge_runtime_binding_contract( ctx.plan, runtimeBindingSnapshot); @@ -6425,7 +6161,30 @@ prepare_build(bool print_fingerprint, : format == "macho" ? "loader_rpath" : "runpath"}, {"closure", closure}, }; + // #418 — the contract each ROLE actually got, after any downgrade. + // + // `CompileFlags::contractByRole` was written and never read: a valuable + // observation with no way out of the process. Since #414 the shared + // library role can legitimately end up on a different contract from the + // binaries beside it, so "which one did my .so actually get?" is a + // question a user has, and the only answer available was to run + // `readelf` and infer. + // + // Recorded as the RESOLVED value, not the requested one — a request + // that was downgraded is exactly the case worth being able to see. + // `compute_flags` is pure in the plan; prepare does not otherwise hold + // the result, and threading it through just for this would widen a + // signature for one field. + const auto roleFlags = mcpp::build::compute_flags(ctx.plan); + nlohmann::json contracts = nlohmann::json::object(); + for (std::size_t i = 0; i < mcpp::build::dist::kRoleCount; ++i) { + contracts[std::string(mcpp::build::dist::to_string( + static_cast(i)))] = + std::string(mcpp::build::dist::to_string(roleFlags.contractByRole[i])); + } + j["runtime"] = { + {"cxx_runtime_by_role", contracts}, {"library_dirs", dirs}, {"dlopen_libs", ctx.plan.runtimeDlopenLibs}, {"capabilities", legacyCaps}, diff --git a/src/build/prepare_inputs.cppm b/src/build/prepare_inputs.cppm new file mode 100644 index 00000000..3e354eee --- /dev/null +++ b/src/build/prepare_inputs.cppm @@ -0,0 +1,375 @@ +// mcpp.build.prepare_inputs — the inputs a build plan is derived FROM, split out +// of mcpp.build.prepare. +// +// WHY. `prepare.cppm` was 6521 lines and 16.4s to compile — 22% of this +// repository's critical build path and its only real outlier. A module that +// large is worth splitting on architecture grounds alone; the build-time effect +// is a bonus and, since the split schedule landed, a smaller one (an interface +// now blocks importers for ~22% of its compile rather than all of it). +// +// ⚠️ WHAT A SPLIT HAS TO BE TO HELP THE CRITICAL PATH. Extracting a piece that +// `prepare` then imports makes the chain LONGER, not shorter: `... -> this -> +// prepare -> ...` is still serial, and prepare only sheds the cost this module +// now pays. It shortens the path only for consumers that can import THIS +// instead of prepare — which is why the pieces chosen here are the ones with no +// dependency on the rest of prepare: cfg() predicate evaluation and the +// fingerprint canonicalisers. +// +// They are re-exported from `mcpp.build.prepare`, so no caller had to change. +export module mcpp.build.prepare_inputs; + +import std; +import mcpp.diag; +import mcpp.manifest; +import mcpp.modgraph.graph; +import mcpp.modgraph.scanner; +import mcpp.platform; +import mcpp.toolchain.model; +import mcpp.toolchain.fingerprint; +import mcpp.toolchain.triple; +import mcpp.ui; + +export namespace mcpp::build { + +// ── L1 platform-conditional config: cfg() predicate evaluation ────────────── +// Context = the RESOLVED target's coordinates. A `[target.'cfg(...)'.build]` +// predicate is evaluated against this (target triple for a cross build, host +// for a native build), so conditional flags follow what the binary will run on +// — not the build host. See the manifest design doc. +namespace cfgpred { + +struct Ctx { std::string os, arch, family, env; }; + +// Derive the cfg context from the resolved --target triple, falling back to +// the host for a native build. Parsing goes through triple.cppm — the single +// triple parser — so the cfg vocabulary IS the canonical triple vocabulary +// (os: linux|macos|windows, arch: GNU spellings, env: gnu|musl|msvc), and +// alias spellings ("x86_64-w64-mingw32") evaluate identically to canonical. +inline Ctx context_for(std::string_view targetTriple) { + namespace triple = mcpp::toolchain::triple; + Ctx c; + auto t = targetTriple.empty() + ? std::optional(triple::host_triple()) + : triple::parse(targetTriple); + if (t) { + c.os = t->os; + c.arch = t->arch; + c.env = t->env; + c.family = t->family(); + } else { + // Escape-hatch triple outside the language: only the leading arch + // segment is derivable; other dimensions stay empty (never match). + auto dash = targetTriple.find('-'); + c.arch = std::string(dash == std::string_view::npos ? targetTriple + : targetTriple.substr(0, dash)); + } + return c; +} + +// Recursive-descent evaluator over the inside of `cfg(...)`: +// expr := all(list) | any(list) | not(expr) | key="value" | bareword +// key ∈ {os, arch, family, env} bareword ∈ {windows, unix, linux, macos} +struct Parser { + std::string_view s; std::size_t i = 0; const Ctx& c; + void ws() { while (i < s.size() && std::isspace((unsigned char)s[i])) ++i; } + bool eat(char ch) { ws(); if (i < s.size() && s[i] == ch) { ++i; return true; } return false; } + std::string ident() { + ws(); std::size_t b = i; + while (i < s.size() && (std::isalnum((unsigned char)s[i]) || s[i] == '_')) ++i; + return std::string(s.substr(b, i - b)); + } + std::string str() { + ws(); if (i >= s.size() || s[i] != '"') return {}; + ++i; std::size_t b = i; while (i < s.size() && s[i] != '"') ++i; + auto v = std::string(s.substr(b, i - b)); if (i < s.size()) ++i; return v; + } + bool match_alias(const std::string& a) { + if (a == "windows") return c.os == "windows"; + if (a == "linux") return c.os == "linux"; + if (a == "macos") return c.os == "macos"; + if (a == "unix") return c.family == "unix"; + return false; // unknown bareword → no match + } + bool match_kv(const std::string& k, const std::string& v) { + if (k == "os") return c.os == v; + if (k == "arch") return c.arch == v; + if (k == "family") return c.family == v; + if (k == "env") return c.env == v; + return false; + } + bool expr() { + std::string id = ident(); + if (id == "all" || id == "any") { + eat('('); + bool acc = (id == "all"); + ws(); + if (!(i < s.size() && s[i] == ')')) { + do { bool r = expr(); acc = (id == "all") ? (acc && r) : (acc || r); } + while (eat(',')); + } + eat(')'); + return acc; + } + if (id == "not") { eat('('); bool r = expr(); eat(')'); return !r; } + ws(); + if (i < s.size() && s[i] == '=') { ++i; return match_kv(id, str()); } + return match_alias(id); + } +}; + +// Evaluate a `[target.]` key. Returns the cfg() result, or — for a +// non-cfg key (a bare triple) — an exact match against the resolved triple. +inline bool matches(const std::string& predicate, const Ctx& c, std::string_view triple) { + std::string_view k = predicate; + if (k.starts_with("cfg(") && k.ends_with(")")) { + Parser p{ k.substr(4, k.size() - 5), 0, c }; + return p.expr(); + } + // Bare OS/family alias sugar: `[target.linux]` ≡ `[target.'cfg(linux)']`. + // These aliases are never valid triples (no dash), so there is no ambiguity + // with the exact-triple namespace. Evaluated as the cfg bareword. + if (predicate == "windows" || predicate == "linux" || + predicate == "macos" || predicate == "unix") { + Parser p{ predicate, 0, c }; + return p.expr(); + } + // Bare-triple match, spelling-independent: a `[target.x86_64-w64-mingw32]` + // key matches a resolved `x86_64-windows-gnu` build (and vice versa) — + // both normalize through triple::parse. Unparseable keys (the explicit- + // section escape hatch) fall back to exact string comparison. + if (triple.empty()) return false; + if (auto p = mcpp::toolchain::triple::parse(predicate)) { + if (auto rt = mcpp::toolchain::triple::parse(triple)) + return p->str() == rt->str(); + } + return predicate == triple; +} + +} // namespace cfgpred + +std::filesystem::path target_dir(const mcpp::toolchain::Toolchain& tc, + const mcpp::toolchain::Fingerprint& fp, + const std::filesystem::path& root) +{ + // Canonical triple names the output directory (D1: `target/ + // x86_64-windows-gnu/`, not the GNU spelling the compiler reports via + // -dumpmachine) — alias inputs land in the same directory. Triples + // outside the language keep their raw spelling. + auto triple = tc.targetTriple.empty() ? std::string{"unknown"} : tc.targetTriple; + if (auto t = mcpp::toolchain::triple::parse(triple)) triple = t->str(); + return root / "target" / triple / fp.hex; +} + + +// Compose a stable canonical compile-flags string for fingerprinting. +// Exported so the "every build-variant knob is in here" invariant is machine- +// checkable: the profile knobs were absent for a long time precisely because +// nothing could assert on this string. +std::string canonical_compile_flags(const mcpp::manifest::Manifest& m) { + std::string s; + s += "-std="; s += m.package.standard; + s += " -fmodules"; + // macOS deployment target changes the effective compile triple + // (arm64-apple-macosxNN) — a std.pcm built for one target cannot be + // loaded by a TU compiled for another. Fold the resolved value + // (env override > [build] macos_deployment_target manifest default) + // into the fingerprint so switching targets rebuilds the BMI cache + // instead of dying with a module config mismatch. + // + // The built-in default floor (rustc-style) lives in the single + // resolver (platform::macos::deployment_target), so this rule, the + // flags and the std-module prebuild always agree — the 0.0.50-era + // attempt to inject a default here alone left the test build's + // std.pcm unstaged (import std failed wholesale on macos CI). + if constexpr (mcpp::platform::is_macos) { + auto dtv = mcpp::platform::macos::deployment_target( + m.buildConfig.macosDeploymentTarget); + if (!dtv.empty()) { + s += " macos_deployment_target="; + s += dtv; + } + } + if (!m.buildConfig.cStandard.empty()) { + s += " c_standard="; + s += m.buildConfig.cStandard; + } + for (auto const& flag : m.buildConfig.cflags) { + s += " cflag:"; + s += flag; + } + for (auto const& flag : m.buildConfig.cxxflags) { + s += " cxxflag:"; + s += flag; + } + // Explicit [build] dialect_cxxflags (auto-promoted ones are already in + // cxxflags above) — they change every BMI in the graph. + for (auto const& flag : m.buildConfig.dialectCxxflags) { + s += " dialect:"; + s += flag; + } + for (auto const& flag : m.buildConfig.ldflags) { + s += " ldflag:"; + s += flag; + } + // Per-glob flags (G4): full ordered serialization — glob + every list — + // so editing any entry (or reordering) re-fingerprints the output dir. + for (auto const& gf : m.buildConfig.globFlags) { + s += " globflags:"; s += gf.glob; + for (auto const& f : gf.cflags) { s += " gc:"; s += f; } + for (auto const& f : gf.cxxflags) { s += " gxx:"; s += f; } + for (auto const& f : gf.asmflags) { s += " gas:"; s += f; } + for (auto const& f : gf.defines) { s += " gd:"; s += f; } + } + // [build] module_extensions changes WHICH FILES ARE MODULE INTERFACES, + // i.e. the shape of the graph: which units emit a BMI, which objects link + // unconditionally, which ninja rule each unit gets. That is a build + // variant, so it belongs in the fingerprint — mcpp.toml's mtime alone only + // protects the fast path within one output dir, not the BMI cache. + // + // Contrast [build] build_program_timeout, which is deliberately absent: + // it changes no edge. See BuildConfig::buildProgramTimeoutSecs. + for (auto const& e : m.buildConfig.moduleExtensions) { + s += " modext:"; + s += e; + } + // The resolved [profile] knobs. These are NOT in cflags/cxxflags: the + // profile block (see the profile resolution below) lands them in + // buildConfig.optLevel/debug/lto/strip and flags.cppm turns them into + // -O/-g/-flto at command-construction time. Leaving them out made + // `--dev`, `--release` and `--profile dist` share ONE fingerprint, hence + // one target/// directory AND one global cache entry — so a + // release build could be served -O0 -g dependency objects. They are + // build-variant by definition; they belong here. + s += " opt="; s += m.buildConfig.optLevel; + s += " debug="; s += m.buildConfig.debug ? "1" : "0"; + s += " lto="; s += m.buildConfig.lto ? "1" : "0"; + s += " strip="; s += m.buildConfig.strip ? "1" : "0"; + return s; +} + +std::string canonical_package_build_metadata( + const std::vector& packages) +{ + std::string s; + for (auto const& pkg : packages) { + s += "\npackage:"; + s += pkg.manifest.package.namespace_; + s += "/"; + s += pkg.manifest.package.name; + s += "@"; + s += pkg.manifest.package.version; + s += " source="; + s += pkg.manifest.package.sourceProvenance; + auto const& runtime = pkg.manifest.runtimeConfig; + for (auto const& requirement : runtime.requirements) { + s += " runtime-need:"; + s += requirement.kind; + s += ':'; + s += requirement.value; + s += ':'; + s += requirement.phase; + s += requirement.required ? ":required" : ":optional"; + } + for (auto const& artifact : runtime.artifacts) { + s += " runtime-artifact:"; + s += artifact.role; + s += ':'; + s += artifact.path.generic_string(); + s += ':'; + s += artifact.provenance; + s += ':'; + s += artifact.abi; + s += ':'; + s += artifact.digest; + s += ':'; + s += artifact.hostFingerprint; + } + for (auto const& value : runtime.linkIntent.libraries) + s += " link-library:" + value; + for (auto const& value : runtime.linkIntent.linkLibraryDirs) + s += " link-dir:" + value.generic_string(); + for (auto const& value : runtime.linkIntent.transitiveNeededDirs) + s += " needed-dir:" + value.generic_string(); + for (auto const& value : runtime.linkIntent.runtimeSearchDirs) + s += " runtime-dir:" + value.generic_string(); + for (auto const& value : runtime.linkIntent.frameworks) + s += " framework:" + value; + for (auto const& value : runtime.linkIntent.deployFiles) + s += " deploy:" + value.generic_string(); + // Legacy fields remain fingerprinted while they are readable. + for (auto const& value : runtime.libraryDirs) + s += " legacy-runtime-dir:" + value.generic_string(); + for (auto const& value : runtime.dlopenLibs) + s += " legacy-soname:" + value; + for (auto const& value : runtime.capabilities) + s += " legacy-capability:" + value; + for (auto const& value : runtime.provides) + s += " legacy-provides:" + value; + for (auto const& [capability, provider] : runtime.providerOverrides) + s += " provider-override:" + capability + '=' + provider; + if (!pkg.manifest.buildConfig.cStandard.empty()) { + s += " c_standard="; + s += pkg.manifest.buildConfig.cStandard; + } + for (auto const& flag : pkg.manifest.buildConfig.cflags) { + s += " cflag:"; + s += flag; + } + for (auto const& flag : pkg.manifest.buildConfig.cxxflags) { + s += " cxxflag:"; + s += flag; + } + for (auto const& flag : pkg.manifest.buildConfig.ldflags) { + s += " ldflag:"; + s += flag; + } + // Per-glob flags — same full ordered serialization as the root-side + // block above. Until #253 dependency globFlags were unfingerprinted + // (held only by "descriptor frozen per version" + "feature toggles + // always change cflags via -DMCPP_FEATURE_*"); feature-folded entries + // make the vector build-variant, so fingerprint it directly. + // featureOrigin is diagnostic-only and deliberately NOT serialized + // (the active feature set is already in cflags above). + for (auto const& gf : pkg.manifest.buildConfig.globFlags) { + s += " globflags:"; s += gf.glob; + for (auto const& f : gf.cflags) { s += " gc:"; s += f; } + for (auto const& f : gf.cxxflags) { s += " gxx:"; s += f; } + for (auto const& f : gf.asmflags) { s += " gas:"; s += f; } + for (auto const& f : gf.defines) { s += " gd:"; s += f; } + } + // Same reason as the root block, and it cannot be skipped on the + // grounds that "a descriptor is frozen per version": path and git + // dependencies are not frozen, and this key changes their products. + for (auto const& e : pkg.manifest.buildConfig.moduleExtensions) { + s += " modext:"; + s += e; + } + if (pkg.usageResolved) { + for (auto const& dir : pkg.privateBuild.includeDirs) { + s += " private_include:"; + s += dir.generic_string(); + } + for (auto const& dir : pkg.publicUsage.includeDirs) { + s += " public_include:"; + s += dir.generic_string(); + } + for (auto const& dir : pkg.privateBuild.includeDirsAfter) { + s += " private_include_after:"; + s += dir.generic_string(); + } + for (auto const& dir : pkg.publicUsage.includeDirsAfter) { + s += " public_include_after:"; + s += dir.generic_string(); + } + } + for (auto const& [path, content] : pkg.manifest.buildConfig.generatedFiles) { + s += " genfile:"; + s += path.generic_string(); + s += "="; + s += content; + } + } + return s; +} + +} // namespace mcpp::build diff --git a/src/build/runtime_validation.cppm b/src/build/runtime_validation.cppm index 5a512e73..88a743c0 100644 --- a/src/build/runtime_validation.cppm +++ b/src/build/runtime_validation.cppm @@ -16,6 +16,7 @@ import mcpp.libs.json; import mcpp.platform; import mcpp.platform.elf_runtime; import mcpp.platform.runtime_binding; +import mcpp.ui; import mcpp.platform.runtime_search; export namespace mcpp::build::runtime_validation { @@ -111,9 +112,14 @@ ArtifactVerdict artifact_identity_verdict( // anyone needing readelf on the box. It is also how "checked and compliant" // stays distinguishable from "never checked": both look identical when the // only output is the absence of a warning. +// +// `before` is the pre-ninja snapshot, same as validate_changed_artifacts takes: +// an artifact whose stat did not move was not produced by this run, so its +// verdict is READ BACK from resolution.json instead of re-derived from the ELF. +// The returned vector still covers every artifact either way. std::vector check_and_record_loader_tags(const mcpp::build::BuildPlan& plan, - const ArtifactSnapshot& produced); + const ArtifactSnapshot& before); } // namespace mcpp::build::runtime_validation @@ -407,6 +413,29 @@ ValidationReport validate_changed_artifacts( } } } + // ⚠️ A BINDING THAT CANNOT BE EVALUATED IS ONE FACT, NOT ONE PER ARTIFACT. + // + // On a brand-new MCPP_HOME the first build finds `binding.loader` and + // `binding.libraryDirs` both empty (the second build has them; #417), and + // rule B then reported that per artifact — two lines each, thirteen + // artifacts, twenty-six lines of the same sentence on a user's very first + // build. The root cause is a separate question and is NOT settled; this is + // the half of the criterion that does not depend on it. + // + // Said once, before the loop, naming what is missing. Rule B still runs: + // it has other inputs (PT_INTERP identity), and suppressing it entirely + // would trade noise for a blind spot. + const bool bindingUnevaluated = + !plan.runtimeBinding.loader.has_value() && plan.runtimeBinding.libraryDirs.empty(); + if (bindingUnevaluated && !before.empty()) { + mcpp::ui::warning(std::format( + "runtime binding {} has no loader path or library directory yet, so " + "rule B cannot decide for this build's artifacts. This is expected on " + "the first build in a fresh MCPP_HOME; a second build resolves it.", + plan.runtimeBinding.runtimeId.empty() ? "" + : plan.runtimeBinding.runtimeId)); + } + for (auto const& [artifact, oldStamp] : before) { auto now = stamp(artifact); if (!now.exists) continue; @@ -523,22 +552,91 @@ ArtifactVerdict artifact_identity_verdict( std::vector check_and_record_loader_tags(const mcpp::build::BuildPlan& plan, - const ArtifactSnapshot& produced) { + const ArtifactSnapshot& before) { namespace loader = mcpp::build::loader; std::vector findings; if constexpr (!mcpp::platform::is_linux) return findings; - for (auto const& [artifact, ignored] : produced) { - (void)ignored; + const auto path = plan.outputDir / "resolution.json"; + nlohmann::json resolution; + { + std::ifstream input(path); + resolution = nlohmann::json::parse(input, nullptr, false); + } + + // What this run actually produced. The parameter used to be spelled + // `produced` and then be given the BEFORE snapshot, so every drive + // re-parsed every link artifact in the plan — and `mcpp test` drives the + // backend once per test on an already-built tree. + // MEASURED on the 83-test suite: 158.7 s of a 190 s hot run, 1.87 s x 85 + // drives, for artifacts that nothing had touched. + // + // An unchanged artifact keeps the verdict already written to + // resolution.json rather than being dropped: a violation must keep being + // reported on every build, and "checked and compliant" must stay + // distinguishable from "never checked" — which is exactly what a shorter + // fix (skip unchanged, record only the fresh ones) would have destroyed. + const auto recorded = [&]() -> nlohmann::json { + auto rt = resolution.is_object() ? resolution.find("runtime") : resolution.end(); + if (rt == resolution.end() || !rt->is_object()) return nlohmann::json::array(); + auto tags = rt->find("loader_tags"); + if (tags == rt->end() || !tags->is_array()) return nlohmann::json::array(); + return *tags; + }(); + auto recorded_entry = [&](const std::string& rel) -> const nlohmann::json* { + for (auto const& e : recorded) + if (e.is_object() && e.value("path", "") == rel) return &e; + return nullptr; + }; + auto required_from = [](std::string_view s) { + if (s == "DT_RPATH") return loader::RequiredTag::Rpath; + if (s == "DT_RUNPATH") return loader::RequiredTag::Runpath; + return loader::RequiredTag::NotApplicable; + }; + auto actual_from = [](std::string_view s) { + using Tag = mcpp::platform::elf::SearchPathTag; + if (s == "DT_RPATH") return Tag::Rpath; + if (s == "DT_RUNPATH") return Tag::Runpath; + if (s == "DT_RPATH+DT_RUNPATH") return Tag::Both; + return Tag::None; + }; + + bool anyFresh = false; + for (auto const& [artifact, oldStamp] : before) { + auto now = stamp(artifact); + if (!now.exists) continue; + + std::error_code ec; + auto rel = std::filesystem::relative(artifact, plan.outputDir, ec); + auto relStr = (ec ? artifact : rel).lexically_normal().generic_string(); + + if (now == oldStamp) { + if (auto const* prev = recorded_entry(relStr)) { + loader::TagFinding f; + f.artifact = artifact; + f.form = prev->value("form", "") == "executable" + ? loader::Form::Executable : loader::Form::SharedLibrary; + f.required = required_from(prev->value("required", "")); + f.actual = actual_from(prev->value("actual", "")); + auto st = prev->value("status", ""); + f.status = st == "ok" ? loader::TagFinding::Status::Ok + : st == "violation" ? loader::TagFinding::Status::Violation + : loader::TagFinding::Status::NotChecked; + findings.push_back(std::move(f)); + continue; + } + // No stored verdict for an unchanged artifact: fall through and + // read it, or the first build after this cache shape changed would + // report "not checked" forever. + } + auto finding = loader::check_artifact(artifact); if (finding.form == loader::Form::NotElf) continue; + anyFresh = true; findings.push_back(std::move(finding)); } - if (findings.empty()) return findings; + if (findings.empty() || !anyFresh) return findings; - const auto path = plan.outputDir / "resolution.json"; - std::ifstream input(path); - auto resolution = nlohmann::json::parse(input, nullptr, false); if (resolution.is_discarded() || !resolution.is_object()) return findings; auto runtime = resolution.find("runtime"); if (runtime == resolution.end() || !runtime->is_object()) return findings; diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm new file mode 100644 index 00000000..2e77ff77 --- /dev/null +++ b/src/build/schedule/detach_codegen.cppm @@ -0,0 +1,655 @@ +// mcpp.build.schedule.detach_codegen — the GCC strategy: let importers start +// when the BMI lands, and let code generation finish off the critical path. +// +// WHY. mcpp's own cold build is 79.9 s and its critical path is 79.73 s — 100% +// of the makespan, at an average of 3.94 concurrent jobs on 32 hardware threads. +// Nothing outside the graph's shape moves it: -j8 → -j32 buys 1.4%, cmake and +// xmake build the same sources in 94.5 s and 94.6 s, and clang only scales the +// constant (32.2 s makespan, 32.15 s critical path — the same 100%). +// +// `-ftime-report` on the chain's heaviest link says where the time goes: +// +// phase opt and generate 14.08s 86% <- code generation +// phase parsing 1.32s 8% +// template instantiation 0.95s 6% +// module import 0.51s 3% +// +// 86% of a module interface compile is code generation, and NO IMPORTER NEEDS A +// BYTE OF IT. +// +// THE STRATEGY IS PER COMPILER, and the two are complementary rather than +// alternatives — see mcpp::toolchain::BmiSplit. This module implements the gcc +// one (DetachCodegen). Clang needs nothing from here: `--precompile` already +// splits the work into two ordinary edges. +// +// WHY WATCHING THE FILE IS SOUND FOR GCC, AND NOT A HEURISTIC. GCC writes the +// BMI to `.gcm~` and rename()s it into place — verified with strace: +// +// openat("gcm.cache/x.gcm~", O_RDWR|O_CREAT|O_TRUNC) = 5 +// rename("gcm.cache/x.gcm~", "gcm.cache/x.gcm") = 0 +// +// so the final path is atomically complete-or-absent. Confirmed three further +// ways: the early snapshot is byte-identical to the finished file, and a real +// downstream importer compiles against it and exits 0. Clang, by contrast, +// writes the BMI straight to the final path with O_TRUNC — which is exactly why +// it gets the other strategy instead of this one. +// +// MEASURED (same build dir, compiler, flags, compiler-concurrency cap and +// sources; the ONLY difference is the graph's shape): +// +// baseline wall=80.51s ninja -j32 compilers<=32 +// split wall=39.23s ninja -j192 compilers<=32 +// +// ⚠️ FOUR HAZARDS, EVERY ONE OF WHICH BIT DURING DEVELOPMENT: +// +// 1. THE COMPILER MUST NOT INHERIT ninja's PIPE. ninja finishes an edge when +// the pipe reaches EOF, NOT when its direct child exits. An inherited pipe +// makes the early exit invisible — every BMI edge is logged with the FULL +// compile duration, and the arm reads as "the idea does not work". The +// compiler's stdio goes to a file, replayed by phase 2. +// 2. ninja's -j MUST EXCEED THE COMPILER CAP. A detached compiler no longer +// holds a ninja slot, so with -j equal to the cap the slots fill with edges +// that are merely sleeping and the ready frontier starves — the schedule +// degenerates to the baseline. Real concurrency is bounded by the semaphore +// below, never by -j. +// 3. FAILURES ARRIVE LATE. A compiler that fails during code generation has +// already had its BMI edge reported successful. Phase 2 must REPLAY that +// failure or it surfaces as undefined symbols at link time. +// 4. THE GRAPH MUST DECLARE ITS SHAPE. build.ninja is shared mutable state and +// the fast path replays it, so "is this graph split" belongs in the +// `# mcpp:graph=` line — see mcpp.build.graph_shape. +// +// NOT POSIX-ONLY. What this needs is a process that outlives the current one, +// which is a spawn, not a fork; the supervisor is `mcpp` itself re-invoked. +// What is compiler-specific is the PREMISE (atomic BMI publication), not the +// platform. +module; + +// The global module fragment is the ONLY place a module interface unit may +// #include. These were briefly written after `module :private;`, which GCC +// rejects with the unhelpful "module already declared". +#if defined(_WIN32) +#include +#else +#include +#include +#include +#include +extern char** environ; +#endif + +export module mcpp.build.schedule.detach_codegen; + +import std; +import mcpp.build.stage; + +export namespace mcpp::build::schedule::detach { + +struct CompileRequest { + // The BMI this compile publishes. Empty for a unit that produces none, in + // which case phase 1 simply waits like an ordinary edge. + std::filesystem::path bmi; + // `.log` and `.rc` live beside this path. + std::filesystem::path slot; + // Absolute path to the mcpp binary, re-invoked as the supervisor. A spawn + // rather than a fork is what keeps this portable. + std::filesystem::path self; + // Directory of concurrency tokens. Empty disables the cap (hazard 2). + std::filesystem::path semaphore; + int maxCompilers{0}; + // The compiler invocation, as ONE shell command line. + // + // Not a token list: the backend has already joined and quoted the flags for + // ninja, and splitting that string back into argv would need to reimplement + // the shell's rules — the exact assumption ("one flag element == one argv + // token") that has been wrong here before. ninja runs every command through + // a shell already, so going through one costs no portability. + std::string command; + // The file `command` was read from; handed to the supervisor unchanged, so + // there is one representation and no re-quoting. + std::filesystem::path commandFile; + // Where this edge's header dependencies come from, and where ninja expects + // to find them. + // + // MEASURED: the compiler's own `-MMD` file is written at 16.39s of a 16.55s + // compile — AFTER the BMI is published at 2.36s. This edge is over before + // it exists, so it cannot use it. The P1689 SCAN has already run and writes + // an equivalent one; compared on real modules the only prerequisites it + // lacks are `.gcm` BMIs, which are ninja's through dyndep. Header coverage + // is exact. + // + // COPIED, not pointed at: ninja DELETES a depfile once it has folded it + // into .ninja_deps, and the scan would not regenerate it unless the scan + // itself reran. + std::filesystem::path depFrom; + std::filesystem::path depTo; +}; + +// Phase 1 — returns 0 as soon as the BMI is published, leaving code generation +// running. Returns the compiler's status if it exits before publishing one. +int compile_release_at_bmi(const CompileRequest& req); + +// The supervisor. Runs the compiler to completion with its output redirected, +// then records the status. Never invoked directly by a build edge. +int supervise(const std::filesystem::path& slot, + const std::filesystem::path& semaphoreToken, + std::string_view command); + +// Phase 2 — blocks until the compiler for `slot` finished, replays what it +// wrote, and propagates its status. `object`, when given, must exist: a +// compiler that reports success without producing its output is a failure this +// must not pass on. +int await_unit(const std::filesystem::path& slot, const std::filesystem::path& object); + +} // namespace mcpp::build::schedule::detach + +// --------------------------------------------------------------------------- + +namespace mcpp::build::schedule::detach { +namespace { + +std::filesystem::path suffixed(const std::filesystem::path& base, std::string_view s) { + return std::filesystem::path{base.string() + std::string(s)}; +} + +bool file_exists(const std::filesystem::path& p) { + std::error_code ec; + return std::filesystem::exists(p, ec); +} + +std::optional read_rc(const std::filesystem::path& slot) { + std::ifstream in(suffixed(slot, ".rc")); + if (!in) return std::nullopt; + int rc = 0; + if (!(in >> rc)) return std::nullopt; + return rc; +} + +// Temp file + rename, so a reader never observes half a number. The same +// guarantee, for the same reason, that makes watching the BMI path sound. +void write_rc(const std::filesystem::path& slot, int rc) { + const auto tmp = suffixed(slot, ".rc.tmp"); + { std::ofstream out(tmp, std::ios::trunc); out << rc << '\n'; } + std::error_code ec; + std::filesystem::rename(tmp, suffixed(slot, ".rc"), ec); +} + +// A counting semaphore made of directories. `mkdir` is atomic on every +// filesystem mcpp targets, it needs no daemon and no shared memory. A holder +// never waits for another token, so this cannot deadlock between holders. +// +// ⚠️ WHAT IT CAN DO IS OUTLIVE ITS HOLDER. A token is released by the +// supervisor that took it; a supervisor killed before its cleanup (Ctrl-C, the +// OOM killer, a reboot) leaves the directory behind, and nothing in this file +// ever reclaims one. The reclaim is therefore done ONCE PER BUILD, in prepare, +// before ninja is spawned — the only moment at which no token can have a live +// owner. Doing it here instead would race every other compiler in the build. +// +// The wait is bounded anyway. If a token is somehow still unreleasable, the +// honest outcome is a failure that names the directory, not a build that stops +// producing output and never returns. +std::filesystem::path acquire_token(const std::filesystem::path& dir, int cap) { + if (dir.empty() || cap <= 0) return {}; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + const auto started = std::chrono::steady_clock::now(); + constexpr auto kLimit = std::chrono::hours(2); + for (;;) { + for (int i = 0; i < cap; ++i) { + const auto tok = dir / std::to_string(i); + if (std::filesystem::create_directory(tok, ec) && !ec) return tok; + } + if (std::chrono::steady_clock::now() - started > kLimit) { + // Proceeds WITHOUT the cap rather than failing: the semaphore bounds + // memory pressure, it is not a correctness property, so an unbounded + // compile is a worse build and a failed one is no build at all. + std::println(std::cerr, + "mcpp: no compiler slot became free in 2h — all {} tokens in {} " + "are held by processes that are gone. Continuing without the " + "concurrency cap; remove that directory to restore it.", + cap, dir.string()); + return {}; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } +} + +// Keep only the first rule of a make-style depfile. +// +// GCC emits several: the real one, then `.PHONY:` entries and a `CXX_IMPORTS` +// section. Handing those to ninja makes it believe in prerequisites that are +// not files. This is the C++ counterpart of the `awk 'NR==1{print;next} +// /^[^ ]/{exit} {print}'` that used to live in the generated command — having +// it here is also what lets the rule stop depending on a POSIX shell. +// `target` REPLACES the one in the source file, and that is not cosmetic: the +// scanner names the OBJECT, this edge's output is the BMI, and ninja silently +// treats an edge whose depfile names something else as permanently dirty. The +// symptom is a no-op build that recompiles all 140 module interfaces in 25s and +// reports success — nothing warns. +void copy_first_rule(const std::filesystem::path& from, const std::filesystem::path& to, + std::string_view target) { + if (from.empty() || to.empty()) return; + std::ifstream in(from); + if (!in) return; + std::ofstream out(to, std::ios::trunc); + std::string line; + bool first = true; + while (std::getline(in, line)) { + if (first) { + const auto colon = line.find(':'); + out << target << (colon == std::string::npos ? ":" : line.substr(colon)) + << '\n'; + first = false; + continue; + } + if (!line.empty() && !std::isspace(static_cast(line[0]))) break; + out << line << '\n'; + } +} + +// The BMI equivalence check, which used to be a POSIX shell one-liner inside the +// generated ninja command — and was therefore skipped entirely on Windows. +// Having it here is what brings cascade suppression to every platform. +// A BMI's IDENTITY, so publication can be detected without the file ever having +// to be absent. +// +// ⚠️ THE OLD DESIGN CREATED THE HOLE IT WAS TRYING TO AVOID. Phase 1 used to +// `rename(bmi, bmi.bak)` before spawning the compiler — the comment said it was +// so "its mere presence can never be mistaken for the new one landing". That is +// a real hazard, but the cure left the module with NO BMI ON DISK from that +// rename until the compiler republished: measured at ~208 ms of a single +// incremental rebuild. Any importer scheduled inside that window dies with +// +// error: failed to read compiled module: No such file or directory +// note: imports must be built before being imported +// +// reproducible at `-j1`, on four of six scenarios of the generated fixture. +// +// The previous BMI is now COPIED aside instead, so the file is continuously +// readable and GCC's own atomic rename is what replaces it. Publication is +// detected by the identity below changing, which is exactly the question the +// existence check was a poor proxy for. +struct BmiIdentity { + bool present{}; + std::uintmax_t size{}; + std::filesystem::file_time_type mtime{}; + bool operator==(const BmiIdentity&) const = default; +}; + +BmiIdentity bmi_identity(const std::filesystem::path& p) { + BmiIdentity id; + if (p.empty()) return id; + std::error_code ec; + // ⚠️ ASSIGN NOTHING BEFORE CHECKING `ec`. `file_size` returns + // `static_cast(-1)` when it fails, so writing it into the struct + // first makes "this file is missing" compare UNEQUAL to a default-built + // identity — which is exactly the sentinel used for "there was no previous + // BMI". Phase 1 then saw a difference on its very first poll and returned + // before the compiler had produced anything, and every object edge failed + // with `no compiler was started … phase 1 did not run`. + const auto size = std::filesystem::file_size(p, ec); + if (ec) return id; + const auto mtime = std::filesystem::last_write_time(p, ec); + if (ec) return id; + id.present = true; + id.size = size; + id.mtime = mtime; + return id; +} + +// Put the previous BMI back. Used when the compile failed: the unit still has +// the BMI it had before, and leaving it parked in `.bak` would strand every +// importer on a file that does not exist. +void restore_backup(const std::filesystem::path& bmi) { + if (bmi.empty()) return; + const auto backup = suffixed(bmi, ".bak"); + if (!file_exists(backup)) return; + std::error_code ec; + std::filesystem::rename(backup, bmi, ec); +} + +void settle_bmi(const std::filesystem::path& bmi) { + if (bmi.empty()) return; + const auto backup = suffixed(bmi, ".bak"); + if (!file_exists(backup)) return; + std::error_code ec; + // Equivalent → put the PREVIOUS file back, so its mtime does not advance and + // ninja's restat stops the cascade. The rename is atomic, so the BMI is + // readable throughout: there is no moment at which importers see nothing. + if (stage::bmi_equivalent(bmi, backup)) + std::filesystem::rename(backup, bmi, ec); + else + std::filesystem::remove(backup, ec); +} + +#if defined(_WIN32) + +std::string join_command(const std::vector& argv) { + std::string cmd; + for (const auto& a : argv) { + if (!cmd.empty()) cmd += ' '; + const bool quote = a.find_first_of(" \t\"") != std::string::npos; + if (!quote) { cmd += a; continue; } + cmd += '"'; + for (char c : a) { if (c == '"') cmd += '\\'; cmd += c; } + cmd += '"'; + } + return cmd; +} + +bool spawn_detached(const std::vector& argv) { + auto cmd = join_command(argv); + STARTUPINFOA si{}; si.cb = sizeof(si); + PROCESS_INFORMATION pi{}; + const BOOL ok = ::CreateProcessA(nullptr, cmd.data(), nullptr, nullptr, FALSE, + DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP, + nullptr, nullptr, &si, &pi); + if (!ok) return false; + ::CloseHandle(pi.hProcess); ::CloseHandle(pi.hThread); + return true; +} + +int run_to_completion(std::string_view command, + const std::filesystem::path& logPath) { + // ⚠️ `cmd.exe /c` DOES NOT USE CreateProcess ARGUMENT QUOTING. + // + // This built the line with `join_command({"cmd.exe", "/c", command})`, which + // treats the whole compiler invocation as ONE argv element: it wraps it in + // quotes and escapes every interior `"` as `\"`. cmd.exe parses none of + // that — its /c rule is about counting quotes in the raw string — so a + // command containing quoted paths came out mangled, and the compiler ran + // with flags missing rather than failing outright. On a windows-host cross + // build that surfaced as + // + // failed: gcm.cache/mcpp.libs.json.gcm + // src/libs/json.cppm:3: fatal error: json.hpp: No such file or directory + // + // i.e. `-I` gone, reported as a missing header (#425). + // + // ninja concatenates verbatim (`subprocess-win32.cc`), and this edge exists + // to run exactly what ninja would have run. Do the same. + std::string line = "cmd.exe /c "; + line += command; + + SECURITY_ATTRIBUTES sa{sizeof(sa), nullptr, TRUE}; + HANDLE log = ::CreateFileA(logPath.string().c_str(), GENERIC_WRITE, + FILE_SHARE_READ, &sa, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + // The command, in the log, before it runs. Phase 2 replays this file, so a + // failure arrives with the exact invocation attached — without it a remote + // failure can only be guessed at, which is how #425 cost a CI cycle to even + // localise. + if (log != INVALID_HANDLE_VALUE) { + const std::string banner = "+ " + line + "\r\n"; + DWORD wrote = 0; + ::WriteFile(log, banner.data(), static_cast(banner.size()), &wrote, nullptr); + } + STARTUPINFOA si{}; si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = si.hStdError = log; + si.hStdInput = ::CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ, &sa, + OPEN_EXISTING, 0, nullptr); + PROCESS_INFORMATION pi{}; + const BOOL ok = ::CreateProcessA(nullptr, line.data(), nullptr, nullptr, TRUE, + 0, nullptr, nullptr, &si, &pi); + if (log != INVALID_HANDLE_VALUE) ::CloseHandle(log); + if (si.hStdInput != INVALID_HANDLE_VALUE) ::CloseHandle(si.hStdInput); + if (!ok) return 127; + ::WaitForSingleObject(pi.hProcess, INFINITE); + DWORD code = 1; + ::GetExitCodeProcess(pi.hProcess, &code); + ::CloseHandle(pi.hProcess); ::CloseHandle(pi.hThread); + return static_cast(code); +} + +#else + +std::vector to_argv(const std::vector& argv) { + std::vector out; + out.reserve(argv.size() + 1); + for (const auto& a : argv) out.push_back(const_cast(a.c_str())); + out.push_back(nullptr); + return out; +} + +bool spawn_detached(const std::vector& argv) { + posix_spawn_file_actions_t fa; + ::posix_spawn_file_actions_init(&fa); + // HAZARD 1 also applies to the supervisor: holding ninja's pipe open would + // keep the edge alive long after this process exits. + ::posix_spawn_file_actions_addopen(&fa, 0, "/dev/null", O_RDONLY, 0); + ::posix_spawn_file_actions_addopen(&fa, 1, "/dev/null", O_WRONLY, 0); + ::posix_spawn_file_actions_adddup2(&fa, 1, 2); + + posix_spawnattr_t at; + ::posix_spawnattr_init(&at); +#ifdef POSIX_SPAWN_SETSID + // Its own session, so a Ctrl-C on the build does not take the supervisor + // with it mid-write and leave a half-written object behind. + ::posix_spawnattr_setflags(&at, POSIX_SPAWN_SETSID); +#endif + auto raw = to_argv(argv); + pid_t pid = 0; + const int rc = ::posix_spawnp(&pid, raw[0], &fa, &at, raw.data(), environ); + ::posix_spawn_file_actions_destroy(&fa); + ::posix_spawnattr_destroy(&at); + return rc == 0; +} + +int run_to_completion(std::string_view command, + const std::filesystem::path& logPath) { + const std::vector argv{"/bin/sh", "-c", std::string(command)}; + posix_spawn_file_actions_t fa; + ::posix_spawn_file_actions_init(&fa); + ::posix_spawn_file_actions_addopen(&fa, 0, "/dev/null", O_RDONLY, 0); + ::posix_spawn_file_actions_addopen(&fa, 1, logPath.c_str(), + O_WRONLY | O_CREAT | O_TRUNC, 0644); + ::posix_spawn_file_actions_adddup2(&fa, 1, 2); + auto raw = to_argv(argv); + pid_t pid = 0; + const int rc = ::posix_spawnp(&pid, raw[0], &fa, nullptr, raw.data(), environ); + ::posix_spawn_file_actions_destroy(&fa); + if (rc != 0) return 127; + int status = 0; + ::waitpid(pid, &status, 0); + return WIFEXITED(status) ? WEXITSTATUS(status) + : 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0); +} + +#endif + +} // namespace + +int compile_release_at_bmi(const CompileRequest& req) { + if (req.command.empty() || req.self.empty()) return 2; + + std::error_code ec; + if (!req.slot.parent_path().empty()) + std::filesystem::create_directories(req.slot.parent_path(), ec); + std::filesystem::remove(suffixed(req.slot, ".rc"), ec); + std::filesystem::remove(suffixed(req.slot, ".rc.tmp"), ec); + + // Keep the previous BMI for the equivalence check, and LEAVE THE ORIGINAL + // IN PLACE — see BmiIdentity for why moving it away is what broke importers. + // Snapshot through the SAME function in both cases, so "no previous BMI" + // and "the BMI as it is now" are directly comparable. + const BmiIdentity before = bmi_identity(req.bmi); + if (!req.bmi.empty()) { + const auto backup = suffixed(req.bmi, ".bak"); + std::filesystem::remove(backup, ec); + if (before.present) { + std::filesystem::copy_file( + req.bmi, backup, std::filesystem::copy_options::overwrite_existing, ec); + // ⚠️ AND CARRY THE MTIME ACROSS. `copy_file` stamps the copy with + // the time of the copy, and `settle_bmi` restores this file when the + // new BMI turns out equivalent — precisely so the mtime does NOT + // advance and ninja's restat stops the cascade. Without this line + // the restore moves the mtime forward instead, every importer is + // rebuilt, and the optimisation is silently off: `touch-hub` came + // back at 12.61s against a 12.37s cold build, with every cell + // reporting `ok`. A status column cannot catch that; the number can. + std::filesystem::last_write_time(backup, before.mtime, ec); + } + } + + const auto token = acquire_token(req.semaphore, req.maxCompilers); + + // The supervisor reads the SAME argv file rather than receiving the command + // on its own command line: one representation, no re-quoting, and no limit + // on how long a compiler command may be. + std::vector sup{req.self.string(), "bmi-supervise", + "--slot", req.slot.string(), + "--command-file", req.commandFile.string()}; + if (!token.empty()) { sup.push_back("--token"); sup.push_back(token.string()); } + if (!spawn_detached(sup)) return 2; + + // BOUNDED, for the same reason `await_unit` is — and this loop waits on the + // SAME supervisor. The spawn succeeding only says a process was created; if + // it then dies without writing `.rc` (the OOM killer is the realistic + // one here, since this strategy deliberately runs ninja at 6x the compiler + // cap and each module compile peaks near a gigabyte), neither exit below can + // ever be taken and the build hangs forever at 2 ms per poll with nothing on + // stdout to say what it is waiting for. + // + // Bounding phase 2 and not phase 1 left the hang reachable from the earlier + // half of the same mechanism. + using clock = std::chrono::steady_clock; + const auto started = clock::now(); + constexpr auto kNoSupervisorGrace = std::chrono::seconds(10); + constexpr auto kHardLimit = std::chrono::hours(2); + + for (;;) { + // Published = the file's identity is no longer the one we snapshotted. + // For a unit with no previous BMI that reduces to "it now exists". + if (!req.bmi.empty() && bmi_identity(req.bmi) != before) { + settle_bmi(req.bmi); + copy_first_rule(req.depFrom, req.depTo, req.bmi.string()); + return 0; // importers may proceed + } + if (const auto rc = read_rc(req.slot)) { + if (*rc != 0) { // failed before publishing a BMI + std::ifstream in(suffixed(req.slot, ".log")); + if (in) std::cerr << in.rdbuf(); + // Nothing was published, so the previous BMI is the truth. Put + // it back rather than leaving the unit with no BMI at all. + restore_backup(req.bmi); + } else { + // ⚠️ THE COMPILER CAN FINISH BETWEEN THE TWO CHECKS ABOVE. + // + // The loop tests `file_exists(bmi)` first and `rc` second, so a + // unit whose compile is shorter than one poll interval lands + // here: no BMI seen, then a zero rc. This path used to return + // success WITHOUT settling — which left the previous BMI parked + // in `.gcm.bak` and skipped the equivalence check + // entirely, so the restat suppression that stops the cascade + // never ran for that unit. + // + // Observed as `.bak` files surviving a completed build, and as + // + // fx.unit_0: error: failed to read compiled module: + // No such file or directory + // fx.unit_0: note: imports must be built before being imported + // + // in an importer — reproducible at `-j1`, so it was never a + // race between compilers, only between this loop's two checks. + settle_bmi(req.bmi); + copy_first_rule(req.depFrom, req.depTo, req.bmi.string()); + } + return *rc; + } + const auto waited = clock::now() - started; + // No log file means no supervisor ever opened one. Distinguished from + // the hard limit because the two need different advice: this one is + // "the process is not there", not "it is taking too long". + if (!file_exists(suffixed(req.slot, ".log")) && waited > kNoSupervisorGrace) { + std::println(std::cerr, + "mcpp: the compiler supervisor for {} never started " + "(no log after {}s) — restoring the previous BMI", + req.slot.string(), + std::chrono::duration_cast(waited).count()); + restore_backup(req.bmi); + return 1; + } + if (waited > kHardLimit) { + std::println(std::cerr, + "mcpp: timed out waiting for {} to publish its BMI", + req.slot.string()); + restore_backup(req.bmi); + return 1; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } +} + +int supervise(const std::filesystem::path& slot, + const std::filesystem::path& semaphoreToken, + std::string_view command) { + // ⚠️ EVERY EXIT FROM HERE MUST LEAVE AN `.rc`, INCLUDING THE ONES THAT DO + // NOT RUN A COMPILER. Phase 1 and phase 2 both wait on that file; a + // supervisor that bails without writing one is indistinguishable from one + // that was never started, and both waiters can only end on a timeout. The + // caller used to return 2 for an unreadable command file before reaching + // this function, which is precisely that case. + if (command.empty()) { + std::println(std::cerr, "mcpp: bmi-supervise got an empty command for {}", + slot.string()); + if (!semaphoreToken.empty()) { + std::error_code ec; + std::filesystem::remove(semaphoreToken, ec); + } + write_rc(slot, 2); + return 0; + } + const int rc = run_to_completion(command, suffixed(slot, ".log")); + if (!semaphoreToken.empty()) { + std::error_code ec; + std::filesystem::remove(semaphoreToken, ec); + } + write_rc(slot, rc); + return 0; // the supervisor's own status is not the compiler's +} + +int await_unit(const std::filesystem::path& slot, const std::filesystem::path& object) { + // BOUNDED. An unbounded wait here turns "phase 1 never started a compiler" + // into a build that hangs forever with no output — which is strictly worse + // than a failure, because nothing says what to look at. If no supervisor + // ever opened the log, there is nothing to wait for and this fails at once. + using clock = std::chrono::steady_clock; + const auto started = clock::now(); + constexpr auto kNoSupervisorGrace = std::chrono::seconds(10); + constexpr auto kHardLimit = std::chrono::hours(2); + + std::optional rc; + while (!(rc = read_rc(slot))) { + const auto waited = clock::now() - started; + if (!file_exists(suffixed(slot, ".log")) && waited > kNoSupervisorGrace) { + std::println(std::cerr, + "mcpp: no compiler was started for {} — phase 1 did not run", + slot.string()); + return 1; + } + if (waited > kHardLimit) { + std::println(std::cerr, "mcpp: timed out waiting for the compiler for {}", + slot.string()); + return 1; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + + // HAZARD 3: the compiler's diagnostics went to a file so phase 1 could exit + // early. Replaying them here is the only thing that keeps them. + { + std::ifstream in(suffixed(slot, ".log")); + if (in) std::cerr << in.rdbuf(); + } + if (*rc != 0) return *rc; + + if (!object.empty() && !file_exists(object)) { + std::println(std::cerr, "mcpp: compiler reported success but {} is missing", + object.string()); + return 1; + } + return 0; +} + +} // namespace mcpp::build::schedule::detach diff --git a/src/build/schedule/policy.cppm b/src/build/schedule/policy.cppm new file mode 100644 index 00000000..e6ed986c --- /dev/null +++ b/src/build/schedule/policy.cppm @@ -0,0 +1,286 @@ +// mcpp.build.schedule.policy — which scheduling shape this build uses, and why. +// +// ONE TABLE, ONE DECISION. The shape of a module build is a function of the +// compiler family and of the host, and both halves of that answer used to be +// implicit: the BMI-equivalence restat was a POSIX shell fragment inside the +// generated ninja command (so Windows silently had none), and the job count was +// whatever ninja defaulted to. Deriving the same decision in two places is how +// the two halves drifted apart. This module is the only place it is derived. +// +// `decide()` IS PURE. No filesystem, no processes, no environment — a caller +// hands it facts and gets a Decision plus the sentence explaining it, so the +// table is unit-testable without a toolchain and the reason can be printed, +// logged and written into build.ninja unchanged. `requested_switch()` is the +// one impure function here, and it is impure on purpose: the switch has to be +// read somewhere, and two callers each doing env-then-manifest in their own +// order is exactly the duplicate derivation this module exists to prevent. +// +// THE MEASUREMENTS BEHIND THE TABLE (2026-08-13, mcpp building itself: 138 +// module interface units, 57k lines, i9-13900K, gcc@16.1.0 / llvm@22.1.8): +// +// The build is 100% critical path. makespan 79.79 s, critical path 79.73 s, +// average parallelism 3.94x of 32 hardware threads. `-j8` → `-j32` buys 1.4%; +// cmake and xmake build the same sources in 94.5 s and 94.6 s. Nothing outside +// the graph's shape moves it. +// +// 86% of a module interface compile is code generation that no importer reads +// (`-ftime-report`: opt-and-generate 14.08 s of 16.2 s). So the lever is: +// unblock importers when the BMI is ready, not when the compiler exits. +// +// HOW that is done differs per compiler, and the two mechanisms are +// COMPLEMENTARY — each family supports exactly one: +// +// clang TwoPhase two ORDINARY edges over the same source: one emits +// only the BMI, one emits only the object. No process +// machinery, portable by construction. +// MEASURED (22.1.8, src/build/prepare.cppm): +// BMI edge 1.67 s vs 7.35 s for the single edge, and +// the object edge is byte-identical to the one the +// single edge produced. +// The object edge recompiles the SOURCE rather than +// reading the BMI back. `-c x.pcm` does work, but only +// against clang's *full* BMI, and publishing those to +// importers makes clang 22.1.8 miscompile a downstream +// TU (see BmiTraits::bmiOnlyFlags). Front-end work is +// therefore done twice — measured on the whole +// project it still wins at every job count tried: +// -j4 56.3 s → 37.6 s, -j8 34.0 s → 25.6 s, +// -j32 32.0 s → 18.0 s. +// clang CANNOT use DetachCodegen — strace shows it +// writes the BMI to the final path with O_TRUNC, so a +// reader can observe a half-written file. +// +// gcc DetachCodegen no cheap BMI-only mode exists (`-fmodule-only` costs +// 99% of a full compile: it skips writing the object, +// not the back end), but gcc publishes the BMI with +// rename(), so the final path appearing is a sound +// signal. BMI ready at ~22%; cold build 80.5 s → 39.2 s. +// +// msvc None unmeasured. `/ifcOnly`'s cost and whether `.ifc` is +// published atomically are both unknown, and guessing +// either wrong fails silently — a half-read BMI is not +// a diagnostic, it is a miscompile. +export module mcpp.build.schedule.policy; + +import std; +import mcpp.toolchain.model; +import mcpp.manifest; +import mcpp.platform.capacity; + +export namespace mcpp::build::schedule { + +enum class Strategy { + None, // one edge per module, importers wait for the compiler to exit + TwoPhase, // BMI edge + object edge, both ordinary compiler invocations + DetachCodegen, // BMI edge exits at publication; code generation continues +}; + +constexpr std::string_view to_string(Strategy s) { + switch (s) { + case Strategy::TwoPhase: return "two-phase"; + case Strategy::DetachCodegen: return "detach-codegen"; + case Strategy::None: break; + } + return "none"; +} + +struct Decision { + Strategy strategy = Strategy::None; + // ALWAYS populated, including for `None`. A scheduler that silently declines + // to optimise is one nobody can debug: the question "why is my build not + // using the fast shape?" has to have an answer that ships with the build. + std::string reason; + // Real concurrency bound. Under DetachCodegen a compiler stops holding a + // ninja slot the moment it publishes, so ninja's -j is no longer a bound on + // how many compilers run — this is (hazard 2 in detach_codegen). + int compilerCap = 0; + // What to hand ninja. MUST exceed compilerCap under DetachCodegen: with the + // two equal, ninja's slots fill with edges that are merely sleeping, the + // ready frontier starves, and the schedule degenerates to the baseline — + // measured, and it is what made the first prototype read as a no-op. + int ninjaJobs = 0; +}; + +// The one place the switch is READ. `decide` above stays pure — a caller hands +// it facts — but the switch itself has to come from somewhere, and having two +// callers each read env-then-manifest in their own order is precisely the +// duplicate-derivation this module exists to prevent. +// +// Precedence matches every other mcpp switch: environment beats manifest. +// +// ALWAYS returns one of "auto" | "on" | "off". Anything else is a typo, and a +// typo must not quietly become "auto" — that is the rule `resolve_jobs` below +// already follows, and this switch was the one place breaking it. `bmi_schedule +// = "ON"` (or "true", or "yes") used to be accepted, mean OFF, and explain +// itself with "the split schedule is opt-in until verified", which reads as +// "you did not ask for it" to someone who just did. +// +// It was also not merely a no-op: prepare.cppm folds this value into the build +// fingerprint whenever it is not "auto", so a typo picked a DIFFERENT build +// directory — a full rebuild — while changing nothing about the schedule. +// Normalising here fixes both halves, because the fingerprint reads this too. +std::string requested_switch(const manifest::Manifest& m, + const std::function& onInvalid = {}); + +// How many compilers this machine should run at once. +// +// Precedence: MCPP_JOBS (where `--jobs` lands) > `[build] jobs` > 0, meaning +// "say nothing" and leave the backend's own default. The default is unchanged +// on purpose: altering everyone's concurrency is a behaviour change. +// +// `auto` is resolved HERE, against the machine doing the build, never frozen +// into a manifest. Measured on this repository: the cold self-build takes 81.0s +// at -j8 and 79.9s at -j32 — 4x the workers for 1.4%, because the build is +// latency-bound — while a single module compile peaks at 0.5–1.0 GB, so the +// extra jobs are pure memory pressure. On a high-core, modest-RAM machine the +// backend default swaps. +// +// `onInvalid` is called with the offending text instead of warning directly, so +// this stays free of any UI dependency and remains testable. +int resolve_jobs(const manifest::Manifest& m, + const std::function& onInvalid = {}); + +// `requested` is the user's switch: "auto" (default), "on", "off". `hostJobs` is +// the already-resolved parallelism (`--jobs` or `[build] jobs`), or 0 meaning +// "the user said nothing, leave the backend's own default". +// +// `autoJobs` is what this machine would choose if asked — `recommended_jobs` +// against the live host. Computed by the CALLER so `decide` stays pure, and +// needed because 0 is not a usable answer for every strategy: +// +// ⚠️ UNDER DetachCodegen, `hostJobs == 0` MEANS NO BOUND AT ALL. A detached +// compiler stops holding a ninja slot the moment it publishes its BMI, so +// ninja's -j is no longer a limit on how many compilers are running — the +// semaphore is. A cap of 0 disables the semaphore (`acquire_token` returns +// immediately), and the graph then goes out with `sched_cap = 0`: ninja keeps +// starting compiles as fast as BMIs appear, with nothing counting them. That is +// hazard 2 in detach_codegen, and it was live in the DEFAULT configuration — +// nobody passing `--jobs` got any bound, on a workload whose single compile +// peaks near a gigabyte. +Decision decide(const toolchain::Toolchain& tc, std::string_view requested, int hostJobs, + int autoJobs = 0); + +// --------------------------------------------------------------------------- + +Decision decide(const toolchain::Toolchain& tc, std::string_view requested, int hostJobs, + int autoJobs) { + Decision d; + const int cap = hostJobs > 0 ? hostJobs : 0; + + if (requested == "off") { + d.reason = "disabled by request"; + d.ninjaJobs = cap; + return d; + } + // `auto` is OFF until the split graph has been through CI on every + // platform. A scheduling change that is wrong is wrong SILENTLY — a missed + // header dependency does not fail, it just stops rebuilding — so this does + // not become the default on the strength of one machine. `on` selects it. + if (requested != "on") { + d.reason = "auto: the split schedule is opt-in until it has been " + "verified on every platform (set bmi_schedule = \"on\")"; + d.ninjaJobs = cap; + return d; + } + + switch (tc.compiler) { + case toolchain::CompilerId::Clang: + d.strategy = Strategy::TwoPhase; + d.reason = "clang: a BMI-only invocation costs ~23% of a full " + "compile, so importers wait on that instead"; + d.compilerCap = cap; + // Two ordinary edges: a compiler always holds a ninja slot, so the + // ordinary job count is still the real bound. + d.ninjaJobs = cap; + return d; + + case toolchain::CompilerId::GCC: { + d.strategy = Strategy::DetachCodegen; + d.reason = "gcc: publishes the BMI with rename() at ~22% of the " + "compile, so importers can start before code generation"; + // The semaphore IS the bound here, so it must exist. Falling back to + // what this machine would pick keeps the default configuration + // bounded; `--jobs` still wins when it is given. Only if the caller + // supplied no fallback either does this end up unbounded, and then + // the reason says so rather than leaving it to be discovered from a + // `sched_cap = 0` in a generated file. + const int effective = cap > 0 ? cap : autoJobs; + if (effective <= 0) + d.reason += " — WARNING: no compiler cap could be resolved, so " + "concurrency is bounded by nothing; pass --jobs"; + d.compilerCap = effective; + // HAZARD 2. 6x is empirical: the prototype starved at 1x and was + // saturated well before 6x (measured -j192 against a cap of 32). + // + // The cap is CLAMPED before multiplying because `--jobs` is only + // checked for `> 0`: `auto` is bounded by recommended_jobs' ceiling + // of 64, but an explicit `--jobs 2000000000` reaches here intact and + // `cap * 6` is then signed overflow — undefined behaviour, i.e. a + // negative `-j` handed to ninja is one of the *better* outcomes. + // 4096 is far above any real machine and far below the overflow. + constexpr int kMaxCap = 4096; + const int bounded = effective > kMaxCap ? kMaxCap : effective; + d.ninjaJobs = bounded > 0 ? bounded * 6 : 0; + return d; + } + + case toolchain::CompilerId::MSVC: + d.reason = "msvc: neither /ifcOnly's cost nor the atomicity of .ifc " + "publication has been measured; guessing either wrong is " + "silent, so the shape stays conservative"; + d.ninjaJobs = cap; + return d; + + case toolchain::CompilerId::Unknown: + break; + } + d.reason = "unknown compiler family"; + d.ninjaJobs = cap; + return d; +} + +int resolve_jobs(const manifest::Manifest& m, + const std::function& onInvalid) { + auto from_text = [&](std::string_view v) -> std::optional { + if (v.empty()) return std::nullopt; + if (v == "auto") { + const auto cap = platform::capacity::host_capacity(); + return platform::capacity::recommended_jobs(cap); + } + int n = 0; + const auto* first = v.data(); + const auto* last = v.data() + v.size(); + if (auto [p, ec] = std::from_chars(first, last, n); + ec == std::errc{} && p == last && n > 0) + return n; + // A malformed value must not silently become "use the default" — that + // is how a typo turns into a build that is mysteriously slower. + if (onInvalid) onInvalid(v); + return std::nullopt; + }; + if (const char* e = std::getenv("MCPP_JOBS")) + if (auto n = from_text(e)) return *n; + if (auto n = from_text(m.buildConfig.jobs)) return *n; + return 0; +} + +std::string requested_switch(const manifest::Manifest& m, + const std::function& onInvalid) { + std::string v; + if (const char* e = std::getenv("MCPP_BMI_SCHEDULE"); e && *e) + v = e; + else if (!m.buildConfig.bmiSchedule.empty()) + v = m.buildConfig.bmiSchedule; + else + return "auto"; + + // Exact match only, no case folding and no synonyms. Accepting "ON" invites + // the next question ("does it take `true`? `1`? `yes`?"), and every answer + // is another spelling of a switch whose value is written into build.ninja + // and compared across runs. One spelling, or a diagnostic. + if (v == "auto" || v == "on" || v == "off") return v; + if (onInvalid) onInvalid(v); + return "auto"; +} + +} // namespace mcpp::build::schedule diff --git a/src/build/stage.cppm b/src/build/stage.cppm index 67fa1280..cba4bad7 100644 --- a/src/build/stage.cppm +++ b/src/build/stage.cppm @@ -79,6 +79,40 @@ std::expected stage_file(const std::filesystem::path& // unreadable or the sizes differ. bool same_content(const std::filesystem::path& a, const std::filesystem::path& b); +// Are two BMIs equivalent for the purpose of "did this module's interface +// change?" — i.e. identical except for the wall clock GCC stamps into them. +// +// WHY THIS EXISTS. The `cxx_module` rule keeps the previous BMI, recompiles, +// and restores the old file when the new one has the same content, so ninja's +// restat sees an unchanged output and does NOT rebuild the importers. That +// mechanism was designed in 2026-05-12 and has NEVER ONCE FIRED, because GCC +// writes +// +// buildtime: 2026/08/12 02:25:01 UTC +// localtime: 2026/08/12 02:25:01 UTC +// +// INTO THE BMI CONTENT. Two compilations of identical source a second apart +// differ by exactly four bytes, so a plain `cmp` always reports "changed". +// Measured on this repository: touching a module with 46 importers and no +// content change cost 73.0 s and re-ran 180 edges — indistinguishable from a +// full rebuild. +// +// The earlier design note anticipated only that GCC would rewrite the FILE +// (mtime churn) and prescribed a content compare as the fix; it did not +// anticipate that the timestamp is part of the content, which is why the fix +// as written could not work. +// +// Deliberately NOT solved with SOURCE_DATE_EPOCH: that pins the epoch for the +// whole compilation and so changes what `__DATE__` and `__TIME__` expand to in +// USER code. Masking the two fields here changes what mcpp considers equal and +// nothing else. +// +// Conservative by construction: if the expected stamps are not found, or the +// two files disagree about where they are, this falls back to a strict +// comparison. It can report "different" for BMIs that are equivalent; it must +// never report "same" for BMIs that are not. +bool bmi_equivalent(const std::filesystem::path& a, const std::filesystem::path& b); + // Parse a --verify / MCPP_STAGE_VERIFY value. Unknown values fall back to the // safe default (Content). Verify parse_verify(std::string_view value); @@ -174,6 +208,83 @@ bool same_content(const std::filesystem::path& a, const std::filesystem::path& b return true; } +namespace { + +// GCC writes the stamp as `YYYY/MM/DD HH:MM:SS UTC`. Fixed width, so a +// match can be masked without re-parsing. +constexpr std::size_t kStampLen = std::string_view("2026/08/12 02:25:01 UTC").size(); + +bool looks_like_stamp(std::string_view v) { + if (v.size() != kStampLen) return false; + auto digit = [&](std::size_t i) { return v[i] >= '0' && v[i] <= '9'; }; + return digit(0) && digit(1) && digit(2) && digit(3) && v[4] == '/' + && digit(5) && digit(6) && v[7] == '/' + && digit(8) && digit(9) && v[10] == ' ' + && digit(11) && digit(12) && v[13] == ':' + && digit(14) && digit(15) && v[16] == ':' + && digit(17) && digit(18) && v.substr(19) == " UTC"; +} + +// GCC writes exactly one `buildtime:` and one `localtime:` into a BMI header — +// verified across BMIs from 10 KiB to 645 KiB, always 2. Anything beyond that +// came from somewhere else (a string literal in user code that happens to look +// like a stamp), and masking it would hide a REAL difference. Finding more than +// this many makes the comparison fall back to strict equality. +constexpr std::size_t kMaxStampSpans = 2; + +// Byte spans to ignore, in ascending order. Only spans whose payload actually +// looks like a timestamp are masked — a prefix that happens to appear in some +// other position is left to compare strictly. +std::vector> stamp_spans(std::string_view data) { + std::vector> spans; + for (std::string_view prefix : {"buildtime: ", "localtime: "}) { + for (std::size_t at = data.find(prefix); at != std::string_view::npos; + at = data.find(prefix, at + 1)) { + const auto start = at + prefix.size(); + if (start + kStampLen > data.size()) continue; + if (!looks_like_stamp(data.substr(start, kStampLen))) continue; + spans.emplace_back(start, start + kStampLen); + } + } + std::ranges::sort(spans); + return spans; +} + +std::optional read_all(const std::filesystem::path& p) { + std::ifstream in(p, std::ios::binary); + if (!in) return std::nullopt; + return std::string((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); +} + +} // namespace + +bool bmi_equivalent(const std::filesystem::path& a, const std::filesystem::path& b) { + auto da = read_all(a); + auto db = read_all(b); + if (!da || !db) return false; + // The stamps are fixed width, so equivalent BMIs always have equal size. A + // size difference is a real difference, never a maskable one. + if (da->size() != db->size()) return false; + + const auto sa = stamp_spans(*da); + const auto sb = stamp_spans(*db); + // Disagreement about WHERE the stamps are is itself a structural + // difference; fall back to strict equality rather than guessing. + if (sa != sb) return *da == *db; + if (sa.empty() || sa.size() > kMaxStampSpans) return *da == *db; + + std::size_t cursor = 0; + for (const auto& [start, end] : sa) { + if (start > cursor + && std::memcmp(da->data() + cursor, db->data() + cursor, start - cursor) != 0) + return false; + cursor = end; + } + return cursor >= da->size() + || std::memcmp(da->data() + cursor, db->data() + cursor, da->size() - cursor) == 0; +} + Verify parse_verify(std::string_view value) { return value == "size" ? Verify::Size : Verify::Content; } diff --git a/src/cli.cppm b/src/cli.cppm index 6c071af6..4441327b 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -89,6 +89,8 @@ void print_usage() { std::println(" --no-cache Deprecated alias for --cache=off (clears the build dir)"); std::println(" --no-color Disable colored output"); std::println(" --offline Never touch the network (also: MCPP_OFFLINE=1)"); + std::println(" --jobs N|auto, -j Concurrent compiles ('auto' = cores + free RAM)"); + std::println(" --toolchain SPEC Use this toolchain for one build (e.g. llvm@22.1.8)"); std::println(""); std::println("Docs: https://github.com/mcpp-community/mcpp/tree/main/docs"); } @@ -105,6 +107,11 @@ int run(int argc, char** argv) { // the App below so they show up in --help and pass schema checks. for (int i = 1; i < argc; ++i) { std::string_view a = argv[i]; + // Everything after a bare `--` belongs to the program being run or the + // test binary being invoked, not to mcpp. Without this, `mcpp run -- -j 4` + // reads the child's flag as mcpp's own concurrency setting — `-j` is a + // common enough flag that this is a matter of when, not whether. + if (a == "--") break; if (a == "--quiet" || a == "-q") mcpp::ui::set_quiet(true); else if (a == "--no-color") mcpp::ui::disable_color(); else if (a == "--verbose" || a == "-v") mcpp::log::set_verbose(true); @@ -114,6 +121,23 @@ int run(int argc, char** argv) { // need a parameter threaded down. Same shape as MCPP_VERBOSE above, and // it makes `MCPP_OFFLINE=1` and `--offline` literally the same switch. else if (a == "--offline") mcpp::platform::env::set("MCPP_OFFLINE", "1"); + // --jobs rides the same side channel as --offline, for the same reason + // recorded there: its consumer is deep in mcpp.build.execute and + // threading a parameter down would touch every caller in between. + // Accepts `--jobs N`, `--jobs=N`, `-j N` and `-jN`. + else if (a == "--jobs" || a == "-j") { + if (i + 1 < argc) mcpp::platform::env::set("MCPP_JOBS", argv[++i]); + } + else if (a.starts_with("--jobs=")) mcpp::platform::env::set("MCPP_JOBS", std::string(a.substr(7))); + // --toolchain rides the same channel, for the same reason: its consumer + // is deep inside prepare's resolution and threading a parameter down + // would touch every caller in between. + else if (a == "--toolchain") { + if (i + 1 < argc) mcpp::platform::env::set("MCPP_TOOLCHAIN", argv[++i]); + } + else if (a.starts_with("--toolchain=")) mcpp::platform::env::set("MCPP_TOOLCHAIN", std::string(a.substr(12))); + else if (a.starts_with("-j") && a.size() > 2) + mcpp::platform::env::set("MCPP_JOBS", std::string(a.substr(2))); } // Decline xlings' linker-wrapper path injection, for this process and // everything it spawns (openxlings/xlings#540). @@ -265,6 +289,10 @@ int run(int argc, char** argv) { .help("Show toolchain fingerprint and 11 inputs")) .option(cl::Option("cache").takes_value().value_name("MODE") .help("Global dependency cache: global (default) | local | off")) + .option(cl::Option("jobs").short_name('j').takes_value().value_name("N") + .help("Concurrent compiles: a number, or 'auto' to size from cores + free RAM")) + .option(cl::Option("toolchain").takes_value().value_name("SPEC") + .help("Build with this toolchain for one build, e.g. llvm@22.1.8")) .option(cl::Option("no-cache") .help("Deprecated alias for --cache=off (also clears the build dir)")) .option(cl::Option("target").takes_value().help( @@ -590,6 +618,9 @@ int run(int argc, char** argv) { .help("BMI cache directory name (default: gcm.cache)")) .option(cl::Option("bmi-ext").takes_value().value_name("EXT") .help("BMI file extension (default: .gcm)")) + .option(cl::Option("split-module") + .help("Also emit a record for the provided BMI (two-phase " + "schedule: BMI and object are separate edges)")) .option(cl::Option("expect-provides").takes_value().value_name("NAME") .help("(verification) planned provided module for this TU")) .option(cl::Option("expect-imports").takes_value().value_name("CSV") @@ -604,6 +635,33 @@ int run(int argc, char** argv) { .option(cl::Option("verify").takes_value().value_name("MODE") .help("Already-staged check: size (default) | content")) .action(wrap_rc(cmd_stage))) + .subcommand(cl::App("bmi-equal") + .description("(internal: invoked by ninja) Compare two BMIs ignoring the compiler's embedded timestamp") + .action(wrap_rc(cmd_bmi_equal))) + // The three edges of the detach-codegen schedule. Internal, and named as + // such: they are only ever invoked by a generated build.ninja. + .subcommand(cl::App("bmi-compile") + .description("(internal) Compile a module interface and return when its BMI is published") + .option(cl::Option("bmi").takes_value().value_name("PATH").help("BMI this unit publishes")) + .option(cl::Option("slot").takes_value().value_name("PATH").help("where .log/.rc are kept")) + .option(cl::Option("self").takes_value().value_name("PATH").help("path to mcpp, re-invoked as supervisor")) + .option(cl::Option("sem").takes_value().value_name("DIR").help("concurrency token directory")) + .option(cl::Option("cap").takes_value().value_name("N").help("max concurrent compilers")) + .option(cl::Option("command-file").takes_value().value_name("PATH").help("file holding the compiler command line")) + .option(cl::Option("dep-from").takes_value().value_name("PATH").help("scanner depfile to adopt")) + .option(cl::Option("dep-to").takes_value().value_name("PATH").help("where ninja expects this edge's depfile")) + .action(wrap_rc(cmd_bmi_compile))) + .subcommand(cl::App("bmi-supervise") + .description("(internal) Run a compiler to completion and record its status") + .option(cl::Option("slot").takes_value().value_name("PATH")) + .option(cl::Option("token").takes_value().value_name("PATH")) + .option(cl::Option("command-file").takes_value().value_name("PATH")) + .action(wrap_rc(cmd_bmi_supervise))) + .subcommand(cl::App("bmi-await") + .description("(internal) Join a detached compiler and replay its diagnostics") + .option(cl::Option("slot").takes_value().value_name("PATH")) + .option(cl::Option("object").takes_value().value_name("PATH")) + .action(wrap_rc(cmd_bmi_await))) ; // The bareword `mcpp help` and `mcpp` (no args) both print the @@ -670,12 +728,16 @@ int run(int argc, char** argv) { { std::string_view first = argv[1]; if (!first.starts_with('-')) { - static constexpr std::array known = { + // Size is deduced, not spelled: an explicit count turns "add a + // command" into "add a command AND remember to bump a number", + // and the compiler only catches the direction that overflows. + static constexpr std::array known = std::to_array({ "new", "build", "run", "test", "clean", "add", "remove", "update", "search", "publish", "pack", "emit", "xpkg", "toolchain", "cache", "index", "self", "explain", - "version", "dyndep", "why", "resolve", "stage", - }; + "version", "dyndep", "why", "resolve", "stage", "bmi-equal", + "bmi-compile", "bmi-supervise", "bmi-await", + }); bool ok = false; for (auto k : known) if (k == first) { ok = true; break; } if (!ok) { diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index c5269b5d..09be73c8 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -14,6 +14,7 @@ import mcpp.build.prepare; import mcpp.build.execute; import mcpp.build.configure; import mcpp.build.stage; +import mcpp.build.schedule.detach_codegen; import mcpp.build.test_targets; import mcpp.dyndep; import mcpp.log; @@ -371,6 +372,7 @@ export int cmd_dyndep(const mcpplibs::cmdline::ParsedArgs& parsed) { opts.bmiDir = bmiDirStorage; if (!bmiExtStorage.empty()) opts.bmiExt = bmiExtStorage; + opts.splitModuleEdges = parsed.is_flag_set("split-module"); std::expected body; if (single) { @@ -461,4 +463,111 @@ export int cmd_stage(const mcpplibs::cmdline::ParsedArgs& parsed) { return 0; } + +// `mcpp bmi-equal A B` — exit 0 when the two BMIs differ only by GCC's embedded +// wall clock. Invoked from the generated `cxx_module` rule in place of `cmp -s`, +// which can never succeed: GCC stamps `buildtime:`/`localtime:` into the BMI +// CONTENT, so two compiles of identical source always differ by four bytes and +// the interface-unchanged fast path never fired. See mcpp.build.stage. +export int cmd_bmi_equal(const mcpplibs::cmdline::ParsedArgs& parsed) { + if (parsed.positional_count() != 2) { + std::println(stderr, "error: bmi-equal requires exactly two paths"); + return 2; + } + const bool same = mcpp::build::stage::bmi_equivalent( + std::filesystem::path{parsed.positional(0)}, + std::filesystem::path{parsed.positional(1)}); + // Exit status IS the answer, so it can drive `if ...; then` in the rule + // exactly the way `cmp -s` did. No output on either path: this runs once per + // module compile and any chatter would land in the build log. + return same ? 0 : 1; +} + +// The three edges of the DetachCodegen shape. They are `mcpp` subcommands +// rather than shell fragments for two reasons: the previous BMI-equivalence +// logic lived in the generated ninja command as POSIX shell and was therefore +// SKIPPED ENTIRELY ON WINDOWS, and a shell fragment cannot outlive its shell — +// which is exactly what phase 1 has to do. +// +// `--` separates mcpp's own options from the compiler command line, so a +// compiler flag can never be mistaken for one of ours. +namespace { + +// The compiler command, one argument per line, read from a file. +// +// NOT `--`: the cmdline parser implements that separator only at the top level, +// so a subcommand receives nothing after it — silently, with an empty argument +// list rather than an error. A file also sidesteps MAX_ARG_STRLEN (128 KiB for +// a single argv entry, which mcpp has hit before on link lines) and needs no +// quoting rules that a compiler flag could violate. +std::string read_command_file(const std::filesystem::path& path) { + std::ifstream in(path, std::ios::binary); + if (!in) return {}; + std::string text((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back(); + return text; +} + +// `option_or_empty(...).value()`, the idiom the rest of this file uses. +// `parsed.value(name)` looks plausible and returns nothing here — the two are +// not interchangeable, and the difference is silent. +std::string opt_value(const mcpplibs::cmdline::ParsedArgs& parsed, std::string_view name) { + return parsed.option_or_empty(name).value(); +} + +} // namespace + +// Phase 1: start the compiler, return when the BMI is published. +export int cmd_bmi_compile(const mcpplibs::cmdline::ParsedArgs& parsed) { + mcpp::build::schedule::detach::CompileRequest req; + req.bmi = std::filesystem::path{opt_value(parsed, "bmi")}; + req.slot = std::filesystem::path{opt_value(parsed, "slot")}; + req.self = std::filesystem::path{opt_value(parsed, "self")}; + req.semaphore = std::filesystem::path{opt_value(parsed, "sem")}; + req.maxCompilers = 0; + if (const auto cap = opt_value(parsed, "cap"); !cap.empty()) + std::from_chars(cap.data(), cap.data() + cap.size(), req.maxCompilers); + req.commandFile = std::filesystem::path{opt_value(parsed, "command-file")}; + req.depFrom = std::filesystem::path{opt_value(parsed, "dep-from")}; + req.depTo = std::filesystem::path{opt_value(parsed, "dep-to")}; + req.command = read_command_file(req.commandFile); + if (req.slot.empty()) { + std::println(stderr, "error: bmi-compile needs --slot"); + return 2; + } + if (req.command.empty()) { + std::println(stderr, "error: bmi-compile got no command from --command-file"); + return 2; + } + return mcpp::build::schedule::detach::compile_release_at_bmi(req); +} + +// The supervisor. Detached by phase 1; never named by a build edge. +export int cmd_bmi_supervise(const mcpplibs::cmdline::ParsedArgs& parsed) { + const std::filesystem::path slot{opt_value(parsed, "slot")}; + const std::filesystem::path token{opt_value(parsed, "token")}; + const auto command = read_command_file(std::filesystem::path{opt_value(parsed, "command-file")}); + // Only `--slot` is checked here. An empty COMMAND is handed to supervise() + // on purpose: it is the one place that can record the failure in the file + // both waiters are polling. Returning early instead left them waiting on an + // `.rc` that nobody would ever write. + if (slot.empty()) { + std::println(stderr, "error: bmi-supervise needs --slot"); + return 2; + } + return mcpp::build::schedule::detach::supervise(slot, token, command); +} + +// Phase 2: join the detached compiler before anything reads its object. +export int cmd_bmi_await(const mcpplibs::cmdline::ParsedArgs& parsed) { + const std::filesystem::path slot{opt_value(parsed, "slot")}; + const std::filesystem::path object{opt_value(parsed, "object")}; + if (slot.empty()) { + std::println(stderr, "error: bmi-await needs --slot"); + return 2; + } + return mcpp::build::schedule::detach::await_unit(slot, object); +} + } // namespace mcpp::cli diff --git a/src/dyndep.cppm b/src/dyndep.cppm index 02d22202..e94dd800 100644 --- a/src/dyndep.cppm +++ b/src/dyndep.cppm @@ -36,6 +36,24 @@ std::string bmi_basename(std::string_view logicalName, struct DyndepOptions { std::string_view bmiDir = "gcm.cache"; std::string_view bmiExt = ".gcm"; + // Emit a record for the BMI *as well as* the primary output. + // + // Needed by the two-phase (clang) schedule, where one source feeds TWO + // independent edges — one emitting the BMI, one emitting the object — and + // both parse the same imports, so both need the same implicit inputs. + // P1689's "primary-output" comes from the scanned command's `-o`, which + // names the object only, and a BMI edge with no record is rejected outright: + // 'pcm.cache/mcpp.version_req.pcm' not mentioned in its dyndep file + // + // Deliberately NOT solved by pointing the scan's `-o` at the BMI: that was + // tried on the GCC side and made the SCAN try to create the BMI directory + // before anything existed (`cc1plus: fatal error: opening output file + // gcm.cache/...`). The scan keeps writing a throwaway object; only the + // dyndep records change. + // + // A unit that provides nothing (implementation unit, plain .cpp) is not + // split, so it keeps its single record. + bool splitModuleEdges = false; }; // Parse a single .ddi JSON body to a UnitInfo. Returns unexpected on JSON error. @@ -179,6 +197,25 @@ std::string bmi_basename(std::string_view logicalName, return out; } +namespace { + +// Which ninja edges this unit's dyndep records augment. Shared by the batch and +// single-unit emitters so the two shapes cannot drift — the failure mode of +// `splitModuleEdges` is a record naming an edge nobody declared (or an edge +// with no record), and ninja reports both as "not mentioned in its dyndep +// file", i.e. pointing at the innocent side. +std::vector dyndep_targets(const UnitInfo& u, const DyndepOptions& opts) { + std::vector t; + if (opts.splitModuleEdges && !u.provides.empty()) { + t.push_back(std::string(opts.bmiDir) + "/" + + bmi_basename(u.provides.front(), opts.bmiExt)); + } + if (!u.primaryOutput.empty()) t.push_back(u.primaryOutput.string()); + return t; +} + +} // namespace + std::expected parse_ddi(std::string_view body) { std::size_t i = 0; skip_ws(body, i); @@ -265,24 +302,24 @@ std::string emit_dyndep(const std::vector& units, std::string out = "ninja_dyndep_version = 1\n"; for (auto& u : units) { - if (u.primaryOutput.empty()) continue; - - std::string line = "build " + u.primaryOutput.string() + ": dyndep"; - - bool firstImplicit = true; - auto add_implicit = [&](const std::string& path) { - if (firstImplicit) { line += " |"; firstImplicit = false; } - line += " " + path; - }; - for (auto& r : u.requires_) { - bool selfProvides = false; - for (auto& p : u.provides) if (p == r) { selfProvides = true; break; } - if (selfProvides) continue; - std::string bmiDir(opts.bmiDir); - add_implicit(bmiDir + "/" + bmi_basename(r, opts.bmiExt)); + for (auto& target : dyndep_targets(u, opts)) { + std::string line = "build " + target + ": dyndep"; + + bool firstImplicit = true; + auto add_implicit = [&](const std::string& path) { + if (firstImplicit) { line += " |"; firstImplicit = false; } + line += " " + path; + }; + for (auto& r : u.requires_) { + bool selfProvides = false; + for (auto& p : u.provides) if (p == r) { selfProvides = true; break; } + if (selfProvides) continue; + std::string bmiDir(opts.bmiDir); + add_implicit(bmiDir + "/" + bmi_basename(r, opts.bmiExt)); + } + line += "\n restat = 1\n"; + out += line; } - line += "\n restat = 1\n"; - out += line; (void)stdImports; } @@ -322,8 +359,8 @@ emit_dyndep_single(const std::filesystem::path& ddiPath, if (!u) return std::unexpected(std::format("{}: {}", ddiPath.string(), u.error())); std::string out = "ninja_dyndep_version = 1\n"; - if (!u->primaryOutput.empty()) { - std::string line = "build " + u->primaryOutput.string() + ": dyndep"; + for (auto& target : dyndep_targets(*u, opts)) { + std::string line = "build " + target + ": dyndep"; bool firstImplicit = true; for (auto& r : u->requires_) { bool selfProvides = false; diff --git a/src/manifest/toml.cppm b/src/manifest/toml.cppm index 17c0bac9..64c57ea1 100644 --- a/src/manifest/toml.cppm +++ b/src/manifest/toml.cppm @@ -1042,6 +1042,13 @@ std::expected parse_string(std::string_view content, } if (auto v = doc->get_string("build.c_standard")) m.buildConfig.cStandard = *v; if (auto v = doc->get_string("build.target")) m.buildConfig.target = *v; + // `jobs` accepts a number or "auto"; both arrive as text and are validated + // where they are used, so a bad value warns at build time instead of making + // the whole manifest unloadable. (A published package carrying an unknown + // key must never break an older mcpp — same rule the dependency keys follow.) + if (auto v = doc->get_string("build.bmi_schedule")) m.buildConfig.bmiSchedule = *v; + if (auto v = doc->get_string("build.jobs")) m.buildConfig.jobs = *v; + else if (auto n = doc->get_int("build.jobs")) m.buildConfig.jobs = std::to_string(*n); if (auto v = doc->get_string("build.default-profile")) m.buildConfig.defaultProfile = *v; else if (auto v = doc->get_string("build.profile")) m.buildConfig.defaultProfile = *v; // accepted alias if (auto v = doc->get_string("build.cache")) m.buildConfig.cacheMode = *v; @@ -1071,10 +1078,10 @@ std::expected parse_string(std::string_view content, // // MUST stay in sync with the `doc->get_*("build.")` reads above. static constexpr std::string_view kKnownBuildKeys[] = { - "allow_host_libs", "build_program_timeout", "c_standard", "cache", - "cflags", "cxxflags", "cxx_runtime", "default-profile", "defines", + "allow_host_libs", "bmi_schedule", "build_program_timeout", "c_standard", + "cache", "cflags", "cxxflags", "cxx_runtime", "default-profile", "defines", "dialect_cxxflags", "flags", "include_dirs", "include_dirs_after", - "ldflags", "macos_deployment_target", "module_extensions", "profile", + "jobs", "ldflags", "macos_deployment_target", "module_extensions", "profile", "sources", "static_stdlib", "target", }; if (auto* bt = doc->get_table("build")) { @@ -1082,13 +1089,26 @@ std::expected parse_string(std::string_view content, bool known = false; for (auto k : kKnownBuildKeys) if (key == k) { known = true; break; } if (!known) { + // ⚠️ THE LIST IN THE MESSAGE IS THE SAME LIST. It used to be a + // THIRD hand-written copy and had already drifted from both + // others: it named neither `jobs` nor `bmi_schedule`, while + // `kKnownBuildKeys` carried a `schedule` that nothing reads and + // omitted the `bmi_schedule` the parser actually looks for. + // + // The user-visible result was the worst possible one: writing + // the documented `bmi_schedule = "on"` produced + // [build] has unsupported key 'bmi_schedule' (ignored) + // which is FALSE — it is read a few lines above — so the only + // way to turn the feature on told you it had been ignored, + // while the typo `schedule` was accepted in silence. + std::string supported; + for (auto k : kKnownBuildKeys) { + if (!supported.empty()) supported += ", "; + supported += k; + } m.schemaWarnings.push_back(std::format( - "[build] has unsupported key '{}' (ignored). Supported keys: " - "sources, module_extensions, cflags, cxxflags, ldflags, " - "defines, flags, include_dirs, include_dirs_after, " - "dialect_cxxflags, c_standard, target, static_stdlib, " - "cxx_runtime, allow_host_libs, cache, profile, " - "build_program_timeout, macos_deployment_target.", key)); + "[build] has unsupported key '{}' (ignored). Supported keys: {}.", + key, supported)); } } } @@ -1367,6 +1387,28 @@ std::expected parse_string(std::string_view content, triple, e.cxxRuntime))); } } + // Unsupported keys are REPORTED, not dropped. `[targets.]` + // has done this since #249; this table did not, so a key that looks + // plausible — `cxx_runtime_tests` was the real one — was accepted + // in silence and had no effect (#418). A configuration key that + // does nothing is worse than one that does not exist. + static constexpr std::string_view kKnownTargetKeys[] = { + "build", "cxx_runtime", "linkage", "toolchain", + }; + for (auto& [key, _] : body) { + bool known = false; + for (auto k : kKnownTargetKeys) if (key == k) { known = true; break; } + if (known) continue; + std::string supported; + for (auto k : kKnownTargetKeys) { + if (!supported.empty()) supported += ", "; + supported += k; + } + m.schemaWarnings.push_back(std::format( + "[target.{}] has unsupported key '{}' (ignored). Supported keys: {}. " + "Per-role contracts go in [build].cxx_runtime's table form.", + triple, key, supported)); + } m.targetOverrides[canon_triple(triple)] = std::move(e); // [target..{build,dependencies,...}] — platform-conditional diff --git a/src/manifest/types.cppm b/src/manifest/types.cppm index 4cda3649..1dcf3afb 100644 --- a/src/manifest/types.cppm +++ b/src/manifest/types.cppm @@ -353,6 +353,30 @@ struct Resources { // is read in ~150 places, and a BuildConfig genuinely IS a set of build // inputs plus the selection axis and resolved policy scalars. struct BuildConfig : BuildInputs { + // `[build] jobs` — how many compiles to run at once. A decimal count, + // "auto", or empty (the default) meaning "let the backend decide". + // + // Kept as TEXT rather than a number so that "auto" survives into the build + // that actually runs: resolving it at parse time would freeze one machine's + // core count into a value that then travels with the manifest. + std::string jobs; + // `[build] bmi_schedule` — when the BMI becomes visible to importers: + // "auto" (default), "on", "off". + // + // "on" publishes each module's BMI as soon as it exists and moves code + // generation onto a separate edge, so downstream units stop waiting for + // work they do not need. The per-compiler strategy that implements it + // (`detach-codegen` for gcc, `two-phase` for clang) is chosen by + // mcpp.build.schedule::decide and reported by `mcpp build --verbose`. + // + // NAMED FOR WHAT IT SCHEDULES. It was `schedule`, which said only that + // something was being scheduled — and disagreed with its own environment + // override, `MCPP_BMI_SCHEDULE`. The two spellings now match. + // + // Text for the same reason `jobs` is: the meaning of "auto" depends on the + // compiler doing the build, and resolving it at parse time would freeze one + // machine's answer into a manifest that travels. + std::string bmiSchedule; // feature name → extra source globs gated by that feature. A glob listed // here is EXCLUDED from the default build and only compiled/linked when the // feature is active for this package (resolved in prepare_build). Lets a @@ -626,7 +650,12 @@ struct TargetEntry { // channel deliberately carries build INPUTS and nothing else // (ConditionalConfig). One axis, one scoping rule. std::string cxxRuntime; - std::string cxxRuntimeTests; + // ⚠️ NO per-role field here. There used to be a `cxxRuntimeTests`, and it was + // parsed nowhere and applied nowhere — a configuration key that looked + // available and did nothing (#418). The per-target channel carries the + // SCALAR contract only; `[build].cxx_runtime`'s table form already covers + // the role split, and an unsupported key in `[target.]` is now + // reported rather than dropped. }; // `[target.'cfg(...)'.build]` — platform-conditional build flags (L1). The diff --git a/src/platform/capacity.cppm b/src/platform/capacity.cppm new file mode 100644 index 00000000..ba35b224 --- /dev/null +++ b/src/platform/capacity.cppm @@ -0,0 +1,272 @@ +// mcpp.platform.capacity — how much machine is actually available. +// +// Exists because `nproc` is the wrong number to build with, in two separate ways +// that both have measurements behind them: +// +// MEMORY. A C++23 module compile is not cheap in RAM. Measured on this +// repository with GCC 16.1 at -O2: +// src/build/prepare.cppm peak RSS 1,057 MB +// src/build/plan.cppm peak RSS 561 MB +// ninja's default job count is `nproc + 2`. On a 64-core / 32 GB machine that +// is 66 concurrent compiles against ~0.5-1 GB each — the machine swaps, and a +// swapping build is far slower than a smaller job count would have been. The +// default is not merely un-tuned there; it is actively harmful. +// +// HETEROGENEITY. An i9-13900K reports 32 logical CPUs, but they are 8 P-cores +// (SMT, 16 threads) plus 16 E-cores. E-cores deliver roughly 40% of a P-core's +// compile throughput and SMT siblings roughly 25%. Treating 32 threads as 32 +// equal workers overestimates usable parallelism by more than 2x. +// +// The interface deliberately names no `std` type: under GCC 16.1 a newly added +// module whose EXPORTS mention std types can poison the BMIs of everything +// downstream of it, and the failures point at unrelated modules. Integers only. +module; + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include // malloc / free — clang does not get these from windows.h +#elif defined(__APPLE__) +#include +#include +#include +#include +#else +#include +#include +#include // atoi +#include +#endif + +export module mcpp.platform.capacity; + +export namespace mcpp::platform::capacity { + +struct HostCapacity { + int logicalCores = 1; + int physicalCores = 1; + // True when the CPU mixes core classes (Intel P/E, Apple performance + + // efficiency). Recorded rather than inferred: every parallelism figure has + // to be read against it. + bool heterogeneous = false; + unsigned long long totalBytes = 0; + unsigned long long availableBytes = 0; // falls back to total when unknown +}; + +HostCapacity host_capacity(); + +// Job count for `auto`. See the module comment for why this is not `nproc`. +// +// cpu_budget = heterogeneous ? physicalCores : logicalCores +// mem_budget = (available - reserve) / per_job +// jobs = clamp(min(cpu, mem), 1, ceiling) +// +// `perJobBytes` and `reserveBytes` are parameters rather than constants so a +// project whose translation units are heavier (or lighter) than mcpp's can say +// so without patching this file. +int recommended_jobs(const HostCapacity& cap, + unsigned long long perJobBytes = 768ull * 1024 * 1024, + unsigned long long reserveBytes = 2ull * 1024 * 1024 * 1024, + int ceiling = 64); + +} // namespace mcpp::platform::capacity + +// ─── Implementation ──────────────────────────────────────────────────────── + +namespace mcpp::platform::capacity { + +#if defined(_WIN32) + +// The processor relationship table needs the two-call pattern: its length is not +// knowable up front, so ask for the size, allocate, then ask again. +static bool win_core_facts(int& physical, bool& hybrid) { + DWORD bytes = 0; + ::GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &bytes); + if (bytes == 0) return false; + auto* buf = static_cast(::malloc(bytes)); + if (!buf) return false; + bool ok = false; + if (::GetLogicalProcessorInformationEx( + RelationProcessorCore, + reinterpret_cast(buf), &bytes)) { + int count = 0, firstClass = -1; + DWORD off = 0; + while (off < bytes) { + auto* info = reinterpret_cast(buf + off); + if (info->Size == 0) break; + ++count; + const int cls = static_cast(info->Processor.EfficiencyClass); + if (firstClass < 0) firstClass = cls; + else if (cls != firstClass) hybrid = true; + off += info->Size; + } + if (count > 0) { physical = count; ok = true; } + } + ::free(buf); + return ok; +} + +HostCapacity host_capacity() { + HostCapacity cap; + SYSTEM_INFO si{}; + ::GetSystemInfo(&si); + if (si.dwNumberOfProcessors > 0) cap.logicalCores = static_cast(si.dwNumberOfProcessors); + cap.physicalCores = cap.logicalCores; + win_core_facts(cap.physicalCores, cap.heterogeneous); + + MEMORYSTATUSEX ms{}; + ms.dwLength = sizeof(ms); + if (::GlobalMemoryStatusEx(&ms)) { + cap.totalBytes = ms.ullTotalPhys; + cap.availableBytes = ms.ullAvailPhys; + } + return cap; +} + +#elif defined(__APPLE__) + +HostCapacity host_capacity() { + HostCapacity cap; + const long n = ::sysconf(_SC_NPROCESSORS_ONLN); + if (n > 0) cap.logicalCores = static_cast(n); + + int value = 0; + size_t len = sizeof(value); + cap.physicalCores = (::sysctlbyname("hw.physicalcpu", &value, &len, nullptr, 0) == 0 && value > 0) + ? value : cap.logicalCores; + + // Apple Silicon is performance + efficiency by construction; hw.nperflevels + // says so directly and is absent on Intel Macs. + value = 0; len = sizeof(value); + if (::sysctlbyname("hw.nperflevels", &value, &len, nullptr, 0) == 0) + cap.heterogeneous = value > 1; + + unsigned long long mem = 0; len = sizeof(mem); + if (::sysctlbyname("hw.memsize", &mem, &len, nullptr, 0) == 0) cap.totalBytes = mem; + + // Free + inactive is the honest "could be handed to a new process" figure on + // Darwin; wired and active are not available in any useful sense. + vm_statistics64_data_t vm{}; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + if (::host_statistics64(::mach_host_self(), HOST_VM_INFO64, + reinterpret_cast(&vm), &count) == KERN_SUCCESS) { + const unsigned long long page = static_cast(::getpagesize()); + cap.availableBytes = (static_cast(vm.free_count) + + static_cast(vm.inactive_count)) * page; + } + if (cap.availableBytes == 0) cap.availableBytes = cap.totalBytes; + return cap; +} + +#else // Linux and other POSIX + +static bool read_meminfo_kb(const char* key, unsigned long long& out) { + FILE* f = ::fopen("/proc/meminfo", "r"); + if (!f) return false; + char line[256]; + bool found = false; + const size_t klen = ::strlen(key); + while (::fgets(line, sizeof(line), f)) { + if (::strncmp(line, key, klen) != 0) continue; + unsigned long long kb = 0; + if (::sscanf(line + klen, " %llu", &kb) == 1) { out = kb * 1024ull; found = true; } + break; + } + ::fclose(f); + return found; +} + +static int read_cpuinfo_cores() { + FILE* f = ::fopen("/proc/cpuinfo", "r"); + if (!f) return 0; + char line[256]; + int cores = 0; + while (::fgets(line, sizeof(line), f)) { + if (::strncmp(line, "cpu cores", 9) != 0) continue; + const char* colon = ::strchr(line, ':'); + if (colon) cores = ::atoi(colon + 1); + break; + } + ::fclose(f); + return cores; +} + +// Hybrid x86 reports differing per-CPU maximum frequencies. Cheapest reliable +// signal short of CPUID; when cpufreq is absent the answer is "cannot tell", +// which must be reported as NOT heterogeneous — a false positive here would +// halve the job count on an ordinary homogeneous server. +static bool detect_hybrid(int logical) { + if (logical <= 1) return false; + long first = -1; + for (int i = 0; i < logical; ++i) { + char path[128]; + ::snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + FILE* f = ::fopen(path, "r"); + if (!f) return false; + long v = 0; + const int got = ::fscanf(f, "%ld", &v); + ::fclose(f); + if (got != 1) return false; + if (first < 0) first = v; + else if (v != first) return true; + } + return false; +} + +HostCapacity host_capacity() { + HostCapacity cap; + const long n = ::sysconf(_SC_NPROCESSORS_ONLN); + if (n > 0) cap.logicalCores = static_cast(n); + + const int cores = read_cpuinfo_cores(); + cap.physicalCores = cores > 0 ? cores : cap.logicalCores; + cap.heterogeneous = detect_hybrid(cap.logicalCores); + + const long pages = ::sysconf(_SC_PHYS_PAGES); + const long psize = ::sysconf(_SC_PAGE_SIZE); + if (pages > 0 && psize > 0) + cap.totalBytes = static_cast(pages) + * static_cast(psize); + + // MemAvailable is the kernel's own estimate of what a new workload can get + // without swapping — strictly better than MemFree, which excludes reclaimable + // page cache and would make every warm machine look starved. + if (!read_meminfo_kb("MemAvailable:", cap.availableBytes)) + cap.availableBytes = cap.totalBytes; + return cap; +} + +#endif + +int recommended_jobs(const HostCapacity& cap, unsigned long long perJobBytes, + unsigned long long reserveBytes, int ceiling) { + // A heterogeneous machine's logical count is not a count of equal workers, + // so fall back to physical cores there rather than pretending E-cores and + // SMT siblings are whole CPUs. + int cpuBudget = cap.heterogeneous ? cap.physicalCores : cap.logicalCores; + if (cpuBudget < 1) cpuBudget = 1; + + int memBudget = cpuBudget; + if (perJobBytes > 0 && cap.availableBytes > reserveBytes) { + const unsigned long long usable = cap.availableBytes - reserveBytes; + const unsigned long long fits = usable / perJobBytes; + memBudget = fits > 0 ? static_cast(fits > 1000000 ? 1000000 : fits) : 1; + } else if (perJobBytes > 0) { + // Less memory available than the reserve: still make progress, but one + // job at a time. Refusing to build would be worse than building slowly. + memBudget = 1; + } + + int jobs = cpuBudget < memBudget ? cpuBudget : memBudget; + if (jobs < 1) jobs = 1; + if (ceiling > 0 && jobs > ceiling) jobs = ceiling; + return jobs; +} + +} // namespace mcpp::platform::capacity diff --git a/src/platform/runtime_search.cppm b/src/platform/runtime_search.cppm index 9936f739..63acea16 100644 --- a/src/platform/runtime_search.cppm +++ b/src/platform/runtime_search.cppm @@ -51,6 +51,21 @@ enum class Origin { // than the toolchain's, so it is ranked separately and can be reported // separately. Package, + // The artifact's own directory — what the linker writes as `$ORIGIN` on + // ELF and `@loader_path` on Mach-O. + // + // ⚠️ IT WAS MISSING, AND THAT MADE THIS ORDERING DECORATIVE. The module + // claims below to be the one place the search order is decided, but + // `$ORIGIN` was emitted by `shared_library_link_flags` on a completely + // separate per-unit channel and never entered the closure — so the ONE + // directory the ordering matters most for was not in it. #414 is exactly + // that failure: the farm outranked `$ORIGIN`, the artifact linked one + // libX11 and loaded another. + // + // Ranked after `Package` and before `SubosFarm`: it is as immutable as the + // artifact it travels with (more so than a farm that any later + // `xlings install` re-points), but a pinned payload still wins. + Artifact, // The SubOS symlink farm — `/lib`. A UNION VIEW of everything // installed into that environment, rewritten whenever the environment is // re-resolved. This is the directory that makes `-lGL` link, and the one @@ -82,10 +97,11 @@ int rank(Origin origin) { switch (origin) { case Origin::Payload: return 0; case Origin::Package: return 1; - case Origin::SubosFarm: return 2; - case Origin::HostDefault: return 3; + case Origin::Artifact: return 2; + case Origin::SubosFarm: return 3; + case Origin::HostDefault: return 4; } - return 3; + return 4; } // Is this directory part of THIS machine's private state? @@ -102,6 +118,12 @@ bool is_machine_local(Origin origin) { case Origin::Payload: return true; case Origin::SubosFarm: return true; case Origin::Package: return true; + // NOT machine-local. `$ORIGIN` is resolved by the loader relative to + // the artifact, so it means the same thing wherever the artifact is + // copied — that is the whole reason `pack` rewrites everything else + // INTO this form. Marking it local would make `pack` reject the one + // entry it actually wants. + case Origin::Artifact: return false; case Origin::HostDefault: return false; } return false; @@ -111,6 +133,7 @@ std::string_view to_string(Origin origin) { switch (origin) { case Origin::Payload: return "payload"; case Origin::Package: return "package"; + case Origin::Artifact: return "artifact"; case Origin::SubosFarm: return "subos_farm"; case Origin::HostDefault: return "host_default"; } diff --git a/src/toolchain/dialect.cppm b/src/toolchain/dialect.cppm index a550b2cd..96d0b780 100644 --- a/src/toolchain/dialect.cppm +++ b/src/toolchain/dialect.cppm @@ -65,6 +65,9 @@ struct CommandDialect { // Static CRT / runtime. On MSVC this is a compile-time CRT model, not a // link mode — there is no /MT equivalent of `-static` for the whole image. std::string_view staticRuntime; // "-static"| "/MT" + // The MSVC CRT model for the OTHER linkage. Only meaningful on the MSVC + // dialect; GNU has no counterpart (the empty string). + std::string_view dynamicRuntime; // "" | "/MD" // Output an executable (linking driver step). std::string_view outputExePrefix; // "-o " | "/Fe:" @@ -91,6 +94,22 @@ struct CommandDialect { // Dialect lookup. GCC / Clang / MinGW → gnu; MSVC → msvc. const CommandDialect& dialect_for(const Toolchain& tc); +// The MSVC CRT model for a given linkage, in ONE place. +// +// ⚠️ cl bakes `_MSVC_MT` / `_MSVC_MD` into every module it produces, so the std +// module and the TUs importing it must agree. They were derived in two places: +// the project's TUs from `flags.cppm` and the std module from cl's own default +// (`/MT`, because the std build passed no flag at all). A project on the default +// dynamic linkage therefore imported a `/MT` std, which cl accepts with a C5050 +// warning and then fails on for real inside the ucrt headers — #422. +// +// Same shape as `macos_deployment_target`, which `stdmod::ensure_built` already +// takes for exactly this reason on the other platform. +constexpr std::string_view msvc_crt_flag(const CommandDialect& d, bool staticLinkage) { + return staticLinkage ? d.staticRuntime : d.dynamicRuntime; +} + + // The two dialect rows, reachable without a Toolchain. Exposed so the MSVC // row — which no build reaches until the cl.exe backend lands — can still be // unit-tested, and so callers that already know the shape they want (the @@ -165,6 +184,7 @@ constexpr CommandDialect kMsvcDialect{ .forceCxxLangArgv = kMsvcForceCxxArgv, .perFileCxxPrefix = "/Tp", .staticRuntime = "/MT", + .dynamicRuntime = "/MD", .outputExePrefix = "/Fe:", .objExt = ".obj", .ninjaDepsMode = "msvc", diff --git a/src/toolchain/lifecycle.cppm b/src/toolchain/lifecycle.cppm index 4db539ec..5ff6db9d 100644 --- a/src/toolchain/lifecycle.cppm +++ b/src/toolchain/lifecycle.cppm @@ -661,8 +661,7 @@ export int toolchain_set_default(const mcpp::config::GlobalConfig& cfg, mcpp::ui::status("Default", std::format( "set to msvc@system (was: {})", cfg.defaultToolchain.empty() ? "" : cfg.defaultToolchain)); - std::println("note: `mcpp build` with native MSVC (cl.exe) is not yet " - "supported — coming in a later release."); + return 0; } diff --git a/src/toolchain/model.cppm b/src/toolchain/model.cppm index 85891073..aa8da0a6 100644 --- a/src/toolchain/model.cppm +++ b/src/toolchain/model.cppm @@ -150,6 +150,32 @@ struct BmiTraits { // // Positional on GNU, so the emitter must place it before `-c $in`. std::string_view moduleInterfaceLangFlag; // " -x c++" | " -x c++-module" | " /interface /TP" + + // Non-empty ⇔ the driver can emit the BMI *and stop*, producing the SAME + // BMI an ordinary compile of that TU would have produced. Both halves + // matter, and the second one is the trap. + // + // MEASURED (clang 22.1.8, src/build/prepare.cppm): + // -fmodule-output= … -c 7.35s BMI 9,102,984 B (reduced) + // --precompile 1.81s BMI 18,402,920 B (FULL) + // --precompile + // -Xclang -emit-reduced- + // module-interface 1.67s BMI 9,102,968 B (reduced) + // + // `--precompile` alone is fast but emits a *full* BMI, because its output + // is meant to be fed back in for codegen. Publishing those to importers is + // not a drop-in substitution: BMIs grow ~16x on small modules, and on + // mcpp's own graph clang 22.1.8 then miscompiles a downstream TU outright — + // error: call to implicitly-deleted default constructor of + // 'formatter, wchar_t>' + // on a narrow format string, from inside `std`. The same TU compiles + // against reduced BMIs. So the reduced form is not an optimisation here, + // it is the contract: this flag must reproduce it byte for byte. + // + // GCC leaves this empty even though `-fmodule-only` exists: MEASURED, it + // does not skip the back end (~99% of a full compile). GCC's split is a + // different mechanism — see Strategy::DetachCodegen. + std::string_view bmiOnlyFlags; }; BmiTraits bmi_traits(const Toolchain& tc); @@ -248,6 +274,7 @@ BmiTraits bmi_traits(const Toolchain& tc) { .moduleOutputPrefix = " -fmodule-output=", .bmiSearchPrefix = " -fprebuilt-module-path=", .moduleInterfaceLangFlag = " -x c++-module", + .bmiOnlyFlags = " --precompile -Xclang -emit-reduced-module-interface", }; } return { diff --git a/src/toolchain/msvc.cppm b/src/toolchain/msvc.cppm index e8b325c8..fe26cb22 100644 --- a/src/toolchain/msvc.cppm +++ b/src/toolchain/msvc.cppm @@ -127,12 +127,17 @@ std::vector build_env_for_cl(const std::filesystem::path& clPath, // std / std.compat module staging commands (single cl step each): // cl /nologo /EHsc /W0 /O2 /c \modules\std.ixx // /ifcOutput \ifc.cache\std.ifc /Fo:\std.obj +// `crtFlag` is the CRT model (`/MT` or `/MD`) the PROJECT'S TUs are compiled +// with. It has to be handed in rather than defaulted, for the same reason +// `macos_deployment_target` is: cl bakes `_MSVC_MT` / `_MSVC_MD` into the +// module, and a TU importing a std built with the other one gets C5050 followed +// by a real C2375 out of the ucrt headers. Empty keeps cl's own default. std::vector std_module_build_commands( const Toolchain& tc, const std::filesystem::path& cacheDir, - std::string_view cppStandardFlag); + std::string_view cppStandardFlag, std::string_view crtFlag = {}); std::vector std_compat_build_commands( const Toolchain& tc, const std::filesystem::path& cacheDir, - std::string_view cppStandardFlag); + std::string_view cppStandardFlag, std::string_view crtFlag = {}); std::filesystem::path std_bmi_path(const std::filesystem::path& cacheDir); std::filesystem::path staged_std_bmi_path(const std::filesystem::path& outputDir); @@ -520,16 +525,18 @@ std::string cl_stage_command(const Toolchain& tc, const std::filesystem::path& source, const std::filesystem::path& ifcOut, std::string_view objName, - std::string_view extraRef) { + std::string_view extraRef, + std::string_view crtFlag) { // cd into the cache dir (relative outputs land there); env (INCLUDE/LIB) // comes from tc.envOverrides via the executor, not the command string. // `/d`: cmd.exe won't change DRIVE without it (workspace on D:, BMI // cache on C: is the real CI layout). return std::format( - "cd /d {} && {} /nologo {} /EHsc /O2 /W0{} /c {} /ifcOutput {} /Fo:{} 2>&1", + "cd /d {} && {} /nologo {}{} /EHsc /O2 /W0{} /c {} /ifcOutput {} /Fo:{} 2>&1", mcpp::xlings::shq(cacheDir.string()), mcpp::xlings::shq(tc.binaryPath.string()), cppStandardFlag, + crtFlag.empty() ? std::string{} : std::format(" {}", crtFlag), extraRef, mcpp::xlings::shq(source.string()), mcpp::xlings::shq(ifcOut.string()), @@ -564,22 +571,22 @@ int std_module_min_level(const Toolchain& tc) { std::vector std_module_build_commands( const Toolchain& tc, const std::filesystem::path& cacheDir, - std::string_view cppStandardFlag) { + std::string_view cppStandardFlag, std::string_view crtFlag) { return { cl_stage_command(tc, cacheDir, cppStandardFlag, tc.stdModuleSource, - std_bmi_path(cacheDir), "std.obj", "") }; + std_bmi_path(cacheDir), "std.obj", "", crtFlag) }; } std::vector std_compat_build_commands( const Toolchain& tc, const std::filesystem::path& cacheDir, - std::string_view cppStandardFlag) { + std::string_view cppStandardFlag, std::string_view crtFlag) { // std.compat imports std — reference the freshly staged std.ifc. auto ref = std::format(" /reference std={}", mcpp::xlings::shq(std_bmi_path(cacheDir).string())); return { cl_stage_command(tc, cacheDir, cppStandardFlag, tc.stdCompatSource, std_compat_bmi_path(cacheDir), "std.compat.obj", - ref) }; + ref, crtFlag) }; } std::expected enrich_toolchain_from_cl(Toolchain& tc) { diff --git a/src/toolchain/stdmod.cppm b/src/toolchain/stdmod.cppm index 47e3878e..9b9c85ef 100644 --- a/src/toolchain/stdmod.cppm +++ b/src/toolchain/stdmod.cppm @@ -66,7 +66,14 @@ std::expected ensure_built( std::string_view cpp_standard, std::string_view cpp_standard_flag, std::string_view macos_deployment_target = {}, - const std::filesystem::path& cache_root = default_cache_root()); + const std::filesystem::path& cache_root = default_cache_root(), + // MSVC only: the CRT model (`/MT` or `/MD`) the project's TUs use. Same + // contract as `macos_deployment_target` above and for the same reason — + // cl bakes `_MSVC_MT` / `_MSVC_MD` into the module, so a std built with the + // other one makes every importing TU fail in the ucrt headers. It also + // enters `std_build_commands`, which is part of the cache identity, so two + // CRT models cannot share a cache directory. + std::string_view msvc_crt_flag = {}); } // namespace mcpp::toolchain @@ -216,7 +223,8 @@ std::expected ensure_built( std::string_view cpp_standard, std::string_view cpp_standard_flag, std::string_view macos_deployment_target, - const std::filesystem::path& cache_root) + const std::filesystem::path& cache_root, + std::string_view msvc_crt_flag) { if (tc.stdModuleSource.empty()) { return std::unexpected(StdModError{ @@ -283,7 +291,7 @@ std::expected ensure_built( d.objectPath = cacheDir / (isMsvc ? "std.obj" : "std.o"); d.stdCommands = isMsvc ? mcpp::toolchain::msvc::std_module_build_commands( - tc, cacheDir, cpp_standard_flag) + tc, cacheDir, cpp_standard_flag, msvc_crt_flag) : is_clang(tc) ? mcpp::toolchain::clang::std_module_build_commands( tc, cacheDir, d.bmiPath, sysroot_flag, cpp_standard_flag) @@ -292,7 +300,7 @@ std::expected ensure_built( if (!tc.stdCompatSource.empty()) { if (isMsvc) { d.compatCommands = mcpp::toolchain::msvc::std_compat_build_commands( - tc, cacheDir, cpp_standard_flag); + tc, cacheDir, cpp_standard_flag, msvc_crt_flag); } else if (is_clang(tc)) { auto compatBmi = mcpp::toolchain::clang::std_compat_bmi_path(cacheDir); d.compatCommands = mcpp::toolchain::clang::std_compat_build_commands( diff --git a/src/version.cppm b/src/version.cppm index cf19c7a8..fd0fe70d 100644 --- a/src/version.cppm +++ b/src/version.cppm @@ -31,6 +31,6 @@ import std; export namespace mcpp { -inline constexpr std::string_view MCPP_VERSION = "2026.8.11.3"; +inline constexpr std::string_view MCPP_VERSION = "2026.8.15.1"; } // namespace mcpp diff --git a/tests/e2e/214_executable_carries_dt_rpath.sh b/tests/e2e/214_executable_carries_dt_rpath.sh index 383c3500..1f403a66 100755 --- a/tests/e2e/214_executable_carries_dt_rpath.sh +++ b/tests/e2e/214_executable_carries_dt_rpath.sh @@ -196,7 +196,49 @@ print("rule E: executable=%s library=%s" % ( by_form["executable"]["actual"], by_form["shared_library"]["actual"])) PY2 +# ── the record must SURVIVE a build that produced nothing ─────────────────── +# +# Rule E used to re-read every link artifact on every backend drive, whether or +# not anything had been relinked. `mcpp test` drives the backend once per test, +# so on the 83-test suite that was 1.87s x 85 = 158.7s of a 190s hot run — +# reading ELF files nothing had touched. +# +# The fix reads back the stored verdict for an artifact whose stat did not +# move. That has its own failure mode, and it is the one asserted here: the +# cheap version of "skip what did not change" also skips RECORDING it, so +# resolution.json shrinks to whatever was rebuilt — and a no-op build empties +# it. An empty record and a compliant build then look identical, which is the +# confusion this whole rule exists to prevent. +count_tags() { + python3 -c "import json,sys;print(len(json.load(open(sys.argv[1]))['runtime']['loader_tags']))" "$1" +} +before_count=$(count_tags "$RES") +[ "$before_count" -ge 2 ] || { echo "FAIL: rule E recorded only $before_count artifact(s)"; exit 1; } + +# (a) a build that relinks NOTHING. +"$MCPP" build > rebuild.log 2>&1 || { cat rebuild.log; exit 1; } +noop_count=$(count_tags "$RES") +[ "$before_count" = "$noop_count" ] || { + echo "FAIL: a no-op build changed the loader-tag record: $before_count -> $noop_count" + cat "$RES"; exit 1 +} + +# (b) THE ONE THAT MATTERS: a build that relinks ONE of the two. `main.cpp` +# belongs to tagbin alone, so the executable is rewritten and the library is +# not. A rule that records only what it re-read drops the library here — and +# (a) alone cannot see that, because a no-op rewrites nothing at all and so +# leaves even a broken record looking intact. +sleep 1 +printf '\n// touch\n' >> src/main.cpp +"$MCPP" build > partial.log 2>&1 || { cat partial.log; exit 1; } +partial_count=$(count_tags "$RES") +[ "$before_count" = "$partial_count" ] || { + echo "FAIL: relinking one artifact shrank the loader-tag record: $before_count -> $partial_count" + cat "$RES"; exit 1 +} +after_count=$partial_count + # and it has to run "$MCPP" run tagbin > run.log 2>&1 || { cat run.log; exit 1; } -echo "PASS: executables carry DT_RPATH, shared libraries keep DT_RUNPATH, and rule E recorded it" +echo "PASS: executables carry DT_RPATH, shared libraries keep DT_RUNPATH, and rule E recorded it ($after_count artifacts, stable across a no-op build)" diff --git a/tests/e2e/219_runtime_search_farm_is_last.sh b/tests/e2e/219_runtime_search_farm_is_last.sh index 31aaff72..fbc07308 100755 --- a/tests/e2e/219_runtime_search_farm_is_last.sh +++ b/tests/e2e/219_runtime_search_farm_is_last.sh @@ -238,6 +238,50 @@ if p.index('\$ORIGIN') > p.index('''$FARM'''): sys.exit(1) PY +# ── invariant 2b: the RECORD and the ARTIFACT agree ITEM BY ITEM (#415) ──── +# +# This is the assertion the record could not support until `$ORIGIN` entered the +# closure. `resolution.json` called itself the one place the search order is +# decided, while the artifact's own directory was emitted on a separate +# per-unit channel and never appeared in the record — so the two lists could not +# be compared at all, and every check here had to be about ORDER RELATIONS +# between entries that happened to be in both. +# +# Now they are the same list in the same order, and this compares them directly. +# `artifact` is the closure's spelling of `$ORIGIN`; nothing else is filtered, +# translated or excused — an exception here would be the gap coming back, not a +# detail of the test. +python3 - "$TMP/closure.txt" "$DT_RPATH" <<'ITEMWISE' || exit 1 +import sys + +rows = [] +for line in open(sys.argv[1], encoding="utf-8"): + line = line.rstrip("\n") + if not line: + continue + origin, _, path = line.partition("\t") + rows.append((origin, path)) + +recorded = ["$ORIGIN" if o == "artifact" else p for o, p in rows] +actual = [e for e in sys.argv[2].split(":") if e] + +if recorded != actual: + print("FAIL: the recorded closure and the artifact's DT_RPATH differ") + print(" recorded (artifact -> $ORIGIN):") + for e in recorded: + print(" " + e) + print(" DT_RPATH:") + for e in actual: + print(" " + e) + print() + print(" These are supposed to BE the same list. A missing $ORIGIN in the") + print(" record means Origin::Artifact stopped entering the closure (#415);") + print(" an $ORIGIN in the record with none in the artifact means it is") + print(" being recorded for a project that emits none.") + sys.exit(1) +print(f"closure == DT_RPATH, {len(actual)} entries, item by item") +ITEMWISE + # ── invariant 3: libc still comes from the payload, not the farm ──────────── # # The point of the ordering. Both directories can hold a libc.so.6 (the farm's diff --git a/tests/e2e/230_bench_harness.sh b/tests/e2e/230_bench_harness.sh new file mode 100755 index 00000000..4d9b7ce2 --- /dev/null +++ b/tests/e2e/230_bench_harness.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# requires: python3 +# Delegates to the bench suite's own test. +# +# The suite lives in `bench/` and is meant to be extractable into its own +# project one day, so its tests live there too — mixing them into mcpp's e2e +# directory would make that separation a rename away from breaking. +# +# This delegator stays because deleting it would silently drop the harness from +# every mcpp PR that does not touch bench/: `bench.yml` is PATH-SCOPED to +# `bench/**`, so a change elsewhere that breaks the suite runs nothing. +set -e +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# The e2e runner exports MCPP as the binary under test and that always wins. +# Filling it in when unset is what makes this script runnable BY HAND, which the +# harness cannot do for itself: it must not fall back to PATH (that resolves to +# the xlings shim, which re-picks a version per working directory), but this +# delegator lives in mcpp's own tree and can simply point at what was built. +if [ -z "${MCPP:-}" ]; then + MCPP="$(bash "$REPO/.github/tools/newest_artifact.sh" "$REPO" mcpp 2>/dev/null || true)" + [ -n "$MCPP" ] || { echo "SKIP: no mcpp binary built yet — run \`mcpp build\` first"; exit 0; } + export MCPP +fi +exec bash "$REPO/bench/tests/harness.sh" diff --git a/tests/e2e/231_jobs_option.sh b/tests/e2e/231_jobs_option.sh new file mode 100755 index 00000000..ae22829b --- /dev/null +++ b/tests/e2e/231_jobs_option.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# `--jobs N|auto` and the `--` boundary that keeps it from eating a program's flags. +# +# Two separate contracts, both easy to break without noticing: +# 1. the option is honoured and a bad value is REPORTED, not silently dropped +# (a typo that quietly restores the default is a build mysteriously slower +# than the user asked for); +# 2. `-j` is a common enough flag on other programs that `mcpp run -- -j 4` +# must reach the child untouched. `--jobs` reaches its consumer through the +# MCPP_JOBS side channel, and that pre-scan used to walk the whole argv. +set -e + +# mcpp's e2e runner exports MCPP as the binary under test. Filled in when unset +# so this file can be run BY HAND — otherwise it dies on +# line 45: : command not found +# which names neither the variable nor the fix. (Same treatment as +# 230_bench_harness.sh; the bench harness itself deliberately refuses to guess, +# because there the binary IS the measurement.) +if [ -z "${MCPP:-}" ]; then + _root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + MCPP="$(bash "$_root/.github/tools/newest_artifact.sh" "$_root" mcpp 2>/dev/null || true)" + [ -n "$MCPP" ] || { echo "SKIP: no mcpp binary built yet — run \`mcpp build\` first"; exit 0; } + case "$MCPP" in /*) ;; *) MCPP="$_root/$MCPP" ;; esac + export MCPP +fi + +TMP=$(mktemp -d) +trap "rm -rf $TMP" EXIT +cd "$TMP" + +cat > mcpp.toml <<'EOF' +[package] +name = "jobsopt" +version = "0.1.0" + +[toolchain] +default = "gcc@16.1.0" +macos = "llvm@22.1.8" +windows = "llvm@20.1.7" +EOF +mkdir -p src +# A module interface unit, not just a .cpp: the split schedule asserted at the +# bottom of this file only produces edges for module interfaces, and a fixture +# without one lets "declares a split schedule, emits no split edges" pass. +cat > src/echo.cppm <<'EOF' +module; +#include +export module jobsopt.echo; +export void echo_arg(const char* s) { std::printf("%s\n", s); } +EOF +cat > src/main.cpp <<'EOF' +import jobsopt.echo; +int main(int argc, char** argv) { + for (int i = 1; i < argc; ++i) echo_arg(argv[i]); +} +EOF + +# 1. A numeric value builds. +"$MCPP" build --release --jobs 2 > "$TMP/j2.txt" 2>&1 \ + || { echo "--jobs 2 failed:"; cat "$TMP/j2.txt"; exit 1; } + +# 2. `auto` builds too. Its value depends on the host, so the assertion is that +# it is ACCEPTED — asserting a particular number would encode this machine. +"$MCPP" build --release --jobs auto > "$TMP/jauto.txt" 2>&1 \ + || { echo "--jobs auto failed:"; cat "$TMP/jauto.txt"; exit 1; } +if grep -qi 'invalid job count' "$TMP/jauto.txt"; then + echo "'auto' was rejected as an invalid job count:"; cat "$TMP/jauto.txt"; exit 1 +fi + +# 3. A bad value must WARN and still build (degrading to the backend default). +# Asserted from both sides: a silent drop and a hard failure are both wrong. +"$MCPP" build --release --jobs bogus > "$TMP/jbad.txt" 2>&1 \ + || { echo "a bad --jobs value should warn, not fail the build:"; cat "$TMP/jbad.txt"; exit 1; } +grep -qi 'invalid job count' "$TMP/jbad.txt" \ + || { echo "a bad --jobs value was accepted silently:"; cat "$TMP/jbad.txt"; exit 1; } + +# 4. THE BOUNDARY. Everything after `--` belongs to the program. +"$MCPP" run -- -j bogus > "$TMP/sep.txt" 2>&1 \ + || { echo "run with trailing program args failed:"; cat "$TMP/sep.txt"; exit 1; } +grep -qx -- '-j' "$TMP/sep.txt" || { echo "'-j' did not reach the program:"; cat "$TMP/sep.txt"; exit 1; } +grep -qx -- 'bogus' "$TMP/sep.txt" || { echo "'bogus' did not reach the program:"; cat "$TMP/sep.txt"; exit 1; } +# ...and mcpp must not have interpreted it as its own concurrency setting. +if grep -qi 'invalid job count' "$TMP/sep.txt"; then + echo "mcpp consumed a flag that belonged to the program:"; cat "$TMP/sep.txt"; exit 1 +fi + +echo "jobs option OK" + +# 10. `--toolchain` selects a toolchain for ONE build, without touching the +# manifest. This is the usable form of "which compiler": on mcpp itself the +# choice is worth 2.5x (gcc 81.8s vs llvm 32.6s), but changing the DEFAULT +# would invalidate every published package's fingerprint, so per-build +# selection is the part that costs nobody anything. +# +# Asserted by OBSERVING THE RESOLUTION, not by timing: a timing assertion on +# CI measures the runner's mood. +# The spec comes from THIS platform's own resolution, not a hard-coded +# `gcc@16.1.0`: the fixture pins llvm on macOS and Windows, and asserting a +# compiler that is not installed there tests the payload index, not the flag. +own=$("$MCPP" build --release 2>&1 | sed -n 's/.*Resolved \([^ ]*\) .*/\1/p' | head -1) +[ -n "$own" ] || { echo "could not learn this platform's toolchain"; exit 1; } +out=$("$MCPP" build --release --toolchain "$own" 2>&1) \ + || { echo "--toolchain $own failed:"; echo "$out"; exit 1; } +echo "$out" | grep -q "Resolved $own" \ + || { echo "--toolchain did not reach toolchain resolution:"; echo "$out"; exit 1; } + +# ...and it must BEAT the manifest, or it is not an override. The fixture pins +# gcc on Linux, so asking for something else has to change what gets resolved. +if [ "$(uname -s)" = "Linux" ]; then + out=$("$MCPP" build --release --toolchain llvm@22.1.8 2>&1) || true + echo "$out" | grep -q 'Resolved llvm@22.1.8' \ + || { echo "--toolchain lost to the manifest pin:"; echo "$out"; exit 1; } +fi + +echo "toolchain override OK" + +# 11. The split module schedule (L2). Default is OFF; `on` selects it. Asserted +# on the GRAPH's own declaration and on the edges it emits — not on timing, +# which on CI measures the runner's mood. +"$MCPP" build --release > /dev/null 2>&1 +ninja_file=$(find target -name build.ninja | head -1) +grep -q 'schedule=none' "$ninja_file" \ + || { echo "default should not use the split schedule:"; head -2 "$ninja_file"; exit 1; } + +# The MANIFEST KEY is asserted as well as the environment override. They are +# two spellings of one switch and only the env one was ever exercised, so +# `[build] bmi_schedule` could be renamed, mistyped or dropped entirely and +# every test would still pass — the key would just silently stop working and +# the build would quietly use the default. (It was called `schedule` until +# it was renamed to agree with `MCPP_BMI_SCHEDULE`; this is what makes the +# next such rename fail loudly.) +rm -rf target +cp mcpp.toml "$TMP/mcpp.toml.bak" +printf '\n[build]\nbmi_schedule = "on"\n' >> mcpp.toml +"$MCPP" build --release > "$TMP/sched-manifest.txt" 2>&1 \ + || { echo "bmi_schedule = \"on\" failed to build:"; cat "$TMP/sched-manifest.txt"; exit 1; } +ninja_file=$(find target -name build.ninja | head -1) +grep -qE 'schedule=(detach-codegen|two-phase)' "$ninja_file" \ + || { echo "[build] bmi_schedule = \"on\" did not select a split schedule:" + head -2 "$ninja_file"; exit 1; } +cp "$TMP/mcpp.toml.bak" mcpp.toml +echo "manifest key bmi_schedule OK" + +rm -rf target +MCPP_BMI_SCHEDULE=on "$MCPP" build --release > "$TMP/sched.txt" 2>&1 \ + || { echo "schedule=on failed to build:"; cat "$TMP/sched.txt"; exit 1; } +ninja_file=$(find target -name build.ninja | head -1) +grep -q 'schedule=detach-codegen\|schedule=two-phase\|schedule=none' "$ninja_file" \ + || { echo "graph does not declare its schedule:"; head -2 "$ninja_file"; exit 1; } + +# Both split shapes must declare their edges. Which one appears is a property of +# the compiler, not of this test, so the assertion is "the shape the graph says +# it has is the shape it emitted" — an empty split (rules present, zero edges) +# has happened twice and looks exactly like a working build. +if grep -q 'schedule=detach-codegen' "$ninja_file"; then + grep -q ': cxx_module_bmi ' "$ninja_file" \ + || { echo "graph declares detach-codegen but emits no BMI edge"; exit 1; } +elif grep -q 'schedule=two-phase' "$ninja_file"; then + grep -q ': cxx_precompile ' "$ninja_file" \ + || { echo "graph declares two-phase but emits no BMI edge"; exit 1; } + grep -q ': cxx_module_object ' "$ninja_file" \ + || { echo "graph declares two-phase but emits no object edge"; exit 1; } +fi + +# The no-op check is GATED on the split shape actually being in effect. It +# exists to catch one specific defect — a dyndep/depfile record that does not +# match the edge's output, which looks exactly like success while recompiling +# everything — and that defect only exists where split edges do. +# The reference mark is taken AFTER the first build, not from its stdout +# redirect: that file's mtime is when the shell opened it, which is before the +# objects exist, so every object counted as "newer" and the comparison measured +# nothing but timestamp ordering. +if grep -qE 'schedule=(detach-codegen|two-phase)' "$ninja_file"; then + sleep 1 + touch "$TMP/mark" + MCPP_BMI_SCHEDULE=on "$MCPP" build --release > /dev/null 2>&1 + rebuilt=$(find target \( -name '*.o' -o -name '*.pcm' -o -name '*.gcm' \) -newer "$TMP/mark") + # Name them. "rebuilt 1 artifact(s)" cost a CI round trip to turn into + # "which one" — and the answer (a generated .c rewritten on every drive, so + # macOS-only) was not guessable from the count. + [ -z "$rebuilt" ] \ + || { echo "second build under the split schedule rebuilt:"; echo "$rebuilt"; exit 1; } +fi + +echo "split schedule OK" diff --git a/tests/e2e/232_workflow_syntax.sh b/tests/e2e/232_workflow_syntax.sh new file mode 100755 index 00000000..c11d1fb7 --- /dev/null +++ b/tests/e2e/232_workflow_syntax.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# requires: python3 +# Every .github/workflows/*.yml must be loadable YAML. +# +# This exists because `bench.yml` was committed with +# +# run: "$BENCH" --list +# +# which YAML reads as a quoted scalar followed by garbage. The file parsed +# nowhere, so the workflow could never start — and NOTHING SAID SO. GitHub still +# lists a broken workflow as "active", a `workflow_dispatch`-only workflow is +# never exercised by a push, and no test looked at it. It was invisible until +# someone tried to load the file by hand. +# +# The check is deliberately syntax-only. Validating the schema would need the +# full Actions grammar; the failure mode that actually happened is a file that +# does not parse, and that costs a few lines to rule out forever. +set -e + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DIR="$REPO/.github/workflows" +[ -d "$DIR" ] || { echo "no workflows directory at $DIR"; exit 1; } + +count=$(find "$DIR" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) | wc -l) +[ "$count" -gt 0 ] || { echo "no workflow files found under $DIR"; exit 1; } + +python3 - "$DIR" <<'PYEOF' +import pathlib, re, sys + +# PyYAML is not everywhere — the macOS runner has none, and a test that +# hard-fails on a missing dev dependency is a test that gets deleted. So: full +# parse where it exists, targeted lint where it does not, and SAY WHICH RAN. A +# fallback that is quietly weaker than the check it replaces is how a green +# stops meaning anything. +try: + import yaml + HAVE_YAML = True +except ImportError: + HAVE_YAML = False + +# The exact failure this test exists for: a scalar that opens with a quote and +# carries more content after the closing one — +# run: "$BENCH" --list +# YAML reads that as a quoted scalar followed by garbage and refuses the file. +TRAILING_AFTER_QUOTED = re.compile(r'^\s*[\w.-]+:\s*"[^"]*"\s*\S') + +bad = [] +files = sorted(p for p in pathlib.Path(sys.argv[1]).iterdir() + if p.suffix in (".yml", ".yaml")) +for p in files: + # encoding is NOT optional: Python on Windows defaults to the ANSI code + # page, and these files are UTF-8 (em dashes in the comments are enough). + # Without it the check dies with + # UnicodeDecodeError: 'charmap' codec can't decode byte 0x8d + # on the runner and nowhere else. + text = p.read_text(encoding="utf-8") + for n, line in enumerate(text.splitlines(), 1): + if TRAILING_AFTER_QUOTED.match(line): + bad.append(f"{p.name}:{n}: content after a quoted scalar: {line.strip()}") + if not HAVE_YAML: + continue + try: + doc = yaml.safe_load(text) + except Exception as e: + bad.append(f"{p.name}: {e}") + continue + # A workflow with no `jobs` parses but can never do anything — the same class + # of silent nothing, so it is reported the same way. + if not isinstance(doc, dict) or not doc.get("jobs"): + bad.append(f"{p.name}: parsed, but declares no jobs") + +if bad: + print("malformed workflow files:") + for b in bad: + print(" " + b) + sys.exit(1) + +mode = "parsed" if HAVE_YAML else "linted (no PyYAML here — quoted-scalar check only)" +print(f"{len(files)} workflow files {mode}") +PYEOF + +# ── every release-archive download goes through the retrying fetcher ──────── +# +# A bare `curl -fsSL -o ` was the single largest source of unexplained +# CI red on this repository: +# +# curl: (52) Empty reply from server +# Error: Process completed with exit code 52 +# +# — the release CDN accepting the connection and closing it with no response. +# It is transient, it hits Windows hardest, and the log carries no test name, so +# it reads like a code failure every time. +# +# Two things make it come back, and this guard catches both: +# * a NEW download added with a plain curl, because the surrounding lines all +# look like that; +# * someone "fixing" it with `--retry` alone, which does NOT cover exit 52 — +# an empty reply is a transport error, not one of the HTTP statuses +# `--retry` knows about. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +[ -x "$ROOT/.github/tools/fetch_release.sh" ] || { echo "FAIL: .github/tools/fetch_release.sh is missing or not executable"; exit 1; } + +bare=$(grep -rn -- '-o "\${WORK}/\|-o "/tmp/' "$ROOT/.github/workflows" "$ROOT/.github/actions" 2>/dev/null | grep 'curl' | grep -v 'retry-all-errors' || true) +if [ -n "$bare" ]; then + echo "FAIL: an archive is downloaded with a bare curl; use .github/tools/fetch_release.sh" + echo " (a plain curl here is the 'curl: (52) Empty reply from server' flake," + echo " and --retry alone does not cover it)" + echo "$bare" + exit 1 +fi +echo "release archive downloads: all via fetch_release.sh" + +echo "workflow syntax OK" diff --git a/tests/e2e/233_bench_matrix.sh b/tests/e2e/233_bench_matrix.sh new file mode 100755 index 00000000..4d290a79 --- /dev/null +++ b/tests/e2e/233_bench_matrix.sh @@ -0,0 +1,771 @@ +#!/usr/bin/env bash +# requires: python3 +# 233_bench_matrix.sh — bench/matrix.json is the ONE place the benchmark matrix +# is written down, and this checks that it stays that way. +# +# The failure this prevents is not a crash. It is a matrix that exists twice — +# once as data and once hard-coded in the workflow — and drifts, because both +# copies keep looking right. The same shape has already cost this repository +# real time elsewhere ("同一决策两处推导"), and a benchmark is the worst place +# for it: the numbers still come out, they are just of something else. +# +# Four things are asserted, and each names a different way of getting it wrong: +# 1. the file parses and every cell draws its coordinates from `axes` +# — a typo'd toolchain plans a job that installs nothing; +# 2. every axis VALUE is one the harness actually accepts +# — the spec is only worth something if it describes the real program; +# 3. every excluded cell carries a reason +# — "not measured" must not quietly become "not applicable"; +# 4. the workflow reads the file instead of repeating it. +set -e + +# ⚠️ EVERY python read below MUST name its encoding, and this is what enforces it. +# +# `open()`, `read_text()` and `subprocess(text=True)` decode with the LOCALE +# encoding, which is UTF-8 on the Linux and macOS runners and cp1252 on the +# Windows one. Every file this test reads — matrix.json, the engine adapters, +# the READMEs — contains non-ASCII, so on Windows the reads either raise +# +# UnicodeDecodeError: 'charmap' codec can't decode byte 0x8f in position 3037 +# +# or, for the bytes cp1252 does happen to map, silently produce mojibake: the +# regex then matches nothing and the guard reports success while guarding +# nothing. That is the same "failure looks like success" shape this whole test +# exists to catch, so it must not be the test's own failure mode. +# +# §1 read matrix.json without an encoding for a while and stayed green purely +# because its non-ASCII bytes missed cp1252's five undefined ones; §7 hit 0x8f +# and turned the whole Windows e2e job red. Both are the same defect. +# +# These two variables turn an unspecified encoding into a hard error, so the +# next one fails on the FIRST machine that runs it rather than only on Windows. +# Ignored by Python < 3.10, which predates EncodingWarning. +export PYTHONWARNDEFAULTENCODING=1 +export PYTHONWARNINGS=error::EncodingWarning + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MATRIX="$ROOT/bench/matrix.json" +RUNNER="$ROOT/bench/run-standard.sh" +SPEC="$ROOT/bench/SPEC.md" + +[ -f "$MATRIX" ] || { echo "FAIL: bench/matrix.json is missing"; exit 1; } +[ -f "$SPEC" ] || { echo "FAIL: bench/SPEC.md is missing"; exit 1; } +[ -f "$RUNNER" ] || { echo "FAIL: bench/run-standard.sh is missing"; exit 1; } + +# ── 1..3: the data ───────────────────────────────────────────────────────── +# ROOT is passed in: this python runs from stdin, so sys.argv[0] is "-" and the +# repository cannot be derived from it. +python3 - "$MATRIX" "$ROOT" <<'PY' +import json, os, re, sys + +m = json.load(open(sys.argv[1], encoding="utf-8")) +axes = m["axes"] +fail = [] + +def check_list(where, field, value, axis): + # `mcpp[schedule=on]` is ONE engine with an option list, not a second engine. + # Splitting on commas alone would also tear `a[x=1,y=2]` in half, so brackets + # are consumed before the split rather than after it. + items = re.findall(r"[^,\[\]]+(?:\[[^\]]*\])?", value) if "[" in value else value.split(",") + for v in items: + v = v.strip() + if not v: + continue + # `native` is the real-project variant: a tree has exactly one form, + # its own, so it is not a generated axis value. + if axis == "variant" and v == "native": + continue + # The axis is the engine NAME; the options modify it. They are checked + # for real by the registry (`engine_option` rejects an unknown one and + # the whole spec with it), so repeating that list here would be a second + # source of truth for it. + base = v.split("[", 1)[0] if axis == "engine" else v + if base not in axes[axis]: + fail.append(f"{where}: {field}='{v}' is not in axes.{axis} {axes[axis]}") + +seen = set() +if not m.get("cells"): + fail.append("the matrix has no cells at all — every per-cell check below " + "would pass by having nothing to check") +for c in m["cells"]: + where = f"{c.get('os')}/{c.get('toolchain')}/{c.get('project')}" + for field, axis in (("os", "os"), ("toolchain", "toolchain"), ("project", "project")): + if c.get(field) not in axes[axis]: + fail.append(f"{where}: {field}='{c.get(field)}' is not in axes.{axis}") + if where in seen: + fail.append(f"{where}: duplicated cell — two jobs would write the same report file") + seen.add(where) + check_list(where, "engines", c["engines"], "engine") + check_list(where, "variants", c["variants"], "variant") + check_list(where, "scenarios", c["scenarios"], "scenario") + +# The baseline must be an engine, and it must actually be IN every cell it is +# supposed to normalise — a ratio against an engine that never ran is not a +# ratio, and the report renders it as bare seconds. +# A cell may OVERRIDE it: the xlings arms are an mcpp-against-mcpp comparison +# because their cmake/xmake arms stop at the link, and normalising against an +# engine that never produced a binary is how a table of bare seconds gets +# published as a comparison. +base = m["baseline"] +if base not in axes["engine"]: + fail.append(f"baseline '{base}' is not one of axes.engine") +for c in m["cells"]: + eff = c.get("baseline", base) + engines = [e.strip() for e in c["engines"].split(",")] + # `mcpp` in a cell's engine list means BOTH mcpp binaries (the built one and + # the released reference), so a reference-version baseline is satisfied by it. + if eff in engines or (eff == m.get("reference_mcpp") and "mcpp" in engines): + continue + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: baseline '{eff}' " + f"is not among its engines — that cell would report bare seconds") + +# An engine may only be waived if it is actually in the cell, and the cell must +# say why. A blanket waiver is how a permanently broken arm stops being noticed. +for c in m["cells"]: + for w in [e.strip() for e in c.get("allow_failed", "").split(",") if e.strip()]: + if w not in [e.strip() for e in c["engines"].split(",")]: + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: allow_failed names " + f"'{w}', which is not one of its engines") + if c.get("allow_failed") and "KNOWN GAP" not in c.get("note", ""): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: allow_failed without a " + f"'KNOWN GAP' note — a waived failure that says nothing is a hidden one") + # ...and the note must say something about EACH waived engine by name. + # + # A note can only be checked for existence, never for truth, so the next + # best thing is to stop one blanket sentence from covering two arms. It + # already went wrong that way: the windows/clang cell waived cmake and xmake + # under a single `import std` explanation, and by the time anyone looked + # xmake was failing with `could not start the process` — not a language- + # feature gap at all but a missing program on the runner, i.e. something + # fixable, hidden behind a reason that was only ever cmake's. + for w in [e.strip() for e in c.get("allow_failed", "").split(",") if e.strip()]: + if w not in c.get("note", ""): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: '{w}' is waived but the " + f"note never mentions it — one reason covering two arms is how a " + f"fixable failure hides behind an unfixable one") + +# 3. Every excluded cell says why, and says something. +for x in m.get("excluded", []): + if len(x.get("reason", "").strip()) < 20: + fail.append(f"excluded {x.get('os')}/{x.get('toolchain')}/{x.get('project','*')}: " + "reason is missing or too short to be one") + +# An exclusion must not also be a cell. `*` is a wildcard, `foo-*` a prefix +# wildcard (the project axis carries pinned versions, so `xlings-*` is the only +# way to say "both styles"), and an exclusion that names an `engine` scopes a +# CAVEAT to one column rather than removing the job — those legitimately coexist +# with the cell. +# +# The prefix form exists because the bare `*` is too big: written as +# `{os: windows, toolchain: clang, project: "*"}` it also claimed the +# windows/clang fixture and mcpp cells, which do run. This check caught that. +def matches(x, c, key): + v = x.get(key) + if v is None or v == "*": return True + if v.endswith("*"): return c[key].startswith(v[:-1]) + return v == c[key] + +for x in m.get("excluded", []): + if x.get("engine"): + continue + for c in m["cells"]: + if all(matches(x, c, k) for k in ("os", "toolchain", "project")): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']} is both a cell and excluded") + +# ── The perturbation targets must EXIST ──────────────────────────────────── +# +# This is the assertion the suite most needed and did not have. `--hub` pointed +# at `src/xlings.cppm` for months after that file stopped existing; the harness +# correctly reported `skipped — points at a file that does not exist`, the +# workflow correctly exited 0, and three CI jobs per run reported success having +# measured precisely nothing. +# +# It is checkable at all only because the trees are now pinned SUBMODULES rather +# than cloned from a moving branch at run time. That is most of the argument for +# pinning them. +root = sys.argv[2] +uninit = set() +for c in m["cells"]: + if c["project"] == "fixture": + if c.get("hub") or c.get("body"): + fail.append(f"{c['os']}/{c['toolchain']}/fixture: hub/body are for real projects; " + "a generated fixture names its own targets") + continue + for field in ("hub", "body"): + if not c.get(field): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: '{field}' is required for a " + "real project — without it every perturbing scenario reports `skipped`") + continue + # EVERY workload is a pinned submodule under bench/projects//, + # including mcpp's own sources. There is no "this checkout" case: the + # engine under test is the binary, the workload must not move with it. + tree = os.path.join(root, "bench", "projects", + c.get("buildfiles", c["project"]), c["project"]) + # A submodule that is DECLARED but not checked out leaves an empty + # directory, which is not the same as a missing one and must not read as + # a broken matrix: only the bench workflow checks submodules out, so + # every other CI job would fail this on a perfectly correct file. + # + # `mcpp.toml` is the marker — every workload here is an mcpp project, and + # its absence means "not initialised" rather than "hub is wrong". + if not os.path.isfile(os.path.join(tree, "mcpp.toml")): + if not os.path.isdir(tree): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: nothing at {tree} — " + "the cell names a workload that is not even declared as a submodule") + else: + uninit.add(c["project"]) + break + target = os.path.join(tree, c[field]) + if not os.path.isfile(target): + fail.append(f"{c['os']}/{c['toolchain']}/{c['project']}: {field}='{c[field]}' does not " + f"exist in the pinned tree — every scenario that perturbs it would be " + f"reported `skipped` and the job would still pass") + +# ── No engine scratch may be tracked ─────────────────────────────────────── +# +# The foreign engines write their state next to the description they are pointed +# at, and ten of xmake's cache files were committed by an over-broad `git add`. +# One of them recorded `builddir = "mcpp-2026.8.11.3/build"` — the path-doubling +# bug this suite was fixed for — in a file CI would have READ, reinstating the +# defect on every runner while the code that caused it was already gone. +# +# Checked here rather than trusted to .gitignore, because the root ignore file +# already had `/.xmake/` and it did not reach `bench/projects/` at all. +import subprocess +tracked = subprocess.run( + ["git", "-C", root, "ls-files", + "bench/projects/*/.xmake*", "bench/projects/*/build/*", + "bench/projects/*/CMakeCache.txt", "bench/projects/*/bazel-*"], + # encoding pinned, not `text=True` alone: that decodes the child's stdout + # with the LOCALE encoding, which on a Windows runner is cp1252. A path (or + # any UTF-8 byte) then either raises or, worse, mojibakes into something + # that no longer matches — a guard that silently stops guarding. + capture_output=True, encoding="utf-8").stdout.split() +if tracked: + fail.append("engine scratch is tracked in git (machine-local state, and one of " + f"these froze a fixed bug into CI): {tracked[:4]}" + + (f" … and {len(tracked)-4} more" if len(tracked) > 4 else "")) + +# ── The tool pins ────────────────────────────────────────────────────────── +# A pin that is absent is a tool resolved from the runner image, which is how +# the matrix ended up measuring cmake 3.31.6 against a suite that needs 4.0. +for t in ("cmake", "xmake", "bazel", "gcc", "llvm"): + v = m.get("tools", {}).get(t, "") + if not re.match(r"^\d+(\.\d+)+$", str(v)): + fail.append(f"tools.{t} = {v!r} is not an exact version; an unpinned tool is a " + "variable the report does not record") +if not re.match(r"^\d+(\.\d+)+$", str(m.get("reference_mcpp", ""))): + fail.append("reference_mcpp must be an exact released version — it is the old-vs-new column") + +# ...and it must be the version the repository already bootstraps from. +# +# They are the same decision written in two files: `.xlings.json` says which +# released mcpp CI installs, and that installed binary IS the reference arm the +# bench compares against. Let them drift and the "old" column silently becomes +# some other release, with every ratio still looking perfectly reasonable. +xlings_pin = os.path.join(root, ".xlings.json") +if os.path.isfile(xlings_pin): + ws = json.load(open(xlings_pin, encoding="utf-8")).get("workspace", {}).get("mcpp") + if ws and ws != m.get("reference_mcpp"): + fail.append(f"reference_mcpp={m.get('reference_mcpp')} but .xlings.json bootstraps " + f"mcpp {ws} — the reference arm IS the bootstrapped binary, so these " + f"two must agree or the old-vs-new column compares the wrong release") + +# ...and so are the COMPILER pins, for the same reason and with a worse failure. +# +# matrix.json's `tools.gcc` / `tools.llvm` decide which toolchain CI INSTALLS. +# bench/src/toolchain.cppm's kGcc / kLlvm decide which payload path the harness +# HANDS EVERY ENGINE via `--compiler payload:*`. Those are one decision written +# in two files — matrix.json's own `_compiler_note` says as much — and nothing +# made them agree. +# +# Drift is silent in the direction that matters: xlings installs the version +# from matrix.json, the harness asks for the payload directory of the version +# from toolchain.cppm, and that directory is simply not there. Every cell then +# fails for a reason that names a path, not a pin. Checked here because this is +# already the file that cross-checks `reference_mcpp` against `.xlings.json`. +tc_src = os.path.join(root, "bench/src/toolchain.cppm") +if os.path.isfile(tc_src): + tc = open(tc_src, encoding="utf-8").read() + for key, const in (("gcc", "kGcc"), ("llvm", "kLlvm"), ("llvm_windows", "kLlvmWindows")): + # `kLlvm` is a prefix of `kLlvmWindows`, so anchor on the whole name. + mm = re.search(rf"\b{const}\b\s*=\s*\"([^\"]+)\"", tc) + if not mm: + fail.append(f"bench/src/toolchain.cppm no longer defines {const} — this check " + f"cannot compare the pins and must not pass silently") + elif mm.group(1) != str(m.get("tools", {}).get(key, "")): + fail.append(f"tools.{key}={m.get('tools', {}).get(key)!r} but toolchain.cppm's " + f"{const} is {mm.group(1)!r} — CI installs one and the harness hands " + f"every engine the other; the cells fail naming a missing path") + +# The READMEs open with a "what is pinned" table whose whole claim is that those +# are the versions the numbers were taken with. It is prose, so nothing made it +# follow `matrix.json` — and it did not: the pin moved to cmake 4.4.2 (a version +# whose `import std` gate is a DIFFERENT UUID) while both tables still said +# 4.0.2, in the one section a reader consults to decide whether to trust the +# data. Rows only; the surrounding prose discusses older versions on purpose. +for doc in ("bench/README.md", "bench/README.zh-CN.md"): + path = os.path.join(root, doc) + if not os.path.isfile(path): + continue + text = open(path, encoding="utf-8").read() + for tool in ("cmake", "xmake", "bazel"): + want = str(m.get("tools", {}).get(tool, "")) + rows = re.findall(rf"^\|\s*{tool}\s*\|\s*\*\*([^*]+)\*\*\s*\|", text, re.M) + if not rows: + fail.append(f"{doc}: no pinned-version row for {tool} — this check " + f"cannot compare anything and must not pass silently") + for got in rows: + if got.strip() != want: + fail.append(f"{doc}: the pinned table says {tool} {got.strip()} but " + f"matrix.json pins {want} — that table is what a reader " + f"uses to decide whether to trust the numbers") + +if fail: + print("FAIL: bench/matrix.json") + for f in fail: + print(" " + f) + raise SystemExit(1) +print(f"matrix: {len(m['cells'])} cells, {len(m.get('excluded', []))} documented exclusions, " + f"baseline={base}, tool pins {m['tools']['cmake']}/{m['tools']['xmake']}/{m['tools']['bazel']}") +if uninit: + # Loud, and named. A silent skip here would mean the check that catches a + # stale `hub` never actually runs anywhere, which is how it got missed in + # the first place. The bench workflow checks submodules out and runs this + # test, so the assertion does execute on every change to the suite. + print(f" NOTE: hub/body existence NOT checked for {', '.join(sorted(uninit))} " + f"— submodule(s) not checked out here (`git submodule update --init`)") +PY + +# ── 2: the axis values are ones the harness accepts ──────────────────────── +# Read out of the harness's own source, not a second list here — the whole +# point of this test is that there is no second list. +python3 - "$MATRIX" "$ROOT/bench/src/spec.cppm" "$ROOT/bench/src/registry.cppm" <<'PY' +import json, os, re, sys + +# A module is its INTERFACE PLUS ITS IMPLEMENTATION UNIT. The suite writes +# declarations in `.cppm` and definitions in `.cpp`, so reading only the +# `.cppm` finds the declaration of `make_engine` and none of the engine names +# inside it — which is exactly how this check started reporting that the harness +# builds no engines at all, one commit after the split. Read the pair. +def module_text(cppm): + text = open(cppm, encoding="utf-8").read() + impl = cppm[:-len(".cppm")] + ".cpp" + if os.path.exists(impl): + text += "\n" + open(impl, encoding="utf-8").read() + return text + +m = json.load(open(sys.argv[1], encoding="utf-8")) +spec = module_text(sys.argv[2]) +registry = module_text(sys.argv[3]) +fail = [] + +# `scenario_from` is the harness's parser: what it accepts IS the axis. +accepted = set(re.findall(r'if \(s == "([a-z-]+)"\)\s*return Scenario::', spec)) +for s in m["axes"]["scenario"]: + if s not in accepted: + fail.append(f"axes.scenario '{s}' is not accepted by bench::scenario_from " + f"(it accepts {sorted(accepted)})") + +# Engines are whatever the registry constructs. +known = set(re.findall(r'make_(\w+)_engine', registry)) | set( + re.findall(r'"(mcpp|cmake|xmake|bazel)"', registry)) +for e in m["axes"]["engine"]: + if e not in known: + fail.append(f"axes.engine '{e}' is not built by bench/src/registry.cppm") + +if fail: + print("FAIL: bench/matrix.json disagrees with the harness") + for f in fail: + print(" " + f) + raise SystemExit(1) +print("axes agree with the harness (scenarios via scenario_from, engines via the registry)") +PY + +# ── 5: every number in the root README exists in the published data ──────── +# +# The tables are generated (`bench/tools/report.py --headline`) precisely so +# nobody types them, but a human still pastes the output — and a pasted number +# that drifts from the run it claims to come from is unfalsifiable by any other +# test. The numbers still print, they are just of something else, which is this +# suite's entire failure mode in miniature. +# +# ⚠️ ONE RUN, ONE FILE. This used to stitch three separate runs together and +# name each column with the file it came from, because no single run measured +# every column. The standard set now measures all of them, so the check reads +# ONE report — and a table that cannot be reproduced from one run is a table +# that should not be published. +# +# BOTH LANGUAGES. The two READMEs must quote the same numbers: a benchmark table +# that says different things in two languages is two claims, and only one of +# them can be checked against the data. +python3 - "$ROOT" <<'PYREADME' || exit 1 +import json, os, re, sys + +root = sys.argv[1] +SOURCE = "bench/results/standard-20260814-linux-x86_64/gcc-mcpp-2026.8.11.3.json" + +path = os.path.join(root, SOURCE) +if not os.path.isfile(path): + print(f"FAIL: {SOURCE} is missing — the root READMEs quote it") + raise SystemExit(1) + +truth = {} +for c in json.load(open(path, encoding="utf-8"))["cells"]: + if c["status"] == "ok": + truth.setdefault(c["engine"], {})[c["scenario"]] = round(c["median_s"], 2) + +checked = 0 +# Anchored on the FIRST column only. The header used to be matched in full +# (`| scenario | what changed |`), so narrowing the table by dropping a column +# made this report "the table did not parse" — which is true but useless: the +# table was fine, the pattern was describing a shape nobody promised to keep. +for doc, header in (("README.md", r"\| scenario \|"), + ("README.zh-CN.md", r"\| 场景 \|")): + text = open(os.path.join(root, doc), encoding="utf-8").read() + m = re.search(header + r".*?(?=\n\n)", text, re.S) + if not m: + print(f"FAIL: {doc}'s benchmark table did not parse — has its shape changed?") + raise SystemExit(1) + + # ⚠️ BOLD IS NOT PART OF THE GRAMMAR. This once required `**Ns**` in a fixed + # column, and silently stopped matching when the bolding moved to whichever + # column is actually faster: two of five rows dropped out and the check went + # on printing a success line for the three that remained. Emphasis is + # stripped first, and the engine names come from the HEADER rather than from + # a list here, so adding an arm cannot leave a column unchecked. + plain = m.group(0).replace("**", "") + lines = plain.splitlines() + engines = re.findall(r"`([^`]+)`", lines[0]) + if not engines: + print(f"FAIL: {doc}: no engine columns in the table header") + raise SystemExit(1) + + # ⚠️ THE HEADER IS A SHORT NAME, THE DATA HOLDS THE FULL LABEL. + # + # Three columns reading `mcpp@2026.8.13.1`, `mcpp@2026.8.13.1+schedule=on` + # and `mcpp@2026.8.11.3` made the table wider than a README renders, so it + # arrived collapsed behind a horizontal scrollbar — a five-column comparison + # showing two. The headers are now `mcpp` / `mcpp +schedule` / + # `mcpp (released)`, and `report.py` emits the mapping as an HTML comment + # above the table so this check reads it instead of keeping a second copy. + # A short name with no mapping would silently stop being verified. + mapping = {} + cm = re.search(r"", text, re.S) + if cm: + for pair in cm.group(1).split(";"): + if "=" in pair: + k, _, v = pair.partition("=") + mapping[k.strip()] = v.strip() + for e in engines: + if e not in mapping and e not in truth: + print(f"FAIL: {doc}: column `{e}` is neither an engine in {SOURCE} nor " + f"mapped by the `` line above the table") + raise SystemExit(1) + + body = [l for l in lines if re.match(r"^\| `[\w-]+` \|", l)] + if not body: + print(f"FAIL: {doc}: the benchmark table has no data rows") + raise SystemExit(1) + + for line in body: + cells = [c.strip() for c in line.strip().strip("|").split("|")] + scenario = cells[0].strip("`") + # ⚠️ COUNT FROM THE RIGHT, not from a fixed offset. This was `cells[2:]` + # ("after scenario and what-changed"), and dropping the what-changed + # column to narrow the table shifted every value one place — which this + # check would have reported as a row whose value count disagrees with the + # header, i.e. the right complaint for the wrong reason. The engine + # columns are always the LAST len(engines) cells, whatever precedes them. + values = cells[-len(engines):] if len(cells) >= len(engines) else cells + if len(values) != len(engines): + print(f"FAIL: {doc}: row `{scenario}` has {len(values)} values for " + f"{len(engines)} engine columns — the check would cover only part of it") + raise SystemExit(1) + for column, value in zip(engines, values): + engine = mapping.get(column, column) + # Cells are code spans (`86.69s · 1.1x`) so a narrow column cannot + # split a number from its ratio; strip the span before parsing. + value = value.strip("`") + if value == "-": + # Declared as not measured. Assert it really is absent, so `-` + # cannot be used to hide a number somebody did not like. + if truth.get(engine, {}).get(scenario) is not None: + print(f"FAIL: {doc}: {scenario}/{engine} is shown as `-` (not measured) " + f"but {SOURCE} has {truth[engine][scenario]}s for it") + raise SystemExit(1) + continue + mm = re.match(r"([\d.]+)s", value) + if not mm: + continue # _failed_ / _unavailable_ carry no number + claimed = float(mm.group(1)) + have = truth.get(engine, {}).get(scenario) + if have is None: + print(f"FAIL: {doc}: {scenario}/{engine}={claimed}s but {SOURCE} " + f"has no ok cell for it") + raise SystemExit(1) + if abs(claimed - have) >= 0.01: + print(f"FAIL: {doc}: {scenario}/{engine}={claimed}s but {SOURCE} says {have}s") + raise SystemExit(1) + checked += 1 + +print(f"root READMEs (en+zh): {checked} quoted medians all match {SOURCE}") +PYREADME + +# ── 4: the runner reads the file, and does not repeat it ─────────────────── +# +# The matrix used to be run by .github/workflows/bench.yml. That workflow is +# gone — 12 of its 32 foreign-engine arms were waived, so a third of the +# comparison never ran while the job went green, and a shared runner measures +# the runner (243s there against 79s on a developer box for the same tree). +# bench/run-standard.sh took its place, and the invariant is unchanged: ONE +# list, read rather than repeated. +# The literal path, not `$MATRIX`: the variable is referenced all over the +# script, so matching it made this check pass even after the assignment was +# repointed at /dev/null — which is exactly what the negative test for it did. +grep -qE '^[^#]*MATRIX=.*bench/matrix\.json' "$RUNNER" \ + || { echo "FAIL: bench/run-standard.sh does not read bench/matrix.json — the matrix has been re-hardcoded"; exit 1; } + +# The standard data set is THREE runs, and the runner must not quietly change it. +# +# n=1 has no dispersion at all, and every table this suite has published so far +# carried an `n=1` caveat asking readers not to compare digits — a caveat that +# is really an admission. Three samples with min/max is the least that makes a +# published number readable. `--runs 1` stays available for a quick check and is +# marked as not publishable in the script's own output. +# +# Checked structurally, not by counting lines: the first version of a check like +# this used `grep -A4` to reach a default and the comment above it pushed the +# default out of range, so it failed on a correct file. +python3 - "$RUNNER" <<'PYRUNS' || exit 1 +import re, sys +text = open(sys.argv[1], encoding="utf-8").read() +m = re.search(r"^RUNS=(\d+)\s*$", text, re.M) +if not m: + print("FAIL: bench/run-standard.sh no longer defines RUNS — this check cannot") + print(" compare anything and must not pass silently") + sys.exit(1) +if m.group(1) != "3": + print(f"FAIL: the standard set takes 3 runs; bench/run-standard.sh defaults to {m.group(1)}") + print(" n=1 has no dispersion, which is why every earlier table needed a") + print(" caveat telling readers not to compare its digits.") + sys.exit(1) +PYRUNS + +# The runner must never hand the harness a BARE `mcpp`. +# +# A bare name resolves through PATH to the xlings shim, which RE-PICKS its +# version from the working directory — and for a `--project` run that directory +# is the measured tree, which carries its own pin. The first full local run +# measured mcpp@2026.8.11.3 in every single cell: the released binary, not the +# branch, and with no old-vs-new column at all. Nothing failed; the report simply +# described a different program. +grep -qE '^[^#]*mcpp=\$MCPP_BIN' "$RUNNER" \ + || { echo "FAIL: bench/run-standard.sh does not pass the mcpp under test as an" + echo " explicit binary. A bare \`mcpp\` is the xlings shim, and it" + echo " re-resolves its version from the measured tree."; exit 1; } + +# The runner must SELECT from matrix.json, never enumerate cells itself. +# +# The workflow this replaced grew an inline platform list once and it had to be +# guarded; a shell script is at least as easy to hard-code into. The tell is a +# project or toolchain name written down here rather than read. +for token in mcpp-2026 xlings-2026 synth- ; do + if grep -qE "^[^#]*${token}" "$RUNNER"; then + echo "FAIL: bench/run-standard.sh mentions '${token}...' outside a comment —" + echo " the cell list belongs to matrix.json and must be read, not repeated" + exit 1 + fi +done + +# SPEC.md must point at the data rather than restate it. A cell list in prose is +# the second copy this whole test exists to prevent. +grep -q 'matrix.json' "$SPEC" \ + || { echo "FAIL: bench/SPEC.md does not reference matrix.json"; exit 1; } + +# §6. An engine may not be scheduled against a project whose build description +# for that engine declares nothing to build. +# +# `bazel build //...` over a package with no rules EXITS 0 having compiled +# nothing, in ~0.2s. That is not a failure anywhere in the stack — bazel +# succeeded, the runner timed it, the report printed it — so it reached the +# matrix as `bazel/clang/release/cold/mcpp-2026.8.11.3 0.43s`, beside mcpp's +# 12s and cmake's 94s, and nothing was red. +# +# Checked statically here (no bazel required) because the adapter's own +# `unbuildable_reason` guard only runs on a machine that HAS bazel, and the +# matrix is edited far more often than the adapter. +python3 - "$ROOT" <<'PY' || exit 1 +import json, pathlib, re, sys +root = pathlib.Path(sys.argv[1]) +m = json.loads((root / "bench/matrix.json").read_text(encoding="utf-8")) +bad = [] +for c in m["cells"]: + proj = c.get("project", "") + bf = root / "bench/projects" / c.get("buildfiles", proj) + for eng in c.get("engines", "").split(","): + if eng != "bazel": + continue + f = bf / "BUILD.bazel" + # No file at all means the description is EMITTED PER RUN by + # bench.fixture.buildfiles (the generated fixture works this way and + # does declare a cc_binary). Only a checked-in description can be + # judged from here; asserting on the generated ones from a static test + # just re-implements the emitter, wrongly — this check's first run + # failed exactly that way. + if not f.exists(): + continue + body = re.sub(r"#.*", "", f.read_text(encoding="utf-8")) + # ANY rule, not `cc_*` specifically. Every bazel rule instantiation + # carries a `name =` attribute; `load()`, `package()` and + # `exports_files()` do not. Matching `cc_binary|cc_library` was wrong: + # the working xlings description declares an `alias`, which is a real + # rule that `bazel query kind(rule, //...)` returns, so the guard would + # have failed a cell that builds perfectly well. The phantom this + # catches is ZERO rules, which is what `Found 0 targets` means. + if not re.search(r"^\s*name\s*=", body, re.M): + bad.append(f"{proj}: engines lists bazel, but {f.relative_to(root)} " + f"declares no cc_binary/cc_library outside comments") +if bad: + print("FAIL: a cell schedules an engine that would build nothing:") + for b in bad: + print(" " + b) + print(" such a cell reports `ok` with a ~0.2s number; drop the engine from the") + print(" cell and record it under matrix.json `excluded`.") + sys.exit(1) +print(f"no cell schedules bazel against a ruleless package ({len(m['cells'])} cells)") +PY + +# §7. No engine adapter may branch on the LITERAL compiler request. +# +# main.cpp resolves `--compiler payload:clang` into an absolute driver path +# before any engine sees it, so `job.compiler == "clang"` is false in exactly +# the cells that mean clang. That rewrite has now broken three separate checks: +# * `payload_toolchain` — --toolchain=mcpp-* was never passed at all +# * `--toolchain=llvm` — xmake fell back to g++ with clang's flags: +# `g++: unrecognized command-line option +# '--no-default-config'`, six fixture cells red +# * (the same shape would hit any new one written the same way) +# +# Each time it looked correct in review, because the string being compared is +# the string the user typed. Adapters must key off the RESOLVED PATH instead. +python3 - "$ROOT" <<'PY' || exit 1 +import pathlib, re, sys +root = pathlib.Path(sys.argv[1]) / "bench/src/engines" + +# ⚠️ AN EMPTY GLOB PASSES THIS CHECK PERFECTLY. Rename the directory, move the +# adapters, and every assertion below iterates zero files and prints its success +# line. That is the failure mode this whole test exists to prevent, so the check +# must first prove it has something to check. Four is the number of adapters +# today (mcpp, cmake, xmake, bazel) plus engine; the floor is deliberately low +# so adding one does not require editing this. +# +# ⚠️ AND THE IMPLEMENTATION UNITS ARE WHERE THE CODE IS. The adapters declare in +# `.cppm` and define in `.cpp`; globbing only `.cppm` leaves this scanning +# signatures, where a `compiler == "clang"` cannot appear — so the check would +# have kept printing its success line while testing nothing at all. It nearly +# did: the split that moved the bodies did not touch this line. +adapters = sorted(root.glob("*.cppm")) + sorted(root.glob("*.cpp")) +if len(adapters) < 6: + print(f"FAIL: only {len(adapters)} engine adapter files found under {root} — " + f"this check cannot mean anything with so few, so something has moved") + sys.exit(1) +if not any(f.suffix == ".cpp" for f in adapters): + print(f"FAIL: no implementation units under {root} — the adapter bodies are " + f"what this check reads, and it is looking at declarations only") + sys.exit(1) + +bad = [] +for f in adapters: + # engine's resolve_cxx() is the NORMALISER — comparing there is how a bare + # `gcc` becomes `g++`, and it runs before any rewrite. Everything else sees + # the resolved path. + if f.stem == "engine": + continue + for n, line in enumerate(f.read_text(encoding="utf-8").splitlines(), 1): + code = line.split("//", 1)[0] + if re.search(r'compiler\s*==\s*"(clang|gcc)"', code): + bad.append(f"{f.name}:{n}: {line.strip()[:90]}") +if bad: + print("FAIL: an engine adapter compares job.compiler to a literal:") + for b in bad: + print(" " + b) + print(" main.cpp rewrites payload:* into a path first, so that test never fires.") + print(" Key off the resolved driver path (see payload_toolchain).") + sys.exit(1) +print("no engine adapter branches on the literal compiler request") +PY + +# §8. Every timing quoted in a bench/README table exists in the published data. +# +# §5 does this for the root README, which carries five rows. bench/README.md +# carries about ninety across seven tables — it is where nearly every number the +# project publishes actually lives, and it had no check at all. +# +# The failure this prevents has already happened once, and was caught by hand +# with one command to spare: the mcpp workload's xmake column was about to be +# published as `cold 0.60s` — a phantom from the run where xmake's `-P`/cwd +# disagreement meant every "cold" build measured an already-up-to-date tree. The +# real figure, 90.30s, lives in a different result file. Nothing about the README +# would have looked wrong; 0.60s is simply a number, and a fast one. +# +# Deliberately a WIDE net rather than a structured parse: any `NN.NNs` inside a +# table row must appear as some cell's median. It does not check that the number +# is in the RIGHT row — §5 does that for the table the most people see — but it +# does make an invented or stale figure impossible, and it needs no per-table +# schema, so it keeps working when a table is added. +# +# Table rows only, and two decimals with no space before the `s`: prose quotes +# measurements from other instruments in other formats ("makespan 79.79 s", +# "0.3 s"), and those are not cells of any bench run. +python3 - "$ROOT" <<'PY' || exit 1 +import json, glob, os, pathlib, re, sys +root = pathlib.Path(sys.argv[1]) + +published = set() +files = sorted(glob.glob(str(root / "bench/results/**/*.json"), recursive=True)) +for f in files: + try: + doc = json.load(open(f, encoding="utf-8")) + except Exception: + continue # hyperfine exports and other shapes + # A bench report is an OBJECT. Engine scratch (compile_commands.json, bazel + # exports) is often an ARRAY, and `.get` on one raised AttributeError right + # out of this loop — a guard that crashes is a guard that stops guarding. + if not isinstance(doc, dict): + continue + for c in doc.get("cells", []): + if c.get("status") == "ok" and isinstance(c.get("median_s"), (int, float)): + published.add(round(float(c["median_s"]), 2)) + +# An empty set would make every README trivially "clean" — the same silent pass +# the whole test exists to prevent, and one `git mv` of bench/results away. +if len(published) < 50: + print(f"FAIL: only {len(published)} published medians found in " + f"bench/results/ ({len(files)} files) — this check cannot mean " + f"anything with so few, so something has moved") + sys.exit(1) + +bad = [] +for name in ["bench/README.md", "bench/README.zh-CN.md"]: + for n, line in enumerate((root / name).read_text(encoding="utf-8").splitlines(), 1): + if not line.lstrip().startswith("|"): + continue + for m in re.finditer(r"(? 20: + print(f" ... and {len(bad) - 20} more") + print(" Either the number is invented, or it comes from a run that was not") + print(" committed under bench/results/. A benchmark whose numbers cannot be") + print(" traced to a run is a claim, not a measurement.") + sys.exit(1) +print(f"bench READMEs: every quoted timing traces to bench/results/ " + f"({len(published)} medians across {len(files)} files)") +PY + +echo "bench matrix OK" diff --git a/tests/e2e/234_bmi_schedule_on.sh b/tests/e2e/234_bmi_schedule_on.sh new file mode 100755 index 00000000..0af1f2c6 --- /dev/null +++ b/tests/e2e/234_bmi_schedule_on.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# `bmi_schedule = "on"` end to end, and the token leak that used to hang it. +# +# The split schedule reorders the module graph: a BMI edge that exits as soon as +# the compiler has published its BMI, plus a join edge that waits for code +# generation. Until now the only coverage was a unit test of `decide()` — which +# checks the TABLE, not the mechanism. Nothing built a project with the feature +# on, so every hazard in detach_codegen.cppm was guarded by review alone. +# +# Two contracts here, and the second is the one that bit: +# +# 1. A build with the feature on produces a working binary, and the graph says +# it is split. If the strategy silently fell back, the timings would look +# like a regression in the feature rather than like the feature being off. +# +# 2. ⚠️ STALE CONCURRENCY TOKENS MUST BE RECLAIMED. Real compiler concurrency is +# bounded by a semaphore of directories under `/.mcpp-sched`, and a +# token is removed by the supervisor holding it. A supervisor that never runs +# its cleanup — Ctrl-C on the build, the OOM killer, a reboot — leaves its +# directory behind, and nothing used to remove it. Each such event lowered +# the cap for that build directory PERMANENTLY, and after `cap` of them the +# next build waited for a token that could never be released: no output, no +# diagnostic, forever. Interrupting a build is an ordinary thing to do. +# +# This test plants a full set of stale tokens and requires the next build to +# finish anyway. +set -e + +_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +if [ -z "${MCPP:-}" ]; then + MCPP="$(bash "$_root/.github/tools/newest_artifact.sh" "$_root" mcpp 2>/dev/null || true)" + [ -n "$MCPP" ] || { echo "SKIP: no mcpp binary built yet — run \`mcpp build\` first"; exit 0; } +fi +# ⚠️ ABSOLUTISE WHATEVER WE GOT, not just what we derived. This test `cd`s into a +# temp directory, so a RELATIVE MCPP — which is what `ls -t target/*/*/bin/mcpp` +# hands you, and what a caller naturally exports — stops resolving the moment we +# leave the repository, with +# line 77: target/.../bin/mcpp: No such file or directory +# reported as "the build with bmi_schedule=on failed". The normalisation used to +# live inside the `if`, so it ran only when the test found the binary itself. +case "$MCPP" in /*) ;; *) MCPP="$_root/$MCPP" ;; esac +[ -x "$MCPP" ] || { echo "FAIL: MCPP=$MCPP is not executable"; exit 1; } +export MCPP + +TMP=$(mktemp -d) +trap "rm -rf $TMP" EXIT +cd "$TMP" + +# ⚠️ PIN PER PLATFORM, not just `default`. A bare `default = "gcc@16.1.0"` sends +# macOS and Windows looking for a gcc payload that does not exist for them, and +# the whole test dies at step 1 with `'xim:gcc@16.1.0' not in current index` — +# reported as "the build with bmi_schedule=on failed", which is a completely +# different diagnosis. Same three-line block every other e2e in this directory +# uses. +cat > mcpp.toml <<'EOF' +[package] +name = "schedon" +version = "0.1.0" + +[toolchain] +default = "gcc@16.1.0" +macos = "llvm@22.1.8" +windows = "llvm@20.1.7" + +[build] +bmi_schedule = "on" +EOF + +mkdir -p src +cat > src/core.cppm <<'EOF' +export module schedon.core; +import std; +export int core_value() { return 7; } +EOF +cat > src/mid.cppm <<'EOF' +export module schedon.mid; +import std; +import schedon.core; +export int mid_value() { return core_value() + 1; } +EOF +cat > src/main.cpp <<'EOF' +import std; +import schedon.mid; +int main() { std::println("{}", mid_value()); return mid_value() == 8 ? 0 : 1; } +EOF + +# detach-codegen is the GCC strategy. On a machine whose default toolchain +# resolves to clang the decision is TwoPhase and there is no semaphore at all, +# so the second contract has nothing to test — say so rather than pass hollowly. +say_skip_reason() { echo "SKIP: $1"; exit 0; } + +echo "== 1. builds with bmi_schedule = \"on\" ==" +"$MCPP" build --release --verbose > build.log 2>&1 || { + echo "FAIL: build with bmi_schedule=on failed"; tail -30 build.log; exit 1; +} + +grep -q 'schedule:' build.log || { + echo "FAIL: --verbose printed no schedule line — the decision is not being reported" + tail -20 build.log; exit 1 +} +sched_line=$(grep -m1 'schedule:' build.log) +echo " $sched_line" + +case "$sched_line" in + *detach-codegen*) strategy=detach ;; + *two-phase*) strategy=twophase ;; + *) say_skip_reason "the schedule resolved to '${sched_line#*schedule: }' on this host; \ +this test covers the split strategies" ;; +esac + +BIN=$(ls -t target/*/*/bin/schedon 2>/dev/null | head -1) +[ -n "$BIN" ] || { echo "FAIL: no binary produced"; exit 1; } +out=$("$BIN") +[ "$out" = "8" ] || { echo "FAIL: binary printed '$out', expected 8"; exit 1; } +echo " built and ran: $out" + +BUILD_DIR=$(dirname "$(dirname "$BIN")") + +echo "== 2. the graph declares its shape ==" +# HAZARD 4 in detach_codegen.cppm: build.ninja is shared mutable state and the +# fast path replays it, so a split graph must say so about itself. Asserting the +# tag exists is what keeps a later fast-path change from replaying a split graph +# as if it were an ordinary one. +grep -q '^# mcpp:graph=' "$BUILD_DIR/build.ninja" || { + echo "FAIL: build.ninja carries no '# mcpp:graph=' line"; exit 1; } +echo " $(grep -m1 '^# mcpp:graph=' "$BUILD_DIR/build.ninja")" + +if [ "$strategy" != detach ]; then + echo "== 3. skipped: no semaphore under the $strategy strategy ==" + echo "234 bmi_schedule OK ($strategy)" + exit 0 +fi + +echo "== 3. an incremental build still works ==" +touch src/core.cppm +"$MCPP" build --release > inc.log 2>&1 || { + echo "FAIL: incremental build after touching an interface failed"; tail -30 inc.log; exit 1; } +echo " ok" + +echo "== 4. a full set of STALE TOKENS does not hang the next build ==" +SEM="$BUILD_DIR/.mcpp-sched" +mkdir -p "$SEM" +# Every slot taken, by nothing. Before the reclaim this made `acquire_token` +# wait forever on the first module edge. +for i in $(seq 0 63); do mkdir -p "$SEM/$i"; done +planted=$(ls "$SEM" | wc -l) +[ "$planted" -gt 0 ] || { echo "FAIL: could not plant tokens"; exit 1; } +echo " planted $planted stale tokens" + +# Force real work, so the build must actually take a token. +printf '\nexport int core_extra() { return 1; }\n' >> src/core.cppm + +# A generous bound that is still far below the in-process 2h fallback: if the +# reclaim is gone this hangs, and the timeout is what turns that into a failure +# a CI log can explain. +# ⚠️ CAPTURE THE STATUS BEFORE ANY TEST TOUCHES IT. Inside `if ! cmd; then`, +# `$?` is the status of `! cmd` — which is 0 exactly when cmd FAILED — so a real +# failure got reported as "exited 0" and the timeout case could never be +# distinguished from any other. Same shape as reading `$?` after `cmd | tail`. +# +# `set -e` has to come off for exactly this command: with errexit on, a failing +# build aborts the script before `rc=$?` runs, which is what pushed the previous +# version into the `if !` form that then could not read the status at all. +set +e +timeout 600 "$MCPP" build --release > stale.log 2>&1 +rc=$? +set -e +if [ "$rc" != 0 ]; then + if [ "$rc" = 124 ]; then + echo "FAIL: the build HUNG with a full set of stale tokens in $SEM" + echo " (the per-build reclaim in prepare.cppm is not running)" + else + echo "FAIL: build after planting stale tokens exited $rc" + fi + tail -30 stale.log + exit 1 +fi +echo " build completed with stale tokens present" + +left=$(ls "$SEM" 2>/dev/null | wc -l) +[ "$left" -lt "$planted" ] || { + echo "FAIL: $left of $planted stale tokens survived — they were not reclaimed," + echo " so the build only succeeded by not needing a slot" + exit 1; } +echo " reclaimed: $planted -> $left" + +BIN=$(ls -t target/*/*/bin/schedon 2>/dev/null | head -1) +out=$("$BIN") +[ "$out" = "8" ] || { echo "FAIL: binary printed '$out' after the stale-token build"; exit 1; } + +echo "234 bmi_schedule OK (detach-codegen, tokens reclaimed)" diff --git a/tests/e2e/235_std_object_only_when_needed.sh b/tests/e2e/235_std_object_only_when_needed.sh new file mode 100755 index 00000000..5908eef6 --- /dev/null +++ b/tests/e2e/235_std_object_only_when_needed.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# requires: elf +# `obj/std.o` is linked into a unit only when that unit actually needs it (#416). +# +# `std.o` holds the `std` module's global initialiser (exactly one symbol, +# `_ZGIW3std` — measured). It used to be appended to EVERY Binary, TestBinary +# and SharedLibrary whenever the toolchain merely HAD a prebuilt std module, +# regardless of whether anything in that unit imported it. +# +# ⚠️ THE TEST THAT MATTERS IS THE TRANSITIVE ONE. A unit that never writes +# `import std` itself still needs the initialiser when a module it imports does. +# Checking only "does this unit's own source say import std" is the same +# "the edge exists but nobody depends on it" mistake as #405, and it fails in +# the direction that breaks builds. Both directions are asserted here: +# +# 1. a project with NO std anywhere → std.o must NOT be linked +# 2. a project that reaches std INDIRECTLY → std.o must be linked, and the +# binary must actually run +# +# The failure mode of getting (2) wrong is loud (undefined `_ZGIW3std` at link +# time), which is why (1) is the one that could regress quietly. +set -e + +_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +if [ -z "${MCPP:-}" ]; then + MCPP="$(bash "$_root/.github/tools/newest_artifact.sh" "$_root" mcpp 2>/dev/null || true)" + [ -n "$MCPP" ] || { echo "SKIP: no mcpp binary built yet"; exit 0; } +fi +case "$MCPP" in /*) ;; *) MCPP="$_root/$MCPP" ;; esac +[ -x "$MCPP" ] || { echo "FAIL: MCPP=$MCPP is not executable"; exit 1; } +export MCPP + +TMP=$(mktemp -d) +trap "rm -rf $TMP || true" EXIT + +toolchain_block() { + printf '[toolchain]\ndefault = "gcc@16.1.0"\nmacos = "llvm@22.1.8"\nwindows = "llvm@20.1.7"\n' +} + +# ── 1. nothing imports std anywhere ──────────────────────────────────────── +mkdir -p "$TMP/nostd/src" +{ printf '[package]\nname = "nostd"\nversion = "0.1.0"\n'; toolchain_block; } > "$TMP/nostd/mcpp.toml" +cat > "$TMP/nostd/src/x.cppm" <<'EOF' +export module nostd.x; +export int v() { return 3; } +EOF +cat > "$TMP/nostd/src/main.cpp" <<'EOF' +import nostd.x; +int main() { return v() - 3; } +EOF + +( cd "$TMP/nostd" && "$MCPP" build --release ) > "$TMP/nostd.log" 2>&1 || { + echo "FAIL: the no-std project did not build"; tail -30 "$TMP/nostd.log"; exit 1; } + +NINJA="$(find "$TMP/nostd/target" -name build.ninja | head -1)" +[ -n "$NINJA" ] || { echo "FAIL: no build.ninja produced"; exit 1; } +if grep -q 'obj/std\.o' "$NINJA"; then + echo "FAIL: std.o is referenced by a project that imports std nowhere." + echo " That object carries the std module's global initialiser; linking" + echo " it into a unit with no std at all is what made a pure-C compat" + echo " package carry one (#416)." + grep -n 'obj/std\.o' "$NINJA" | head -5 + exit 1 +fi +echo " no-std project: std.o absent from the graph" + +# ── 2. std is reached INDIRECTLY, through one module in between ───────────── +mkdir -p "$TMP/trans/src" +{ printf '[package]\nname = "trans"\nversion = "0.1.0"\n'; toolchain_block; } > "$TMP/trans/mcpp.toml" +# a imports std; b imports a; main imports b. Nothing but `a` mentions std. +cat > "$TMP/trans/src/a.cppm" <<'EOF' +export module trans.a; +import std; +export std::string greet() { return "hi"; } +EOF +cat > "$TMP/trans/src/b.cppm" <<'EOF' +export module trans.b; +import trans.a; +export int n() { return static_cast(greet().size()); } +EOF +cat > "$TMP/trans/src/main.cpp" <<'EOF' +import trans.b; +int main() { return n() - 2; } +EOF + +( cd "$TMP/trans" && "$MCPP" build --release ) > "$TMP/trans.log" 2>&1 || { + echo "FAIL: the transitive project did not build." + echo " An undefined \`_ZGIW3std\` here means the reachability walk did" + echo " not follow the import edge a <- b <- main." + tail -30 "$TMP/trans.log"; exit 1; } + +NINJA="$(find "$TMP/trans/target" -name build.ninja | head -1)" +grep -q 'obj/std\.o' "$NINJA" || { + echo "FAIL: std.o is NOT in the graph, but this project reaches std through" + echo " trans.b -> trans.a -> std. The predicate is not transitive." + exit 1; } +echo " transitive project: std.o present" + +# Running it is what proves the initialiser is not merely referenced but works: +# `greet()` returns a std::string, so a std module that never ran its global +# initialiser is a crash rather than a link error. +BIN="$(find "$TMP/trans/target" -path '*/bin/trans' -type f | head -1)" +[ -n "$BIN" ] || { echo "FAIL: no binary produced by the transitive project"; exit 1; } +"$BIN" || { echo "FAIL: the transitive binary exited $?"; exit 1; } +echo " transitive project: binary runs" + +echo "235 std.o linked only when needed OK" diff --git a/tests/e2e/236_module_extensions_default_toolchain.sh b/tests/e2e/236_module_extensions_default_toolchain.sh new file mode 100755 index 00000000..1e3bd4e4 --- /dev/null +++ b/tests/e2e/236_module_extensions_default_toolchain.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# `[build] module_extensions` on the platform's DEFAULT toolchain (#412). +# +# ⚠️ WHY THIS EXISTS SEPARATELY FROM 217. `217_module_extensions.sh` declares +# `# requires: gcc`, and `run_all.sh` deliberately does not grant `gcc` on +# Windows or macOS: +# +# # Windows runners may have g++.exe (MinGW/Strawberry) in PATH but it's +# # not a proper mcpp-compatible GCC. Don't add gcc capability. +# # macOS g++ is Apple Clang, not real GCC — don't add gcc capability. +# +# So `module_extensions` has only ever been exercised on Linux. The specific +# combination "declare `.ixx` and build it with MSVC" — the one a user porting +# an MSVC project actually takes — has never run anywhere. It follows by +# reasoning (`.ixx` is cl's own convention and mcpp emits `/interface /TP` +# unconditionally), but reasoning is not measurement: this is the same shape as +# mcpp#272, where `# requires: elf gcc` meant the Clang and MSVC legs were never +# reached at all. +# +# This test grants itself no capability. It uses whatever toolchain the platform +# defaults to — MSVC on Windows, LLVM on macOS, GCC on Linux — so it runs +# EVERYWHERE and covers the two legs 217 cannot reach. +set -e + +_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +if [ -z "${MCPP:-}" ]; then + MCPP="$(bash "$_root/.github/tools/newest_artifact.sh" "$_root" mcpp 2>/dev/null || true)" + [ -n "$MCPP" ] || { echo "SKIP: no mcpp binary built yet"; exit 0; } +fi +case "$MCPP" in /*|?:[/\\]*) ;; *) MCPP="$_root/$MCPP" ;; esac +[ -x "$MCPP" ] || { echo "FAIL: MCPP=$MCPP is not executable"; exit 1; } +export MCPP + +TMP=$(mktemp -d) +trap "rm -rf $TMP || true" EXIT +mkdir -p "$TMP/proj/src" +cd "$TMP/proj" + +# `.ixx` is the extension this is about: cl's own spelling, and the one an MSVC +# project arrives with. The toolchain block gives each platform its default — +# the point is that no leg is skipped, not that they all use the same compiler. +cat > mcpp.toml <<'EOF' +[package] +name = "extdefault" +version = "0.1.0" + +[toolchain] +default = "gcc@16.1.0" +macos = "llvm@22.1.8" +windows = "llvm@20.1.7" + +[build] +module_extensions = [".ixx"] +EOF + +cat > src/greet.ixx <<'EOF' +export module extdefault.greet; +import std; +export auto greet() -> std::string { return "ixx-ok"; } +EOF + +cat > src/main.cpp <<'EOF' +import std; +import extdefault.greet; +int main() { std::println("{}", greet()); return greet() == "ixx-ok" ? 0 : 1; } +EOF + +echo "== build ==" +"$MCPP" build --release > build.log 2>&1 || { + echo "FAIL: build with module_extensions = [\".ixx\"] failed on this platform's" + echo " default toolchain. This is the leg 217 cannot reach." + tail -40 build.log + exit 1 +} + +# ⚠️ A BUILD THAT EXITS 0 IS NOT ENOUGH, and this is the exact trap the +# extension work was full of: Clang hands an unrecognised `.ixx` to the LINKER, +# warns, and exits 0 having produced NO BMI. So assert the interface really was +# compiled as one — a BMI exists for the module. +BMI="$(find target -type f \( -name '*.gcm' -o -name '*.pcm' -o -name '*.ifc' \) 2>/dev/null \ + | grep -iE 'greet' | head -1)" +[ -n "$BMI" ] || { + echo "FAIL: no BMI produced for the .ixx interface — it was not treated as a" + echo " module interface unit at all. A zero exit code does not prove" + echo " compilation here: an unrecognised extension is a LINKER input." + find target -type f \( -name '*.gcm' -o -name '*.pcm' -o -name '*.ifc' \) | head -10 + exit 1 +} +echo " BMI: $BMI" + +echo "== run ==" +BIN="$(find target -type f -path '*/bin/*' \( -name 'extdefault' -o -name 'extdefault.exe' \) | head -1)" +[ -n "$BIN" ] || { echo "FAIL: no binary produced"; find target -path '*/bin/*' -type f | head; exit 1; } +OUT="$("$BIN")" || { echo "FAIL: the binary exited $?"; exit 1; } +[ "$OUT" = "ixx-ok" ] || { echo "FAIL: binary printed '$OUT', expected 'ixx-ok'"; exit 1; } +echo " ran: $OUT" + +echo "236 module_extensions on the default toolchain OK" diff --git a/tests/e2e/95_msvc_system_toolchain.sh b/tests/e2e/95_msvc_system_toolchain.sh index 5b737925..9e67a485 100755 --- a/tests/e2e/95_msvc_system_toolchain.sh +++ b/tests/e2e/95_msvc_system_toolchain.sh @@ -6,7 +6,6 @@ # - `toolchain list` shows the detected MSVC in a System section, starred # - version pin-verify: `msvc@99` mismatches the detected install # - `toolchain remove/install msvc`: mcpp never manages MSVC itself -# - `mcpp build` fails with the owned "not yet supported" gate message set -e # This test flips the global default toolchain; save + restore it so later diff --git a/tests/unit/test_bmi_equivalent.cpp b/tests/unit/test_bmi_equivalent.cpp new file mode 100644 index 00000000..0b8ffc8f --- /dev/null +++ b/tests/unit/test_bmi_equivalent.cpp @@ -0,0 +1,133 @@ +// mcpp.build.stage::bmi_equivalent — the comparison that decides whether an +// importer must be rebuilt. +// +// Asserted from BOTH sides on purpose. A function that always returned `true` +// would pass every "equivalent BMIs compare equal" test while silently +// suppressing every legitimate cascade — which is a far worse bug than the one +// it replaces. Half of these cases exist to catch that. +#include + +#include +#include +#include + +import mcpp.build.stage; + +namespace { + +std::filesystem::path tmpdir() { + auto d = std::filesystem::temp_directory_path() / + ("mcpp-bmi-eq-" + std::to_string(::getpid())); + std::filesystem::create_directories(d); + return d; +} + +std::filesystem::path write(const std::filesystem::path& p, const std::string& bytes) { + std::ofstream out(p, std::ios::binary | std::ios::trunc); + out.write(bytes.data(), static_cast(bytes.size())); + return p; +} + +// A stand-in for the shape GCC actually emits, verified against a real .gcm: +// ...export.repository: gcm.cache\0buildtime: 2026/08/12 02:25:01 UTC\0... +std::string bmi_like(const std::string& stamp, const std::string& payload = "PAYLOAD") { + return "GCM\x01" + payload + std::string("\0", 1) + + "buildtime: " + stamp + std::string("\0", 1) + + "localtime: " + stamp + std::string("\0", 1) + "TAIL"; +} + +} // namespace + +TEST(BmiEquivalent, IdenticalFilesAreEquivalent) { + const auto d = tmpdir(); + auto a = write(d / "a.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + auto b = write(d / "b.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + EXPECT_TRUE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +// The whole reason this function exists: GCC stamps a wall clock into the BMI, +// so two compiles of identical source differ by a few bytes and `cmp` reports +// "changed" every single time. +TEST(BmiEquivalent, DifferingOnlyInTheEmbeddedTimestamp) { + const auto d = tmpdir(); + auto a = write(d / "t1.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + auto b = write(d / "t2.gcm", bmi_like("2026/08/12 02:25:33 UTC")); + EXPECT_TRUE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +// The side that must NOT be lost: a real interface change still cascades. +TEST(BmiEquivalent, DifferingPayloadIsNotEquivalent) { + const auto d = tmpdir(); + auto a = write(d / "p1.gcm", bmi_like("2026/08/12 02:25:01 UTC", "PAYLOAD")); + auto b = write(d / "p2.gcm", bmi_like("2026/08/12 02:25:01 UTC", "PAYLOAX")); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +// A payload difference must not be masked just because a timestamp is nearby. +TEST(BmiEquivalent, PayloadDifferenceIsNotHiddenByATimestampDifference) { + const auto d = tmpdir(); + auto a = write(d / "m1.gcm", bmi_like("2026/08/12 02:25:01 UTC", "PAYLOAD")); + auto b = write(d / "m2.gcm", bmi_like("2026/08/12 09:59:59 UTC", "PAYLOAX")); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +TEST(BmiEquivalent, DifferentSizesAreNeverEquivalent) { + const auto d = tmpdir(); + auto a = write(d / "s1.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + auto b = write(d / "s2.gcm", bmi_like("2026/08/12 02:25:01 UTC") + "EXTRA"); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +// No stamps at all → strict comparison, which is the old behaviour. This is the +// conservative fallback that keeps the function from ever being MORE permissive +// than a byte compare on inputs it does not understand. +TEST(BmiEquivalent, WithoutStampsItIsAStrictCompare) { + const auto d = tmpdir(); + auto a = write(d / "n1.gcm", "no stamps here"); + auto b = write(d / "n2.gcm", "no stamps here"); + auto c = write(d / "n3.gcm", "no stamps HERE"); + EXPECT_TRUE(mcpp::build::stage::bmi_equivalent(a, b)); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, c)); +} + +// A value that merely follows the prefix but is not a timestamp must not be +// masked — otherwise an attacker-shaped or simply unusual BMI could hide a real +// difference behind the literal text "buildtime: ". +TEST(BmiEquivalent, NonTimestampAfterThePrefixIsNotMasked) { + const auto d = tmpdir(); + auto a = write(d / "f1.gcm", "buildtime: not-a-timestamp-xxxxxA"); + auto b = write(d / "f2.gcm", "buildtime: not-a-timestamp-xxxxxB"); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, b)); +} + +TEST(BmiEquivalent, MissingFileIsNotEquivalent) { + const auto d = tmpdir(); + auto a = write(d / "e1.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(a, d / "does-not-exist.gcm")); +} + +// A real BMI carries exactly two stamps — one `buildtime:`, one `localtime:` — +// checked across BMIs from 10 KiB to 645 KiB. A third one did not come from +// GCC's header, so it is not mcpp's to ignore: masking a timestamp-shaped +// string literal in user code would hide a change the importers must see. +// +// Asserted from BOTH sides. Checking only that the extra span is not masked +// would also pass an implementation that masks nothing at all, which is the +// bug this whole function exists to fix. +TEST(BmiEquivalent, MoreStampsThanGccEmitsFallsBackToStrictCompare) { + const auto d = tmpdir(); + // Two stamps: masked, so the differing clock does not count. + auto two_a = write(d / "t1.gcm", bmi_like("2026/08/12 02:25:01 UTC")); + auto two_b = write(d / "t2.gcm", bmi_like("2026/08/12 02:25:09 UTC")); + EXPECT_TRUE(mcpp::build::stage::bmi_equivalent(two_a, two_b)); + + // The same two, plus a third stamp-shaped string in the payload that + // differs. It must NOT be masked, so the two files are different. + auto three_a = write(d / "u1.gcm", + bmi_like("2026/08/12 02:25:01 UTC", + "buildtime: 2020/01/01 00:00:00 UTC")); + auto three_b = write(d / "u2.gcm", + bmi_like("2026/08/12 02:25:09 UTC", + "buildtime: 2020/01/02 00:00:00 UTC")); + EXPECT_FALSE(mcpp::build::stage::bmi_equivalent(three_a, three_b)); +} diff --git a/tests/unit/test_capacity_jobs.cpp b/tests/unit/test_capacity_jobs.cpp new file mode 100644 index 00000000..34d01a9b --- /dev/null +++ b/tests/unit/test_capacity_jobs.cpp @@ -0,0 +1,83 @@ +// mcpp.platform.capacity::recommended_jobs — the `--jobs auto` formula. +// +// Asserted against SYNTHETIC capacities, not the machine running the test: a +// test that asked the host how many cores it has would assert nothing (it would +// just restate the answer) and would produce a different verdict on every CI +// runner. +#include + +import mcpp.platform.capacity; + +using mcpp::platform::capacity::HostCapacity; +using mcpp::platform::capacity::recommended_jobs; + +namespace { +constexpr unsigned long long GiB = 1024ull * 1024 * 1024; +} + +// A homogeneous machine with ample RAM should use its logical CPUs: SMT siblings +// still contribute, just less than a full core. +TEST(RecommendedJobs, HomogeneousAndAmpleMemoryUsesLogicalCores) { + HostCapacity cap{.logicalCores = 16, .physicalCores = 8, .heterogeneous = false, + .totalBytes = 64 * GiB, .availableBytes = 60 * GiB}; + EXPECT_EQ(recommended_jobs(cap), 16); +} + +// The case that motivated the whole function: 32 "cores" on a 13900K are 8 +// P-cores + 16 E-cores, and treating them as 32 equal workers overestimates +// usable parallelism by more than 2x. +TEST(RecommendedJobs, HeterogeneousFallsBackToPhysicalCores) { + HostCapacity cap{.logicalCores = 32, .physicalCores = 24, .heterogeneous = true, + .totalBytes = 64 * GiB, .availableBytes = 60 * GiB}; + EXPECT_EQ(recommended_jobs(cap), 24); +} + +// The dangerous shape: many cores, little RAM. ninja's default here would be 66 +// concurrent compiles against ~0.75 GB each and the machine would swap. +TEST(RecommendedJobs, MemoryBoundMachineIsCappedByRam) { + HostCapacity cap{.logicalCores = 64, .physicalCores = 64, .heterogeneous = false, + .totalBytes = 32 * GiB, .availableBytes = 32 * GiB}; + // (32 - 2) / 0.75 = 40 → below the 64 CPUs, so memory decides. + EXPECT_EQ(recommended_jobs(cap), 40); + EXPECT_LT(recommended_jobs(cap), cap.logicalCores); +} + +// Must still make progress rather than refusing to build. +TEST(RecommendedJobs, TinyMemoryStillYieldsOneJob) { + HostCapacity cap{.logicalCores = 8, .physicalCores = 8, .heterogeneous = false, + .totalBytes = 2 * GiB, .availableBytes = 1 * GiB}; + EXPECT_EQ(recommended_jobs(cap), 1); +} + +TEST(RecommendedJobs, NeverExceedsTheCeiling) { + HostCapacity cap{.logicalCores = 256, .physicalCores = 256, .heterogeneous = false, + .totalBytes = 1024 * GiB, .availableBytes = 1000 * GiB}; + EXPECT_EQ(recommended_jobs(cap), 64); + EXPECT_EQ(recommended_jobs(cap, 768ull * 1024 * 1024, 2 * GiB, /*ceiling=*/8), 8); +} + +TEST(RecommendedJobs, NeverReturnsZero) { + HostCapacity cap{.logicalCores = 0, .physicalCores = 0, .heterogeneous = false, + .totalBytes = 0, .availableBytes = 0}; + EXPECT_GE(recommended_jobs(cap), 1); +} + +// A project with heavier translation units can say so without patching mcpp. +TEST(RecommendedJobs, PerJobEstimateIsAParameter) { + HostCapacity cap{.logicalCores = 32, .physicalCores = 32, .heterogeneous = false, + .totalBytes = 32 * GiB, .availableBytes = 32 * GiB}; + const int light = recommended_jobs(cap, 256ull * 1024 * 1024); + const int heavy = recommended_jobs(cap, 4ull * 1024 * 1024 * 1024); + EXPECT_GT(light, heavy); + EXPECT_LE(heavy, 8); +} + +// The real machine must at least produce something sane — a weak assertion on +// purpose, since the value is host-dependent. +TEST(RecommendedJobs, HostQueryIsPlausible) { + const auto cap = mcpp::platform::capacity::host_capacity(); + EXPECT_GE(cap.logicalCores, 1); + EXPECT_GE(cap.physicalCores, 1); + EXPECT_LE(cap.physicalCores, cap.logicalCores); + EXPECT_GE(recommended_jobs(cap), 1); +} diff --git a/tests/unit/test_dyndep.cpp b/tests/unit/test_dyndep.cpp index c477413b..59abe564 100644 --- a/tests/unit/test_dyndep.cpp +++ b/tests/unit/test_dyndep.cpp @@ -80,6 +80,75 @@ TEST(Dyndep, EmitDyndepSelfProvideFiltered) { EXPECT_EQ(body.find("gcm.cache/foo.gcm"), std::string::npos); } +// Two-phase (clang): one source, two edges — a BMI edge and an object edge — +// and BOTH parse the same imports, so both need the same implicit inputs. +// P1689 only knows about the object (`primary-output` comes from the scanned +// command's `-o`), and an edge with no record is not a warning: ninja refuses +// the whole graph with "'…pcm' not mentioned in its dyndep file", naming the +// edge rather than the missing record. +TEST(Dyndep, SplitModuleEmitsARecordForBothEdges) { + std::vector units = { + { "obj/lib.m.o", {"myapp.lib"}, {"std"} }, + }; + DyndepOptions opts; + opts.bmiDir = "pcm.cache"; + opts.bmiExt = ".pcm"; + opts.splitModuleEdges = true; + auto body = emit_dyndep(units, {}, opts); + + EXPECT_NE(body.find("build pcm.cache/myapp.lib.pcm: dyndep | pcm.cache/std.pcm\n"), + std::string::npos) << body; + EXPECT_NE(body.find("build obj/lib.m.o: dyndep | pcm.cache/std.pcm\n"), + std::string::npos) << body; +} + +// A unit that provides nothing is NOT split — there is only one edge, so a +// second record would name something nobody declared. Same failure text as the +// missing-record case, from the opposite mistake. +TEST(Dyndep, SplitModuleLeavesNonProvidingUnitsAlone) { + std::vector units = { + { "obj/main.o", {}, {"myapp.lib"} }, + }; + DyndepOptions opts; + opts.splitModuleEdges = true; + auto body = emit_dyndep(units, {}, opts); + + // Count the records, not the lines: one record is `build …` plus its + // `restat = 1`, so a line count says nothing about how many there are. + std::size_t records = 0; + for (auto pos = body.find("build "); pos != std::string::npos; + pos = body.find("build ", pos + 1)) + ++records; + EXPECT_EQ(records, 1u) << body; + EXPECT_NE(body.find("build obj/main.o: dyndep | gcm.cache/myapp.lib.gcm\n"), + std::string::npos) << body; +} + +// The single-unit path is what ninja actually runs (one .ddi → one .dd), and it +// used to be a separate hand-written copy of the loop above. Pinned together so +// a change to one shape cannot silently leave the other behind. +TEST(Dyndep, SplitModuleSingleFileMatchesTheBatchShape) { + auto tmp = std::filesystem::temp_directory_path() + / std::format("mcpp_dyndep_split_{}", std::random_device{}()); + std::filesystem::create_directories(tmp); + auto p = tmp / "lib.ddi"; + std::ofstream(p) << R"({ +"rules":[{"primary-output":"obj/lib.m.o","provides":[{"logical-name":"myapp.lib","is-interface":true}],"requires":[{"logical-name":"std"}]}] +})"; + DyndepOptions opts; + opts.bmiDir = "pcm.cache"; + opts.bmiExt = ".pcm"; + opts.splitModuleEdges = true; + + auto single = emit_dyndep_single(p, opts); + ASSERT_TRUE(single) << single.error(); + + std::vector units = { { "obj/lib.m.o", {"myapp.lib"}, {"std"} } }; + EXPECT_EQ(*single, emit_dyndep(units, {}, opts)); + + std::filesystem::remove_all(tmp); +} + TEST(Dyndep, EmitDyndepFromFiles) { auto tmp = std::filesystem::temp_directory_path() / std::format("mcpp_dyndep_test_{}", std::random_device{}()); diff --git a/tests/unit/test_loader_contract.cpp b/tests/unit/test_loader_contract.cpp index 1e55d3cc..0ef34d20 100644 --- a/tests/unit/test_loader_contract.cpp +++ b/tests/unit/test_loader_contract.cpp @@ -82,12 +82,33 @@ TEST(GraphShape, HeaderAndReaderAgree) { ~Cleanup() { std::error_code ec; std::filesystem::remove_all(d, ec); } } cleanup{dir}; + // The line now carries the module-edge schedule too. Round-tripping both + // fields together is the point: the schedule was added to this line rather + // than to a second file precisely so the two cannot disagree. for (auto shape : {GraphShape::Normal, GraphShape::WithTests}) { + for (std::string_view sched : {"none", "two-phase", "detach-codegen"}) { + auto p = dir / "build.ninja"; + { std::ofstream out(p, std::ios::trunc); + out << header_line(shape, sched) << "\n"; } + auto read = read_shape(p); + ASSERT_TRUE(read.has_value()); + EXPECT_EQ(*read, shape); + EXPECT_EQ(read_schedule(p), sched); + } + } + + // A graph written before the schedule field existed still reads as its + // shape — an older file must degrade, not become "unknown" — but its + // schedule reads as empty, which is NOT "none": callers that care have to + // be able to tell "this file predates the field" from "this file chose to + // do nothing". + { auto p = dir / "build.ninja"; - { std::ofstream out(p, std::ios::trunc); out << header_line(shape) << "\n"; } + { std::ofstream out(p, std::ios::trunc); out << "# mcpp:graph=normal\n"; } auto read = read_shape(p); ASSERT_TRUE(read.has_value()); - EXPECT_EQ(*read, shape); + EXPECT_EQ(*read, GraphShape::Normal); + EXPECT_TRUE(read_schedule(p).empty()); } } diff --git a/tests/unit/test_manifest.cpp b/tests/unit/test_manifest.cpp index 44eb679e..95e5ef86 100644 --- a/tests/unit/test_manifest.cpp +++ b/tests/unit/test_manifest.cpp @@ -3476,3 +3476,62 @@ cxx_runtime = "host-coupled" EXPECT_TRUE(m->buildConfig.cxxRuntimeShared.empty()); EXPECT_TRUE(m->schemaWarnings.empty()); } + +TEST(Manifest, BmiScheduleKey) { + // The KEY SPELLING, not just the field. `[build] bmi_schedule` reaches the + // build only through this one string in mcpp.manifest.toml, and nothing + // downstream can tell a mistyped key from an absent one: the value silently + // stays empty, `requested_switch` returns "auto", and the build quietly uses + // the default schedule. Every other test would still pass. + // + // It was called `schedule` until it was renamed to agree with its own + // environment override, `MCPP_BMI_SCHEDULE`. This is what makes the next + // such rename fail loudly instead of silently. + constexpr auto src = R"( +[package] +name = "x" +version = "0.1.0" +[build] +bmi_schedule = "on" +)"; + auto m = mcpp::manifest::parse_string(src); + ASSERT_TRUE(m.has_value()) << m.error().format(); + EXPECT_EQ(m->buildConfig.bmiSchedule, "on"); + + // And the old spelling must NOT still work: leaving it accepted would mean + // two keys for one switch, which is how a rename ends up half-done. + constexpr auto old_key = R"( +[package] +name = "x" +version = "0.1.0" +[build] +schedule = "on" +)"; + auto o = mcpp::manifest::parse_string(old_key); + ASSERT_TRUE(o.has_value()) << o.error().format(); + EXPECT_TRUE(o->buildConfig.bmiSchedule.empty()) + << "the pre-rename key `schedule` is still being read"; + + // ⚠️ AND THE PARSER MUST NOT WARN ABOUT ITS OWN KEY. + // + // Checking only the VALUE is what let this ship broken. The rename reached + // the read (`build.bmi_schedule`) but not the accepted-key list, which kept + // the old `schedule` — so the assertions above passed while a user writing + // the one documented way to enable the feature got + // + // [build] has unsupported key 'bmi_schedule' (ignored) + // + // and that message is FALSE: the value is read. The only way to turn the + // feature on announced that it had been ignored, and the dead `schedule` + // was accepted in silence. Both halves of the rename are pinned now. + for (const auto& w : m->schemaWarnings) + EXPECT_EQ(w.find("bmi_schedule"), std::string::npos) + << "the parser reads `bmi_schedule` but also warns about it: " << w; + + EXPECT_TRUE(std::ranges::any_of(o->schemaWarnings, + [](const std::string& w) { + return w.find("schedule") != std::string::npos; + })) + << "the dead key `schedule` is accepted silently — an unread key that " + "produces no diagnostic is a typo that costs a debugging session"; +} diff --git a/tests/unit/test_runtime_search.cpp b/tests/unit/test_runtime_search.cpp index 459b3ddb..bd43b8fe 100644 --- a/tests/unit/test_runtime_search.cpp +++ b/tests/unit/test_runtime_search.cpp @@ -24,6 +24,14 @@ TEST(RuntimeSearch, PayloadOutranksFarm) { EXPECT_LT(search::rank(Origin::Payload), search::rank(Origin::SubosFarm)); EXPECT_LT(search::rank(Origin::Package), search::rank(Origin::SubosFarm)); EXPECT_LT(search::rank(Origin::SubosFarm), search::rank(Origin::HostDefault)); + + // #415 — the artifact's own directory (`$ORIGIN`) sits between the packages + // and the farm. This is the ordering #414 was about: with the farm ahead of + // `$ORIGIN` an artifact linked one libX11 and loaded another. A pinned + // payload still wins, because that is the one thing nothing re-points. + EXPECT_LT(search::rank(Origin::Package), search::rank(Origin::Artifact)); + EXPECT_LT(search::rank(Origin::Artifact), search::rank(Origin::SubosFarm)); + EXPECT_LT(search::rank(Origin::Payload), search::rank(Origin::Artifact)); } // `mcpp pack` asks this to decide what may not be baked into a distributable. @@ -35,6 +43,13 @@ TEST(RuntimeSearch, MachineLocalIsEverythingButTheHostDefaults) { EXPECT_TRUE(search::is_machine_local(Origin::Package)); EXPECT_TRUE(search::is_machine_local(Origin::SubosFarm)); EXPECT_FALSE(search::is_machine_local(Origin::HostDefault)); + + // ⚠️ `$ORIGIN` is NOT machine-local, and getting this backwards would be + // worse than the gap it closes: it is resolved by the loader relative to + // the artifact, so it means the same thing wherever that artifact is + // copied — `pack` REWRITES everything else into this form. Marking it + // local would make pack reject the one entry it is trying to produce. + EXPECT_FALSE(search::is_machine_local(Origin::Artifact)); } // These strings are PUBLISHED — they are the `origin` field of every entry in @@ -44,6 +59,7 @@ TEST(RuntimeSearch, MachineLocalIsEverythingButTheHostDefaults) { TEST(RuntimeSearch, OriginNamesArePublishedAndStable) { EXPECT_EQ(search::to_string(Origin::Payload), "payload"); EXPECT_EQ(search::to_string(Origin::Package), "package"); + EXPECT_EQ(search::to_string(Origin::Artifact), "artifact"); EXPECT_EQ(search::to_string(Origin::SubosFarm), "subos_farm"); EXPECT_EQ(search::to_string(Origin::HostDefault), "host_default"); } diff --git a/tests/unit/test_schedule_policy.cpp b/tests/unit/test_schedule_policy.cpp new file mode 100644 index 00000000..cdc25b2e --- /dev/null +++ b/tests/unit/test_schedule_policy.cpp @@ -0,0 +1,225 @@ +// The build-shape policy: one table, asserted from both sides. +// +// `decide()` is pure precisely so this file needs no toolchain, no filesystem +// and no compiler — the table can be wrong in a way that only shows up as a +// slower build, which is the kind of wrong that never gets noticed. + +#include +#include + +import std; +import mcpp.build.schedule.policy; +import mcpp.toolchain.model; +import mcpp.manifest; + +using mcpp::build::schedule::Strategy; +using mcpp::build::schedule::decide; +using mcpp::build::schedule::requested_switch; +using mcpp::toolchain::CompilerId; +using mcpp::toolchain::Toolchain; + +namespace { +Toolchain with(CompilerId id) { + Toolchain tc; + tc.compiler = id; + return tc; +} + +// MCPP_BMI_SCHEDULE outranks the manifest, so a stray one in the developer's +// shell would decide these tests instead of the code under test. +class ScopedVar { +public: + ScopedVar(std::string name, const char* value) : name_(std::move(name)) { + if (const char* old = std::getenv(name_.c_str()); old) { had_ = true; old_ = old; } + apply(value); + } + ~ScopedVar() { apply(had_ ? old_.c_str() : nullptr); } + ScopedVar(const ScopedVar&) = delete; + ScopedVar& operator=(const ScopedVar&) = delete; +private: + void apply(const char* v) { +#if defined(_WIN32) + ::_putenv_s(name_.c_str(), v ? v : ""); +#else + if (v) ::setenv(name_.c_str(), v, 1); else ::unsetenv(name_.c_str()); +#endif + } + std::string name_; + bool had_ = false; + std::string old_; +}; + +mcpp::manifest::Manifest with_schedule(std::string v) { + mcpp::manifest::Manifest m; + m.buildConfig.bmiSchedule = std::move(v); + return m; +} +} // namespace + +// The two mechanisms are COMPLEMENTARY, not interchangeable, and getting them +// backwards is silent: clang writes its BMI to the final path with O_TRUNC, so +// detach-codegen would hand importers a half-written file; gcc has no cheap +// BMI-only mode, so two-phase would just compile everything twice. +TEST(SchedulePolicy, EachCompilerGetsItsOwnMechanism) { + EXPECT_EQ(decide(with(CompilerId::Clang), "on", 8).strategy, Strategy::TwoPhase); + EXPECT_EQ(decide(with(CompilerId::GCC), "on", 8).strategy, Strategy::DetachCodegen); +} + +// Unmeasured means None. A guess here is not a slow build, it is a miscompile: +// a BMI read while it is still being written is not a diagnostic. +// `auto` is OFF for now — pinned, because it is a decision rather than a gap. +TEST(SchedulePolicy, AutoIsOptInForNow) { + EXPECT_EQ(decide(with(CompilerId::GCC), "auto", 8).strategy, Strategy::None); + EXPECT_NE(decide(with(CompilerId::GCC), "on", 8).strategy, Strategy::None); +} + +TEST(SchedulePolicy, UnmeasuredCompilersStayConservative) { + EXPECT_EQ(decide(with(CompilerId::MSVC), "on", 8).strategy, Strategy::None); + EXPECT_EQ(decide(with(CompilerId::Unknown), "on", 8).strategy, Strategy::None); +} + +// Asserted from BOTH sides: that "off" disables, and that the same input with +// "auto" does NOT. Checking only the first would pass an implementation that +// never enables anything at all. +TEST(SchedulePolicy, OffDisablesAndAutoDoesNot) { + EXPECT_EQ(decide(with(CompilerId::GCC), "off", 8).strategy, Strategy::None); + EXPECT_NE(decide(with(CompilerId::GCC), "on", 8).strategy, Strategy::None); +} + +// HAZARD 2, encoded. Under detach-codegen a compiler stops holding a ninja slot +// the moment it publishes its BMI, so ninja's -j is no longer a bound on how +// many compilers run. With the two equal, ninja's slots fill with edges that +// are merely sleeping, the ready frontier starves, and the schedule degenerates +// to the baseline — which is exactly what the first prototype measured. +TEST(SchedulePolicy, DetachCodegenGivesNinjaMoreSlotsThanCompilers) { + const auto d = decide(with(CompilerId::GCC), "on", 32); + EXPECT_EQ(d.compilerCap, 32); + EXPECT_GT(d.ninjaJobs, d.compilerCap); +} + +// Two-phase runs ordinary compilers that hold their slot for the whole compile, +// so inflating -j there would only oversubscribe the machine. +TEST(SchedulePolicy, TwoPhaseLeavesTheJobCountAlone) { + const auto d = decide(with(CompilerId::Clang), "on", 32); + EXPECT_EQ(d.ninjaJobs, d.compilerCap); +} + +// A scheduler that silently declines to optimise cannot be debugged: "why is my +// build not using the fast shape?" has to have an answer that ships with the +// build. Every branch, including the ones that choose None. +TEST(SchedulePolicy, EveryDecisionCarriesAReason) { + for (auto id : {CompilerId::GCC, CompilerId::Clang, CompilerId::MSVC, + CompilerId::Unknown}) { + EXPECT_FALSE(decide(with(id), "auto", 8).reason.empty()) + << "no reason for compiler id " << static_cast(id); + EXPECT_FALSE(decide(with(id), "off", 8).reason.empty()) + << "no reason when disabled, compiler id " << static_cast(id); + } +} + +// A host that reports nothing must not turn into "-j0" or a negative cap. +TEST(SchedulePolicy, ZeroJobsStaysZeroRatherThanBecomingNonsense) { + const auto d = decide(with(CompilerId::GCC), "on", 0); + EXPECT_EQ(d.compilerCap, 0); + EXPECT_EQ(d.ninjaJobs, 0); + // ...and it must SAY so. Under detach-codegen a cap of 0 disables the + // semaphore, which is the only bound on how many compilers run at once, so + // this state is not a neutral default — it is unbounded concurrency. + EXPECT_NE(d.reason.find("bounded by nothing"), std::string::npos) + << "a zero cap silently means no bound at all: " << d.reason; +} + +// ⚠️ THE DEFAULT CONFIGURATION MUST STILL BE BOUNDED. +// +// `resolve_jobs` returns 0 when the user passed neither `--jobs` nor +// `[build] jobs` — "say nothing, leave the backend's default". For every other +// strategy that is fine, because ninja's -j is a real bound. Under +// DetachCodegen it is NOT: a compiler stops holding its ninja slot the moment +// it publishes a BMI, so the semaphore is the only counter, and a cap of 0 +// turns `acquire_token` into a no-op. +// +// This shipped: a plain `mcpp build` with `bmi_schedule = "on"` generated +// `sched_cap = 0`, i.e. ninja starting compiles as fast as BMIs appeared with +// nothing limiting them, on a workload whose single compile peaks near a +// gigabyte. The caller now resolves what the host would pick and hands it in. +TEST(SchedulePolicy, DetachCodegenFallsBackToTheHostWhenNoJobCountWasGiven) { + const auto d = decide(with(CompilerId::GCC), "on", /*hostJobs=*/0, /*autoJobs=*/24); + EXPECT_EQ(d.strategy, Strategy::DetachCodegen); + EXPECT_EQ(d.compilerCap, 24) << "the semaphore would be disabled"; + EXPECT_GT(d.ninjaJobs, d.compilerCap) << "hazard 2: -j must exceed the cap"; + + // An explicit job count still wins over the fallback. + const auto e = decide(with(CompilerId::GCC), "on", /*hostJobs=*/8, /*autoJobs=*/24); + EXPECT_EQ(e.compilerCap, 8); + + // The fallback is only for the strategy that needs it; two-phase uses + // ordinary edges, where ninja's -j is already the bound. + const auto c = decide(with(CompilerId::Clang), "on", /*hostJobs=*/0, /*autoJobs=*/24); + EXPECT_EQ(c.ninjaJobs, 0) << "clang must still defer to the backend default"; +} + +// `auto` is bounded by recommended_jobs' ceiling of 64, but `--jobs N` is only +// checked for `> 0`, so an absurd N reaches decide() intact and `cap * 6` was +// signed overflow — undefined behaviour, with a NEGATIVE `-j` handed to ninja as +// one of the friendlier outcomes. Asserted as "still positive and still greater +// than the cap" rather than against the clamp constant, so tuning the clamp does +// not require editing the test that exists to stop it going negative. +TEST(SchedulePolicy, AnAbsurdJobCountDoesNotOverflowIntoANegativeOne) { + const auto d = decide(with(CompilerId::GCC), "on", 2000000000); + EXPECT_GT(d.ninjaJobs, 0) << "ninja -j went non-positive"; + EXPECT_GT(d.ninjaJobs, 1) << "hazard 2: ninja must still outnumber the compilers"; +} + +// ─── requested_switch: a typo is a diagnostic, never a silent "auto" ─────── +// +// This is the rule resolve_jobs already followed and this switch did not. +// `bmi_schedule = "ON"` was accepted, meant OFF, and explained itself with +// "the split schedule is opt-in until verified" — which reads as "you did not +// ask for it" to someone who just did. +TEST(SchedulePolicy, RequestedSwitchPassesTheThreeSpellingsThrough) { + ScopedVar clear("MCPP_BMI_SCHEDULE", nullptr); + EXPECT_EQ(requested_switch(with_schedule("on")), "on"); + EXPECT_EQ(requested_switch(with_schedule("off")), "off"); + EXPECT_EQ(requested_switch(with_schedule("auto")), "auto"); + EXPECT_EQ(requested_switch(with_schedule("")), "auto"); // unset +} + +TEST(SchedulePolicy, RequestedSwitchReportsATypoInsteadOfSwallowingIt) { + ScopedVar clear("MCPP_BMI_SCHEDULE", nullptr); + for (const char* typo : {"ON", "On", "true", "yes", "1", "enabled"}) { + std::string seen; + const auto v = requested_switch(with_schedule(typo), + [&](std::string_view bad) { seen = bad; }); + EXPECT_EQ(v, "auto") << typo << " must fall back to the default"; + EXPECT_EQ(seen, typo) << typo << " was accepted silently"; + } +} + +// Both directions. Checking only that a typo warns would pass an implementation +// that warns about everything, including the spellings that are correct. +TEST(SchedulePolicy, RequestedSwitchStaysQuietForValidValues) { + ScopedVar clear("MCPP_BMI_SCHEDULE", nullptr); + for (const char* ok : {"on", "off", "auto"}) { + bool warned = false; + requested_switch(with_schedule(ok), [&](std::string_view) { warned = true; }); + EXPECT_FALSE(warned) << ok << " is valid but was reported as invalid"; + } +} + +// The environment outranks the manifest — and is validated on the same terms. +// A typo'd MCPP_BMI_SCHEDULE must not fall through to the manifest either: +// silently honouring `[build] bmi_schedule = "on"` when the environment asked +// for something unparseable would make the warning a lie. +TEST(SchedulePolicy, EnvironmentBeatsManifestAndIsValidatedToo) { + { + ScopedVar on("MCPP_BMI_SCHEDULE", "off"); + EXPECT_EQ(requested_switch(with_schedule("on")), "off"); + } + { + ScopedVar bad("MCPP_BMI_SCHEDULE", "ON"); + std::string seen; + EXPECT_EQ(requested_switch(with_schedule("on"), + [&](std::string_view b) { seen = b; }), "auto"); + EXPECT_EQ(seen, "ON"); + } +} diff --git a/tests/unit/test_toolchain_msvc.cpp b/tests/unit/test_toolchain_msvc.cpp index 5b9123f1..af6a25e9 100644 --- a/tests/unit/test_toolchain_msvc.cpp +++ b/tests/unit/test_toolchain_msvc.cpp @@ -4,6 +4,7 @@ import std; import mcpp.toolchain.model; import mcpp.toolchain.msvc; import mcpp.toolchain.registry; +import mcpp.toolchain.dialect; using namespace mcpp::toolchain; @@ -129,3 +130,68 @@ TEST(MsvcStdModule, MinLevelFollowsStlUnblockVersion) { EXPECT_EQ(msvc::std_module_min_level(tc_of("")), 23); EXPECT_EQ(msvc::std_module_min_level(tc_of("unknown")), 23); } + +// #422 — the std module must be built with the SAME CRT model as the TUs that +// import it. +// +// cl bakes `_MSVC_MT` / `_MSVC_MD` into every module it produces. The std build +// passed no `/M` flag at all, so it took cl's default (`/MT`), while a project +// on the default (dynamic) linkage compiles `/MD`. cl accepts the mismatch with +// a C5050 warning and then fails for real inside the ucrt headers: +// +// corecrt_malloc.h(89): error C2375: 'free': redefinition; different linkage +// +// Two properties are pinned here, and the second is the one that makes the +// first stay true: the flag must be IN the command (so the module is right), +// and it must be in the command STRING (so it enters `std_build_commands`, +// which is part of the std cache identity — two CRT models then cannot share a +// cache directory and silently serve each other's module). +TEST(ToolchainMsvc, StdModuleCarriesTheProjectCrtModel) { + Toolchain tc; + tc.compiler = CompilerId::MSVC; + tc.binaryPath = "C:/vc/bin/cl.exe"; + tc.stdModuleSource = "C:/vc/modules/std.ixx"; + tc.stdCompatSource = "C:/vc/modules/std.compat.ixx"; + + const std::filesystem::path cache = "C:/cache/std/key"; + + for (std::string_view crt : {std::string_view("/MT"), std::string_view("/MD")}) { + auto cmds = msvc::std_module_build_commands(tc, cache, "/std:c++23", crt); + ASSERT_EQ(cmds.size(), 1u); + EXPECT_NE(cmds[0].find(crt), std::string::npos) + << crt << " missing from the std build command: " << cmds[0]; + + auto compat = msvc::std_compat_build_commands(tc, cache, "/std:c++23", crt); + ASSERT_EQ(compat.size(), 1u); + EXPECT_NE(compat[0].find(crt), std::string::npos) + << crt << " missing from the std.compat build command: " << compat[0]; + } + + // The two models must produce DIFFERENT command strings. Equal strings would + // mean equal cache keys, i.e. the exact silent divergence this fixes. + EXPECT_NE(msvc::std_module_build_commands(tc, cache, "/std:c++23", "/MT")[0], + msvc::std_module_build_commands(tc, cache, "/std:c++23", "/MD")[0]); + + // Empty keeps cl's own default, so non-MSVC callers are unaffected. + auto bare = msvc::std_module_build_commands(tc, cache, "/std:c++23"); + EXPECT_EQ(bare[0].find("/MT"), std::string::npos); + EXPECT_EQ(bare[0].find("/MD"), std::string::npos); +} + +// The mapping linkage -> CRT model lives in ONE place. It was derived twice — +// flags.cppm for the project's TUs, and cl's default for the std module — which +// is how they disagreed. +TEST(ToolchainMsvc, CrtFlagHasASingleDerivation) { + Toolchain cl; + cl.compiler = CompilerId::MSVC; + const auto& msvcDialect = dialect_for(cl); + EXPECT_EQ(msvc_crt_flag(msvcDialect, /*staticLinkage=*/true), "/MT"); + EXPECT_EQ(msvc_crt_flag(msvcDialect, /*staticLinkage=*/false), "/MD"); + + // GNU has no counterpart; the helper must yield nothing rather than invent + // a flag that would be passed to gcc. + Toolchain gcc; + gcc.compiler = CompilerId::GCC; + const auto& gnu = dialect_for(gcc); + EXPECT_TRUE(msvc_crt_flag(gnu, false).empty()); +}