diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a460d..e7c1298 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Structure gates (banned-pattern + layering) run: bash tests/check_structure_gates.sh + - name: POSIX conformance coverage (only-up gate) + run: tests/posix/coverage.sh + - name: clang-format dry-run (advisory — version skew until toolchain pinned) continue-on-error: true run: clang-format --dry-run --Werror $(find src include tests -name '*.cpp' -o -name '*.hpp') diff --git a/document/notes/2026-07-06-posix-coverage.md b/document/notes/2026-07-06-posix-coverage.md new file mode 100644 index 0000000..65f9bce --- /dev/null +++ b/document/notes/2026-07-06-posix-coverage.md @@ -0,0 +1,64 @@ +# Phase 4 起步 — POSIX 符合度覆盖率标尺 + +> 2026-07-06。Phase 4(生产质量门禁)第一块基建:`tests/posix/coverage.sh` —— +> 客观度量 cfbox 对 POSIX.1-2017 XCU 的符合度,CI gate「只升不降」。Phase 3 网络 +> 闭环合并后立即开建,作为 Phase 4「POSIX 子集验收」的第一步。 + +## 触及文件 + +- [tests/posix/coverage.sh](../../tests/posix/coverage.sh) — 双维度度量脚本 +- [tests/posix/baseline](../../tests/posix/baseline) — 当前覆盖率下限(utility 61 / builtin 10) +- [.github/workflows/ci.yml](../../.github/workflows/ci.yml) — native 阶段加 POSIX coverage gate(structure gates 后) + +## 双维度方法论 + +1. **utility 维度(静态)**:POSIX.1-2017 XCU mandatory utility list(剔除 shell builtin 后 77 个) + vs `cfbox --list`。独立 applet 实现的覆盖。 +2. **builtin 维度(动态)**:POSIX mandatory shell builtin(20 个)每个跑最小行为探针 + (`cfbox sh -c ''`),判输出是否含 `": command not found"`。 + **不靠声明,靠真实行为** —— 避免「applet list 有但行为缺」的假绿。 + +## 当前覆盖率(baseline) + +| 维度 | 覆盖 | % | +|------|------|---| +| utility (applet) | 61/77 | 79% | +| builtin (sh) | 10/20 | 50% | +| **total** | **71/97** | **73%** | + +## 高价值发现(gap,单独排批) + +**utility 缺(16)**: +- **`dd`** —— P0 高频 POSIX mandatory,coreutils 必备。**真硬伤**,优先补。 +- `stty file logger csplit getconf pr pathchk strings strip tput unexpand uudecode uuencode locale what` + —— 长尾,按场景驱动排期。 + +**builtin 缺(10)**:`command type exec getopts hash umask ulimit wait times unalias` +- `type` / `command` —— POSIX sh 高频 builtin,脚本里 `type ls` / `command -v` 用得多 +- `exec` / `wait` —— shell 核心 builtin,影响脚本语义 +- `umask` / `ulimit` —— 嵌入式/资源控制常用 +- `getopts` —— 脚本选项解析标配 +- `hash` / `times` / `unalias` —— 较低频 + +注:探针确认了 `read`(管道)、`pwd`、`cd`、`export`、`unset`、`shift`、`eval`、`trap`、 +`return`、`exit`、`set` 工作(10 个支持)。 + +## CI gate 机制 + +`tests/posix/coverage.sh` 退出码: +- 当前覆盖 ≥ baseline → exit 0(PASS) +- 任一维度 < baseline → exit 1(**退化报警,CI 红**) +- 高于 baseline → exit 0 + 提示「run with --update-baseline to lock in」 + +补 applet/builtin 后跑 `tests/posix/coverage.sh --update-baseline` 锁新下限。 + +## 后续排批建议(Phase 4 增量) + +1. **dd**(P0,独立批次,~1 commit)—— bs/conv/iflag 非平凡,但高频值得做 +2. **sh builtin 扩展**(command/type/exec/getopts/hash/umask/ulimit/wait/times/unalias) + —— cfbox sh 一批补齐 10 个 builtin,builtin 覆盖率 50%→100% +3. **行为符合度 spot check**(Phase 4 Part 1 差异测试)—— 对 P0 utility 跑 BusyBox + testsuite 的 `.tests` 文件(`competition/busybox/testsuite/`,206 个),对照输出 +4. **XSI/optional utility** 扩展(awk/bc/vi/yacc 等,扩大 list) + +baseline 先锁 73%,每补一项只升不降。 diff --git a/tests/posix/baseline b/tests/posix/baseline new file mode 100644 index 0000000..b73d318 --- /dev/null +++ b/tests/posix/baseline @@ -0,0 +1,2 @@ +utility 61 +builtin 10 diff --git a/tests/posix/coverage.sh b/tests/posix/coverage.sh new file mode 100755 index 0000000..f674215 --- /dev/null +++ b/tests/posix/coverage.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# tests/posix/coverage.sh — POSIX.1-2017 XCU conformance coverage for cfbox. +# +# Two dimensions: +# 1. utility — POSIX mandatory utilities cfbox ships as standalone applets +# (static: compared against `cfbox --list`) +# 2. builtin — POSIX mandatory shell builtins cfbox sh actually supports +# (dynamic: behavior probe via `cfbox sh -c`, NOT declaration) +# +# CI gate: coverage must not regress below tests/posix/baseline — the +# COVERAGE.md "only up, never down" spirit. Exits non-zero on regression. +# When coverage rises, run with --update-baseline to lock the new floor. +# +# Usage: tests/posix/coverage.sh # measure + gate +# tests/posix/coverage.sh --update-baseline +# CFBOX=./build/cfbox tests/posix/coverage.sh +set -euo pipefail + +script_dir="$(cd "$(dirname "$0")" && pwd)" +project_dir="$(cd "$script_dir/../.." && pwd)" +baseline_file="$script_dir/baseline" + +# locate cfbox binary (CI builds into build/, local size-opt into build-size/) +CFBOX="${CFBOX:-}" +if [[ -z "$CFBOX" ]]; then + for cand in "$project_dir/build/cfbox" "$project_dir/build-size/cfbox"; do + [[ -x "$cand" ]] && CFBOX="$cand" && break + done +fi +[[ -x "$CFBOX" ]] || { echo "ERROR: no cfbox binary found (build it first)"; exit 2; } + +# POSIX.1-2017 XCU mandatory utilities — standalone, with the shell builtins +# removed (cd/command/type/ulimit/... are measured in dimension 2, not here). +POSIX_UTILITIES=(basename cat chgrp chmod chown cksum cmp comm cp csplit cut + date dd df diff dirname du echo env expand expr false file find fold getconf + grep head id kill link ln locale logger logname ls mkdir mkfifo mv nice nl + nohup od paste patch pathchk pr printf pwd renice rm rmdir sed sleep + sort split strings strip stty tail tee test touch tput tr true tsort tty + uname unexpand uniq unlink uudecode uuencode wc what xargs) + +# POSIX.1-2017 mandatory shell builtins, each with a minimal behavior probe. +# A builtin is "supported" if the probe does NOT print ": command not found". +declare -A BUILTIN_PROBE=( + [cd]='cd /tmp && pwd' + [command]='command echo ok' + [eval]="eval 'echo evaled'" + [exec]='exec echo done' + [exit]='exit 0' + [export]='export X=hi; echo $X' + [getopts]='while getopts ab: opt -a; do echo $opt; break; done' + [hash]='hash; echo ok' + [read]='echo hi | (read x; echo $x)' + [return]='f(){ return 3; }; f; echo $?' + [set]='set -- a b; echo $#' + [shift]='set -- a b c; shift; echo $1' + [times]='times' + [trap]="trap 'echo trapped' EXIT" + [type]='type sh' + [ulimit]='ulimit' + [umask]='umask 022; umask' + [unalias]='unalias cfbox_no_such_alias' + [unset]='x=1; unset x; echo "[${x:-empty}]"' + [wait]='true & wait; echo done' +) + +# --- dimension 1: utility coverage (static) ----------------------------------- +applet_set=" $("$CFBOX" --list | awk '{print $1}' | tr '\n' ' ') " +util_covered=0 +util_missing=() +for u in "${POSIX_UTILITIES[@]}"; do + if [[ "$applet_set" == *" $u "* ]]; then + util_covered=$((util_covered + 1)) + else + util_missing+=("$u") + fi +done +util_total=${#POSIX_UTILITIES[@]} + +# --- dimension 2: builtin coverage (behavior probe) --------------------------- +builtin_covered=0 +builtin_missing=() +for b in "${!BUILTIN_PROBE[@]}"; do + out=$("$CFBOX" sh -c "${BUILTIN_PROBE[$b]}" 2>&1 || true) + if echo "$out" | grep -qE "(^|[^a-z])$b: command not found"; then + builtin_missing+=("$b") + else + builtin_covered=$((builtin_covered + 1)) + fi +done +builtin_total=${#BUILTIN_PROBE[@]} + +# --- report ------------------------------------------------------------------- +util_pct=$((util_covered * 100 / util_total)) +builtin_pct=$((builtin_covered * 100 / builtin_total)) +total_covered=$((util_covered + builtin_covered)) +total_total=$((util_total + builtin_total)) +total_pct=$((total_covered * 100 / total_total)) + +echo "=== POSIX.1-2017 XCU conformance coverage ===" +echo "cfbox: $CFBOX" +echo "" +printf " utility (applet): %d/%d (%d%%)\n" "$util_covered" "$util_total" "$util_pct" +printf " builtin (sh): %d/%d (%d%%)\n" "$builtin_covered" "$builtin_total" "$builtin_pct" +printf " total: %d/%d (%d%%)\n" "$total_covered" "$total_total" "$total_pct" +echo "" +if [[ ${#util_missing[@]} -gt 0 ]]; then + echo "utility missing (${#util_missing[@]}): ${util_missing[*]}" +fi +if [[ ${#builtin_missing[@]} -gt 0 ]]; then + echo "builtin missing (${#builtin_missing[@]}): ${builtin_missing[*]}" +fi + +# --- baseline gate (only-up) -------------------------------------------------- +if [[ "${1:-}" == "--update-baseline" ]]; then + printf 'utility %d\nbuiltin %d\n' "$util_covered" "$builtin_covered" > "$baseline_file" + echo "" + echo "baseline written: utility=$util_covered builtin=$builtin_covered → $baseline_file" + exit 0 +fi + +if [[ ! -f "$baseline_file" ]]; then + echo "" + echo "WARNING: no baseline file ($baseline_file). Run '$0 --update-baseline' to set the floor." + exit 0 +fi + +base_util=$(awk '/^utility /{print $2}' "$baseline_file") +base_builtin=$(awk '/^builtin /{print $2}' "$baseline_file") +echo "" +echo "=== baseline gate (only-up; floor: utility=$base_util builtin=$base_builtin) ===" +fail=0 +if (( util_covered < base_util )); then + echo "FAIL: utility $util_covered < $base_util (regression)"; fail=1 +fi +if (( builtin_covered < base_builtin )); then + echo "FAIL: builtin $builtin_covered < $base_builtin (regression)"; fail=1 +fi +if [[ $fail -eq 0 ]]; then + if (( util_covered > base_util || builtin_covered > base_builtin )); then + echo "PASS — coverage rose above baseline; run with --update-baseline to lock it in" + else + echo "PASS" + fi +fi +exit $fail