From 7ef7ee3821959c17240d4577bce4fa605c07f058 Mon Sep 17 00:00:00 2001 From: Asgeir Frimannsson Date: Sun, 13 Sep 2026 17:06:12 +0200 Subject: [PATCH 1/3] feat: report did-not-run checks as their own result kapi check exits 4 when a check did not run. The check step still fails with that code, and now sets result=did_not_run and did_not_run_cause and names the cause in its error annotation and job summary. The cause is read from the report kapi printed: the JSON or YAML key, or the "(cause)" that ends the text sentence. result is set for every check outcome; exit 0, 3 and other codes behave as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VRid8i4qnuNfE7Lio73Uu4 --- .github/workflows/test.yml | 145 +++++++++++++++++- README.md | 20 ++- action.yml | 95 +++++++++++- test/check-outcomes.sh | 114 ++++++++++++++ test/check-output/checker_invalid.json | 53 +++++++ test/check-output/checker_invalid.txt | 7 + .../content_not_checked.ship.json | 36 +++++ .../check-output/content_not_checked.ship.txt | 12 ++ test/check-output/failed.txt | 12 ++ test/check-output/future_cause.txt | 7 + test/check-output/nothing_to_check.color.txt | 7 + test/check-output/nothing_to_check.json | 111 ++++++++++++++ test/check-output/nothing_to_check.txt | 7 + test/check-output/nothing_to_check.yaml | 83 ++++++++++ test/check-output/passed.txt | 5 + test/check-output/unreadable_cause.json | 15 ++ test/stub/kapi | 14 ++ 17 files changed, 730 insertions(+), 13 deletions(-) create mode 100755 test/check-outcomes.sh create mode 100644 test/check-output/checker_invalid.json create mode 100644 test/check-output/checker_invalid.txt create mode 100644 test/check-output/content_not_checked.ship.json create mode 100644 test/check-output/content_not_checked.ship.txt create mode 100644 test/check-output/failed.txt create mode 100644 test/check-output/future_cause.txt create mode 100644 test/check-output/nothing_to_check.color.txt create mode 100644 test/check-output/nothing_to_check.json create mode 100644 test/check-output/nothing_to_check.txt create mode 100644 test/check-output/nothing_to_check.yaml create mode 100644 test/check-output/passed.txt create mode 100644 test/check-output/unreadable_cause.json create mode 100755 test/stub/kapi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c4bd3a..a2b98ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,10 @@ name: Test # and check need no provider or server credentials. The CLI version is pinned # to the first release that ships `kapi up` / `check --ship`; bump # deliberately. +# +# The check-outcome jobs put a kapi stub (test/stub/kapi) on PATH instead, so +# each exit code kapi check can end with, and each did-not-run cause, is +# exercised whatever the pinned release can produce. on: push: @@ -72,10 +76,12 @@ jobs: env: STEP_OUTCOME: ${{ steps.gate.outcome }} GATE: ${{ steps.gate.outputs.gate }} + RESULT: ${{ steps.gate.outputs.result }} run: | - echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}" + echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}, result: ${RESULT}" test "${STEP_OUTCOME}" = "failure" test "${GATE}" = "fail" + test "${RESULT}" = "failed" test-check-file: name: "check (file-mode pass)" @@ -98,7 +104,142 @@ jobs: - name: Verify pass env: GATE: ${{ steps.check.outputs.gate }} - run: test "${GATE}" = "pass" + RESULT: ${{ steps.check.outputs.result }} + run: | + test "${GATE}" = "pass" + test "${RESULT}" = "passed" + + test-check-outcomes: + name: "check outcomes through the action (kapi stub)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Put the kapi stub on PATH + run: echo "${GITHUB_WORKSPACE}/test/stub" >> "$GITHUB_PATH" + + # One run of the action per outcome. Every run but the first fails its + # step on purpose; the last step reads each one's outcome and outputs. + - name: "exit 0: passed" + id: passed + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/passed.txt + KAPI_STUB_EXIT: "0" + with: + command: check + + - name: "exit 3: failed" + id: failed + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/failed.txt + KAPI_STUB_EXIT: "3" + with: + command: check + args: "--ship" + + - name: "exit 4: checker_invalid" + id: checker_invalid + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/checker_invalid.txt + KAPI_STUB_EXIT: "4" + with: + command: check + + - name: "exit 4: checker_invalid (JSON)" + id: checker_invalid_json + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/checker_invalid.json + KAPI_STUB_EXIT: "4" + with: + command: check + args: "--json" + + - name: "exit 4: nothing_to_check" + id: nothing_to_check + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/nothing_to_check.txt + KAPI_STUB_EXIT: "4" + with: + command: check + + - name: "exit 4: content_not_checked" + id: content_not_checked + continue-on-error: true + uses: ./ + env: + KAPI_STUB_OUTPUT: test/check-output/content_not_checked.ship.txt + KAPI_STUB_EXIT: "4" + with: + command: check + args: "--ship --gate voice" + + - name: "exit 4: no cause in the output" + id: unknown + continue-on-error: true + uses: ./ + env: + KAPI_STUB_EXIT: "4" + with: + command: check + + - name: "exit 1: operational error" + id: error + continue-on-error: true + uses: ./ + env: + KAPI_STUB_EXIT: "1" + with: + command: check + + - name: Verify each outcome and its outputs + env: + STEPS: ${{ toJSON(steps) }} + run: | + status=0 + # expect STEP OUTCOME RESULT GATE CAUSE + expect() { + local got want + got="$(jq -r --arg id "$1" '.[$id] | [.outcome, (.outputs.result // ""), (.outputs.gate // ""), (.outputs.did_not_run_cause // "")] | join(" | ")' <<< "$STEPS")" + want="$2 | $3 | $4 | $5" + if [ "$got" = "$want" ]; then + echo "ok $1: $got" + else + echo "::error::$1: got '$got', want '$want'" + status=1 + fi + } + expect passed success passed pass "" + expect failed failure failed fail "" + expect checker_invalid failure did_not_run "" checker_invalid + expect checker_invalid_json failure did_not_run "" checker_invalid + expect nothing_to_check failure did_not_run "" nothing_to_check + expect content_not_checked failure did_not_run "" content_not_checked + expect unknown failure did_not_run "" unknown + expect error failure error "" "" + exit "$status" + + test-check-annotations: + name: "check outcomes: exit codes, outputs and annotations (kapi stub)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + # Runs the action's own step script once per outcome, including the + # did-not-run causes read from text, JSON, YAML and colored output. + - name: Run the check-outcome harness + run: | + yq --version + test/check-outcomes.sh test-run-keyless: name: "pseudo-translate (general runner)" diff --git a/README.md b/README.md index b0b334d..cb4bd08 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ jobs: ### Gate pull requests on content quality -`command: check` with `--ship` is the release bar: the project's bound quality gates (brand, terminology, QA) plus its ship/source coverage gates. An unmet gate exits `3`, which the Action surfaces as a distinct **"gate unmet"** annotation (not a generic failure) and as the `gate` output: +`command: check` with `--ship` is the release bar: the project's bound quality gates (brand, terminology, QA) plus its ship/source coverage gates. An unmet gate exits `3`, which the Action surfaces as a distinct **"gate unmet"** annotation (not a generic failure), as `gate: fail` and as `result: failed`: ```yaml name: Ship gate @@ -176,12 +176,24 @@ jobs: with: command: check args: "--ship" - commit: "false" pr-comment: "true" ``` Ordinary builds never fail on target-language drift — a locale that is behind is pending work, not an error. `check --ship` is the explicit, opt-in enforcement point. +#### When a check does not run + +A check can also end without a verdict. kapi exits `4` when the check did not run: it checked no content, or it could not show that its checkers are able to fail. The Action fails the step and reports this as its own result: `result` is `did_not_run`, `gate` stays empty, and `did_not_run_cause` carries the cause kapi named. The step's error annotation and the job summary state the cause in words. + +| `did_not_run_cause` | What it means | +|---|---| +| `checker_invalid` | A checker failed its canary, so the run's result cannot be trusted. Read it as neither a pass nor a gate failure, and fix or report the checker. | +| `nothing_to_check` | There was nothing in scope to check. | +| `content_not_checked` | Content in scope was not checked, for example a gate named with `--gate` that has nothing bound in the recipe, or a changed file whose blocks could not be located. | +| `unknown` | kapi's output named no cause. | + +The Action reads the cause from the report kapi printed, in whichever format `args` asks for (text, `--json`, or `--output-format yaml`), and passes your command line through unchanged. kapi releases that predate the did-not-run verdict never exit `4`; with them the step behaves as before, and `result` is `passed`, `failed`, or `error`. + ### Run any other kapi command `command` takes any kapi subcommand — the Action stays a general runner. The loop outputs (`outcome`, `passes`, `parked-locales`) are only populated for `up`. @@ -232,7 +244,9 @@ Server-connected projects don't need this — the project state lives on the ser | `outcome` | With `command: up`: `converged` or `parked` (a failed run fails the step, so it never reaches an output) | | `passes` | With `command: up`: how many reconciliation passes the run took | | `parked-locales` | With `command: up`: comma-separated locales still short of their gate | -| `gate` | With `command: check`: `pass` or `fail` | +| `gate` | With `command: check`: `pass` or `fail` (empty when the check did not run or errored) | +| `result` | With `command: check`: `passed` (exit 0), `failed` (exit 3), `did_not_run` (exit 4), or `error` (any other exit code) | +| `did_not_run_cause` | With `command: check`, when `result` is `did_not_run`: `checker_invalid`, `nothing_to_check`, `content_not_checked`, or `unknown` | | `plan-missing` / `plan-tm-exact` / `plan-ai-remaining` / `plan-token-estimate` | With `plan: true`: the plan totals | | `has-changes` | Whether the run left changes in the working tree for your delivery step | | `changed-files` | Newline-separated paths the run changed | diff --git a/action.yml b/action.yml index 87e4f29..9df978b 100644 --- a/action.yml +++ b/action.yml @@ -62,8 +62,18 @@ outputs: description: "With `command: up`: comma-separated locales still short of their gate" value: ${{ steps.run-kapi.outputs.parked-locales }} gate: - description: "With `command: check`: `pass` or `fail`" + description: "With `command: check`: `pass` or `fail` (empty when the check did not run or errored)" value: ${{ steps.run-kapi.outputs.gate }} + result: + description: "With `command: check`: `passed`, `failed`, `did_not_run`, or `error`" + value: ${{ steps.run-kapi.outputs.result }} + did_not_run_cause: + description: >- + With `command: check`, when `result` is `did_not_run`: `checker_invalid` + (a checker failed its canary, so the result cannot be trusted), + `nothing_to_check`, `content_not_checked`, or `unknown` when kapi's + output named no cause + value: ${{ steps.run-kapi.outputs.did_not_run_cause }} plan-missing: description: "With `plan: true`: pending units across all locales" value: ${{ steps.run-kapi.outputs.plan-missing }} @@ -140,22 +150,29 @@ runs: exit 0 fi - # ---------- check: the quality gate, with a distinct exit-3 signal ---------- + # ---------- check: the quality gate, with distinct exit-3 and exit-4 signals ---------- if [ "$COMMAND" = "check" ]; then printf 'Running:'; printf ' %q' "${cmd[@]}"; echo - # || rc=$? keeps the gate's exit code without tripping errexit — the - # annotation, summary, and outputs must still publish on a failing gate. + # || rc=$? keeps the check's exit code without tripping errexit, so the + # annotation, summary, and outputs still publish when it fails. + CHECK_OUTPUT="${RUNNER_TEMP}/kapi-check-output.txt" rc=0 - "${cmd[@]}" | tee "${RUNNER_TEMP}/kapi-check-output.txt" || rc=$? + "${cmd[@]}" | tee "$CHECK_OUTPUT" || rc=$? if [ "$rc" -eq 0 ]; then - echo "gate=pass" >> "$GITHUB_OUTPUT" + { + echo "gate=pass" + echo "result=passed" + } >> "$GITHUB_OUTPUT" { echo "## kapi check" echo echo "✅ **Gate: pass**" } >> "$GITHUB_STEP_SUMMARY" elif [ "$rc" -eq 3 ]; then - echo "gate=fail" >> "$GITHUB_OUTPUT" + { + echo "gate=fail" + echo "result=failed" + } >> "$GITHUB_OUTPUT" { echo "## kapi check" echo @@ -163,8 +180,70 @@ runs: } >> "$GITHUB_STEP_SUMMARY" echo "::error::kapi gate unmet (exit 3) — content quality gates failed; read the findings and fix them. This is 'not on-spec yet', not a crash." exit 3 + elif [ "$rc" -eq 4 ]; then + # Exit 4: the check did not run, so it reached no verdict. kapi names + # the cause in the report it printed: `did_not_run_cause` in JSON or + # YAML, and a text sentence ending in "()". The command line + # is the caller's, so read whichever format it produced, with color + # codes removed. Older kapi releases never exit 4. + PLAIN_OUTPUT="${RUNNER_TEMP}/kapi-check-output.plain.txt" + esc="$(printf '\033')" + sed "s/${esc}\[[0-9;]*m//g" "$CHECK_OUTPUT" > "$PLAIN_OUTPUT" + CAUSE="" + if jq -e 'type == "object"' "$PLAIN_OUTPUT" > /dev/null 2>&1; then + CAUSE="$(jq -r '.did_not_run_cause // empty | strings' "$PLAIN_OUTPUT")" + fi + if [ -z "$CAUSE" ]; then + CAUSE="$(sed -n -E \ + -e 's/^did_not_run_cause:[[:space:]]*"?([a-z_]+)"?[[:space:]]*$/\1/p' \ + -e 's/.*Did not run: .*\(([a-z_]+)\)[[:space:]]*$/\1/p' \ + "$PLAIN_OUTPUT" | tail -n 1)" + fi + case "$CAUSE" in + "" | *[!a-z_]*) CAUSE="unknown" ;; + esac + case "$CAUSE" in + checker_invalid) + SUMMARY="a checker failed its canary, so this run's result cannot be trusted" + ADVICE="Read it as neither a pass nor a gate failure: fix or report the checker the log names, then re-run." + ;; + nothing_to_check) + SUMMARY="there was nothing in scope to check" + ADVICE="No content reached the check. Confirm the recipe's content paths or the files passed in args." + ;; + content_not_checked) + SUMMARY="content in scope was not checked" + ADVICE="A requested gate or analyzer had nothing bound, or a changed file could not be checked. The log names which." + ;; + unknown) + SUMMARY="kapi's output named no cause" + ADVICE="Read the job log for the reason." + ;; + *) + SUMMARY="kapi gave a cause this version of the action does not describe" + ADVICE="Read the job log for the reason." + ;; + esac + { + echo "result=did_not_run" + echo "did_not_run_cause=${CAUSE}" + } >> "$GITHUB_OUTPUT" + ICON="⚠️" + if [ "$CAUSE" = "checker_invalid" ]; then + ICON="❌" + fi + { + echo "## kapi check" + echo + echo "${ICON} **Did not run** (\`${CAUSE}\`): ${SUMMARY}. ${ADVICE}" + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::kapi check did not run (exit 4, ${CAUSE}): ${SUMMARY}. ${ADVICE}" + exit 4 else - echo "run_status=failed" >> "$GITHUB_OUTPUT" + { + echo "run_status=failed" + echo "result=error" + } >> "$GITHUB_OUTPUT" echo "::error::kapi check failed operationally (exit ${rc})" exit "$rc" fi diff --git a/test/check-outcomes.sh b/test/check-outcomes.sh new file mode 100755 index 0000000..0401dfe --- /dev/null +++ b/test/check-outcomes.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Runs the action's "Run kapi" step with `command: check` against the kapi stub +# in test/stub, once per check outcome, and asserts the step's exit code, its +# outputs and its error annotation. The step script is read out of action.yml, +# so the code under test is the code the action ships. Needs yq (v4) and jq. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(dirname "$here")" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +yq '.runs.steps[] | select(.id == "run-kapi") | .run' "$root/action.yml" > "$work/step.sh" +if ! grep -q 'kapi-check-output.txt' "$work/step.sh"; then + echo "could not read the run-kapi step out of action.yml" >&2 + exit 1 +fi + +failures=0 + +# output_value KEY FILE prints the last value the step wrote for KEY. +output_value() { + sed -n "s/^$1=//p" "$2" | tail -n 1 +} + +fail() { + echo "FAIL $1: $2" + failures=$((failures + 1)) +} + +# run_case NAME ARGS OUTPUT EXIT WANT_RC WANT_RESULT WANT_GATE WANT_CAUSE WANT_ERROR +# +# OUTPUT names a file in test/check-output for the stub to print (empty prints +# nothing). WANT_ERROR is a substring the step's ::error:: line must contain; +# empty means the step must print no ::error:: line. +run_case() { + local name="$1" args="$2" output="$3" code="$4" want_rc="$5" + local want_result="$6" want_gate="$7" want_cause="$8" want_error="$9" + local dir="$work/cases/$name" + mkdir -p "$dir/tmp" + : > "$dir/output" + : > "$dir/summary" + + local stub_output="" + if [ -n "$output" ]; then + stub_output="$here/check-output/$output" + fi + + local rc=0 + env \ + PATH="$here/stub:$PATH" \ + COMMAND=check ARGS="$args" PROJECT="" PLAN=false FAIL_ON_PARKED=false \ + RUNNER_TEMP="$dir/tmp" GITHUB_OUTPUT="$dir/output" GITHUB_STEP_SUMMARY="$dir/summary" \ + KAPI_STUB_OUTPUT="$stub_output" KAPI_STUB_EXIT="$code" KAPI_STUB_ARGV="$dir/argv" \ + bash "$work/step.sh" > "$dir/stdout" 2>&1 || rc=$? + + local before="$failures" + [ "$rc" = "$want_rc" ] || fail "$name" "exit code: got '$rc', want '$want_rc'" + + local got + got="$(output_value result "$dir/output")" + [ "$got" = "$want_result" ] || fail "$name" "result: got '$got', want '$want_result'" + got="$(output_value gate "$dir/output")" + [ "$got" = "$want_gate" ] || fail "$name" "gate: got '$got', want '$want_gate'" + got="$(output_value did_not_run_cause "$dir/output")" + [ "$got" = "$want_cause" ] || fail "$name" "did_not_run_cause: got '$got', want '$want_cause'" + + local error_line + error_line="$(grep '^::error::' "$dir/stdout" || true)" + if [ -z "$want_error" ]; then + [ -z "$error_line" ] || fail "$name" "unexpected annotation: $error_line" + else + case "$error_line" in + *"$want_error"*) ;; + *) fail "$name" "annotation: got '$error_line', want it to contain '$want_error'" ;; + esac + fi + + # The action passes the caller's command line through untouched. + local want_argv + # shellcheck disable=SC2086 # split ARGS into words the way the step does + want_argv="$(printf '%s\n' check $args)" + got="$(cat "$dir/argv")" + [ "$got" = "$want_argv" ] || fail "$name" "kapi argv: got '$(tr '\n' ' ' <<< "$got")', want '$(tr '\n' ' ' <<< "$want_argv")'" + + if [ "$failures" = "$before" ]; then + echo "ok $name" + else + sed 's/^/ | /' "$dir/stdout" + fi +} + +# name args output exit rc result gate cause annotation +run_case passed "content/en.json" passed.txt 0 0 passed pass "" "" +run_case failed "--ship" failed.txt 3 3 failed fail "" "kapi gate unmet (exit 3)" +run_case checker_invalid-text "content/en.json" checker_invalid.txt 4 4 did_not_run "" checker_invalid "kapi check did not run (exit 4, checker_invalid): a checker failed its canary, so this run's result cannot be trusted." +run_case checker_invalid-json "content/en.json --json" checker_invalid.json 4 4 did_not_run "" checker_invalid "this run's result cannot be trusted" +run_case nothing_to_check-text "empty.json" nothing_to_check.txt 4 4 did_not_run "" nothing_to_check "kapi check did not run (exit 4, nothing_to_check): there was nothing in scope to check." +run_case nothing_to_check-json "empty.json --output-format json" nothing_to_check.json 4 4 did_not_run "" nothing_to_check "there was nothing in scope to check" +run_case nothing_to_check-yaml "empty.json --output-format yaml" nothing_to_check.yaml 4 4 did_not_run "" nothing_to_check "there was nothing in scope to check" +run_case nothing_to_check-color "empty.json --color always" nothing_to_check.color.txt 4 4 did_not_run "" nothing_to_check "there was nothing in scope to check" +run_case content_not_checked-ship "--ship --gate voice" content_not_checked.ship.txt 4 4 did_not_run "" content_not_checked "kapi check did not run (exit 4, content_not_checked): content in scope was not checked." +run_case content_not_checked-ship-json "--ship --gate voice --json" content_not_checked.ship.json 4 4 did_not_run "" content_not_checked "content in scope was not checked" +run_case future-cause "content/en.json" future_cause.txt 4 4 did_not_run "" some_new_cause "(exit 4, some_new_cause): kapi gave a cause this version of the action does not describe." +run_case unreadable-cause-json "content/en.json --json" unreadable_cause.json 4 4 did_not_run "" unknown "(exit 4, unknown): kapi's output named no cause." +run_case no-cause "content/en.json" "" 4 4 did_not_run "" unknown "(exit 4, unknown): kapi's output named no cause." +run_case operational-error "content/en.json" "" 1 1 error "" "" "kapi check failed operationally (exit 1)" +run_case usage-error "--no-such-flag" "" 2 2 error "" "" "kapi check failed operationally (exit 2)" + +if [ "$failures" -gt 0 ]; then + echo "$failures assertion(s) failed" + exit 1 +fi +echo "all check outcomes passed" diff --git a/test/check-output/checker_invalid.json b/test/check-output/checker_invalid.json new file mode 100644 index 0000000..4a695b3 --- /dev/null +++ b/test/check-output/checker_invalid.json @@ -0,0 +1,53 @@ +{ + "schema": "kapi.check/v1", + "pass": false, + "verdict": "did_not_run", + "did_not_run": [ + "hygiene reported no finding on its canary on content/en.json, so its result cannot be trusted" + ], + "did_not_run_cause": "checker_invalid", + "target": { + "kind": "file", + "file": "content/en.json", + "blocks": 3 + }, + "summary": { + "findings": 0, + "critical": 0, + "major": 0, + "minor": 0, + "neutral": 0, + "score": 100 + }, + "gate": { + "max_critical": 0, + "max_major": -1, + "max_minor": -1, + "min_score": 0, + "failed": [] + }, + "findings": [], + "execution": { + "analyzers": [ + { + "id": "hygiene", + "status": "invalid", + "file": "content/en.json", + "required": true, + "findings": 0, + "duration_ms": 0.062791, + "canary": { + "status": "missed", + "probes": 1 + } + } + ], + "timings": { + "context_ms": 0.004375, + "extraction_ms": 0.0975, + "analyzers_ms": 0.062791, + "report_ms": 0.001334, + "total_ms": 0.189416 + } + } +} diff --git a/test/check-output/checker_invalid.txt b/test/check-output/checker_invalid.txt new file mode 100644 index 0000000..b24d2a6 --- /dev/null +++ b/test/check-output/checker_invalid.txt @@ -0,0 +1,7 @@ + No findings. + +DID NOT RUN configured checks: score 100/100 · 0 finding(s) (0 critical, 0 major, 0 minor) + Did not run: a checker failed its canary, so this run's result cannot be trusted. (checker_invalid) + Coverage: 0 completed, 6 not run or unsupported, 1 missed a canary. Score covers reported findings only. + did not run: hygiene reported no finding on its canary on content/en.json, so its result cannot be trusted + Context: content/en.json; voice none (none); terms none loaded diff --git a/test/check-output/content_not_checked.ship.json b/test/check-output/content_not_checked.ship.json new file mode 100644 index 0000000..848b212 --- /dev/null +++ b/test/check-output/content_not_checked.ship.json @@ -0,0 +1,36 @@ +{ + "pass": false, + "verdict": "did_not_run", + "did_not_run": [ + "voice: --gate voice was requested but the project binds no defaults.voice" + ], + "did_not_run_cause": "content_not_checked", + "gates": [ + { + "gate": "voice", + "pass": false, + "verdict": "did_not_run", + "did_not_run": [ + "--gate voice was requested but the project binds no defaults.voice" + ], + "did_not_run_cause": "content_not_checked", + "findings": [ + { + "gate": "voice", + "severity": "error", + "message": "voice gate was requested with --gate voice but the project binds no defaults.voice, so there is nothing to check", + "suggestion": "add defaults.voice to the kapi.yaml recipe, or drop --gate voice to skip this gate" + } + ] + } + ], + "summary": { + "gates": 1, + "passed": 0, + "failed": 0, + "did_not_run": 1, + "findings": 1, + "errors": 1, + "warnings": 0 + } +} diff --git a/test/check-output/content_not_checked.ship.txt b/test/check-output/content_not_checked.ship.txt new file mode 100644 index 0000000..4849a02 --- /dev/null +++ b/test/check-output/content_not_checked.ship.txt @@ -0,0 +1,12 @@ + gate result findings content checked + voice DID NOT RUN 1 + +voice: + + severity location message + ERROR — voice gate was requested with --gate voice but the project binds no defaults.voice, so there is nothing to check + ↳ add defaults.voice to the kapi.yaml recipe, or drop --gate voice to skip this gate + +DID NOT RUN: 1 gate(s), 0 passed, 0 failed, 1 did not run, 1 finding(s) (1 error, 0 warning) + Did not run: content in scope was not checked. (content_not_checked) + did not run: voice: --gate voice was requested but the project binds no defaults.voice diff --git a/test/check-output/failed.txt b/test/check-output/failed.txt new file mode 100644 index 0000000..af82e97 --- /dev/null +++ b/test/check-output/failed.txt @@ -0,0 +1,12 @@ + gate result findings content checked + checks FAIL 2 0 file(s), 0 block(s) + +checks: + + severity location message + ERROR content/de.json [de] target file is missing, so the content is untranslated + ↳ translate the source content for this locale + ERROR content/fr.json [fr] target file is missing, so the content is untranslated + ↳ translate the source content for this locale + +FAIL: 1 gate(s), 0 passed, 1 failed, 0 did not run, 2 finding(s) (2 error, 0 warning) diff --git a/test/check-output/future_cause.txt b/test/check-output/future_cause.txt new file mode 100644 index 0000000..706408a --- /dev/null +++ b/test/check-output/future_cause.txt @@ -0,0 +1,7 @@ + No findings. + +DID NOT RUN configured checks: score 100/100 · 0 finding(s) (0 critical, 0 major, 0 minor) + Did not run: a cause a later kapi release introduced. (some_new_cause) + Coverage: 1 completed, 6 not run or unsupported. Score covers reported findings only. + did not run: a reason a later kapi release gives + Context: content/en.json; voice none (none); terms none loaded diff --git a/test/check-output/nothing_to_check.color.txt b/test/check-output/nothing_to_check.color.txt new file mode 100644 index 0000000..8d0fd9d --- /dev/null +++ b/test/check-output/nothing_to_check.color.txt @@ -0,0 +1,7 @@ + No findings. + +DID NOT RUN configured checks: score 100/100 · 0 finding(s) (0 critical, 0 major, 0 minor) + Did not run: there was nothing in scope to check. (nothing_to_check) + Coverage: 1 completed, 6 not run or unsupported. Score covers reported findings only. + did not run: no content blocks were checked + Context: content/empty.json; voice none (none); terms none loaded diff --git a/test/check-output/nothing_to_check.json b/test/check-output/nothing_to_check.json new file mode 100644 index 0000000..1ccd8c3 --- /dev/null +++ b/test/check-output/nothing_to_check.json @@ -0,0 +1,111 @@ +{ + "schema": "kapi.check/v1", + "pass": false, + "verdict": "did_not_run", + "did_not_run": [ + "no content blocks were checked" + ], + "did_not_run_cause": "nothing_to_check", + "target": { + "kind": "file", + "file": "content/empty.json", + "blocks": 0 + }, + "summary": { + "findings": 0, + "critical": 0, + "major": 0, + "minor": 0, + "neutral": 0, + "score": 100 + }, + "gate": { + "max_critical": 0, + "max_major": -1, + "max_minor": -1, + "min_score": 0, + "failed": [] + }, + "findings": [], + "execution": { + "analyzers": [ + { + "id": "reader.validation", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "Reader validation was not requested." + }, + { + "id": "hygiene", + "status": "passed", + "file": "content/empty.json", + "required": true, + "findings": 0, + "duration_ms": 0.056083, + "canary": { + "status": "caught", + "probes": 1 + } + }, + { + "id": "length", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "No length limit was configured." + }, + { + "id": "pattern", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "No explicit patterns were configured." + }, + { + "id": "voice.rules", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "No voice profile or project terms were bound." + }, + { + "id": "voice.similarity", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "Similarity analysis was not requested." + }, + { + "id": "voice.llm", + "status": "not_requested", + "file": "content/empty.json", + "required": false, + "findings": 0, + "reason": "This command does not run semantic review." + } + ], + "timings": { + "context_ms": 0.004374, + "extraction_ms": 0.105916, + "analyzers_ms": 0.056083, + "report_ms": 0.001042, + "total_ms": 0.195875 + }, + "contexts": [ + { + "file": "content/empty.json", + "voice": { + "selection": "none", + "applied": false + }, + "terms_applied": false + } + ] + } +} diff --git a/test/check-output/nothing_to_check.txt b/test/check-output/nothing_to_check.txt new file mode 100644 index 0000000..dd908d1 --- /dev/null +++ b/test/check-output/nothing_to_check.txt @@ -0,0 +1,7 @@ + No findings. + +DID NOT RUN configured checks: score 100/100 · 0 finding(s) (0 critical, 0 major, 0 minor) + Did not run: there was nothing in scope to check. (nothing_to_check) + Coverage: 1 completed, 6 not run or unsupported. Score covers reported findings only. + did not run: no content blocks were checked + Context: content/empty.json; voice none (none); terms none loaded diff --git a/test/check-output/nothing_to_check.yaml b/test/check-output/nothing_to_check.yaml new file mode 100644 index 0000000..a474402 --- /dev/null +++ b/test/check-output/nothing_to_check.yaml @@ -0,0 +1,83 @@ +did_not_run: + - no content blocks were checked +did_not_run_cause: nothing_to_check +execution: + analyzers: + - file: content/empty.json + findings: 0 + id: reader.validation + reason: Reader validation was not requested. + required: false + status: not_requested + - canary: + probes: 1 + status: caught + duration_ms: 0.045791 + file: content/empty.json + findings: 0 + id: hygiene + required: true + status: passed + - file: content/empty.json + findings: 0 + id: length + reason: No length limit was configured. + required: false + status: not_requested + - file: content/empty.json + findings: 0 + id: pattern + reason: No explicit patterns were configured. + required: false + status: not_requested + - file: content/empty.json + findings: 0 + id: voice.rules + reason: No voice profile or project terms were bound. + required: false + status: not_requested + - file: content/empty.json + findings: 0 + id: voice.similarity + reason: Similarity analysis was not requested. + required: false + status: not_requested + - file: content/empty.json + findings: 0 + id: voice.llm + reason: This command does not run semantic review. + required: false + status: not_requested + contexts: + - file: content/empty.json + terms_applied: false + voice: + applied: false + selection: none + timings: + analyzers_ms: 0.045791 + context_ms: 0.004291 + extraction_ms: 0.086792 + report_ms: 0.001125 + total_ms: 0.1615 +findings: [] +gate: + failed: [] + max_critical: 0 + max_major: -1 + max_minor: -1 + min_score: 0 +pass: false +schema: kapi.check/v1 +summary: + critical: 0 + findings: 0 + major: 0 + minor: 0 + neutral: 0 + score: 100 +target: + blocks: 0 + file: content/empty.json + kind: file +verdict: did_not_run diff --git a/test/check-output/passed.txt b/test/check-output/passed.txt new file mode 100644 index 0000000..da10642 --- /dev/null +++ b/test/check-output/passed.txt @@ -0,0 +1,5 @@ + No findings. + +PASS configured checks: score 100/100 · 0 finding(s) (0 critical, 0 major, 0 minor) + Coverage: 1 completed, 6 not run or unsupported. Score covers reported findings only. + Context: content/en.json; voice none (none); terms none loaded diff --git a/test/check-output/unreadable_cause.json b/test/check-output/unreadable_cause.json new file mode 100644 index 0000000..f14ed2e --- /dev/null +++ b/test/check-output/unreadable_cause.json @@ -0,0 +1,15 @@ +{ + "schema": "kapi.check/v1", + "pass": false, + "verdict": "did_not_run", + "did_not_run": [ + "a cause that is not a cause code, including a line break that would write a second output" + ], + "did_not_run_cause": "Nothing To Check\nresult=passed", + "target": { + "kind": "file", + "file": "content/en.json", + "blocks": 0 + }, + "findings": [] +} diff --git a/test/stub/kapi b/test/stub/kapi new file mode 100755 index 0000000..2ba7a5d --- /dev/null +++ b/test/stub/kapi @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# A stand-in for the kapi CLI, for testing how the action maps kapi's exit code +# and output to its own outputs. It prints the file named by KAPI_STUB_OUTPUT +# and exits with KAPI_STUB_EXIT. When KAPI_STUB_ARGV is set, it records the +# arguments it was called with there, one per line. +set -euo pipefail + +if [ -n "${KAPI_STUB_ARGV:-}" ]; then + printf '%s\n' "$@" > "$KAPI_STUB_ARGV" +fi +if [ -n "${KAPI_STUB_OUTPUT:-}" ]; then + cat "$KAPI_STUB_OUTPUT" +fi +exit "${KAPI_STUB_EXIT:-0}" From 9e7317a4eb6badeadc3b2e62c5d3508b09d7497c Mon Sep 17 00:00:00 2001 From: Asgeir Frimannsson Date: Sun, 13 Sep 2026 17:07:24 +0200 Subject: [PATCH 2/3] test: send exit 4 to the generic error to prove the tests fail Temporary, reverted in the next commit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VRid8i4qnuNfE7Lio73Uu4 --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 9df978b..2549fa8 100644 --- a/action.yml +++ b/action.yml @@ -180,7 +180,7 @@ runs: } >> "$GITHUB_STEP_SUMMARY" echo "::error::kapi gate unmet (exit 3) — content quality gates failed; read the findings and fix them. This is 'not on-spec yet', not a crash." exit 3 - elif [ "$rc" -eq 4 ]; then + elif [ "$rc" -eq 999 ]; then # Exit 4: the check did not run, so it reached no verdict. kapi names # the cause in the report it printed: `did_not_run_cause` in JSON or # YAML, and a text sentence ending in "()". The command line From d90b2ebc5e8c5ed1afb801b5fcd96ae5c7039dad Mon Sep 17 00:00:00 2001 From: Asgeir Frimannsson Date: Sun, 13 Sep 2026 17:08:32 +0200 Subject: [PATCH 3/3] test: restore the exit-4 branch after the must-fail proof Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VRid8i4qnuNfE7Lio73Uu4 --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 2549fa8..9df978b 100644 --- a/action.yml +++ b/action.yml @@ -180,7 +180,7 @@ runs: } >> "$GITHUB_STEP_SUMMARY" echo "::error::kapi gate unmet (exit 3) — content quality gates failed; read the findings and fix them. This is 'not on-spec yet', not a crash." exit 3 - elif [ "$rc" -eq 999 ]; then + elif [ "$rc" -eq 4 ]; then # Exit 4: the check did not run, so it reached no verdict. kapi names # the cause in the report it printed: `did_not_run_cause` in JSON or # YAML, and a text sentence ending in "()". The command line