feat: report did-not-run checks as their own result

.github/workflows/test.yml

@@ -4,6 +4,10 @@ name: Test
 # and check need no provider or server credentials. The CLI version is pinned
 # to the first release that ships `kapi up` / `check --ship`; bump
 # deliberately.
+#
+# The check-outcome jobs put a kapi stub (test/stub/kapi) on PATH instead, so
+# each exit code kapi check can end with, and each did-not-run cause, is
+# exercised whatever the pinned release can produce.
 
 on:
   push:
@@ -72,10 +76,12 @@ jobs:
         env:
           STEP_OUTCOME: ${{ steps.gate.outcome }}
           GATE: ${{ steps.gate.outputs.gate }}
+          RESULT: ${{ steps.gate.outputs.result }}
         run: |
-          echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}"
+          echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}, result: ${RESULT}"
           test "${STEP_OUTCOME}" = "failure"
           test "${GATE}" = "fail"
+          test "${RESULT}" = "failed"
 
   test-check-file:
     name: "check <files> (file-mode pass)"
@@ -98,7 +104,142 @@ jobs:
       - name: Verify pass
         env:
           GATE: ${{ steps.check.outputs.gate }}
-        run: test "${GATE}" = "pass"
+          RESULT: ${{ steps.check.outputs.result }}
+        run: |
+          test "${GATE}" = "pass"
+          test "${RESULT}" = "passed"
+
+  test-check-outcomes:
+    name: "check outcomes through the action (kapi stub)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Put the kapi stub on PATH
+        run: echo "${GITHUB_WORKSPACE}/test/stub" >> "$GITHUB_PATH"
+
+      # One run of the action per outcome. Every run but the first fails its
+      # step on purpose; the last step reads each one's outcome and outputs.
+      - name: "exit 0: passed"
+        id: passed
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/passed.txt
+          KAPI_STUB_EXIT: "0"
+        with:
+          command: check
+
+      - name: "exit 3: failed"
+        id: failed
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/failed.txt
+          KAPI_STUB_EXIT: "3"
+        with:
+          command: check
+          args: "--ship"
+
+      - name: "exit 4: checker_invalid"
+        id: checker_invalid
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/checker_invalid.txt
+          KAPI_STUB_EXIT: "4"
+        with:
+          command: check
+
+      - name: "exit 4: checker_invalid (JSON)"
+        id: checker_invalid_json
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/checker_invalid.json
+          KAPI_STUB_EXIT: "4"
+        with:
+          command: check
+          args: "--json"
+
+      - name: "exit 4: nothing_to_check"
+        id: nothing_to_check
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/nothing_to_check.txt
+          KAPI_STUB_EXIT: "4"
+        with:
+          command: check
+
+      - name: "exit 4: content_not_checked"
+        id: content_not_checked
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_OUTPUT: test/check-output/content_not_checked.ship.txt
+          KAPI_STUB_EXIT: "4"
+        with:
+          command: check
+          args: "--ship --gate voice"
+
+      - name: "exit 4: no cause in the output"
+        id: unknown
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_EXIT: "4"
+        with:
+          command: check
+
+      - name: "exit 1: operational error"
+        id: error
+        continue-on-error: true
+        uses: ./
+        env:
+          KAPI_STUB_EXIT: "1"
+        with:
+          command: check
+
+      - name: Verify each outcome and its outputs
+        env:
+          STEPS: ${{ toJSON(steps) }}
+        run: |
+          status=0
+          # expect STEP OUTCOME RESULT GATE CAUSE
+          expect() {
+            local got want
+            got="$(jq -r --arg id "$1" '.[$id] | [.outcome, (.outputs.result // ""), (.outputs.gate // ""), (.outputs.did_not_run_cause // "")] | join(" | ")' <<< "$STEPS")"
+            want="$2 | $3 | $4 | $5"
+            if [ "$got" = "$want" ]; then
+              echo "ok   $1: $got"
+            else
+              echo "::error::$1: got '$got', want '$want'"
+              status=1
+            fi
+          }
+          expect passed               success passed      pass ""
+          expect failed               failure failed      fail ""
+          expect checker_invalid      failure did_not_run ""   checker_invalid
+          expect checker_invalid_json failure did_not_run ""   checker_invalid
+          expect nothing_to_check     failure did_not_run ""   nothing_to_check
+          expect content_not_checked  failure did_not_run ""   content_not_checked
+          expect unknown              failure did_not_run ""   unknown
+          expect error                failure error       ""   ""
+          exit "$status"
+
+  test-check-annotations:
+    name: "check outcomes: exit codes, outputs and annotations (kapi stub)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      # Runs the action's own step script once per outcome, including the
+      # did-not-run causes read from text, JSON, YAML and colored output.
+      - name: Run the check-outcome harness
+        run: |
+          yq --version
+          test/check-outcomes.sh
 
   test-run-keyless:
     name: "pseudo-translate (general runner)"

README.md

@@ -155,7 +155,7 @@ jobs:
 
 ### Gate pull requests on content quality
 
-`command: check` with `--ship` is the release bar: the project's bound quality gates (brand, terminology, QA) plus its ship/source coverage gates. An unmet gate exits `3`, which the Action surfaces as a distinct **"gate unmet"** annotation (not a generic failure) and as the `gate` output:
+`command: check` with `--ship` is the release bar: the project's bound quality gates (brand, terminology, QA) plus its ship/source coverage gates. An unmet gate exits `3`, which the Action surfaces as a distinct **"gate unmet"** annotation (not a generic failure), as `gate: fail` and as `result: failed`:
 
 ```yaml
 name: Ship gate
@@ -176,12 +176,24 @@ jobs:
         with:
           command: check
           args: "--ship"
-          commit: "false"
           pr-comment: "true"
 ```
 
 Ordinary builds never fail on target-language drift — a locale that is behind is pending work, not an error. `check --ship` is the explicit, opt-in enforcement point.
 
+#### When a check does not run
+
+A check can also end without a verdict. kapi exits `4` when the check did not run: it checked no content, or it could not show that its checkers are able to fail. The Action fails the step and reports this as its own result: `result` is `did_not_run`, `gate` stays empty, and `did_not_run_cause` carries the cause kapi named. The step's error annotation and the job summary state the cause in words.
+
+| `did_not_run_cause` | What it means |
+|---|---|
+| `checker_invalid` | A checker failed its canary, so the run's result cannot be trusted. Read it as neither a pass nor a gate failure, and fix or report the checker. |
+| `nothing_to_check` | There was nothing in scope to check. |
+| `content_not_checked` | Content in scope was not checked, for example a gate named with `--gate` that has nothing bound in the recipe, or a changed file whose blocks could not be located. |
+| `unknown` | kapi's output named no cause. |
+
+The Action reads the cause from the report kapi printed, in whichever format `args` asks for (text, `--json`, or `--output-format yaml`), and passes your command line through unchanged. kapi releases that predate the did-not-run verdict never exit `4`; with them the step behaves as before, and `result` is `passed`, `failed`, or `error`.
+
 ### Run any other kapi command
 
 `command` takes any kapi subcommand — the Action stays a general runner. The loop outputs (`outcome`, `passes`, `parked-locales`) are only populated for `up`.
@@ -232,7 +244,9 @@ Server-connected projects don't need this — the project state lives on the ser
 | `outcome` | With `command: up`: `converged` or `parked` (a failed run fails the step, so it never reaches an output) |
 | `passes` | With `command: up`: how many reconciliation passes the run took |
 | `parked-locales` | With `command: up`: comma-separated locales still short of their gate |
-| `gate` | With `command: check`: `pass` or `fail` |
+| `gate` | With `command: check`: `pass` or `fail` (empty when the check did not run or errored) |
+| `result` | With `command: check`: `passed` (exit 0), `failed` (exit 3), `did_not_run` (exit 4), or `error` (any other exit code) |
+| `did_not_run_cause` | With `command: check`, when `result` is `did_not_run`: `checker_invalid`, `nothing_to_check`, `content_not_checked`, or `unknown` |
 | `plan-missing` / `plan-tm-exact` / `plan-ai-remaining` / `plan-token-estimate` | With `plan: true`: the plan totals |
 | `has-changes` | Whether the run left changes in the working tree for your delivery step |
 | `changed-files` | Newline-separated paths the run changed |

action.yml

@@ -62,8 +62,18 @@ outputs:
     description: "With `command: up`: comma-separated locales still short of their gate"
     value: ${{ steps.run-kapi.outputs.parked-locales }}
   gate:
-    description: "With `command: check`: `pass` or `fail`"
+    description: "With `command: check`: `pass` or `fail` (empty when the check did not run or errored)"
     value: ${{ steps.run-kapi.outputs.gate }}
+  result:
+    description: "With `command: check`: `passed`, `failed`, `did_not_run`, or `error`"
+    value: ${{ steps.run-kapi.outputs.result }}
+  did_not_run_cause:
+    description: >-
+      With `command: check`, when `result` is `did_not_run`: `checker_invalid`
+      (a checker failed its canary, so the result cannot be trusted),
+      `nothing_to_check`, `content_not_checked`, or `unknown` when kapi's
+      output named no cause
+    value: ${{ steps.run-kapi.outputs.did_not_run_cause }}
   plan-missing:
     description: "With `plan: true`: pending units across all locales"
     value: ${{ steps.run-kapi.outputs.plan-missing }}
@@ -140,31 +150,100 @@ runs:
           exit 0
         fi
 
-        # ---------- check: the quality gate, with a distinct exit-3 signal ----------
+        # ---------- check: the quality gate, with distinct exit-3 and exit-4 signals ----------
         if [ "$COMMAND" = "check" ]; then
           printf 'Running:'; printf ' %q' "${cmd[@]}"; echo
-          # || rc=$? keeps the gate's exit code without tripping errexit — the
-          # annotation, summary, and outputs must still publish on a failing gate.
+          # || rc=$? keeps the check's exit code without tripping errexit, so the
+          # annotation, summary, and outputs still publish when it fails.
+          CHECK_OUTPUT="${RUNNER_TEMP}/kapi-check-output.txt"
           rc=0
-          "${cmd[@]}" | tee "${RUNNER_TEMP}/kapi-check-output.txt" || rc=$?
+          "${cmd[@]}" | tee "$CHECK_OUTPUT" || rc=$?
           if [ "$rc" -eq 0 ]; then
-            echo "gate=pass" >> "$GITHUB_OUTPUT"
+            {
+              echo "gate=pass"
+              echo "result=passed"
+            } >> "$GITHUB_OUTPUT"
             {
               echo "## kapi check"
               echo
               echo "✅ **Gate: pass**"
             } >> "$GITHUB_STEP_SUMMARY"
           elif [ "$rc" -eq 3 ]; then
-            echo "gate=fail" >> "$GITHUB_OUTPUT"
+            {
+              echo "gate=fail"
+              echo "result=failed"
+            } >> "$GITHUB_OUTPUT"
             {
               echo "## kapi check"
               echo
               echo "❌ **Gate: unmet** — a brand, terminology, QA, or coverage gate failed. See the findings in the job log."
             } >> "$GITHUB_STEP_SUMMARY"
             echo "::error::kapi gate unmet (exit 3) — content quality gates failed; read the findings and fix them. This is 'not on-spec yet', not a crash."
             exit 3
+          elif [ "$rc" -eq 4 ]; then
+            # Exit 4: the check did not run, so it reached no verdict. kapi names
+            # the cause in the report it printed: `did_not_run_cause` in JSON or
+            # YAML, and a text sentence ending in "(<cause>)". The command line
+            # is the caller's, so read whichever format it produced, with color
+            # codes removed. Older kapi releases never exit 4.
+            PLAIN_OUTPUT="${RUNNER_TEMP}/kapi-check-output.plain.txt"
+            esc="$(printf '\033')"
+            sed "s/${esc}\[[0-9;]*m//g" "$CHECK_OUTPUT" > "$PLAIN_OUTPUT"
+            CAUSE=""
+            if jq -e 'type == "object"' "$PLAIN_OUTPUT" > /dev/null 2>&1; then
+              CAUSE="$(jq -r '.did_not_run_cause // empty | strings' "$PLAIN_OUTPUT")"
+            fi
+            if [ -z "$CAUSE" ]; then
+              CAUSE="$(sed -n -E \
+                -e 's/^did_not_run_cause:[[:space:]]*"?([a-z_]+)"?[[:space:]]*$/\1/p' \
+                -e 's/.*Did not run: .*\(([a-z_]+)\)[[:space:]]*$/\1/p' \
+                "$PLAIN_OUTPUT" | tail -n 1)"
+            fi
+            case "$CAUSE" in
+              "" | *[!a-z_]*) CAUSE="unknown" ;;
+            esac
+            case "$CAUSE" in
+              checker_invalid)
+                SUMMARY="a checker failed its canary, so this run's result cannot be trusted"
+                ADVICE="Read it as neither a pass nor a gate failure: fix or report the checker the log names, then re-run."
+                ;;
+              nothing_to_check)
+                SUMMARY="there was nothing in scope to check"
+                ADVICE="No content reached the check. Confirm the recipe's content paths or the files passed in args."
+                ;;
+              content_not_checked)
+                SUMMARY="content in scope was not checked"
+                ADVICE="A requested gate or analyzer had nothing bound, or a changed file could not be checked. The log names which."
+                ;;
+              unknown)
+                SUMMARY="kapi's output named no cause"
+                ADVICE="Read the job log for the reason."
+                ;;
+              *)
+                SUMMARY="kapi gave a cause this version of the action does not describe"
+                ADVICE="Read the job log for the reason."
+                ;;
+            esac
+            {
+              echo "result=did_not_run"
+              echo "did_not_run_cause=${CAUSE}"
+            } >> "$GITHUB_OUTPUT"
+            ICON="⚠️"
+            if [ "$CAUSE" = "checker_invalid" ]; then
+              ICON="❌"
+            fi
+            {
+              echo "## kapi check"
+              echo
+              echo "${ICON} **Did not run** (\`${CAUSE}\`): ${SUMMARY}. ${ADVICE}"
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::error::kapi check did not run (exit 4, ${CAUSE}): ${SUMMARY}. ${ADVICE}"
+            exit 4
           else
-            echo "run_status=failed" >> "$GITHUB_OUTPUT"
+            {
+              echo "run_status=failed"
+              echo "result=error"
+            } >> "$GITHUB_OUTPUT"
             echo "::error::kapi check failed operationally (exit ${rc})"
             exit "$rc"
           fi