Skip to content

fix: run the reporting steps after a failing check #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asgeirf merged 3 commits into from
Sep 13, 2026
Merged

fix: run the reporting steps after a failing check #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fdb839c
fix: run the reporting steps after a failing check
asgeirf Sep 13, 2026
f151550
test: revert the reporting conditions to prove the tests fail
asgeirf Sep 13, 2026
318309a
test: restore the reporting conditions after the must-fail proof
asgeirf Sep 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 129 additions & 14 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ name: Test

# Keyless self-tests against the fixture project in test/fixture — plan mode
# and check need no provider or server credentials. The CLI version is pinned
# to the first release that ships `kapi up` / `check --ship`; bump
# deliberately.
# to a release that reads the fixture's `collections:` recipe (1.2.0-rc17 or
# later); bump deliberately.
#
# The check-outcome jobs put a kapi stub (test/stub/kapi) on PATH instead, so
# each exit code kapi check can end with, and each did-not-run cause, is
# exercised whatever the pinned release can produce.
# The check-outcome and reporting jobs put stubs (test/stub) on PATH instead:
# a kapi that exits with a chosen code, and a gh that records the sticky PR
# comment rather than posting it. Every exit code kapi check can end with, and
# each did-not-run cause, is exercised whatever the pinned release can produce.

on:
push:
Expand All @@ -19,7 +20,7 @@ on:
workflow_dispatch:

env:
KAPI_VERSION: "1.2.0-rc14"
KAPI_VERSION: "1.2.0-rc29"

jobs:
test-plan:
Expand All @@ -30,7 +31,7 @@ jobs:

- uses: neokapi/setup-kapi@v1
with:
version: "1.2.0-rc14"
version: ${{ env.KAPI_VERSION }}
plugins: ""

- name: Run plan
Expand All @@ -57,7 +58,7 @@ jobs:

- uses: neokapi/setup-kapi@v1
with:
version: "1.2.0-rc14"
version: ${{ env.KAPI_VERSION }}
plugins: ""

# The fixture is deliberately untranslated: the ship gate must FAIL the
Expand All @@ -77,11 +78,13 @@ jobs:
STEP_OUTCOME: ${{ steps.gate.outcome }}
GATE: ${{ steps.gate.outputs.gate }}
RESULT: ${{ steps.gate.outputs.result }}
STATUS: ${{ steps.gate.outputs.status }}
run: |
echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}, result: ${RESULT}"
echo "step outcome: ${STEP_OUTCOME}, gate: ${GATE}, result: ${RESULT}, status: ${STATUS}"
test "${STEP_OUTCOME}" = "failure"
test "${GATE}" = "fail"
test "${RESULT}" = "failed"
test "${STATUS}" = "failed"

test-check-file:
name: "check <files> (file-mode pass)"
Expand All @@ -91,7 +94,7 @@ jobs:

- uses: neokapi/setup-kapi@v1
with:
version: "1.2.0-rc14"
version: ${{ env.KAPI_VERSION }}
plugins: ""

- name: Gate the fixture source file
Expand Down Expand Up @@ -209,7 +212,7 @@ jobs:
# expect STEP OUTCOME RESULT GATE CAUSE
expect() {
local got want
got="$(jq -r --arg id "$1" '.[$id] | [.outcome, (.outputs.result // ""), (.outputs.gate // ""), (.outputs.did_not_run_cause // "")] | join(" | ")' <<< "$STEPS")"
got="$(jq -r --arg id "$1" '.[$id] | [.outcome, (.outputs.result // ""), (.outputs.gate // ""), (.outputs["did-not-run-cause"] // "")] | join(" | ")' <<< "$STEPS")"
want="$2 | $3 | $4 | $5"
if [ "$got" = "$want" ]; then
echo "ok $1: $got"
Expand All @@ -228,14 +231,126 @@ jobs:
expect error failure error "" ""
exit "$status"

test-check-reporting:
name: "reporting after a failing check (kapi and gh stubs)"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6

- name: Put the stubs on PATH
run: echo "${GITHUB_WORKSPACE}/test/stub" >> "$GITHUB_PATH"

# Each run asks for the sticky PR comment, and the gh stub records the
# body instead of posting it. The action comments only on pull_request
# events, so on a push or a schedule the verify step expects no comment.
- name: "exit 0 with pr-comment"
id: passed
continue-on-error: true
uses: ./
env:
KAPI_STUB_OUTPUT: test/check-output/passed.txt
KAPI_STUB_EXIT: "0"
GH_STUB_LOG: ${{ runner.temp }}/gh-passed
with:
command: check
pr-comment: "true"

- name: "exit 3 with pr-comment"
id: failed
continue-on-error: true
uses: ./
env:
KAPI_STUB_OUTPUT: test/check-output/failed.txt
KAPI_STUB_EXIT: "3"
GH_STUB_LOG: ${{ runner.temp }}/gh-failed
with:
command: check
args: "--ship"
pr-comment: "true"

- name: "exit 4 with pr-comment"
id: did_not_run
continue-on-error: true
uses: ./
env:
KAPI_STUB_OUTPUT: test/check-output/checker_invalid.txt
KAPI_STUB_EXIT: "4"
GH_STUB_LOG: ${{ runner.temp }}/gh-did_not_run
with:
command: check
pr-comment: "true"

- name: Verify the reports, and that the check still failed
env:
STEPS: ${{ toJSON(steps) }}
EVENT_NAME: ${{ github.event_name }}
GH_LOGS: ${{ runner.temp }}
run: |
status=0
expect() {
if [ "$2" = "$3" ]; then
echo "ok $1: $2"
else
echo "::error::$1: got '$2', want '$3'"
status=1
fi
}
# value STEP FILTER reads one field of a step from the steps context.
value() {
jq -r --arg id "$1" ".[\$id] | $2 // \"\"" <<< "$STEPS"
}
# comment STEP TEXT: on a pull_request event the body the action
# posted must contain TEXT; on any other event it must post nothing.
comment() {
local body="${GH_LOGS}/gh-$1/body.md"
if [ "$EVENT_NAME" != "pull_request" ]; then
if [ -e "$body" ]; then
echo "::error::$1: a comment was posted on a ${EVENT_NAME} event"
status=1
else
echo "ok $1: no comment on a ${EVENT_NAME} event"
fi
elif [ ! -f "$body" ]; then
echo "::error::$1: no comment was posted"
status=1
elif grep -qF -- "$2" "$body"; then
echo "ok $1: the comment contains: $2"
else
echo "::error::$1: the comment lacks: $2"
sed 's/^/ > /' "$body"
status=1
fi
}

# The action step itself fails on exit 3 and exit 4, which is what
# fails a job that does not set continue-on-error.
expect "passed outcome" "$(value passed .outcome)" success
expect "failed outcome" "$(value failed .outcome)" failure
expect "did_not_run outcome" "$(value did_not_run .outcome)" failure

# The reporting steps still ran after the failing check.
expect "passed status" "$(value passed .outputs.status)" no-changes
expect "failed status" "$(value failed .outputs.status)" failed
expect "did_not_run status" "$(value did_not_run .outputs.status)" failed
expect "passed has-changes" "$(value passed '.outputs["has-changes"]')" false
expect "failed has-changes" "$(value failed '.outputs["has-changes"]')" false
expect "did_not_run has-changes" "$(value did_not_run '.outputs["has-changes"]')" false
expect "did_not_run did-not-run-cause" "$(value did_not_run '.outputs["did-not-run-cause"]')" checker_invalid

comment passed "✅ **Gate: pass**"
comment failed "❌ **Gate: unmet**"
comment did_not_run "❌ **Did not run** (\`checker_invalid\`): a checker failed its canary, so this run's result cannot be trusted."
exit "$status"

test-check-annotations:
name: "check outcomes: exit codes, outputs and annotations (kapi stub)"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6

# Runs the action's own step script once per outcome, including the
# did-not-run causes read from text, JSON, YAML and colored output.
# Runs the action's own step scripts once per outcome, including the
# did-not-run causes read from text, JSON, YAML and colored output, and
# the sticky PR comment each outcome renders.
- name: Run the check-outcome harness
run: |
yq --version
Expand All @@ -249,7 +364,7 @@ jobs:

- uses: neokapi/setup-kapi@v1
with:
version: "1.2.0-rc14"
version: ${{ env.KAPI_VERSION }}
plugins: ""

- name: Pseudo-translate the fixture
Expand Down
12 changes: 7 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,11 +181,13 @@ jobs:

Ordinary builds never fail on target-language drift — a locale that is behind is pending work, not an error. `check --ship` is the explicit, opt-in enforcement point.

A failing gate still reports: `status` is `failed`, `has-changes` is set, and with `pr-comment` the sticky comment shows the result. The step keeps the gate's exit code, so the job fails.

#### When a check does not run

A check can also end without a verdict. kapi exits `4` when the check did not run: it checked no content, or it could not show that its checkers are able to fail. The Action fails the step and reports this as its own result: `result` is `did_not_run`, `gate` stays empty, and `did_not_run_cause` carries the cause kapi named. The step's error annotation and the job summary state the cause in words.
A check can also end without a verdict. kapi exits `4` when the check did not run: it checked no content, or it could not show that its checkers are able to fail. The Action fails the step and reports this as its own result: `result` is `did_not_run`, `gate` stays empty, `status` is `failed`, and `did-not-run-cause` carries the cause kapi named. The step's error annotation, the job summary and, with `pr-comment`, the sticky PR comment state the cause in words.

| `did_not_run_cause` | What it means |
| `did-not-run-cause` | What it means |
|---|---|
| `checker_invalid` | A checker failed its canary, so the run's result cannot be trusted. Read it as neither a pass nor a gate failure, and fix or report the checker. |
| `nothing_to_check` | There was nothing in scope to check. |
Expand Down Expand Up @@ -232,21 +234,21 @@ Server-connected projects don't need this — the project state lives on the ser
| `project` | | Path to the `kapi.yaml` recipe (`-p` flag) |
| `plan` | `false` | With `command: up`: dry run — pending work, TM leverage, token estimate; no writes, no provider calls |
| `fail-on-parked` | `false` | With `command: up`, fail the workflow when the run parks instead of reporting partial progress |
| `pr-comment` | `false` | Sticky report comment on pull-request events |
| `pr-comment` | `false` | Sticky report comment on pull-request events, including for a failed gate or a check that did not run |
| `token` | `${{ github.token }}` | Token for the sticky PR comment |
| `paths` | | Space-separated paths to scan for changes (whole working tree if empty) |

## Outputs

| Output | Description |
|---|---|
| `status` | `success`, `no-changes`, or `failed` |
| `status` | `success`, `no-changes`, or `failed` (with `command: check`: a failed gate or a check that did not run) |
| `outcome` | With `command: up`: `converged` or `parked` (a failed run fails the step, so it never reaches an output) |
| `passes` | With `command: up`: how many reconciliation passes the run took |
| `parked-locales` | With `command: up`: comma-separated locales still short of their gate |
| `gate` | With `command: check`: `pass` or `fail` (empty when the check did not run or errored) |
| `result` | With `command: check`: `passed` (exit 0), `failed` (exit 3), `did_not_run` (exit 4), or `error` (any other exit code) |
| `did_not_run_cause` | With `command: check`, when `result` is `did_not_run`: `checker_invalid`, `nothing_to_check`, `content_not_checked`, or `unknown` |
| `did-not-run-cause` | With `command: check`, when `result` is `did_not_run`: `checker_invalid`, `nothing_to_check`, `content_not_checked`, or `unknown` |
| `plan-missing` / `plan-tm-exact` / `plan-ai-remaining` / `plan-token-estimate` | With `plan: true`: the plan totals |
| `has-changes` | Whether the run left changes in the working tree for your delivery step |
| `changed-files` | Newline-separated paths the run changed |
Expand Down
41 changes: 33 additions & 8 deletions action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ inputs:
pr-comment:
description: >-
On pull-request events, post one sticky comment with the report (plan,
`kapi up` outcome, or gate result) that re-runs update in place.
`kapi up` outcome, or check result, including a failed gate or a check
that did not run) that re-runs update in place.
Needs `permissions: pull-requests: write`.
required: false
default: "false"
Expand All @@ -50,7 +51,7 @@ inputs:

outputs:
status:
description: "Run result: success, no-changes, or failed"
description: "Run result: success, no-changes, or failed (with `command: check`: a failed gate or a check that did not run)"
value: ${{ steps.set-outputs.outputs.status }}
outcome:
description: "With `command: up`: `converged` or `parked` (empty for other commands)"
Expand All @@ -67,13 +68,13 @@ outputs:
result:
description: "With `command: check`: `passed`, `failed`, `did_not_run`, or `error`"
value: ${{ steps.run-kapi.outputs.result }}
did_not_run_cause:
did-not-run-cause:
description: >-
With `command: check`, when `result` is `did_not_run`: `checker_invalid`
(a checker failed its canary, so the result cannot be trusted),
`nothing_to_check`, `content_not_checked`, or `unknown` when kapi's
output named no cause
value: ${{ steps.run-kapi.outputs.did_not_run_cause }}
value: ${{ steps.run-kapi.outputs.did-not-run-cause }}
plan-missing:
description: "With `plan: true`: pending units across all locales"
value: ${{ steps.run-kapi.outputs.plan-missing }}
Expand Down Expand Up @@ -226,7 +227,8 @@ runs:
esac
{
echo "result=did_not_run"
echo "did_not_run_cause=${CAUSE}"
echo "did-not-run-cause=${CAUSE}"
echo "did-not-run-summary=${SUMMARY}. ${ADVICE}"
} >> "$GITHUB_OUTPUT"
ICON="⚠️"
if [ "$CAUSE" = "checker_invalid" ]; then
Expand Down Expand Up @@ -330,9 +332,15 @@ runs:
fi
fi

# The reporting steps below also run after the check step failed with a
# verdict (a failed gate, or a check that did not run), because reporting
# that outcome is what they are for. The job still fails: the check step
# already did, and a later step that succeeds does not undo it. A run that
# failed any other way reports nothing, so a delivery step is never handed
# the partial work of a broken `kapi up`.
- name: Check for changes
id: check-changes
if: inputs.plan != 'true'
if: ${{ !cancelled() && inputs.plan != 'true' && (success() || steps.run-kapi.outputs.result == 'failed' || steps.run-kapi.outputs.result == 'did_not_run') }}
shell: bash
env:
PATHS: ${{ inputs.paths }}
Expand Down Expand Up @@ -371,7 +379,8 @@ runs:
fi

- name: Sticky PR comment
if: inputs.pr-comment == 'true' && github.event_name == 'pull_request'
id: pr-comment
if: ${{ !cancelled() && inputs.pr-comment == 'true' && github.event_name == 'pull_request' && (success() || steps.run-kapi.outputs.result == 'failed' || steps.run-kapi.outputs.result == 'did_not_run') }}
shell: bash
env:
GH_TOKEN: ${{ inputs.token }}
Expand All @@ -382,6 +391,9 @@ runs:
PASSES: ${{ steps.run-kapi.outputs.passes }}
PARKED: ${{ steps.run-kapi.outputs.parked-locales }}
GATE: ${{ steps.run-kapi.outputs.gate }}
RESULT: ${{ steps.run-kapi.outputs.result }}
CAUSE: ${{ steps.run-kapi.outputs.did-not-run-cause }}
DID_NOT_RUN_SUMMARY: ${{ steps.run-kapi.outputs.did-not-run-summary }}
PLAN_MISSING: ${{ steps.run-kapi.outputs.plan-missing }}
PLAN_TM: ${{ steps.run-kapi.outputs.plan-tm-exact }}
PLAN_AI: ${{ steps.run-kapi.outputs.plan-ai-remaining }}
Expand Down Expand Up @@ -413,6 +425,15 @@ runs:
## kapi check

${HEAD}"
elif [ "$RESULT" = "did_not_run" ]; then
ICON="⚠️"
if [ "$CAUSE" = "checker_invalid" ]; then
ICON="❌"
fi
BODY="${MARKER}
## kapi check

${ICON} **Did not run** (\`${CAUSE}\`): ${DID_NOT_RUN_SUMMARY} See the [run log](${RUN_URL}) for the reasons."
elif [ -n "$OUTCOME" ]; then
BODY="${MARKER}
## kapi up report
Expand Down Expand Up @@ -440,12 +461,16 @@ runs:

- name: Set outputs
id: set-outputs
if: ${{ !cancelled() && (success() || steps.run-kapi.outputs.result == 'failed' || steps.run-kapi.outputs.result == 'did_not_run') }}
shell: bash
env:
HAS_CHANGES: ${{ steps.check-changes.outputs.has_changes }}
RESULT: ${{ steps.run-kapi.outputs.result }}
run: |
set -euo pipefail
if [ "$HAS_CHANGES" = "true" ]; then
if [ "$RESULT" = "failed" ] || [ "$RESULT" = "did_not_run" ]; then
echo "status=failed" >> "$GITHUB_OUTPUT"
elif [ "$HAS_CHANGES" = "true" ]; then
echo "status=success" >> "$GITHUB_OUTPUT"
else
echo "status=no-changes" >> "$GITHUB_OUTPUT"
Expand Down
Loading
Loading