From d51d920e5a15b06fab48215b9a8201dd17987eb0 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:13:56 -0500 Subject: [PATCH] fix(source-control): bind a review re-run after an unposted timeout A review run whose first attempt failed with failure-class=timeout (every review while the reviews credential was rejected at its usage limit, run 41626484307) left its successful second attempt unbound, so the merge gate refused six lanes App PRs with rerun-not-allowed. A timeout now binds the second attempt under the same unposted proof no-outcome uses; a lane that posted on the head still refuses, and the gate never starts the re-run. Co-Authored-By: Claude Opus 5.5 --- ...fix-review-timeout-rerun-binds-7c41e2a9.md | 7 +++ .../skills/babysit-prs/reference/safety.md | 7 +-- .../scripts/babysit_review_evidence.py | 8 +++- .../tests/test_babysit_review_evidence.py | 44 +++++++++++++++++++ 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 .changes/source-control/fix-review-timeout-rerun-binds-7c41e2a9.md diff --git a/.changes/source-control/fix-review-timeout-rerun-binds-7c41e2a9.md b/.changes/source-control/fix-review-timeout-rerun-binds-7c41e2a9.md new file mode 100644 index 00000000000..1ca218e601d --- /dev/null +++ b/.changes/source-control/fix-review-timeout-rerun-binds-7c41e2a9.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- The merge gate's review evidence binds an AI review run's second attempt when the first failed with `failure-class=timeout` and the lane posted nothing on the head, the same unposted proof `no-outcome` uses. The gate still never starts that re-run itself. diff --git a/plugins/source-control/skills/babysit-prs/reference/safety.md b/plugins/source-control/skills/babysit-prs/reference/safety.md index 0b2577975d8..78515ee6489 100644 --- a/plugins/source-control/skills/babysit-prs/reference/safety.md +++ b/plugins/source-control/skills/babysit-prs/reference/safety.md @@ -896,9 +896,10 @@ stopped the first attempt (class `upstream-degraded`): the caller's `kill-switch its lane's `an upstream outage stops this review (` (or `security review (`) error and the first attempt has no lane job. The gate never starts that rerun; the lost-run reconciler reruns the run's failed jobs once GitHub Actions is operational and the upstream gate would let it run. A first -attempt the evidence classifies as `NO_OUTCOME` (`babysit_review_evidence.py`) binds a second -attempt only when the lane posted nothing on the head, by the lost-run rule below. The gate never -starts that rerun: a defect in the lane fails every run until its fix ships. +attempt the evidence classifies as `NO_OUTCOME` or `TIMEOUT` (`babysit_review_evidence.py`) +binds a second attempt only when the lane posted nothing on the head, by the lost-run rule below. +The gate never starts that rerun: a defect in the lane, or a spent credential, fails every run +until it is fixed. The gate also reruns, at once and under the same head, job and attempt rules, an AI review check whose run an outage lost: its job and the run's first attempt concluded `cancelled` or `timed_out` diff --git a/plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py index 7f18984b10a..bf2bf9ce266 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py @@ -219,8 +219,14 @@ def main_review_name(lane: str, pr: int, tree: str) -> str: # re-run but does not start one (GATE_RERUN_CLASSES): a defect fails every run # until its fix ships, and a re-run before then would spend the one attempt. NO_OUTCOME = "no-outcome" +# Or when the review leg reached its time limit (class `timeout`), as every +# review did while its subscription credential was rejected at a usage limit +# (run 41626484307). The same unposted proof applies: attempt 1 left no verdict +# on the head, so attempt 2 is the only review. +TIMEOUT = "timeout" NO_OUTCOME_RE = re.compile( - rf"(?P[\w-]+): failure-class={re.escape(NO_OUTCOME)}: " + rf"(?P[\w-]+): failure-class=" + rf"(?:{re.escape(NO_OUTCOME)}|{re.escape(TIMEOUT)}): " ) # Or when a kill switch stopped attempt 1 before the review started: the # caller's `kill-switch` job succeeded and its `Report the kill switch` step diff --git a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_review_evidence.py b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_review_evidence.py index c53b088b186..b1412ad832b 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_review_evidence.py +++ b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_review_evidence.py @@ -2083,6 +2083,50 @@ def test_the_gate_does_not_rerun_a_no_outcome(self) -> None: self.assertNotIn(ev.NO_OUTCOME, ev.GATE_RERUN_CLASSES) +def timeout_first(lane: str = "claude-review") -> dict[str, Any]: + """Attempt 1 whose review leg timed out on a rejected credential, as the + status job of run 41626484307 attempt 1 recorded it.""" + lines = [ + dict( + OTHER_FAILURE, + title="", + message=f"{lane}: failure-class=timeout:" + " the review leg reached its time limit", + ), + dict( + OTHER_FAILURE, + annotation_level="warning", + title="run-claude-cli usage", + message="REVIEWS_CLAUDE_CODE_OAUTH_TOKEN is near a subscription usage" + " limit: window=seven_day status=rejected resets=2026-11-04T07:00:00Z", + ), + ] + return no_outcome(lane, lines) + + +class TimeoutRerunTests(Base): + """Attempt 2 after an attempt 1 whose review leg timed out binds only when + that lane posted nothing on the head.""" + + def test_second_attempt_after_an_unposted_timeout_binds(self) -> None: + for lane in LANES: + with self.subTest(lane=lane): + item = run(lane, S, attempts=[timeout_first(lane), rerun_ok(lane)]) + result = self.bind([item]) + self.assertTrue(result[lane].bound) + + def test_a_timeout_whose_lane_posted_does_not_bind(self) -> None: + mark = dict(marker("claude-review", S), created_at="2026-01-01T00:40:00Z") + item = run("claude-review", S, attempts=[timeout_first(), rerun_ok()]) + result = self.bind([item], [*both_markers(), mark], []) + self.assertUnbound(result["claude-review"], "rerun-not-allowed") + + def test_the_gate_does_not_rerun_a_timeout(self) -> None: + job = timeout_first()["jobs"][0] + self.assertEqual(ev.rerun_classes("claude-review", job), frozenset()) + self.assertNotIn(ev.TIMEOUT, ev.GATE_RERUN_CLASSES) + + # The caller workflows' `Report the kill switch` notice # (pr-review-hosted.yml, pr-review-security-hosted.yml). KILL_NOTICE = {