From a99b2ef5726f92d4922244e61a62bb852ec2b6cd Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:23:14 -0500 Subject: [PATCH 1/2] fix(ci): re-send a resolve-conflicts, fix-ci or dependency round a head-lag stop lost, once check-trusted-trigger's head-lag stop posts a neutral superseded-sha skip on the head the run read. Every other superseded-sha check lands on a head the PR has left, so on the head a dispatcher reads, the PR's current one, it is a stop that did no work. The tick's dispatchers read it as handling the head, so the round was lost until a push. limit-stop.mjs gains headLagStop and a lagStopsRelease opt-in that reads such a stop as it reads a kill-switch stop: it handles nothing and counts toward limit-stops-per-head. A newer run still going or finished decides the head, so each stop is sent again once. - dispatch.mjs: resolve-conflicts and sync-shared-copies opt in. - fix-ci-dispatch.mjs: the stop is a paused run, retried under the per-head cap without spending a restart. - dependency-dispatch.mjs: opts in. TERMINAL_SKIPS keeps superseded-sha for the select job's own skip, which lands on the head it was sent for; a lag stop on the current head is no longer recorded handled. Each logs a `refire: #: head 's was stopped head-lag` line. address-feedback keeps its own path (sweepRefused). Co-Authored-By: Claude Opus 5.5 --- .../pr-unblock/dependency-dispatch.mjs | 17 +++++--- .../pr-unblock/dependency-dispatch.test.mjs | 24 +++++++++++ .github/scripts/pr-unblock/dispatch.mjs | 18 ++++++-- .github/scripts/pr-unblock/dispatch.test.mjs | 42 +++++++++++++++++++ .../scripts/pr-unblock/fix-ci-dispatch.mjs | 12 ++++-- .../pr-unblock/fix-ci-dispatch.test.mjs | 29 +++++++++++++ .github/scripts/pr-unblock/limit-stop.mjs | 38 +++++++++++------ .../scripts/pr-unblock/limit-stop.test.mjs | 18 +++++++- 8 files changed, 172 insertions(+), 26 deletions(-) diff --git a/.github/scripts/pr-unblock/dependency-dispatch.mjs b/.github/scripts/pr-unblock/dependency-dispatch.mjs index b385e908c17..79c50f76462 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.mjs @@ -26,7 +26,7 @@ import { DEPENDABOT_ID, DEPENDABOT_LOGIN, RED } from "./dependency-classify.mjs"; import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; import { LIMIT_DEFAULTS } from "../work-resolve-issue/limits.mjs"; -import { anyHandles, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, headLagStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; import { ciStatusOf, classifyPull, commitFilesReader } from "./review-dependency.mjs"; export const DEFAULTS = Object.freeze({ @@ -46,7 +46,11 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/; // The neutral titles review-dependency-not-run posts from the select job's // skip reasons (review-dependency.mjs select): no verdict job follows them, -// so no no-model-turn verdict can release the head. +// so no no-model-turn verdict can release the head. The select job's +// superseded-sha lands on the head it was sent for, which the PR has left; +// the same title on the head read here, the PR's current one, is the gate's +// head-lag stop, which did no work, so lagStopsRelease sends it again +// (limit-stop.mjs) and it is not recorded. const TERMINAL_SKIPS = new Set(["Skipped: awaiting-human", "Skipped: not-applicable", "Skipped: superseded-sha"]); // NaN for a run with no readable start: it compares false either way. @@ -141,7 +145,7 @@ export async function dispatchDependencyReviews({ // not released that way, though its round's verdict also finds no model // turn: it holds the head until its limit resets. let verdicts = []; - if (anyHandles(existing, { now, limits })) { + if (anyHandles(existing, { now, limits, lagStopsRelease: true })) { verdicts = await api.listCheckRuns({ sha: node.headRefOid, checkName: verdictCheckName, appId }); const newest = verdicts.reduce((a, b) => (startOf(b) > startOf(a) ? b : a), verdicts[0]); if (newest?.status === "completed" && newest.conclusion === "neutral" && newest.output?.title === "Skipped: no-model-turn") { @@ -163,7 +167,7 @@ export async function dispatchDependencyReviews({ const newestCheck = Math.max(...existing.map(startOf)); const last = existing.find((run) => startOf(run) === newestCheck); const terminal = last?.status === "completed" && last.conclusion === "neutral" && TERMINAL_SKIPS.has(last.output?.title); - if (permanentlyHandles(existing, { limits }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) { + if (permanentlyHandles(existing, { limits, lagStopsRelease: true }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) { markHandled(record, checkName, node.headRefOid); } } catch (error) { @@ -171,10 +175,13 @@ export async function dispatchDependencyReviews({ log(`::warning::#${node.number}: could not read check runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`); continue; } - if (anyHandles(existing, { now, limits })) { + if (anyHandles(existing, { now, limits, lagStopsRelease: true })) { log(`skip #${node.number}: head ${node.headRefOid} already has a ${checkName} check run`); continue; } + if (existing.some(headLagStop)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } sent += 1; try { const { workflowRunId } = await api.dispatchWorkflow({ diff --git a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs index b322640a6ff..31fe4d990b4 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs @@ -259,6 +259,30 @@ test("a head the kill switch stopped dispatches again, and is not recorded as ha } }); +// The gate stopped the review head-lag: report-check-run posts a superseded-sha +// skip on the head the run read. On the head the dispatcher reads, the PR's +// current one, it did no work, so it is sent again once and not recorded, +// unlike the select job's superseded-sha, which lands on a head the PR left. +const lagged = (started) => roundRun("review-dependency", "neutral", "Skipped: superseded-sha", started); + +test("a review the gate stopped head-lag on the current head is sent again once, under the cap, and not recorded", async () => { + const node = pr(62); + const record = emptyRecord(); + const lines = []; + const once = await run(fakeApi({ nodes: [node], checks: headOf(node, [lagged("2026-10-17T10:00:00Z")]) }), { record, log: (line) => lines.push(line) }); + assert.deepEqual(once.dispatched, [{ number: 62, workflowRunId: "7062" }]); + assert.equal(record.handled.get(CHECK), undefined); + assert.ok(lines.includes(`refire: #62: head ${node.headRefOid}'s ${CHECK} was stopped head-lag; sending it again`), lines.join("\n")); + for (const [label, checks] of [ + ["a newer review passed", [lagged("2026-10-17T10:00:00Z"), roundRun("review-dependency", "success", "Passed", "2026-10-17T10:30:00Z")]], + ["a newer review is still going", [lagged("2026-10-17T10:00:00Z"), { ...roundRun("review-dependency", null, "", "2026-10-17T10:30:00Z"), status: "in_progress" }]], + ["the head is at the cap", [lagged("2026-10-17T10:00:00Z"), lagged("2026-10-17T10:30:00Z"), lagged("2026-10-17T11:00:00Z")]], + ]) { + const { dispatched } = await run(fakeApi({ nodes: [node], checks: headOf(node, checks) })); + assert.deepEqual(dispatched, [], label); + } +}); + // A review-dependency run the usage limit stopped at `started`, its lane record // naming a reset an hour later. const limitStop = (started) => { diff --git a/.github/scripts/pr-unblock/dispatch.mjs b/.github/scripts/pr-unblock/dispatch.mjs index 0db9f2481b7..5e9590b50a7 100644 --- a/.github/scripts/pr-unblock/dispatch.mjs +++ b/.github/scripts/pr-unblock/dispatch.mjs @@ -13,7 +13,7 @@ // limit stop, a usage limit or a refused credential, stops counting once its // limit resets, and the skip names its class and reset; a failed run until // limit-stops-per-head failures stand on the head, a skip while held once -// the hold is gone, and a select job's own lane-stopped skip once the base tip +// the hold is gone, a head-lag stop (the gate's lagging PR read) at once, and a select job's own lane-stopped skip once the base tip // was committed after it started, since the head then merges another base), // and dispatches pr-unblock.yml on main for at most maxDispatch of them, // lowest PR number first, heads retried on a moved base last. Authors are @@ -56,7 +56,7 @@ import { prHold, reviewReader } from "./gate-source.mjs"; import { runTitle, shareWorkflowRuns } from "./feedback-dispatch.mjs"; import { DEFAULTS as CI_FIX_DEFAULTS, dispatchCiFixes, humanGatedLabel } from "./fix-ci-dispatch.mjs"; import { DEFAULTS as DOCS_FIX_DEFAULTS, dispatchDocsFixes, HISTORY_QUERY as DOCS_FIX_HISTORY_QUERY } from "./fix-docs-dispatch.mjs"; -import { anyHandles, cappedAt, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, cappedAt, headLagStop, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; import { budgetedFetch } from "./request-budget.mjs"; import { verifyCapReader } from "./verify-cap.mjs"; import { readConfig, readPolicy } from "./review-dependency.mjs"; @@ -85,6 +85,14 @@ export const DEFAULTS = Object.freeze({ // The run id form the record's readers accept. const RUN_ID = /^[1-9][0-9]{0,14}$/; + +// The log line that names a head-lag stop on the head as the cause of this +// dispatch (limit-stop.mjs lagStopsRelease). +function lagRefire(log, node, checkName, existing, handled) { + if (!handled && existing.some(headLagStop)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } +} const TRUSTED_ACTORS_PATH = ".github/standards/trusted-actors/trusted-actors.json"; // A merge commit on a Dependabot head stops Dependabot's own rebases, which @@ -246,7 +254,7 @@ export async function dispatchConflicting({ continue; } let existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); - const options = { limits, failuresRetry: true, heldSkipsRelease: true }; + const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true }; // A push job that stopped on a reason routed to a failing status // releases the head's earlier checks: the next run retries the same // head, a fresh model session, until limit-stops-per-head such stops @@ -268,6 +276,7 @@ export async function dispatchConflicting({ existing = existing.filter((run) => !(laneStop(run) && Date.parse(run.started_at ?? "") < committedAt)); } handled = anyHandles(existing, { now, ...options }); + lagRefire(log, node, checkName, existing, handled); // Handled only because a limit stop has not reset: the log names it. waiting = handled && !anyHandles(existing, { now: Infinity, ...options }) ? pendingStop(existing, { now, limits }) : null; if (permanentlyHandles(existing, options)) { @@ -401,7 +410,7 @@ export async function dispatchSharedCopies({ continue; } const existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); - const options = { limits, failuresRetry: true, heldSkipsRelease: true }; + const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true }; if (permanentlyHandles(existing, options)) { const capped = cappedAt(existing, options); if (capped > 0) { @@ -419,6 +428,7 @@ export async function dispatchSharedCopies({ log(`skip #${node.number}: ${workflow} run ${inFlight.id} for ${activity} is not done yet`); continue; } + lagRefire(log, node, checkName, existing, false); } catch (error) { log(`::warning::#${node.number}: could not read its refusal, check runs or runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`); continue; diff --git a/.github/scripts/pr-unblock/dispatch.test.mjs b/.github/scripts/pr-unblock/dispatch.test.mjs index 2b79658f395..61d0a3daf14 100644 --- a/.github/scripts/pr-unblock/dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dispatch.test.mjs @@ -1696,6 +1696,48 @@ test("sync-shared-copies: a failed select job's failure is retried until limit-s assert.deepEqual(next.calls.checkReads, [], "recorded at the cap: nothing read again"); }); +// The gate stopped the run head-lag (GET /pulls and the branch ref never +// agreed): report-check-run posts a superseded-sha skip on the head the run +// read. On the head the dispatcher reads, the PR's current one, the run did +// no work there, so it is sent again once, under limit-stops-per-head. +const lagStop = (time) => ({ status: "completed", conclusion: "neutral", output: { title: "Skipped: superseded-sha" }, started_at: at(time), completed_at: at(time) }); + +test("resolve-conflicts: a head-lag stop on the current head is sent again once, under the cap", async () => { + const head = pr(1).headRefOid; + const cap = LIMIT_DEFAULTS["limit-stops-per-head"]; + const once = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) }); + await once.run(); + assert.deepEqual(dispatchedNumbers(once.api), ["1"]); + assert.ok(once.lines.includes(`refire: #1: head ${head}'s ${CHECK_NAME} was stopped head-lag; sending it again`), once.lines.join("\n")); + for (const [label, options] of [ + ["the round it sent is still going", { checked: new Map([[head, [lagStop("10:00")]]]), workflowRuns: [{ id: 1, status: "queued", display_title: pr_unblockTitle("resolve-conflicts", 1) }] }], + ["a newer round resolved it", { checked: new Map([[head, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" }, started_at: at("10:30") }]]]) }], + ["the head is at the cap", { checked: new Map([[head, Array.from({ length: cap }, (_, i) => lagStop(`1${i}:00`))]]) }], + ["the stop is on a head the PR left", { pages: [[pr(1)]], checked: new Map([["e".repeat(40), [lagStop("10:00")]], [head, [{ status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) }], + ]) { + const { api, run } = harness({ pages: [[pr(1)]], ...options }); + await run(); + assert.deepEqual(dispatchedNumbers(api), [], label); + } +}); + +test("sync-shared-copies: a head-lag stop on the current head is sent again once, under the cap", async () => { + const node = copyPr(45); + const timelines = new Map([[45, copyRefusal(node.headRefOid)]]); + const cap = LIMIT_DEFAULTS["limit-stops-per-head"]; + const lines = []; + const once = copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) }); + assert.equal((await copyDispatch(once, [node], { log: (line) => lines.push(line) })).dispatched.length, 1); + assert.ok(lines.includes(`refire: #45: head ${node.headRefOid}'s pr-unblock / sync-shared-copies was stopped head-lag; sending it again`), lines.join("\n")); + for (const [label, api] of [ + ["the round it sent is still going", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]), runs: [{ id: 9, status: "queued", display_title: pr_unblockTitle("sync-shared-copies", 45) }] })], + ["a newer round passed", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) })], + ["the head is at the cap", copyApi({ timelines, checks: new Map([[node.headRefOid, Array.from({ length: cap }, () => lagStop("10:00"))]]) })], + ]) { + assert.equal((await copyDispatch(api, [node])).dispatched.length, 0, label); + } +}); + // A select job's own stop (pr-unblock.yml resolve-conflicts-not-run) judged // the base it merged: once main moves, the same conflicting head is tried // again, and an unchanged head and base stay skipped. diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index 231f2770db6..3e38e56b983 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -59,7 +59,7 @@ import { STOP_RESOLVED } from "./escalation-routes.mjs"; import { HOLD_LINE, runTitle } from "./feedback-dispatch.mjs"; import { failingChecks } from "./fix-ci-context.mjs"; import { prHold, reviewReader } from "./gate-source.mjs"; -import { KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs"; +import { HEAD_LAG_STOP, KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs"; import { serverFailure } from "./request-budget.mjs"; import { final as capFinal, deadlineWarning } from "./verify-cap.mjs"; @@ -189,9 +189,10 @@ export function withStops(runs, stopRuns) { const counted = (runs) => runs.filter((run) => !(run.conclusion === "NEUTRAL" && String(run.title ?? "").startsWith("Skipped: untrusted-trigger"))); // A run that failed, found no failing check to fix, skipped on a PR held -// then, or that the kill switch stopped (paused) may be retried; any other -// run (going, pushed a fix, escalated) handles the head. -const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP); +// then, that the kill switch stopped, or that the gate stopped head-lag on +// this head, the PR's current one (paused), may be retried; any other run +// (going, pushed a fix, escalated) handles the head. +const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP || run.title === HEAD_LAG_STOP); const retryable = (run) => run.status === "COMPLETED" && (run.conclusion === "FAILURE" || paused(run) || (run.conclusion === "NEUTRAL" && NOTHING_TO_FIX.has(run.title))); // The number of counted fix-ci model runs when every attempt may be retried @@ -655,6 +656,9 @@ export async function dispatchCiFixes({ } continue; } + if (fixCiRuns.some((run) => run.conclusion === "NEUTRAL" && run.title === HEAD_LAG_STOP)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } sent += 1; try { const { workflowRunId } = await api.dispatchWorkflow({ diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs index 3b025344621..91d7e62b76d 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs @@ -1395,6 +1395,35 @@ test("a fix-ci run the kill switch stopped is retried on the same head without s assert.ok(own.lines.some((line) => line.startsWith("dispatched fix-ci #7 ")), own.lines.join("\n")); }); +// The gate stopped the fix-ci run head-lag (GET /pulls and the branch ref +// never agreed): report-check-run posts a superseded-sha skip on the head the +// run read. On the head the dispatcher reads, the PR's current one, the run +// did no work there, so it is sent again once, under the per-head cap. +test("a fix-ci run the gate stopped head-lag on the current head is sent again once, under the cap", async () => { + const head = "7".padStart(40, "a"); + const at = (time) => `2026-10-22T${time}:00Z`; + const lagged = (time) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "neutral", output: { title: "Skipped: superseded-sha" }, started_at: at(time), completed_at: at(time) }); + const lagCheck = check("pr-unblock / fix-ci", "NEUTRAL", { title: "Skipped: superseded-sha" }); + const lagPr = () => pr(7, { checks: [lanes("FAILURE"), lagCheck] }); + const once = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00")]]]) }); + assert.deepEqual(once.result.dispatched.map((d) => d.number), [7], once.lines.join("\n")); + assert.ok(once.lines.includes(`refire: #7: head ${head}'s pr-unblock / fix-ci was stopped head-lag; sending it again`), once.lines.join("\n")); + // The round it sent is still going: not sent twice. + const going = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00")]]]), workflowRuns: [{ id: 99, status: "queued", display_title: "pr-unblock fix-ci #7" }] }); + assert.deepEqual(going.result.dispatched, [], going.lines.join("\n")); + // A newer run pushed a fix: the head is handled. + const fixed = { name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title: "Fixed" }, started_at: at("10:30"), completed_at: at("10:30") }; + const newer = await run({ pages: [[pr(7, { checks: [lanes("FAILURE"), check("pr-unblock / fix-ci", "SUCCESS", { title: "Fixed" })] })]], history: new Map([[head, [lagged("10:00"), fixed]]]) }); + assert.deepEqual(newer.result.dispatched, [], newer.lines.join("\n")); + // Lag stops count toward the per-head cap. + const capped = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00"), lagged("10:30"), lagged("11:00")]]]) }); + assert.deepEqual(capped.result.dispatched, [], capped.lines.join("\n")); + assert.ok(capped.lines.some((line) => line.includes("ran 3 time(s), the most one head gets")), capped.lines.join("\n")); + // The stop sits on a head the PR has left: the current head has no fix-ci run, so it is dispatched as any head with none, with no refire line. + const moved = await run({ pages: [[pr(7)]], history: new Map([["8".padStart(40, "a"), [lagged("10:00")]]]) }); + assert.ok(!moved.lines.some((line) => line.startsWith("refire:")), moved.lines.join("\n")); +}); + // fix-ci and the in-run fix-ci-rung-up each post a fix-ci check in one // workflow run (the run id in each check's external_id; the API returns each // check's details_url as the check run's own page). When the rung-up's diff --git a/.github/scripts/pr-unblock/limit-stop.mjs b/.github/scripts/pr-unblock/limit-stop.mjs index 7997c5ac286..1cdfc8e0b6b 100644 --- a/.github/scripts/pr-unblock/limit-stop.mjs +++ b/.github/scripts/pr-unblock/limit-stop.mjs @@ -107,13 +107,27 @@ const untrustedSkip = (run) => // `Skipped: ` title). const heldSkip = (run) => run.status === "completed" && run.conclusion === "neutral" && run.output?.title === "Skipped: awaiting-human"; -// Two opt-ins, for a dispatcher that checks the PR's holds itself before it +// The gate's head-lag stop (check-trusted-trigger.mjs fetchPull), which +// report-check-run posts as a superseded-sha skip (gate-skip-reasons.json) on +// the head the run read. Every other superseded-sha check lands on a head the +// PR has left (a cancel, a moved tip before the mint or the push), so on the +// head a dispatcher reads, the PR's current one, it is a lag stop that did no +// work there. +export const HEAD_LAG_STOP = "Skipped: superseded-sha"; +export const headLagStop = (run) => run?.status === "completed" && run.conclusion === "neutral" && run.output?.title === HEAD_LAG_STOP; + +// Three opt-ins, for a dispatcher that checks the PR's holds itself before it // reads the check runs: with `heldSkipsRelease` a held skip stops handling the // head, so a head skipped while held is dispatched once the hold is gone; with -// `failuresRetry` a failed run of any kind does too. Either way the head is -// handled once `limit-stops-per-head` of them, limit stops included, stand on -// it, so a dispatcher and a run that disagree about a hold stop there. -const released = (run, heldSkipsRelease) => (heldSkipsRelease && heldSkip(run)) || killSwitchStop(run); +// `lagStopsRelease` a head-lag stop on the head read does too, so a round the +// gate stopped on a lagging API read is sent again; with `failuresRetry` a +// failed run of any kind does too. Either way the head is handled once +// `limit-stops-per-head` of them, limit stops included, stand on it, so a +// dispatcher and a run that disagree about a hold stop there. A newer run +// still going handles the head, and a newer finished one decides it, so each +// stop is sent again once. +const released = (run, { heldSkipsRelease = false, lagStopsRelease = false } = {}) => + (heldSkipsRelease && heldSkip(run)) || (lagStopsRelease && headLagStop(run)) || killSwitchStop(run); // The neutral check a run the lanes kill switch stopped posts: its own // kill-switch step, or the broker refusing its mint for a switch turned on @@ -132,13 +146,13 @@ export const releasesHead = (run) => laneStop(run) || killSwitchStop(run); // Whether `runs` handle the head: any run that is not an untrusted skip, a // kill-switch stop or a reset limit stop does, and so do // `limit-stops-per-head` such stops. -export function anyHandles(runs, { now, limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false }) { +export function anyHandles(runs, { now, limits = LIMIT_DEFAULTS, failuresRetry = false, ...opt }) { let stops = 0; for (const run of runs) { if (untrustedSkip(run)) { continue; } - if (released(run, heldSkipsRelease)) { + if (released(run, opt)) { stops += 1; continue; } @@ -167,9 +181,9 @@ export function permanentlyHandles(runs, options = {}) { return done.some((run) => !retryable(run)) || cappedAt(runs, options) > 0; } -function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false }) { +function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, ...opt }) { const done = runs.filter((run) => run.status === "completed" && !untrustedSkip(run)); - const retryable = (run) => released(run, heldSkipsRelease) || (failuresRetry ? run.conclusion === "failure" : limitResetsAt(run, limits) !== null); + const retryable = (run) => released(run, opt) || (failuresRetry ? run.conclusion === "failure" : limitResetsAt(run, limits) !== null); return { done, retryable }; } @@ -177,11 +191,11 @@ function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, h // they reach `limit-stops-per-head`, the head's hard end, else 0: the part of // permanentlyHandles a dispatcher reports as a stop. A head with any other // finished run (a success, an escalation) is done, not capped. -export function cappedAt(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false } = {}) { - if (!failuresRetry && !heldSkipsRelease) { +export function cappedAt(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false, lagStopsRelease = false } = {}) { + if (!failuresRetry && !heldSkipsRelease && !lagStopsRelease) { return 0; } - const { done, retryable } = completedRuns(runs, { limits, failuresRetry, heldSkipsRelease }); + const { done, retryable } = completedRuns(runs, { limits, failuresRetry, heldSkipsRelease, lagStopsRelease }); return done.length >= limits["limit-stops-per-head"] && done.length > 0 && done.every(retryable) ? done.length : 0; } diff --git a/.github/scripts/pr-unblock/limit-stop.test.mjs b/.github/scripts/pr-unblock/limit-stop.test.mjs index a9242f35e90..450973a223d 100644 --- a/.github/scripts/pr-unblock/limit-stop.test.mjs +++ b/.github/scripts/pr-unblock/limit-stop.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { anyHandles, KILL_SWITCH_STOP, killSwitchStop, limitResetsAt, pendingStop, permanentlyHandles, releasedBy, releasesHead, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, cappedAt, HEAD_LAG_STOP, headLagStop, KILL_SWITCH_STOP, killSwitchStop, limitResetsAt, pendingStop, permanentlyHandles, releasedBy, releasesHead, stopRelease } from "./limit-stop.mjs"; const HOUR = 3_600_000; const ENDED = "2026-10-17T10:00:00Z"; @@ -184,6 +184,22 @@ test("with held skips released, an awaiting-human skip does not handle the head" assert.equal(anyHandles([heldSkip, run(1), run(2)], { now, heldSkipsRelease: true, failuresRetry: true }), true, "held skips and failures share the cap"); }); +test("with lag stops released, a head-lag stop does not handle the head until the cap; by default it does", () => { + const now = ended + HOUR; + const lagged = (id) => ({ ...heldSkip, id, output: { title: HEAD_LAG_STOP } }); + assert.equal(HEAD_LAG_STOP, "Skipped: superseded-sha"); + assert.equal(headLagStop(lagged(1)), true); + assert.equal(headLagStop({ ...lagged(1), conclusion: "failure" }), false); + assert.equal(anyHandles([lagged(1)], { now }), true, "by default it handles the head"); + assert.equal(anyHandles([lagged(1)], { now, lagStopsRelease: true }), false); + assert.equal(permanentlyHandles([lagged(1)], { lagStopsRelease: true }), false); + assert.equal(anyHandles([lagged(1), { ...lagged(2), status: "in_progress", conclusion: null }], { now, lagStopsRelease: true }), true, "a newer run going handles it"); + assert.equal(anyHandles([lagged(1), { ...lagged(2), conclusion: "success", output: { title: "Passed" } }], { now, lagStopsRelease: true }), true, "a newer run that acted handles it"); + const three = [lagged(1), lagged(2), lagged(3)]; + assert.equal(anyHandles(three, { now, lagStopsRelease: true }), true, "the cap ends it"); + assert.equal(cappedAt(three, { lagStopsRelease: true }), 3); +}); + // A run the lanes kill switch stopped did no work: its own kill-switch step, // or a broker refusal of its mint (request-lane-token kill-switch-ok), posts // neutral kill-switch (decide.mjs: "Skipped: "). From a7dfcf8eeff462faf8d93b309d8a6de6afa9cb4a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:33:51 -0500 Subject: [PATCH 2/2] fix(ci): drop dispatch records that hold a head-lag-stopped head A version 5 record could hold a resolve-conflicts, sync-shared-copies or review-dependency head as handled for good when its only check was a head-lag stop, and each dispatcher's isHandled skip read it before the lagStopsRelease opt-in could. RECORD_VERSION 6 makes parseRecord read an older record as empty, so each such head is read once more under the new rule. The cache key does not name the version, so no workflow changes. Co-Authored-By: Claude Opus 5.5 --- .../pr-unblock/dependency-dispatch.test.mjs | 10 ++++++++++ .github/scripts/pr-unblock/dispatch-record.mjs | 6 ++++-- .github/scripts/pr-unblock/dispatch.test.mjs | 16 ++++++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs index 31fe4d990b4..0261927f845 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs @@ -445,6 +445,16 @@ test("a version 2 record that holds a limit-stopped head is ignored, and the hea assert.deepEqual(dispatched, [{ number: 67, workflowRunId: "7067" }]); }); +// A version 5 record held a head-lag-stopped head as reviewed (its +// superseded-sha skip was terminal); that record's version is read as empty. +test("a version 5 record that holds a head-lag-stopped head is ignored, and the head dispatches again", async () => { + const node = pr(68); + const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK]: [node.headRefOid] }, dispatched: {} })); + assert.equal(record.handled.get(CHECK), undefined, "read as empty"); + const checks = headOf(node, [roundRun("review-dependency", "neutral", "Skipped: superseded-sha", "2026-10-17T10:00:00Z")]); + assert.deepEqual((await run(fakeApi({ nodes: [node], checks }), { record })).dispatched, [{ number: 68, workflowRunId: "7068" }]); +}); + // Reads spent only where they change the decision. Each count beside the // assertion is what the dispatcher read before it skipped them. const RED_CI = [{ id: 1, app: { slug: "github-actions" }, status: "completed", conclusion: "failure" }]; diff --git a/.github/scripts/pr-unblock/dispatch-record.mjs b/.github/scripts/pr-unblock/dispatch-record.mjs index 1730dd17b3b..642ab617925 100644 --- a/.github/scripts/pr-unblock/dispatch-record.mjs +++ b/.github/scripts/pr-unblock/dispatch-record.mjs @@ -20,10 +20,12 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/; // version 3 a review-dependency head a limit stop holds until its reset, and // version 4 a resolve-conflicts head a failed run or a held skip left, and // version 5 a resolve-conflicts head a select job's stop left under an older -// base. A fix-ci +// base, and version 6 a resolve-conflicts, sync-shared-copies or +// review-dependency head a head-lag stop left (limit-stop.mjs +// lagStopsRelease). A fix-ci // head recorded at its per-head cap needs no new version: a record with no // fix-ci entry reads the same either way; nor do fix-docs runs, read as none. -export const RECORD_VERSION = 5; +export const RECORD_VERSION = 6; // The fix-docs runs kept per head, newest last: the reader drops the settled // runs that do not count, so more than this many is a tampered entry. const MAX_DOCS_RUNS = 20; diff --git a/.github/scripts/pr-unblock/dispatch.test.mjs b/.github/scripts/pr-unblock/dispatch.test.mjs index 61d0a3daf14..defc850b6ec 100644 --- a/.github/scripts/pr-unblock/dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dispatch.test.mjs @@ -1443,13 +1443,13 @@ test("main carries the record in DISPATCH_RECORD and writes it only when it chan DISPATCH_RECORD: path, }; // A recorded head no open PR carries any more is dropped. - writeFileSync(path, JSON.stringify({ version: 5, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} })); + writeFileSync(path, JSON.stringify({ version: 6, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} })); const options = () => ({ api: fakeApi({ pages: [[pr(1)]], checked: new Set([pr(1).headRefOid]) }), sleep: async () => {}, log: () => {}, loadLimits: async () => LIMIT_DEFAULTS, loadAdoption: async () => ({}), loadOwnedBranches: async () => [] }); try { writeFileSync(output, ""); assert.equal(await main(env, options()), 0); assert.equal(readFileSync(output, "utf8"), "record-changed=true\n"); - assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 5, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} }); + assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 6, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} }); writeFileSync(output, ""); assert.equal(await main(env, options()), 0); assert.equal(readFileSync(output, "utf8"), ""); @@ -1721,6 +1721,18 @@ test("resolve-conflicts: a head-lag stop on the current head is sent again once, } }); +test("a version 5 record that holds a head-lag-stopped head is ignored, and resolve-conflicts and sync-shared-copies send it again", async () => { + const head = pr(1).headRefOid; + const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK_NAME]: [head], "pr-unblock / sync-shared-copies": [copyPr(46).headRefOid] }, bases: {}, dispatched: {} })); + assert.equal(record.handled.size, 0, "read as empty"); + const { api, run } = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) }, { record }); + await run(); + assert.deepEqual(dispatchedNumbers(api), ["1"]); + const node = copyPr(46); + const copies = copyApi({ timelines: new Map([[46, copyRefusal(node.headRefOid)]]), checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) }); + assert.equal((await copyDispatch(copies, [node], { record })).dispatched.length, 1); +}); + test("sync-shared-copies: a head-lag stop on the current head is sent again once, under the cap", async () => { const node = copyPr(45); const timelines = new Map([[45, copyRefusal(node.headRefOid)]]);