diff --git a/.github/scripts/pr-unblock/dependency-dispatch.mjs b/.github/scripts/pr-unblock/dependency-dispatch.mjs index b385e908c17..79c50f76462 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.mjs @@ -26,7 +26,7 @@ import { DEPENDABOT_ID, DEPENDABOT_LOGIN, RED } from "./dependency-classify.mjs"; import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; import { LIMIT_DEFAULTS } from "../work-resolve-issue/limits.mjs"; -import { anyHandles, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, headLagStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; import { ciStatusOf, classifyPull, commitFilesReader } from "./review-dependency.mjs"; export const DEFAULTS = Object.freeze({ @@ -46,7 +46,11 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/; // The neutral titles review-dependency-not-run posts from the select job's // skip reasons (review-dependency.mjs select): no verdict job follows them, -// so no no-model-turn verdict can release the head. +// so no no-model-turn verdict can release the head. The select job's +// superseded-sha lands on the head it was sent for, which the PR has left; +// the same title on the head read here, the PR's current one, is the gate's +// head-lag stop, which did no work, so lagStopsRelease sends it again +// (limit-stop.mjs) and it is not recorded. const TERMINAL_SKIPS = new Set(["Skipped: awaiting-human", "Skipped: not-applicable", "Skipped: superseded-sha"]); // NaN for a run with no readable start: it compares false either way. @@ -141,7 +145,7 @@ export async function dispatchDependencyReviews({ // not released that way, though its round's verdict also finds no model // turn: it holds the head until its limit resets. let verdicts = []; - if (anyHandles(existing, { now, limits })) { + if (anyHandles(existing, { now, limits, lagStopsRelease: true })) { verdicts = await api.listCheckRuns({ sha: node.headRefOid, checkName: verdictCheckName, appId }); const newest = verdicts.reduce((a, b) => (startOf(b) > startOf(a) ? b : a), verdicts[0]); if (newest?.status === "completed" && newest.conclusion === "neutral" && newest.output?.title === "Skipped: no-model-turn") { @@ -163,7 +167,7 @@ export async function dispatchDependencyReviews({ const newestCheck = Math.max(...existing.map(startOf)); const last = existing.find((run) => startOf(run) === newestCheck); const terminal = last?.status === "completed" && last.conclusion === "neutral" && TERMINAL_SKIPS.has(last.output?.title); - if (permanentlyHandles(existing, { limits }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) { + if (permanentlyHandles(existing, { limits, lagStopsRelease: true }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) { markHandled(record, checkName, node.headRefOid); } } catch (error) { @@ -171,10 +175,13 @@ export async function dispatchDependencyReviews({ log(`::warning::#${node.number}: could not read check runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`); continue; } - if (anyHandles(existing, { now, limits })) { + if (anyHandles(existing, { now, limits, lagStopsRelease: true })) { log(`skip #${node.number}: head ${node.headRefOid} already has a ${checkName} check run`); continue; } + if (existing.some(headLagStop)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } sent += 1; try { const { workflowRunId } = await api.dispatchWorkflow({ diff --git a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs index b322640a6ff..0261927f845 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs @@ -259,6 +259,30 @@ test("a head the kill switch stopped dispatches again, and is not recorded as ha } }); +// The gate stopped the review head-lag: report-check-run posts a superseded-sha +// skip on the head the run read. On the head the dispatcher reads, the PR's +// current one, it did no work, so it is sent again once and not recorded, +// unlike the select job's superseded-sha, which lands on a head the PR left. +const lagged = (started) => roundRun("review-dependency", "neutral", "Skipped: superseded-sha", started); + +test("a review the gate stopped head-lag on the current head is sent again once, under the cap, and not recorded", async () => { + const node = pr(62); + const record = emptyRecord(); + const lines = []; + const once = await run(fakeApi({ nodes: [node], checks: headOf(node, [lagged("2026-10-17T10:00:00Z")]) }), { record, log: (line) => lines.push(line) }); + assert.deepEqual(once.dispatched, [{ number: 62, workflowRunId: "7062" }]); + assert.equal(record.handled.get(CHECK), undefined); + assert.ok(lines.includes(`refire: #62: head ${node.headRefOid}'s ${CHECK} was stopped head-lag; sending it again`), lines.join("\n")); + for (const [label, checks] of [ + ["a newer review passed", [lagged("2026-10-17T10:00:00Z"), roundRun("review-dependency", "success", "Passed", "2026-10-17T10:30:00Z")]], + ["a newer review is still going", [lagged("2026-10-17T10:00:00Z"), { ...roundRun("review-dependency", null, "", "2026-10-17T10:30:00Z"), status: "in_progress" }]], + ["the head is at the cap", [lagged("2026-10-17T10:00:00Z"), lagged("2026-10-17T10:30:00Z"), lagged("2026-10-17T11:00:00Z")]], + ]) { + const { dispatched } = await run(fakeApi({ nodes: [node], checks: headOf(node, checks) })); + assert.deepEqual(dispatched, [], label); + } +}); + // A review-dependency run the usage limit stopped at `started`, its lane record // naming a reset an hour later. const limitStop = (started) => { @@ -421,6 +445,16 @@ test("a version 2 record that holds a limit-stopped head is ignored, and the hea assert.deepEqual(dispatched, [{ number: 67, workflowRunId: "7067" }]); }); +// A version 5 record held a head-lag-stopped head as reviewed (its +// superseded-sha skip was terminal); that record's version is read as empty. +test("a version 5 record that holds a head-lag-stopped head is ignored, and the head dispatches again", async () => { + const node = pr(68); + const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK]: [node.headRefOid] }, dispatched: {} })); + assert.equal(record.handled.get(CHECK), undefined, "read as empty"); + const checks = headOf(node, [roundRun("review-dependency", "neutral", "Skipped: superseded-sha", "2026-10-17T10:00:00Z")]); + assert.deepEqual((await run(fakeApi({ nodes: [node], checks }), { record })).dispatched, [{ number: 68, workflowRunId: "7068" }]); +}); + // Reads spent only where they change the decision. Each count beside the // assertion is what the dispatcher read before it skipped them. const RED_CI = [{ id: 1, app: { slug: "github-actions" }, status: "completed", conclusion: "failure" }]; diff --git a/.github/scripts/pr-unblock/dispatch-record.mjs b/.github/scripts/pr-unblock/dispatch-record.mjs index 1730dd17b3b..642ab617925 100644 --- a/.github/scripts/pr-unblock/dispatch-record.mjs +++ b/.github/scripts/pr-unblock/dispatch-record.mjs @@ -20,10 +20,12 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/; // version 3 a review-dependency head a limit stop holds until its reset, and // version 4 a resolve-conflicts head a failed run or a held skip left, and // version 5 a resolve-conflicts head a select job's stop left under an older -// base. A fix-ci +// base, and version 6 a resolve-conflicts, sync-shared-copies or +// review-dependency head a head-lag stop left (limit-stop.mjs +// lagStopsRelease). A fix-ci // head recorded at its per-head cap needs no new version: a record with no // fix-ci entry reads the same either way; nor do fix-docs runs, read as none. -export const RECORD_VERSION = 5; +export const RECORD_VERSION = 6; // The fix-docs runs kept per head, newest last: the reader drops the settled // runs that do not count, so more than this many is a tampered entry. const MAX_DOCS_RUNS = 20; diff --git a/.github/scripts/pr-unblock/dispatch.mjs b/.github/scripts/pr-unblock/dispatch.mjs index 0db9f2481b7..5e9590b50a7 100644 --- a/.github/scripts/pr-unblock/dispatch.mjs +++ b/.github/scripts/pr-unblock/dispatch.mjs @@ -13,7 +13,7 @@ // limit stop, a usage limit or a refused credential, stops counting once its // limit resets, and the skip names its class and reset; a failed run until // limit-stops-per-head failures stand on the head, a skip while held once -// the hold is gone, and a select job's own lane-stopped skip once the base tip +// the hold is gone, a head-lag stop (the gate's lagging PR read) at once, and a select job's own lane-stopped skip once the base tip // was committed after it started, since the head then merges another base), // and dispatches pr-unblock.yml on main for at most maxDispatch of them, // lowest PR number first, heads retried on a moved base last. Authors are @@ -56,7 +56,7 @@ import { prHold, reviewReader } from "./gate-source.mjs"; import { runTitle, shareWorkflowRuns } from "./feedback-dispatch.mjs"; import { DEFAULTS as CI_FIX_DEFAULTS, dispatchCiFixes, humanGatedLabel } from "./fix-ci-dispatch.mjs"; import { DEFAULTS as DOCS_FIX_DEFAULTS, dispatchDocsFixes, HISTORY_QUERY as DOCS_FIX_HISTORY_QUERY } from "./fix-docs-dispatch.mjs"; -import { anyHandles, cappedAt, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, cappedAt, headLagStop, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; import { budgetedFetch } from "./request-budget.mjs"; import { verifyCapReader } from "./verify-cap.mjs"; import { readConfig, readPolicy } from "./review-dependency.mjs"; @@ -85,6 +85,14 @@ export const DEFAULTS = Object.freeze({ // The run id form the record's readers accept. const RUN_ID = /^[1-9][0-9]{0,14}$/; + +// The log line that names a head-lag stop on the head as the cause of this +// dispatch (limit-stop.mjs lagStopsRelease). +function lagRefire(log, node, checkName, existing, handled) { + if (!handled && existing.some(headLagStop)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } +} const TRUSTED_ACTORS_PATH = ".github/standards/trusted-actors/trusted-actors.json"; // A merge commit on a Dependabot head stops Dependabot's own rebases, which @@ -246,7 +254,7 @@ export async function dispatchConflicting({ continue; } let existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); - const options = { limits, failuresRetry: true, heldSkipsRelease: true }; + const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true }; // A push job that stopped on a reason routed to a failing status // releases the head's earlier checks: the next run retries the same // head, a fresh model session, until limit-stops-per-head such stops @@ -268,6 +276,7 @@ export async function dispatchConflicting({ existing = existing.filter((run) => !(laneStop(run) && Date.parse(run.started_at ?? "") < committedAt)); } handled = anyHandles(existing, { now, ...options }); + lagRefire(log, node, checkName, existing, handled); // Handled only because a limit stop has not reset: the log names it. waiting = handled && !anyHandles(existing, { now: Infinity, ...options }) ? pendingStop(existing, { now, limits }) : null; if (permanentlyHandles(existing, options)) { @@ -401,7 +410,7 @@ export async function dispatchSharedCopies({ continue; } const existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); - const options = { limits, failuresRetry: true, heldSkipsRelease: true }; + const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true }; if (permanentlyHandles(existing, options)) { const capped = cappedAt(existing, options); if (capped > 0) { @@ -419,6 +428,7 @@ export async function dispatchSharedCopies({ log(`skip #${node.number}: ${workflow} run ${inFlight.id} for ${activity} is not done yet`); continue; } + lagRefire(log, node, checkName, existing, false); } catch (error) { log(`::warning::#${node.number}: could not read its refusal, check runs or runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`); continue; diff --git a/.github/scripts/pr-unblock/dispatch.test.mjs b/.github/scripts/pr-unblock/dispatch.test.mjs index 2b79658f395..defc850b6ec 100644 --- a/.github/scripts/pr-unblock/dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dispatch.test.mjs @@ -1443,13 +1443,13 @@ test("main carries the record in DISPATCH_RECORD and writes it only when it chan DISPATCH_RECORD: path, }; // A recorded head no open PR carries any more is dropped. - writeFileSync(path, JSON.stringify({ version: 5, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} })); + writeFileSync(path, JSON.stringify({ version: 6, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} })); const options = () => ({ api: fakeApi({ pages: [[pr(1)]], checked: new Set([pr(1).headRefOid]) }), sleep: async () => {}, log: () => {}, loadLimits: async () => LIMIT_DEFAULTS, loadAdoption: async () => ({}), loadOwnedBranches: async () => [] }); try { writeFileSync(output, ""); assert.equal(await main(env, options()), 0); assert.equal(readFileSync(output, "utf8"), "record-changed=true\n"); - assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 5, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} }); + assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 6, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} }); writeFileSync(output, ""); assert.equal(await main(env, options()), 0); assert.equal(readFileSync(output, "utf8"), ""); @@ -1696,6 +1696,60 @@ test("sync-shared-copies: a failed select job's failure is retried until limit-s assert.deepEqual(next.calls.checkReads, [], "recorded at the cap: nothing read again"); }); +// The gate stopped the run head-lag (GET /pulls and the branch ref never +// agreed): report-check-run posts a superseded-sha skip on the head the run +// read. On the head the dispatcher reads, the PR's current one, the run did +// no work there, so it is sent again once, under limit-stops-per-head. +const lagStop = (time) => ({ status: "completed", conclusion: "neutral", output: { title: "Skipped: superseded-sha" }, started_at: at(time), completed_at: at(time) }); + +test("resolve-conflicts: a head-lag stop on the current head is sent again once, under the cap", async () => { + const head = pr(1).headRefOid; + const cap = LIMIT_DEFAULTS["limit-stops-per-head"]; + const once = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) }); + await once.run(); + assert.deepEqual(dispatchedNumbers(once.api), ["1"]); + assert.ok(once.lines.includes(`refire: #1: head ${head}'s ${CHECK_NAME} was stopped head-lag; sending it again`), once.lines.join("\n")); + for (const [label, options] of [ + ["the round it sent is still going", { checked: new Map([[head, [lagStop("10:00")]]]), workflowRuns: [{ id: 1, status: "queued", display_title: pr_unblockTitle("resolve-conflicts", 1) }] }], + ["a newer round resolved it", { checked: new Map([[head, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" }, started_at: at("10:30") }]]]) }], + ["the head is at the cap", { checked: new Map([[head, Array.from({ length: cap }, (_, i) => lagStop(`1${i}:00`))]]) }], + ["the stop is on a head the PR left", { pages: [[pr(1)]], checked: new Map([["e".repeat(40), [lagStop("10:00")]], [head, [{ status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) }], + ]) { + const { api, run } = harness({ pages: [[pr(1)]], ...options }); + await run(); + assert.deepEqual(dispatchedNumbers(api), [], label); + } +}); + +test("a version 5 record that holds a head-lag-stopped head is ignored, and resolve-conflicts and sync-shared-copies send it again", async () => { + const head = pr(1).headRefOid; + const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK_NAME]: [head], "pr-unblock / sync-shared-copies": [copyPr(46).headRefOid] }, bases: {}, dispatched: {} })); + assert.equal(record.handled.size, 0, "read as empty"); + const { api, run } = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) }, { record }); + await run(); + assert.deepEqual(dispatchedNumbers(api), ["1"]); + const node = copyPr(46); + const copies = copyApi({ timelines: new Map([[46, copyRefusal(node.headRefOid)]]), checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) }); + assert.equal((await copyDispatch(copies, [node], { record })).dispatched.length, 1); +}); + +test("sync-shared-copies: a head-lag stop on the current head is sent again once, under the cap", async () => { + const node = copyPr(45); + const timelines = new Map([[45, copyRefusal(node.headRefOid)]]); + const cap = LIMIT_DEFAULTS["limit-stops-per-head"]; + const lines = []; + const once = copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) }); + assert.equal((await copyDispatch(once, [node], { log: (line) => lines.push(line) })).dispatched.length, 1); + assert.ok(lines.includes(`refire: #45: head ${node.headRefOid}'s pr-unblock / sync-shared-copies was stopped head-lag; sending it again`), lines.join("\n")); + for (const [label, api] of [ + ["the round it sent is still going", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]), runs: [{ id: 9, status: "queued", display_title: pr_unblockTitle("sync-shared-copies", 45) }] })], + ["a newer round passed", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) })], + ["the head is at the cap", copyApi({ timelines, checks: new Map([[node.headRefOid, Array.from({ length: cap }, () => lagStop("10:00"))]]) })], + ]) { + assert.equal((await copyDispatch(api, [node])).dispatched.length, 0, label); + } +}); + // A select job's own stop (pr-unblock.yml resolve-conflicts-not-run) judged // the base it merged: once main moves, the same conflicting head is tried // again, and an unchanged head and base stay skipped. diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index 231f2770db6..3e38e56b983 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -59,7 +59,7 @@ import { STOP_RESOLVED } from "./escalation-routes.mjs"; import { HOLD_LINE, runTitle } from "./feedback-dispatch.mjs"; import { failingChecks } from "./fix-ci-context.mjs"; import { prHold, reviewReader } from "./gate-source.mjs"; -import { KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs"; +import { HEAD_LAG_STOP, KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs"; import { serverFailure } from "./request-budget.mjs"; import { final as capFinal, deadlineWarning } from "./verify-cap.mjs"; @@ -189,9 +189,10 @@ export function withStops(runs, stopRuns) { const counted = (runs) => runs.filter((run) => !(run.conclusion === "NEUTRAL" && String(run.title ?? "").startsWith("Skipped: untrusted-trigger"))); // A run that failed, found no failing check to fix, skipped on a PR held -// then, or that the kill switch stopped (paused) may be retried; any other -// run (going, pushed a fix, escalated) handles the head. -const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP); +// then, that the kill switch stopped, or that the gate stopped head-lag on +// this head, the PR's current one (paused), may be retried; any other run +// (going, pushed a fix, escalated) handles the head. +const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP || run.title === HEAD_LAG_STOP); const retryable = (run) => run.status === "COMPLETED" && (run.conclusion === "FAILURE" || paused(run) || (run.conclusion === "NEUTRAL" && NOTHING_TO_FIX.has(run.title))); // The number of counted fix-ci model runs when every attempt may be retried @@ -655,6 +656,9 @@ export async function dispatchCiFixes({ } continue; } + if (fixCiRuns.some((run) => run.conclusion === "NEUTRAL" && run.title === HEAD_LAG_STOP)) { + log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`); + } sent += 1; try { const { workflowRunId } = await api.dispatchWorkflow({ diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs index 3b025344621..91d7e62b76d 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs @@ -1395,6 +1395,35 @@ test("a fix-ci run the kill switch stopped is retried on the same head without s assert.ok(own.lines.some((line) => line.startsWith("dispatched fix-ci #7 ")), own.lines.join("\n")); }); +// The gate stopped the fix-ci run head-lag (GET /pulls and the branch ref +// never agreed): report-check-run posts a superseded-sha skip on the head the +// run read. On the head the dispatcher reads, the PR's current one, the run +// did no work there, so it is sent again once, under the per-head cap. +test("a fix-ci run the gate stopped head-lag on the current head is sent again once, under the cap", async () => { + const head = "7".padStart(40, "a"); + const at = (time) => `2026-10-22T${time}:00Z`; + const lagged = (time) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "neutral", output: { title: "Skipped: superseded-sha" }, started_at: at(time), completed_at: at(time) }); + const lagCheck = check("pr-unblock / fix-ci", "NEUTRAL", { title: "Skipped: superseded-sha" }); + const lagPr = () => pr(7, { checks: [lanes("FAILURE"), lagCheck] }); + const once = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00")]]]) }); + assert.deepEqual(once.result.dispatched.map((d) => d.number), [7], once.lines.join("\n")); + assert.ok(once.lines.includes(`refire: #7: head ${head}'s pr-unblock / fix-ci was stopped head-lag; sending it again`), once.lines.join("\n")); + // The round it sent is still going: not sent twice. + const going = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00")]]]), workflowRuns: [{ id: 99, status: "queued", display_title: "pr-unblock fix-ci #7" }] }); + assert.deepEqual(going.result.dispatched, [], going.lines.join("\n")); + // A newer run pushed a fix: the head is handled. + const fixed = { name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title: "Fixed" }, started_at: at("10:30"), completed_at: at("10:30") }; + const newer = await run({ pages: [[pr(7, { checks: [lanes("FAILURE"), check("pr-unblock / fix-ci", "SUCCESS", { title: "Fixed" })] })]], history: new Map([[head, [lagged("10:00"), fixed]]]) }); + assert.deepEqual(newer.result.dispatched, [], newer.lines.join("\n")); + // Lag stops count toward the per-head cap. + const capped = await run({ pages: [[lagPr()]], history: new Map([[head, [lagged("10:00"), lagged("10:30"), lagged("11:00")]]]) }); + assert.deepEqual(capped.result.dispatched, [], capped.lines.join("\n")); + assert.ok(capped.lines.some((line) => line.includes("ran 3 time(s), the most one head gets")), capped.lines.join("\n")); + // The stop sits on a head the PR has left: the current head has no fix-ci run, so it is dispatched as any head with none, with no refire line. + const moved = await run({ pages: [[pr(7)]], history: new Map([["8".padStart(40, "a"), [lagged("10:00")]]]) }); + assert.ok(!moved.lines.some((line) => line.startsWith("refire:")), moved.lines.join("\n")); +}); + // fix-ci and the in-run fix-ci-rung-up each post a fix-ci check in one // workflow run (the run id in each check's external_id; the API returns each // check's details_url as the check run's own page). When the rung-up's diff --git a/.github/scripts/pr-unblock/limit-stop.mjs b/.github/scripts/pr-unblock/limit-stop.mjs index 7997c5ac286..1cdfc8e0b6b 100644 --- a/.github/scripts/pr-unblock/limit-stop.mjs +++ b/.github/scripts/pr-unblock/limit-stop.mjs @@ -107,13 +107,27 @@ const untrustedSkip = (run) => // `Skipped: ` title). const heldSkip = (run) => run.status === "completed" && run.conclusion === "neutral" && run.output?.title === "Skipped: awaiting-human"; -// Two opt-ins, for a dispatcher that checks the PR's holds itself before it +// The gate's head-lag stop (check-trusted-trigger.mjs fetchPull), which +// report-check-run posts as a superseded-sha skip (gate-skip-reasons.json) on +// the head the run read. Every other superseded-sha check lands on a head the +// PR has left (a cancel, a moved tip before the mint or the push), so on the +// head a dispatcher reads, the PR's current one, it is a lag stop that did no +// work there. +export const HEAD_LAG_STOP = "Skipped: superseded-sha"; +export const headLagStop = (run) => run?.status === "completed" && run.conclusion === "neutral" && run.output?.title === HEAD_LAG_STOP; + +// Three opt-ins, for a dispatcher that checks the PR's holds itself before it // reads the check runs: with `heldSkipsRelease` a held skip stops handling the // head, so a head skipped while held is dispatched once the hold is gone; with -// `failuresRetry` a failed run of any kind does too. Either way the head is -// handled once `limit-stops-per-head` of them, limit stops included, stand on -// it, so a dispatcher and a run that disagree about a hold stop there. -const released = (run, heldSkipsRelease) => (heldSkipsRelease && heldSkip(run)) || killSwitchStop(run); +// `lagStopsRelease` a head-lag stop on the head read does too, so a round the +// gate stopped on a lagging API read is sent again; with `failuresRetry` a +// failed run of any kind does too. Either way the head is handled once +// `limit-stops-per-head` of them, limit stops included, stand on it, so a +// dispatcher and a run that disagree about a hold stop there. A newer run +// still going handles the head, and a newer finished one decides it, so each +// stop is sent again once. +const released = (run, { heldSkipsRelease = false, lagStopsRelease = false } = {}) => + (heldSkipsRelease && heldSkip(run)) || (lagStopsRelease && headLagStop(run)) || killSwitchStop(run); // The neutral check a run the lanes kill switch stopped posts: its own // kill-switch step, or the broker refusing its mint for a switch turned on @@ -132,13 +146,13 @@ export const releasesHead = (run) => laneStop(run) || killSwitchStop(run); // Whether `runs` handle the head: any run that is not an untrusted skip, a // kill-switch stop or a reset limit stop does, and so do // `limit-stops-per-head` such stops. -export function anyHandles(runs, { now, limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false }) { +export function anyHandles(runs, { now, limits = LIMIT_DEFAULTS, failuresRetry = false, ...opt }) { let stops = 0; for (const run of runs) { if (untrustedSkip(run)) { continue; } - if (released(run, heldSkipsRelease)) { + if (released(run, opt)) { stops += 1; continue; } @@ -167,9 +181,9 @@ export function permanentlyHandles(runs, options = {}) { return done.some((run) => !retryable(run)) || cappedAt(runs, options) > 0; } -function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false }) { +function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, ...opt }) { const done = runs.filter((run) => run.status === "completed" && !untrustedSkip(run)); - const retryable = (run) => released(run, heldSkipsRelease) || (failuresRetry ? run.conclusion === "failure" : limitResetsAt(run, limits) !== null); + const retryable = (run) => released(run, opt) || (failuresRetry ? run.conclusion === "failure" : limitResetsAt(run, limits) !== null); return { done, retryable }; } @@ -177,11 +191,11 @@ function completedRuns(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, h // they reach `limit-stops-per-head`, the head's hard end, else 0: the part of // permanentlyHandles a dispatcher reports as a stop. A head with any other // finished run (a success, an escalation) is done, not capped. -export function cappedAt(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false } = {}) { - if (!failuresRetry && !heldSkipsRelease) { +export function cappedAt(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, heldSkipsRelease = false, lagStopsRelease = false } = {}) { + if (!failuresRetry && !heldSkipsRelease && !lagStopsRelease) { return 0; } - const { done, retryable } = completedRuns(runs, { limits, failuresRetry, heldSkipsRelease }); + const { done, retryable } = completedRuns(runs, { limits, failuresRetry, heldSkipsRelease, lagStopsRelease }); return done.length >= limits["limit-stops-per-head"] && done.length > 0 && done.every(retryable) ? done.length : 0; } diff --git a/.github/scripts/pr-unblock/limit-stop.test.mjs b/.github/scripts/pr-unblock/limit-stop.test.mjs index a9242f35e90..450973a223d 100644 --- a/.github/scripts/pr-unblock/limit-stop.test.mjs +++ b/.github/scripts/pr-unblock/limit-stop.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { anyHandles, KILL_SWITCH_STOP, killSwitchStop, limitResetsAt, pendingStop, permanentlyHandles, releasedBy, releasesHead, stopRelease } from "./limit-stop.mjs"; +import { anyHandles, cappedAt, HEAD_LAG_STOP, headLagStop, KILL_SWITCH_STOP, killSwitchStop, limitResetsAt, pendingStop, permanentlyHandles, releasedBy, releasesHead, stopRelease } from "./limit-stop.mjs"; const HOUR = 3_600_000; const ENDED = "2026-10-17T10:00:00Z"; @@ -184,6 +184,22 @@ test("with held skips released, an awaiting-human skip does not handle the head" assert.equal(anyHandles([heldSkip, run(1), run(2)], { now, heldSkipsRelease: true, failuresRetry: true }), true, "held skips and failures share the cap"); }); +test("with lag stops released, a head-lag stop does not handle the head until the cap; by default it does", () => { + const now = ended + HOUR; + const lagged = (id) => ({ ...heldSkip, id, output: { title: HEAD_LAG_STOP } }); + assert.equal(HEAD_LAG_STOP, "Skipped: superseded-sha"); + assert.equal(headLagStop(lagged(1)), true); + assert.equal(headLagStop({ ...lagged(1), conclusion: "failure" }), false); + assert.equal(anyHandles([lagged(1)], { now }), true, "by default it handles the head"); + assert.equal(anyHandles([lagged(1)], { now, lagStopsRelease: true }), false); + assert.equal(permanentlyHandles([lagged(1)], { lagStopsRelease: true }), false); + assert.equal(anyHandles([lagged(1), { ...lagged(2), status: "in_progress", conclusion: null }], { now, lagStopsRelease: true }), true, "a newer run going handles it"); + assert.equal(anyHandles([lagged(1), { ...lagged(2), conclusion: "success", output: { title: "Passed" } }], { now, lagStopsRelease: true }), true, "a newer run that acted handles it"); + const three = [lagged(1), lagged(2), lagged(3)]; + assert.equal(anyHandles(three, { now, lagStopsRelease: true }), true, "the cap ends it"); + assert.equal(cappedAt(three, { lagStopsRelease: true }), 3); +}); + // A run the lanes kill switch stopped did no work: its own kill-switch step, // or a broker refusal of its mint (request-lane-token kill-switch-ok), posts // neutral kill-switch (decide.mjs: "Skipped: ").