Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions .github/scripts/pr-unblock/dependency-dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import { DEPENDABOT_ID, DEPENDABOT_LOGIN, RED } from "./dependency-classify.mjs";
import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs";
import { LIMIT_DEFAULTS } from "../work-resolve-issue/limits.mjs";
import { anyHandles, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs";
import { anyHandles, headLagStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs";
import { ciStatusOf, classifyPull, commitFilesReader } from "./review-dependency.mjs";

export const DEFAULTS = Object.freeze({
Expand All @@ -46,7 +46,11 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/;

// The neutral titles review-dependency-not-run posts from the select job's
// skip reasons (review-dependency.mjs select): no verdict job follows them,
// so no no-model-turn verdict can release the head.
// so no no-model-turn verdict can release the head. The select job's
// superseded-sha lands on the head it was sent for, which the PR has left;
// the same title on the head read here, the PR's current one, is the gate's
// head-lag stop, which did no work, so lagStopsRelease sends it again
// (limit-stop.mjs) and it is not recorded.
const TERMINAL_SKIPS = new Set(["Skipped: awaiting-human", "Skipped: not-applicable", "Skipped: superseded-sha"]);

// NaN for a run with no readable start: it compares false either way.
Expand Down Expand Up @@ -141,7 +145,7 @@ export async function dispatchDependencyReviews({
// not released that way, though its round's verdict also finds no model
// turn: it holds the head until its limit resets.
let verdicts = [];
if (anyHandles(existing, { now, limits })) {
if (anyHandles(existing, { now, limits, lagStopsRelease: true })) {
verdicts = await api.listCheckRuns({ sha: node.headRefOid, checkName: verdictCheckName, appId });
const newest = verdicts.reduce((a, b) => (startOf(b) > startOf(a) ? b : a), verdicts[0]);
if (newest?.status === "completed" && newest.conclusion === "neutral" && newest.output?.title === "Skipped: no-model-turn") {
Expand All @@ -163,18 +167,21 @@ export async function dispatchDependencyReviews({
const newestCheck = Math.max(...existing.map(startOf));
const last = existing.find((run) => startOf(run) === newestCheck);
const terminal = last?.status === "completed" && last.conclusion === "neutral" && TERMINAL_SKIPS.has(last.output?.title);
if (permanentlyHandles(existing, { limits }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) {
if (permanentlyHandles(existing, { limits, lagStopsRelease: true }) && (terminal || verdicts.some((run) => startOf(run) > newestCheck))) {
markHandled(record, checkName, node.headRefOid);
}
} catch (error) {
// A read that fails, whatever GitHub answered, holds the PR for a later run.
log(`::warning::#${node.number}: could not read check runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`);
continue;
}
if (anyHandles(existing, { now, limits })) {
if (anyHandles(existing, { now, limits, lagStopsRelease: true })) {
log(`skip #${node.number}: head ${node.headRefOid} already has a ${checkName} check run`);
continue;
}
if (existing.some(headLagStop)) {
log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`);
}
sent += 1;
try {
const { workflowRunId } = await api.dispatchWorkflow({
Expand Down
34 changes: 34 additions & 0 deletions .github/scripts/pr-unblock/dependency-dispatch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,30 @@ test("a head the kill switch stopped dispatches again, and is not recorded as ha
}
});

// The gate stopped the review head-lag: report-check-run posts a superseded-sha
// skip on the head the run read. On the head the dispatcher reads, the PR's
// current one, it did no work, so it is sent again once and not recorded,
// unlike the select job's superseded-sha, which lands on a head the PR left.
const lagged = (started) => roundRun("review-dependency", "neutral", "Skipped: superseded-sha", started);

test("a review the gate stopped head-lag on the current head is sent again once, under the cap, and not recorded", async () => {
const node = pr(62);
const record = emptyRecord();
const lines = [];
const once = await run(fakeApi({ nodes: [node], checks: headOf(node, [lagged("2026-10-17T10:00:00Z")]) }), { record, log: (line) => lines.push(line) });
assert.deepEqual(once.dispatched, [{ number: 62, workflowRunId: "7062" }]);
assert.equal(record.handled.get(CHECK), undefined);
assert.ok(lines.includes(`refire: #62: head ${node.headRefOid}'s ${CHECK} was stopped head-lag; sending it again`), lines.join("\n"));
for (const [label, checks] of [
["a newer review passed", [lagged("2026-10-17T10:00:00Z"), roundRun("review-dependency", "success", "Passed", "2026-10-17T10:30:00Z")]],
["a newer review is still going", [lagged("2026-10-17T10:00:00Z"), { ...roundRun("review-dependency", null, "", "2026-10-17T10:30:00Z"), status: "in_progress" }]],
["the head is at the cap", [lagged("2026-10-17T10:00:00Z"), lagged("2026-10-17T10:30:00Z"), lagged("2026-10-17T11:00:00Z")]],
]) {
const { dispatched } = await run(fakeApi({ nodes: [node], checks: headOf(node, checks) }));
assert.deepEqual(dispatched, [], label);
}
});

// A review-dependency run the usage limit stopped at `started`, its lane record
// naming a reset an hour later.
const limitStop = (started) => {
Expand Down Expand Up @@ -421,6 +445,16 @@ test("a version 2 record that holds a limit-stopped head is ignored, and the hea
assert.deepEqual(dispatched, [{ number: 67, workflowRunId: "7067" }]);
});

// A version 5 record held a head-lag-stopped head as reviewed (its
// superseded-sha skip was terminal); that record's version is read as empty.
test("a version 5 record that holds a head-lag-stopped head is ignored, and the head dispatches again", async () => {
const node = pr(68);
const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK]: [node.headRefOid] }, dispatched: {} }));
assert.equal(record.handled.get(CHECK), undefined, "read as empty");
const checks = headOf(node, [roundRun("review-dependency", "neutral", "Skipped: superseded-sha", "2026-10-17T10:00:00Z")]);
assert.deepEqual((await run(fakeApi({ nodes: [node], checks }), { record })).dispatched, [{ number: 68, workflowRunId: "7068" }]);
});

// Reads spent only where they change the decision. Each count beside the
// assertion is what the dispatcher read before it skipped them.
const RED_CI = [{ id: 1, app: { slug: "github-actions" }, status: "completed", conclusion: "failure" }];
Expand Down
6 changes: 4 additions & 2 deletions .github/scripts/pr-unblock/dispatch-record.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/;
// version 3 a review-dependency head a limit stop holds until its reset, and
// version 4 a resolve-conflicts head a failed run or a held skip left, and
// version 5 a resolve-conflicts head a select job's stop left under an older
// base. A fix-ci
// base, and version 6 a resolve-conflicts, sync-shared-copies or
// review-dependency head a head-lag stop left (limit-stop.mjs
// lagStopsRelease). A fix-ci
// head recorded at its per-head cap needs no new version: a record with no
// fix-ci entry reads the same either way; nor do fix-docs runs, read as none.
export const RECORD_VERSION = 5;
export const RECORD_VERSION = 6;
// The fix-docs runs kept per head, newest last: the reader drops the settled
// runs that do not count, so more than this many is a tampered entry.
const MAX_DOCS_RUNS = 20;
Expand Down
18 changes: 14 additions & 4 deletions .github/scripts/pr-unblock/dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// limit stop, a usage limit or a refused credential, stops counting once its
// limit resets, and the skip names its class and reset; a failed run until
// limit-stops-per-head failures stand on the head, a skip while held once
// the hold is gone, and a select job's own lane-stopped skip once the base tip
// the hold is gone, a head-lag stop (the gate's lagging PR read) at once, and a select job's own lane-stopped skip once the base tip
// was committed after it started, since the head then merges another base),
// and dispatches pr-unblock.yml on main for at most maxDispatch of them,
// lowest PR number first, heads retried on a moved base last. Authors are
Expand Down Expand Up @@ -56,7 +56,7 @@ import { prHold, reviewReader } from "./gate-source.mjs";
import { runTitle, shareWorkflowRuns } from "./feedback-dispatch.mjs";
import { DEFAULTS as CI_FIX_DEFAULTS, dispatchCiFixes, humanGatedLabel } from "./fix-ci-dispatch.mjs";
import { DEFAULTS as DOCS_FIX_DEFAULTS, dispatchDocsFixes, HISTORY_QUERY as DOCS_FIX_HISTORY_QUERY } from "./fix-docs-dispatch.mjs";
import { anyHandles, cappedAt, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs";
import { anyHandles, cappedAt, headLagStop, laneStop, limitStop, pendingStop, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs";
import { budgetedFetch } from "./request-budget.mjs";
import { verifyCapReader } from "./verify-cap.mjs";
import { readConfig, readPolicy } from "./review-dependency.mjs";
Expand Down Expand Up @@ -85,6 +85,14 @@ export const DEFAULTS = Object.freeze({

// The run id form the record's readers accept.
const RUN_ID = /^[1-9][0-9]{0,14}$/;

// The log line that names a head-lag stop on the head as the cause of this
// dispatch (limit-stop.mjs lagStopsRelease).
function lagRefire(log, node, checkName, existing, handled) {
if (!handled && existing.some(headLagStop)) {
log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`);
}
}
const TRUSTED_ACTORS_PATH = ".github/standards/trusted-actors/trusted-actors.json";

// A merge commit on a Dependabot head stops Dependabot's own rebases, which
Expand Down Expand Up @@ -246,7 +254,7 @@ export async function dispatchConflicting({
continue;
}
let existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId });
const options = { limits, failuresRetry: true, heldSkipsRelease: true };
const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true };
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved
// A push job that stopped on a reason routed to a failing status
// releases the head's earlier checks: the next run retries the same
// head, a fresh model session, until limit-stops-per-head such stops
Expand All @@ -268,6 +276,7 @@ export async function dispatchConflicting({
existing = existing.filter((run) => !(laneStop(run) && Date.parse(run.started_at ?? "") < committedAt));
}
handled = anyHandles(existing, { now, ...options });
lagRefire(log, node, checkName, existing, handled);
// Handled only because a limit stop has not reset: the log names it.
waiting = handled && !anyHandles(existing, { now: Infinity, ...options }) ? pendingStop(existing, { now, limits }) : null;
if (permanentlyHandles(existing, options)) {
Expand Down Expand Up @@ -401,7 +410,7 @@ export async function dispatchSharedCopies({
continue;
}
const existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId });
const options = { limits, failuresRetry: true, heldSkipsRelease: true };
const options = { limits, failuresRetry: true, heldSkipsRelease: true, lagStopsRelease: true };
if (permanentlyHandles(existing, options)) {
const capped = cappedAt(existing, options);
if (capped > 0) {
Expand All @@ -419,6 +428,7 @@ export async function dispatchSharedCopies({
log(`skip #${node.number}: ${workflow} run ${inFlight.id} for ${activity} is not done yet`);
continue;
}
lagRefire(log, node, checkName, existing, false);
} catch (error) {
log(`::warning::#${node.number}: could not read its refusal, check runs or runs for ${node.headRefOid}: ${error.message}; skipped, a later run retries`);
continue;
Expand Down
58 changes: 56 additions & 2 deletions .github/scripts/pr-unblock/dispatch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1443,13 +1443,13 @@ test("main carries the record in DISPATCH_RECORD and writes it only when it chan
DISPATCH_RECORD: path,
};
// A recorded head no open PR carries any more is dropped.
writeFileSync(path, JSON.stringify({ version: 5, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} }));
writeFileSync(path, JSON.stringify({ version: 6, handled: { [CHECK_NAME]: ["f".repeat(40)] }, bases: { ["f".repeat(40)]: "b".repeat(40) }, dispatched: {} }));
const options = () => ({ api: fakeApi({ pages: [[pr(1)]], checked: new Set([pr(1).headRefOid]) }), sleep: async () => {}, log: () => {}, loadLimits: async () => LIMIT_DEFAULTS, loadAdoption: async () => ({}), loadOwnedBranches: async () => [] });
try {
writeFileSync(output, "");
assert.equal(await main(env, options()), 0);
assert.equal(readFileSync(output, "utf8"), "record-changed=true\n");
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 5, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} });
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { version: 6, handled: { [CHECK_NAME]: [pr(1).headRefOid] }, bases: {}, dispatched: {}, docsRuns: {} });
writeFileSync(output, "");
assert.equal(await main(env, options()), 0);
assert.equal(readFileSync(output, "utf8"), "");
Expand Down Expand Up @@ -1696,6 +1696,60 @@ test("sync-shared-copies: a failed select job's failure is retried until limit-s
assert.deepEqual(next.calls.checkReads, [], "recorded at the cap: nothing read again");
});

// The gate stopped the run head-lag (GET /pulls and the branch ref never
// agreed): report-check-run posts a superseded-sha skip on the head the run
// read. On the head the dispatcher reads, the PR's current one, the run did
// no work there, so it is sent again once, under limit-stops-per-head.
const lagStop = (time) => ({ status: "completed", conclusion: "neutral", output: { title: "Skipped: superseded-sha" }, started_at: at(time), completed_at: at(time) });

test("resolve-conflicts: a head-lag stop on the current head is sent again once, under the cap", async () => {
const head = pr(1).headRefOid;
const cap = LIMIT_DEFAULTS["limit-stops-per-head"];
const once = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) });
await once.run();
assert.deepEqual(dispatchedNumbers(once.api), ["1"]);
assert.ok(once.lines.includes(`refire: #1: head ${head}'s ${CHECK_NAME} was stopped head-lag; sending it again`), once.lines.join("\n"));
for (const [label, options] of [
["the round it sent is still going", { checked: new Map([[head, [lagStop("10:00")]]]), workflowRuns: [{ id: 1, status: "queued", display_title: pr_unblockTitle("resolve-conflicts", 1) }] }],
["a newer round resolved it", { checked: new Map([[head, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" }, started_at: at("10:30") }]]]) }],
["the head is at the cap", { checked: new Map([[head, Array.from({ length: cap }, (_, i) => lagStop(`1${i}:00`))]]) }],
["the stop is on a head the PR left", { pages: [[pr(1)]], checked: new Map([["e".repeat(40), [lagStop("10:00")]], [head, [{ status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) }],
]) {
const { api, run } = harness({ pages: [[pr(1)]], ...options });
await run();
assert.deepEqual(dispatchedNumbers(api), [], label);
}
});

test("a version 5 record that holds a head-lag-stopped head is ignored, and resolve-conflicts and sync-shared-copies send it again", async () => {
const head = pr(1).headRefOid;
const record = parseRecord(JSON.stringify({ version: 5, handled: { [CHECK_NAME]: [head], "pr-unblock / sync-shared-copies": [copyPr(46).headRefOid] }, bases: {}, dispatched: {} }));
assert.equal(record.handled.size, 0, "read as empty");
const { api, run } = harness({ pages: [[pr(1)]], checked: new Map([[head, [lagStop("10:00")]]]) }, { record });
await run();
assert.deepEqual(dispatchedNumbers(api), ["1"]);
const node = copyPr(46);
const copies = copyApi({ timelines: new Map([[46, copyRefusal(node.headRefOid)]]), checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) });
assert.equal((await copyDispatch(copies, [node], { record })).dispatched.length, 1);
});

test("sync-shared-copies: a head-lag stop on the current head is sent again once, under the cap", async () => {
const node = copyPr(45);
const timelines = new Map([[45, copyRefusal(node.headRefOid)]]);
const cap = LIMIT_DEFAULTS["limit-stops-per-head"];
const lines = [];
const once = copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]) });
assert.equal((await copyDispatch(once, [node], { log: (line) => lines.push(line) })).dispatched.length, 1);
assert.ok(lines.includes(`refire: #45: head ${node.headRefOid}'s pr-unblock / sync-shared-copies was stopped head-lag; sending it again`), lines.join("\n"));
for (const [label, api] of [
["the round it sent is still going", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00")]]]), runs: [{ id: 9, status: "queued", display_title: pr_unblockTitle("sync-shared-copies", 45) }] })],
["a newer round passed", copyApi({ timelines, checks: new Map([[node.headRefOid, [lagStop("10:00"), { status: "completed", conclusion: "success", output: { title: "Passed" } }]]]) })],
["the head is at the cap", copyApi({ timelines, checks: new Map([[node.headRefOid, Array.from({ length: cap }, () => lagStop("10:00"))]]) })],
]) {
assert.equal((await copyDispatch(api, [node])).dispatched.length, 0, label);
}
});

// A select job's own stop (pr-unblock.yml resolve-conflicts-not-run) judged
// the base it merged: once main moves, the same conflicting head is tried
// again, and an unchanged head and base stay skipped.
Expand Down
12 changes: 8 additions & 4 deletions .github/scripts/pr-unblock/fix-ci-dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ import { STOP_RESOLVED } from "./escalation-routes.mjs";
import { HOLD_LINE, runTitle } from "./feedback-dispatch.mjs";
import { failingChecks } from "./fix-ci-context.mjs";
import { prHold, reviewReader } from "./gate-source.mjs";
import { KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs";
import { HEAD_LAG_STOP, KILL_SWITCH_STOP, killSwitchStop, laneStop, releasesHead } from "./limit-stop.mjs";
import { serverFailure } from "./request-budget.mjs";
import { final as capFinal, deadlineWarning } from "./verify-cap.mjs";

Expand Down Expand Up @@ -189,9 +189,10 @@ export function withStops(runs, stopRuns) {
const counted = (runs) => runs.filter((run) => !(run.conclusion === "NEUTRAL" && String(run.title ?? "").startsWith("Skipped: untrusted-trigger")));

// A run that failed, found no failing check to fix, skipped on a PR held
// then, or that the kill switch stopped (paused) may be retried; any other
// run (going, pushed a fix, escalated) handles the head.
const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP);
// then, that the kill switch stopped, or that the gate stopped head-lag on
// this head, the PR's current one (paused), may be retried; any other run
// (going, pushed a fix, escalated) handles the head.
const paused = (run) => run.conclusion === "NEUTRAL" && (run.title === HELD_SKIP || run.title === KILL_SWITCH_STOP || run.title === HEAD_LAG_STOP);
const retryable = (run) => run.status === "COMPLETED" && (run.conclusion === "FAILURE" || paused(run) || (run.conclusion === "NEUTRAL" && NOTHING_TO_FIX.has(run.title)));

// The number of counted fix-ci model runs when every attempt may be retried
Expand Down Expand Up @@ -655,6 +656,9 @@ export async function dispatchCiFixes({
}
continue;
}
if (fixCiRuns.some((run) => run.conclusion === "NEUTRAL" && run.title === HEAD_LAG_STOP)) {
log(`refire: #${node.number}: head ${node.headRefOid}'s ${checkName} was stopped head-lag; sending it again`);
}
sent += 1;
try {
const { workflowRunId } = await api.dispatchWorkflow({
Expand Down
Loading
Loading