Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/scripts/maintenance-monitor-signals/signals.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ export function ceilingFindings({ now, budgetReadings = [], jobsInUse = null })
}

// Broker refusals from request-lane-token's annotations. workflow-modified is
// left out: pr-merge re-fires those runs on its schedule.
// left out: pr-merge re-fires those runs on its schedule, and the feedback
// sweep sends a refused address-feedback round again.
const RATE_LIMITED = /github-rate-limited|rate limit|\b429\b/i;
const BROKER_DENY = /^lane-(?:token|check)-denied: the broker answered (\d{3}) with reason ([a-z]+(?:-[a-z]+)*)/;
const BROKER_UNREACHABLE = /^broker-unreachable:/;
Expand Down
31 changes: 24 additions & 7 deletions .github/scripts/pr-merge/refire-refused.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
// on its current head failed at "Request the lane token" because the token
// broker refused it as workflow-modified. A rerun keeps its first event's
// workflow files, so no rerun can pass; after the new head, the next
// pr-unblock run uses main's files. No model.
// pr-unblock run uses main's files. No model. An address-feedback run refused
// that way is left to the feedback sweep, which dispatches it again on main's
// files at the same head (the lane's `dispatcherRefires`).
//
// The new head is an empty commit: the head's own tree, the head as its only
// parent, made with the lanes App token through the Git Data API and moved
Expand Down Expand Up @@ -113,6 +115,11 @@ const LANES = Object.freeze({
// review lanes finish a head, and a failed check run there holds the head
// (pr-unblock/feedback-dispatch.mjs).
transientActivities: ["address-feedback"],
// The activities whose workflow-modified refusal the half-hourly tick's
// feedback sweep sends again as a fresh dispatch on main
// (pr-unblock/feedback-dispatch.mjs sweepRefused), so no empty commit
// restarts both review lanes for them.
dispatcherRefires: ["address-feedback"],
transientStarts:
" The push starts no run itself: both review lanes review the new head, and an address-feedback run starts after them if an unresolved thread is left for it.",
},
Expand Down Expand Up @@ -252,17 +259,27 @@ async function pullOfRun({ actions, repo, run, lane, byNumber, ids, appId, mainS
}

// Why this run's token step failed, when an empty commit can help, or null:
// `{kind: "workflow-modified", path}` for the workflow file the broker named,
// else, for a run of one of the lane's `transientActivities`,
// `{kind: "transient", label}` for a TRANSIENT failure.
// tokenRefusal's answer, except a workflow-modified refusal of one of the
// lane's `dispatcherRefires`, which a dispatcher sends again on main.
async function refusedFile({ actions, repo, run, lane }) {
const activity = DISPATCH_TITLE.exec(String(run.display_title ?? ""))?.[1];
const transientOk = (lane.transientActivities ?? []).includes(activity);
let transient = null;
const jobs = (await actions("GET", `/repos/${repo}/actions/runs/${run.id}/jobs?filter=latest&per_page=100`))?.jobs;
if (!Array.isArray(jobs)) {
throw new Error(`run ${run.id}'s jobs have an unexpected shape`);
}
const readAnnotations = (id) => actions("GET", `/repos/${repo}/check-runs/${id}/annotations?per_page=100`);
const why = await tokenRefusal({ jobs, readAnnotations, lane, activity });
return why?.kind === "workflow-modified" && (lane.dispatcherRefires ?? []).includes(activity) ? null : why;
}

// Why a lane run's token step failed, read from its latest `jobs` and each
// failed write job's annotations (`readAnnotations(job id)`), or null:
// `{kind: "workflow-modified", path}` for the workflow file the broker named,
// else, for a run of one of the lane's `transientActivities`,
// `{kind: "transient", label}` for a TRANSIENT failure.
export async function tokenRefusal({ jobs, readAnnotations, lane = LANES["pr-unblock.yml"], activity }) {
const transientOk = (lane.transientActivities ?? []).includes(activity);
let transient = null;
for (const job of jobs) {
// Only a write runner job requests the token. The jobs API can list a
// completed job's steps empty or cut short: such a job whose list lacks
Expand All @@ -273,7 +290,7 @@ async function refusedFile({ actions, repo, run, lane }) {
if (!failedAtToken || !Number.isSafeInteger(job.id)) {
continue;
}
const annotations = await actions("GET", `/repos/${repo}/check-runs/${job.id}/annotations?per_page=100`);
const annotations = await readAnnotations(job.id);
for (const annotation of Array.isArray(annotations) ? annotations : []) {
if (annotation.annotation_level !== "failure") {
continue;
Expand Down
7 changes: 7 additions & 0 deletions .github/scripts/pr-merge/refire-refused.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,13 @@ test("a token 403 for any reason other than workflow-modified is not re-fired, a
}
});

test("an address-feedback run refused as workflow-modified is left to the feedback sweep, which dispatches it again on main", async () => {
const w = refusedWorld([pr(9)], { runOptions: { title: FEEDBACK.title, jobName: FEEDBACK.jobName } });
assert.equal(await main({ env, github: w.github, actions: w.actions, now: NOW }), 0);
assert.deepEqual(pushes(w.calls), []);
assert.deepEqual(posts(w.calls), []);
});

test("a transient token failure of an activity pr-unblock-dispatch retries itself is left to that tick", async () => {
for (const activity of ["resolve-conflicts", "review-dependency", "fix-ci"]) {
const w = refusedWorld([pr(9)], { runOptions: { title: `pr-unblock ${activity} #9`, messages: [DENIED_5XX] } });
Expand Down
69 changes: 63 additions & 6 deletions .github/scripts/pr-unblock/feedback-dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@
// EARLIER_FAIL_READS cap may have left it. Nothing records an answer, so a
// head answered "no earlier FAIL" may be read again inside that window.
//
// Before every other sweep, the tick sends again a round the token broker
// refused as workflow-modified, once per refused run (sweepRefused): a round
// in flight when a lane workflow file changes on main is refused its push or
// reply token, a re-run keeps the old file, and the head's check runs would
// otherwise hold it.
//
// A run of this file an outage lost, or whose dispatch job failed, dispatched
// nothing the lanes trust when re-run: check-trusted-trigger's dispatcher
// check accepts a dispatcher at attempt 1 or re-run by a listed human, so
Expand Down Expand Up @@ -177,7 +183,9 @@ import { pathToFileURL } from "node:url";

import { commitPulls, detailSuffix, errorLabel, responseDetail } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs";
import { countRequests } from "../../actions/log-api-budget/rate-guard.mjs";
import { postingRunId } from "../../actions/report-check-run/posting-run.mjs";
import { LOOKBACK_HOURS, LOST } from "../maintenance-reconcile-lost-runs/infra-failure.mjs";
import { tokenRefusal } from "../pr-merge/refire-refused.mjs";
import { anyListed, fixRequesters, HOLD_LINE, humanGatedLabelFrom, isFixRequest, MARKER_AUTHOR_ID, NO_ADOPTION, onReviewedHead, ownerAdoption, ownerAuthored, readTriggers, REQUEST_HISTORY_QUERY, requestHistoryOf, requestSkip, requestSkipLine, restMentions, reviewedHeads, reviewerIdsFrom, threadAuthors, trustedHumansFrom, VERIFY_NOTE_MARKER } from "./address-feedback.mjs";

// fix-ci-dispatch.mjs imports HOLD_LINE from here.
Expand Down Expand Up @@ -837,8 +845,9 @@ export const SWEEP_TICK_CAP = 5;
// 2.5 minutes of margin (judgment).
export const SWEEP_DECIDE_BUDGET_MS = 5 * 60_000;

// The sweep, in pass order: each open lanes App PR into main with a fix
// request newer than its head's last round (sweepFixRequest); each whose
// The sweep, in pass order: each open PR whose newest round the token broker
// refused as workflow-modified (sweepRefused); each open lanes App PR into
// main with a fix request newer than its head's last round (sweepFixRequest); each whose
// head's newest reply check took a releasing skip, under SWEEP_HEAD_CAP such
// checks (sweepOne); then sweepStale, sweepUnsent and sweepIdleOwner. Each PR
// is decided at most once a tick, until SWEEP_TICK_CAP PRs are decided or
Expand All @@ -856,6 +865,7 @@ export async function sweepReleased({ api, repository, appSenderId, log, setting
const taken = new Set();
const started = clock();
const passes = [
[sweepRefused, listed, "sweep: no round the broker refused as workflow-modified to send again"],
[sweepFixRequest, pulls, "sweep: no head with a fix request newer than its last round"],
[sweepOne, pulls, "sweep: no head whose last round left it for another round to dispatch"],
[sweepStale, pulls, "sweep: no head with a failing address-feedback stop and nothing left to fix"],
Expand Down Expand Up @@ -892,10 +902,10 @@ export async function sweepReleased({ api, repository, appSenderId, log, setting
}
}
const sent = [];
for (const { number, send, upstreamRunId } of decided) {
for (const { number, send, upstreamRunId, refireOf } of decided) {
let result;
try {
result = { ...(await send()), upstreamRunId };
result = { ...(await send()), upstreamRunId, ...(refireOf ? { refireOf } : {}) };
} catch (error) {
// A POST the API rejects starts no round to hold the PR: log it and
// send the rounds after it, as the per-PR sweep did.
Expand Down Expand Up @@ -1176,6 +1186,49 @@ async function dispatchAtHead({ api, pull, repository, appSenderId, log, setting
return dispatched === null ? null : { ...dispatched, upstreamRunId: upstream.id };
}

// A round the token broker refused as workflow-modified (refire-refused.mjs
// tokenRefusal): it was in flight when a lane workflow file changed on main,
// so its push or reply job's token was refused, and a re-run keeps the old
// file. The tick, before its other sweeps, sends the activity again as a
// fresh attempt-1 dispatch on main's files at the same head, with the head's
// newest completed review run as the upstream, as every sweep dispatch.
// Once per refused run: only the newest run of the activity for the PR
// counts, so the run this sends, or any later one, ends it. It counts only
// when it failed within FAILED_RUN_WINDOW_HOURS, posted a check run on the
// PR's current head (one head per refusal), and that head has fewer than
// SWEEP_HEAD_CAP address-feedback check runs, which bounds a chain of
// refusals. Check runs started before the refused run ended stop counting
// (requestSince), but a limit stop keeps its reset hold. A refusal for any
// other reason is left alone. The result names the refused run (refireOf),
// which the dispatched-run record keeps.
async function sweepRefused(context) {
const { api, pull, log, settings, rest } = context;
const head = pull.head.sha;
const since = (rest.now ?? Date.now()) - FAILED_RUN_WINDOW_HOURS * 3_600_000;
const [last] = (await api.listWorkflowRuns(settings.workflow))
.filter((run) => run.display_title === runTitle(settings.activity, pull.number))
.sort((a, b) => Date.parse(b.created_at ?? "") - Date.parse(a.created_at ?? ""));
if (last?.status !== "completed" || last.conclusion !== "failure" || !RUN_ID.test(String(last.id)) || !(Date.parse(last.updated_at ?? "") >= since)) {
return null;
}
const rounds = await api.listCheckRuns({ sha: head, checkName: settings.checkName, appId: settings.appId });
const posted = (checks) => checks.some((check) => postingRunId(check) === String(last.id));
if (!posted(rounds) && !posted(await api.listCheckRuns({ sha: head, checkName: settings.replyCheckName, appId: settings.appId }))) {
return null;
}
const why = await tokenRefusal({ jobs: await api.listRunJobs(last.id), readAnnotations: api.listAnnotations, activity: settings.activity });
if (why?.kind !== "workflow-modified") {
return null;
}
if (rounds.length >= SWEEP_HEAD_CAP) {
log(`skip: #${pull.number}: run ${last.id} was refused as workflow-modified (${why.path}), and head ${head} has ${rounds.length} rounds, the sweep's cap`);
return null;
}
log(`refire: #${pull.number}: run ${last.id} was refused as workflow-modified (${why.path}) at head ${head}; sending ${settings.activity} again on main`);
const dispatched = await dispatchAtHead(context, { requestSince: Date.parse(last.updated_at) });
return dispatched === null ? null : { ...dispatched, refireOf: String(last.id) };
}

// This file's workflow, whose lost runs the tick recovers.
export const DISPATCH_WORKFLOW = ".github/workflows/pr-feedback-dispatch.yml";

Expand Down Expand Up @@ -1559,6 +1612,8 @@ export function createGitHubApi({ token, repository, apiUrl, graphqlUrl, fetch:
}
return listing.jobs;
},
// A job's check run annotations, one page: the token step's refusal.
listAnnotations: (id) => request(`${repo}/check-runs/${encodeURIComponent(id)}/annotations?per_page=100`),
async dispatchWorkflow({ workflow, ref, inputs }) {
const result = await request(`${repo}/actions/workflows/${workflow}/dispatches`, "POST", {
ref,
Expand Down Expand Up @@ -1658,12 +1713,14 @@ export async function main(env, { api, github, resolveSlot, verifyCapped, log =
}
// Each send's outputs are written as it is sent, so a later send's
// failure still leaves the earlier ones recorded: the first as run-id,
// the nth as run-id-<n>.
// the nth as run-id-<n>. A round sent again after a workflow-modified
// refusal also names the refused run, as refire-of-<n>.
let sends = 0;
const write = (swept) => {
sends += 1;
const suffix = sends === 1 ? "" : `-${sends}`;
appendFileSync(env.GITHUB_OUTPUT, `run-id${suffix}=${swept.workflowRunId}\nupstream-run-id${suffix}=${swept.upstreamRunId}\n`);
const refire = swept.refireOf ? `refire-of${suffix}=${swept.refireOf}\n` : "";
appendFileSync(env.GITHUB_OUTPUT, `run-id${suffix}=${swept.workflowRunId}\nupstream-run-id${suffix}=${swept.upstreamRunId}\n${refire}`);
};
// A lost run is recovered only on a tick the sweep sent nothing on.
const sweep = async () => (await sweepReleased({ ...common, onSent: write })) ?? recoverLost({ ...common, now });
Expand Down
72 changes: 72 additions & 0 deletions .github/scripts/pr-unblock/feedback-dispatch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2303,6 +2303,78 @@ test("a lost run whose dispatch job never started or was not lost, or this run i
}
});

// A round in flight when a lane workflow file changed on main: its push job's
// token step got the broker's workflow-modified refusal (request-lane-token's
// annotation), after its address-feedback check posted on the head.
const REFUSED_RUN = 700;
const WORKFLOW_MODIFIED = "lane-token-denied: the broker answered 403 with reason workflow-modified: .github/workflows/pr-unblock.yml differs from refs/heads/main";
const refusedRun = (overrides = {}) => ({
id: REFUSED_RUN,
display_title: TITLE_42,
status: "completed",
conclusion: "failure",
created_at: "2026-10-17T13:00:00Z",
updated_at: "2026-10-17T13:40:00Z",
...overrides,
});
const refusedRound = (run = REFUSED_RUN) => ({
...roundRun("address-feedback", "success", "Passed", "2026-10-17T13:05:00Z"),
external_id: `pr-unblock:address-feedback:${run}:1`,
});
const tokenJob = { id: 9001, name: "apply-feedback-patch / run", conclusion: "failure", steps: [{ name: "Request the lane token", conclusion: "failure" }] };
function refusedApi({ runs = [refusedRun()], checks = [refusedRound()], message = WORKFLOW_MODIFIED, ...options } = {}) {
const api = sweepApi({ checksAt: { [HEAD]: checks }, workflowRuns: runs, ...options });
api.jobReads = [];
api.listRunJobs = async (id) => {
api.jobReads.push(id);
return [tokenJob];
};
api.listAnnotations = async (id) => (id === tokenJob.id ? [{ annotation_level: "notice", message: "token=github-token" }, { annotation_level: "failure", message }] : []);
return api;
}

test("a tick sends a round the broker refused as workflow-modified again once, on main, and the record names the refused run", async (t) => {
const api = refusedApi();
const { output, lines } = await sweepMain(t, api, TICK, NOW);
assert.deepEqual(api.jobReads, [REFUSED_RUN]);
assert.deepEqual(api.calls.dispatch, [
{ workflow: "pr-unblock.yml", ref: "refs/heads/main", inputs: { "pr-number": "42", activity: "address-feedback", "dispatcher-run-id": RUN_ID } },
]);
assert.equal(output, `run-id=38779200001\nupstream-run-id=13\nrefire-of=${REFUSED_RUN}\n`);
assert.ok(lines.some((line) => line.startsWith("refire: #42: run 700 was refused as workflow-modified (.github/workflows/pr-unblock.yml)")), lines.join("\n"));
});

test("a refused round already sent again, by the tick or a person, is not sent a second time", async (t) => {
for (const [label, later] of [
["the refire is still going", refusedRun({ id: 701, status: "in_progress", conclusion: null, created_at: "2026-10-17T13:45:00Z" })],
["the refire was refused as well", refusedRun({ id: 701, created_at: "2026-10-17T13:45:00Z", updated_at: "2026-10-17T13:50:00Z" })],
["the refire passed", refusedRun({ id: 701, conclusion: "success", created_at: "2026-10-17T13:45:00Z" })],
]) {
const api = refusedApi({ runs: [later, refusedRun()] });
const { output } = await sweepMain(t, api, TICK, NOW);
assert.deepEqual(api.jobReads, [], label);
assert.deepEqual(api.calls.dispatch, [], label);
assert.equal(output, "", label);
}
});

test("a round refused for another reason, outside the window, on another head or at the cap is not sent again", async (t) => {
for (const [label, options, reads] of [
["a kill-switch 403", { message: "lane-token-denied: the broker answered 403 with reason kill-switch" }, [REFUSED_RUN]],
["a broker 5xx", { message: "lane-token-denied: the broker answered 503 with reason broker-error" }, [REFUSED_RUN]],
["another workflow file", { message: WORKFLOW_MODIFIED.replace("pr-unblock.yml", "pr-review-hosted.yml") }, [REFUSED_RUN]],
["refused hours ago", { runs: [refusedRun({ updated_at: "2026-10-17T11:00:00Z" })] }, []],
["the head moved since", { checks: [] }, []],
["the head is at the cap", { checks: [refusedRound(), answeredRound, answeredRound] }, [REFUSED_RUN]],
]) {
const api = refusedApi(options);
const { output } = await sweepMain(t, api, TICK, NOW);
assert.deepEqual(api.jobReads, reads, label);
assert.deepEqual(api.calls.dispatch, [], label);
assert.equal(output, "", label);
}
});

// A verdict the kill switch deferred: the reconciler re-ran post-verify-notes
// (attempt 2, github-actions[bot]), which posted the notes or found the stale
// ones and uploaded pr-feedback-deferred-<head>-<verdict>, since the gate
Expand Down
Loading
Loading