diff --git a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py index 31d8dd7b6e..88df0f5279 100755 --- a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py +++ b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py @@ -39,19 +39,45 @@ def fetch_log(job_name, build_id, job_id, output_path): log_url = f"https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/{build_id}/jobs/{job_id}/download.txt" # Check if this is a GitHub Actions job - gh_match = re.search(r"github\.com/.*/job/(\d+)", log_url) + gh_repo_match = re.search( + r"github\.com/([^/]+/[^/]+)/actions/runs/\d+/job/(\d+)", log_url + ) + gh_match = gh_repo_match or re.search(r"github\.com/.*/job/(\d+)", log_url) if not gh_match and "github" in job_name.lower() and re.match(r"^\d+$", job_id): gh_match = re.match(r"^(\d+)$", job_id) if gh_match: - gh_job_id = gh_match.group(1) + if gh_repo_match: + gh_repo = gh_repo_match.group(1) + gh_job_id = gh_repo_match.group(2) + else: + gh_repo = "bazel-contrib/rules_python" + gh_job_id = gh_match.group(1) print(f"đŸ“Ĩ Fetching GitHub Action log for job {gh_job_id} using gh CLI...") - cmd = ["gh", "run", "view", "--job", gh_job_id, "--log"] + api_cmd = ["gh", "api", f"repos/{gh_repo}/actions/jobs/{gh_job_id}/logs"] + try: + res = subprocess.run(api_cmd, capture_output=True, text=True, check=True) + if res.stdout.strip(): + with open(output_path, "w") as f: + f.write(res.stdout) + return True + except Exception as e: + print( + f"⚠ī¸ Failed to fetch GitHub log via gh api for job {gh_job_id}: {e}", + file=sys.stderr, + ) + + cmd = ["gh", "run", "view", "--job", gh_job_id, "--repo", gh_repo, "--log"] try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) - with open(output_path, "w") as f: - f.write(res.stdout) - return True + if ( + res.stdout.strip() + and "is still in progress; logs will be available when it is complete" + not in res.stdout + ): + with open(output_path, "w") as f: + f.write(res.stdout) + return True except Exception as e: print( f"⚠ī¸ Failed to fetch GitHub log via gh CLI for job {gh_job_id}: {e}", @@ -118,6 +144,7 @@ def parse_log(log_path): "connection reset by peer", "input/output error", "errno 5", + "unable to find bazelisk version", ] ): if clean_line: @@ -168,6 +195,13 @@ def create_plan(job_name, log_path, errors): ): is_flake = True flake_reason = "Transient runner host disk / Darwin sandbox I/O error (OSError: [Errno 5] Input/output error). This is an infrastructure flake, not a codebase defect." + elif any("unable to find bazelisk version" in e.lower() for e in errors): + is_flake = True + flake_reason = ( + "Transient network error fetching Bazelisk release metadata during " + "GitHub Actions `setup-bazel` step (`Unable to find Bazelisk version`). " + "This is an infrastructure network flake, not a codebase bug." + ) classification = ( "⚡ **Classification**: **Infrastructure / Flake Issue** (Not a codebase bug)" @@ -215,7 +249,8 @@ def main(): os.makedirs(scratch_dir, exist_ok=True) safe_jname = re.sub(r"[^a-zA-Z0-9]", "_", args.job_name) - log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{args.job_id}.log") + safe_jid = re.sub(r"[^a-zA-Z0-9_-]", "_", args.job_id) + log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{safe_jid}.log") fetch_log(args.job_name, args.build_id, args.job_id, log_path) diff --git a/.agents/skills/buildkite-retry-job/SKILL.md b/.agents/skills/buildkite-retry-job/SKILL.md index d01707cb01..7acbc62148 100644 --- a/.agents/skills/buildkite-retry-job/SKILL.md +++ b/.agents/skills/buildkite-retry-job/SKILL.md @@ -1,21 +1,32 @@ --- name: buildkite-retry-job -description: Retry failed Buildkite jobs, unblock manual approval steps, or - rebuild full builds +description: Retry failed Buildkite or GitHub Actions jobs, unblock manual + approval steps, or rebuild full builds --- -Use `scripts/retry_buildkite_jobs.py` to retry individual jobs, unblock manual -approval steps (such as `.bazelci/presubmit.yml` changes), or rebuild full -Buildkite builds. +Use `scripts/retry_buildkite_jobs.py` to retry individual Buildkite or GitHub +Actions jobs, unblock manual approval steps (such as `.bazelci/presubmit.yml` +changes), or rebuild full builds. + +When a GitHub Actions job fails early (for example, `Setup Bazel` failing with +`Error: Unable to find Bazelisk version ...` due to a transient network error) +while sibling matrix jobs in the same workflow run are still running, GitHub +rejects rerun requests until the parent workflow run completes. The script +automatically waits for an in-progress workflow run to finish before triggering +`gh run rerun`. ### Retrying & Unblocking Modes -1. **Default Retry & Unblock**: - Unblocks any blocked manual approval steps (`bk job unblock - --no-input`) and retries specific failing jobs (`bk job retry `): +1. **Default Retry & Unblock (by PR, Buildkite URL, or GitHub Actions URL)**: + Unblocks any blocked Buildkite manual approval steps (`bk job unblock + --no-input`), retries specific failing Buildkite jobs (`bk job + retry `), and reruns failing GitHub Actions jobs/runs (`gh run + rerun`): ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + "https://github.com///actions/runs//job/" ``` 2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**: @@ -27,16 +38,21 @@ Buildkite builds. ``` 3. **Retry Specific Jobs by Name or ID**: + Retries matching Buildkite or GitHub Actions jobs by name/regex pattern, + Buildkite job UUID, or numeric GitHub Actions job ID: ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ --jobs "compile_pip_requirements" ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --job-id + --jobs "Build linux" + ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ + --job-id ``` 4. **Rebuild Full Build**: - Re-runs the entire pipeline build from scratch via `bk build rebuild`: + Re-runs the entire pipeline/workflow build from scratch via `bk build + rebuild` or `gh run rerun`: ```bash ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --rebuild + --rebuild ``` diff --git a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py index 97e5fb5a8c..168b1dacc4 100755 --- a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py +++ b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py @@ -5,6 +5,12 @@ import re import subprocess import sys +import time + +DEFAULT_GH_REPO = "bazel-contrib/rules_python" +GH_ACTIONS_URL_RE = re.compile( + r"github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/job/(\d+))?" +) def check_cli(cmd_name, install_url): @@ -24,21 +30,194 @@ def check_cli(cmd_name, install_url): sys.exit(1) -def get_build_url_from_pr(pr_number): +def get_pr_checks(pr_number): check_cli("gh", "https://cli.github.com/") - cmd = ["gh", "pr", "checks", str(pr_number), "--json", "name,link"] + cmd = ["gh", "pr", "checks", str(pr_number), "--json", "name,link,state,bucket"] + res = subprocess.run(cmd, capture_output=True, text=True) + out = res.stdout + if "[" not in out: + print( + f"❌ Error fetching PR checks for PR #{pr_number}: {res.stderr}", + file=sys.stderr, + ) + sys.exit(1) + json_str = out[out.find("[") : out.rfind("]") + 1] + return json.loads(json_str) + + +def parse_github_actions_url(target): + m = GH_ACTIONS_URL_RE.search(target) + if m: + return m.group(1), m.group(2), m.group(3) + return None + + +def get_github_run_data(repo, run_id): + cmd = [ + "gh", + "run", + "view", + str(run_id), + "--repo", + repo, + "--json", + "status,conclusion,jobs", + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + out = res.stdout + json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" + return json.loads(json_str) + + +def wait_for_github_run_completion( + repo, run_id, job_id=None, poll_interval=15, max_attempts=60 +): + for attempt in range(max_attempts): + try: + data = get_github_run_data(repo, str(run_id)) + except Exception as e: + print( + f"⚠ī¸ Warning: Could not inspect GitHub Actions run {run_id}: {e}", + file=sys.stderr, + ) + return None + + status = (data.get("status") or "").lower() + jobs = data.get("jobs", []) + + if job_id is not None: + active_job_ids = {str(j.get("databaseId")) for j in jobs} + if str(job_id) not in active_job_ids: + # The job ID from an earlier attempt is no longer in the active jobs + # list, meaning a newer run_attempt is already queued/running/done. + if status != "completed" or data.get("conclusion") == "success": + print( + f"ℹī¸ GitHub Actions job {job_id} has already been superseded " + f"by a newer attempt in run {run_id} (status: {status})." + ) + return data + + if status == "completed": + return data + + print( + f"âŗ GitHub Actions run {run_id} is currently '{status}' " + f"(attempt {attempt + 1}/{max_attempts}); waiting {poll_interval}s " + f"for run to complete before rerunning..." + ) + time.sleep(poll_interval) + + print( + f"⚠ī¸ Timed out waiting for GitHub Actions run {run_id} to complete.", + file=sys.stderr, + ) + return None + + +def retry_github_actions(repo, run_id, job_id=None, job_pattern=None, rebuild=False): + check_cli("gh", "https://cli.github.com/") + data = wait_for_github_run_completion(repo, run_id, job_id=job_id) + jobs = data.get("jobs", []) if isinstance(data, dict) else [] + + if rebuild: + print(f"🚀 Re-running entire GitHub Actions run {run_id} ({repo})...") + res = subprocess.run(["gh", "run", "rerun", str(run_id), "--repo", repo]) + return res.returncode + + if job_pattern: + matched = [] + for j in jobs: + name = j.get("name", "") + jid = j.get("databaseId") + conclusion = j.get("conclusion") or j.get("status") or "" + if jid and ( + re.search(job_pattern, name, re.IGNORECASE) + or job_pattern.lower() in name.lower() + ): + matched.append((str(jid), name, conclusion)) + + if not matched: + print( + f"⚠ī¸ No GitHub Actions jobs found matching '{job_pattern}' in run {run_id}.", + file=sys.stderr, + ) + return 1 + + rc = 0 + for jid, name, conclusion in matched: + print( + f"🚀 Retrying GitHub Actions job '{name}' " + f"(ID: {jid}, conclusion: {conclusion})..." + ) + res = subprocess.run(["gh", "run", "rerun", "--job", jid, "--repo", repo]) + if res.returncode != 0: + print( + f"⚠ī¸ Single-job rerun failed for {jid}; falling back to " + f"'gh run rerun {run_id} --failed'..." + ) + res = subprocess.run( + ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] + ) + return res.returncode + return rc + + if job_id: + active_jobs_by_id = {str(j.get("databaseId")): j for j in jobs} + if data is not None and str(job_id) not in active_jobs_by_id: + status = (data.get("status") or "").lower() + conclusion = (data.get("conclusion") or "").lower() + if status != "completed" or conclusion == "success": + return 0 + + target_job = active_jobs_by_id.get(str(job_id)) + if target_job and (target_job.get("conclusion") or "").lower() == "success": + print( + f"ℹī¸ GitHub Actions job '{target_job.get('name')}' " + f"(ID: {job_id}) already succeeded." + ) + return 0 + + print(f"🚀 Retrying GitHub Actions job {job_id} in run {run_id} ({repo})...") + res = subprocess.run( + ["gh", "run", "rerun", "--job", str(job_id), "--repo", repo] + ) + if res.returncode == 0: + return 0 + print( + f"⚠ī¸ Single-job rerun for {job_id} exited with {res.returncode}; " + f"falling back to rerunning failed jobs in run {run_id}..." + ) + + if data is not None and (data.get("conclusion") or "").lower() == "success": + print(f"ℹī¸ GitHub Actions run {run_id} already succeeded.") + return 0 + + print(f"🚀 Retrying failed jobs in GitHub Actions run {run_id} ({repo})...") + res = subprocess.run( + ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] + ) + return res.returncode + + +def retry_github_job_by_id(job_id, repo=DEFAULT_GH_REPO): + check_cli("gh", "https://cli.github.com/") + cmd = ["gh", "api", f"repos/{repo}/actions/jobs/{job_id}"] try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) - checks = json.loads(res.stdout) - for c in checks: - link = c.get("link", "") - if "buildkite.com" in link: - return link.split("#")[0] - print(f"❌ No Buildkite checks found for PR #{pr_number}.", file=sys.stderr) - sys.exit(1) - except subprocess.CalledProcessError as e: - print(f"❌ Error fetching PR checks: {e.stderr}", file=sys.stderr) - sys.exit(1) + out = res.stdout + json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" + job_data = json.loads(json_str) + run_id = job_data.get("run_id") + if run_id: + return retry_github_actions(repo, str(run_id), job_id=str(job_id)) + except Exception as e: + print( + f"⚠ī¸ Could not resolve run_id for GitHub job {job_id} via API: {e}", + file=sys.stderr, + ) + print(f"🚀 Retrying GitHub Actions job ID: {job_id}...") + res = subprocess.run(["gh", "run", "rerun", "--job", str(job_id), "--repo", repo]) + return res.returncode def normalize_build_target(target): @@ -57,7 +236,7 @@ def normalize_build_target(target): def retry_job_by_uuid(job_id): check_cli("bk", "https://github.com/buildkite/cli") - print(f"🚀 Retrying individual job UUID: {job_id}") + print(f"🚀 Retrying individual Buildkite job UUID: {job_id}") return subprocess.run(["bk", "job", "retry", job_id]) @@ -75,7 +254,7 @@ def unblock_manual_jobs(jobs): return unblocked -def retry_jobs_by_name(pipeline, build_number, job_pattern): +def retry_jobs_by_name(pipeline, build_number, job_pattern, exit_on_missing=True): check_cli("bk", "https://github.com/buildkite/cli") cmd = ["bk", "build", "view", str(build_number), "-p", pipeline, "--json"] try: @@ -94,78 +273,37 @@ def retry_jobs_by_name(pipeline, build_number, job_pattern): matched.append((jid, name, state)) if not matched: - print( - f"⚠ī¸ No jobs found matching pattern '{job_pattern}' in build #{build_number}", - file=sys.stderr, - ) - sys.exit(1) + if exit_on_missing: + print( + f"⚠ī¸ No jobs found matching pattern '{job_pattern}' in build #{build_number}", + file=sys.stderr, + ) + sys.exit(1) + return False for jid, name, state in matched: print(f"🚀 Retrying job '{name}' (ID: {jid}, status: {state})...") subprocess.run(["bk", "job", "retry", jid]) + return True except Exception as e: print(f"❌ Error fetching build jobs: {e}", file=sys.stderr) - sys.exit(1) - + if exit_on_missing: + sys.exit(1) + return False -def main(): - parser = argparse.ArgumentParser( - description="Retry Buildkite jobs individually using 'bk job retry' or rebuild the whole build using 'bk build rebuild'." - ) - parser.add_argument( - "args", - nargs="+", - help="Target build (PR#, Buildkite URL, or org/pipeline/build)", - ) - parser.add_argument( - "--jobs", - "--job-name", - dest="job_name", - help="Specific job name or regex pattern to retry individually via 'bk job retry'", - ) - parser.add_argument( - "--job-id", - dest="job_id", - help="Specific job UUID to retry individually via 'bk job retry'", - ) - parser.add_argument( - "--rebuild", - action="store_true", - help="Rebuild the entire build afresh via 'bk build rebuild'", - ) - parser.add_argument( - "--unblock", - action="store_true", - help="Unblock manual approval steps (e.g. .bazelci/presubmit.yml changes) via 'bk job unblock'", - ) - args = parser.parse_args() +def retry_buildkite_build( + pipeline, + build_number, + job_name=None, + unblock=False, + rebuild=False, + rebuild_if_no_failed_jobs=True, +): check_cli("bk", "https://github.com/buildkite/cli") - - if args.job_id: - res = retry_job_by_uuid(args.job_id) - sys.exit(res.returncode) - - if len(args.args) == 3: - target = f"{args.args[0]}/{args.args[1]}/{args.args[2]}" - elif len(args.args) == 1: - target = args.args[0] - else: - print( - "❌ Error: Invalid arguments. Provide a single target (PR#, URL, or org/pipeline/build).", - file=sys.stderr, - ) - sys.exit(1) - - if target.isdigit() and len(target) < 10: - print(f"🔍 Inspecting PR #{target} via gh to find Buildkite URL...") - target = get_build_url_from_pr(target) - - pipeline, build_number = normalize_build_target(target) - - if args.job_name: - retry_jobs_by_name(pipeline, build_number, args.job_name) - elif args.unblock: + if job_name: + return retry_jobs_by_name(pipeline, build_number, job_name) + elif unblock: cmd = [ "bk", "build", @@ -183,14 +321,14 @@ def main(): f"ℹī¸ No blocked manual approval jobs found in build #{build_number}.", file=sys.stderr, ) - elif args.rebuild: + return True + elif rebuild: print(f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}") res = subprocess.run( ["bk", "build", "rebuild", build_number, "-p", pipeline, "--yes"] ) - sys.exit(res.returncode) + return res.returncode == 0 else: - # Default behavior: unblock manual approval steps and retry failed jobs if any, otherwise rebuild cmd = [ "bk", "build", @@ -208,7 +346,10 @@ def main(): failed_jobs = [ j for j in jobs - if j.get("state") in ("failed", "broken", "timed_out") and j.get("id") + if j.get("state") in ("failed", "broken", "timed_out") + and j.get("id") + and not j.get("retried_in_job_uuid") + and not j.get("retried_at") ] if failed_jobs: for j in failed_jobs: @@ -216,7 +357,10 @@ def main(): jname = j.get("name") or j.get("label") or jid print(f"🚀 Retrying failed job '{jname}' (ID: {jid})...") subprocess.run(["bk", "job", "retry", jid]) - elif not unblocked_jobs: + return True + elif unblocked_jobs: + return True + elif rebuild_if_no_failed_jobs: print( f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" ) @@ -231,23 +375,240 @@ def main(): "--yes", ] ) - sys.exit(res.returncode) + return res.returncode == 0 + return False except Exception: + if rebuild_if_no_failed_jobs: + print( + f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" + ) + res = subprocess.run( + [ + "bk", + "build", + "rebuild", + build_number, + "-p", + pipeline, + "--yes", + ] + ) + return res.returncode == 0 + return False + + +def handle_pr_target(pr_number, job_name=None, unblock=False, rebuild=False): + print(f"🔍 Inspecting PR #{pr_number} checks via gh...") + checks = get_pr_checks(pr_number) + + bk_checks = [] + gh_checks = [] + for c in checks: + link = c.get("link", "") + if "buildkite.com" in link: + bk_checks.append(c) + elif parse_github_actions_url(link): + gh_checks.append(c) + + if unblock: + if not bk_checks: + print(f"ℹī¸ No Buildkite checks found for PR #{pr_number}.", file=sys.stderr) + return 0 + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + retry_buildkite_build(pipeline, build_number, unblock=True) + return 0 + + if job_name: + matched_any = False + matched_gh_runs = {} + for c in gh_checks: + cname = c.get("name", "") + parsed = parse_github_actions_url(c.get("link", "")) + if parsed and ( + re.search(job_name, cname, re.IGNORECASE) + or job_name.lower() in cname.lower() + ): + repo, run_id, job_id = parsed + matched_gh_runs.setdefault((repo, run_id), []).append(job_id) + + for (repo, run_id), job_ids in matched_gh_runs.items(): + matched_any = True + if len(job_ids) == 1 and job_ids[0]: + retry_github_actions(repo, run_id, job_id=job_ids[0]) + else: + retry_github_actions(repo, run_id, job_pattern=job_name) + + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + if retry_jobs_by_name( + pipeline, build_number, job_name, exit_on_missing=False + ): + matched_any = True + + if not matched_any: print( - f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" + f"⚠ī¸ No GitHub Actions or Buildkite jobs found matching " + f"'{job_name}' for PR #{pr_number}.", + file=sys.stderr, + ) + return 1 + return 0 + + # Retry failed GitHub Actions checks (or rebuild all failed runs if --rebuild) + failed_gh_runs = {} + for c in gh_checks: + state = (c.get("state") or "").upper() + bucket = (c.get("bucket") or "").lower() + is_failed = ( + state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE") + or bucket == "fail" + ) + if is_failed or rebuild: + parsed = parse_github_actions_url(c.get("link", "")) + if parsed: + repo, run_id, job_id = parsed + if is_failed: + failed_gh_runs.setdefault((repo, run_id), []).append(job_id) + + gh_retried = False + for (repo, run_id), job_ids in failed_gh_runs.items(): + gh_retried = True + if rebuild: + retry_github_actions(repo, run_id, rebuild=True) + elif len(job_ids) == 1 and job_ids[0]: + retry_github_actions(repo, run_id, job_id=job_ids[0]) + else: + retry_github_actions(repo, run_id) + + # Check Buildkite builds + for c in bk_checks: + pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) + state = (c.get("state") or "").upper() + bucket = (c.get("bucket") or "").lower() + bk_failed = ( + state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT") or bucket == "fail" + ) + retry_buildkite_build( + pipeline, + build_number, + rebuild=rebuild, + rebuild_if_no_failed_jobs=(bk_failed and not gh_retried), + ) + + if not bk_checks and not gh_retried: + print( + f"ℹī¸ No failing GitHub Actions or Buildkite checks found for PR #{pr_number}." + ) + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Retry failed Buildkite or GitHub Actions jobs individually, " + "unblock Buildkite approval steps, or rebuild entire builds." + ) + ) + parser.add_argument( + "args", + nargs="*", + help=( + "Target build (PR#, Buildkite URL, GitHub Actions run/job URL, " + "or org/pipeline/build)" + ), + ) + parser.add_argument( + "--jobs", + "--job-name", + dest="job_name", + help="Specific job name or regex pattern to retry individually", + ) + parser.add_argument( + "--job-id", + dest="job_id", + help=( + "Specific Buildkite job UUID or GitHub Actions job ID to retry individually" + ), + ) + parser.add_argument( + "--rebuild", + action="store_true", + help="Rebuild the entire build afresh", + ) + parser.add_argument( + "--unblock", + action="store_true", + help=( + "Unblock manual approval steps (e.g. .bazelci/presubmit.yml " + "changes) via 'bk job unblock'" + ), + ) + args = parser.parse_args() + + if args.job_id: + gh_parsed = parse_github_actions_url(args.job_id) + if gh_parsed: + repo, run_id, job_id = gh_parsed + sys.exit( + retry_github_actions( + repo, + run_id, + job_id=job_id, + job_pattern=args.job_name, + rebuild=args.rebuild, + ) + ) + if args.job_id.isdigit(): + sys.exit(retry_github_job_by_id(args.job_id)) + res = retry_job_by_uuid(args.job_id) + sys.exit(res.returncode) + + if len(args.args) == 3: + target = f"{args.args[0]}/{args.args[1]}/{args.args[2]}" + elif len(args.args) == 1: + target = args.args[0] + else: + print( + "❌ Error: Invalid arguments. Provide a single target " + "(PR#, Buildkite URL, GitHub Actions URL, or org/pipeline/build).", + file=sys.stderr, + ) + sys.exit(1) + + gh_parsed = parse_github_actions_url(target) + if gh_parsed: + repo, run_id, job_id = gh_parsed + sys.exit( + retry_github_actions( + repo, + run_id, + job_id=job_id, + job_pattern=args.job_name, + rebuild=args.rebuild, ) - res = subprocess.run( - [ - "bk", - "build", - "rebuild", - build_number, - "-p", - pipeline, - "--yes", - ] + ) + + if target.isdigit() and len(target) < 10: + sys.exit( + handle_pr_target( + target, + job_name=args.job_name, + unblock=args.unblock, + rebuild=args.rebuild, ) - sys.exit(res.returncode) + ) + + pipeline, build_number = normalize_build_target(target) + ok = retry_buildkite_build( + pipeline, + build_number, + job_name=args.job_name, + unblock=args.unblock, + rebuild=args.rebuild, + rebuild_if_no_failed_jobs=True, + ) + sys.exit(0 if ok else 1) if __name__ == "__main__": diff --git a/.agents/skills/merge-pr/SKILL.md b/.agents/skills/merge-pr/SKILL.md index f410692a2b..99f8176442 100644 --- a/.agents/skills/merge-pr/SKILL.md +++ b/.agents/skills/merge-pr/SKILL.md @@ -25,7 +25,8 @@ this PR", or monitor its merge): - Have the subagent use the **`buildkite-retry-job`** skill (`retry_buildkite_jobs.py `) to automatically retry any transient network flakes (e.g., HTTP 504 gateway timeouts, downloader - errors) and unblock manual approval steps. + errors, `setup-bazel` `Unable to find Bazelisk version` errors) and + unblock manual approval steps. - **Blocked Builds (`.bazelci/presubmit.yml` Approval)**: When `.bazelci/presubmit.yml` is modified, Buildkite blocks the build until manually approved. Approve and unblock it immediately via diff --git a/.agents/skills/monitor-ci-results/SKILL.md b/.agents/skills/monitor-ci-results/SKILL.md index b5a0af7873..29cba5d2c3 100644 --- a/.agents/skills/monitor-ci-results/SKILL.md +++ b/.agents/skills/monitor-ci-results/SKILL.md @@ -50,7 +50,8 @@ or when monitoring CI after PR updates: `analyze-ci-failure` skill on the reported failure. 5. **Autonomous Flake Retry**: If the CI Failure Analyzer subagent confirms that the failure is a transient infrastructure or network flake (e.g. disk - I/O error, 504 gateway, sandbox initialization failure), **immediately and + I/O error, 504 gateway, sandbox initialization failure, or `setup-bazel` + failing with `Unable to find Bazelisk version`), **immediately and autonomously** launch a separate subagent using `invoke_subagent` with the role "CI Job Retrier" and the prompt template in `.agents/skills/monitor-ci-results/retry-job-prompt.md` to run the diff --git a/.agents/skills/monitor-ci-results/retry-job-prompt.md b/.agents/skills/monitor-ci-results/retry-job-prompt.md index 4d2edad650..9d948b24a5 100644 --- a/.agents/skills/monitor-ci-results/retry-job-prompt.md +++ b/.agents/skills/monitor-ci-results/retry-job-prompt.md @@ -1,3 +1,5 @@ -Use the `buildkite-retry-job` skill to retry the failed Buildkite job(s) for PR {pr_number}. -Follow the instructions in `.agents/skills/buildkite-retry-job/SKILL.md` to retry the failing jobs using the skill. +Use the `buildkite-retry-job` skill to retry the failed Buildkite or GitHub +Actions job(s) for PR {pr_number}. +Follow the instructions in `.agents/skills/buildkite-retry-job/SKILL.md` to +retry the failing jobs using the skill. Once the retry request completes, report back to the parent conversation. diff --git a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py index 19a7f119c9..669d117922 100755 --- a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py +++ b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py @@ -324,26 +324,30 @@ def main(): elif ( state.upper() in ["FAILURE", "FAIL", "CANCELLED", "TIMED_OUT"] or bucket == "fail" - ) and name not in monitored: - print(f"🚨 Notifying failure for GitHub check '{name}'...") - job_id = link.split("/")[-1] if "/" in link else link - msg = ( - f"⚠ī¸ Remote CI GitHub Check '{name}' completed with errors!\n\n" - f"Link: {link}\n\n" - f"Action required: Start a subagent to run the analyze-ci-failure skill for '{name}' with link '{link}' and job ID '{job_id}'." - ) - subprocess.run( - [ - "agentapi", - "send-message", - "--title=CI Check Failed", - args.conv_id, - msg, - ] - ) - monitored[name] = time.time() - with open(state_file, "w") as f: - json.dump(monitored, f) + ): + job_id = link.split("/")[-1].split("?")[0] if "/" in link else link + gh_key = f"gh_{job_id}" if job_id else name + if gh_key not in monitored: + print(f"🚨 Notifying failure for GitHub check '{name}'...") + msg = ( + f"⚠ī¸ Remote CI GitHub Check '{name}' completed with errors!\n\n" + f"Link: {link}\n\n" + f"Action required: Start a subagent to run the " + f"analyze-ci-failure skill for '{name}' with link " + f"'{link}' and job ID '{job_id}'." + ) + subprocess.run( + [ + "agentapi", + "send-message", + "--title=CI Check Failed", + args.conv_id, + msg, + ] + ) + monitored[gh_key] = time.time() + with open(state_file, "w") as f: + json.dump(monitored, f) time.sleep(args.interval)