Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions .agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,45 @@ def fetch_log(job_name, build_id, job_id, output_path):
log_url = f"https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/{build_id}/jobs/{job_id}/download.txt"

# Check if this is a GitHub Actions job
gh_match = re.search(r"github\.com/.*/job/(\d+)", log_url)
gh_repo_match = re.search(
r"github\.com/([^/]+/[^/]+)/actions/runs/\d+/job/(\d+)", log_url
)
gh_match = gh_repo_match or re.search(r"github\.com/.*/job/(\d+)", log_url)
if not gh_match and "github" in job_name.lower() and re.match(r"^\d+$", job_id):
gh_match = re.match(r"^(\d+)$", job_id)

if gh_match:
gh_job_id = gh_match.group(1)
if gh_repo_match:
gh_repo = gh_repo_match.group(1)
gh_job_id = gh_repo_match.group(2)
else:
gh_repo = "bazel-contrib/rules_python"
gh_job_id = gh_match.group(1)
print(f"📥 Fetching GitHub Action log for job {gh_job_id} using gh CLI...")
cmd = ["gh", "run", "view", "--job", gh_job_id, "--log"]
api_cmd = ["gh", "api", f"repos/{gh_repo}/actions/jobs/{gh_job_id}/logs"]
try:
res = subprocess.run(api_cmd, capture_output=True, text=True, check=True)
if res.stdout.strip():
with open(output_path, "w") as f:
f.write(res.stdout)
return True
except Exception as e:
print(
f"⚠️ Failed to fetch GitHub log via gh api for job {gh_job_id}: {e}",
file=sys.stderr,
)

cmd = ["gh", "run", "view", "--job", gh_job_id, "--repo", gh_repo, "--log"]
try:
res = subprocess.run(cmd, capture_output=True, text=True, check=True)
with open(output_path, "w") as f:
f.write(res.stdout)
return True
if (
res.stdout.strip()
and "is still in progress; logs will be available when it is complete"
not in res.stdout
):
with open(output_path, "w") as f:
f.write(res.stdout)
return True
except Exception as e:
print(
f"⚠️ Failed to fetch GitHub log via gh CLI for job {gh_job_id}: {e}",
Expand Down Expand Up @@ -118,6 +144,7 @@ def parse_log(log_path):
"connection reset by peer",
"input/output error",
"errno 5",
"unable to find bazelisk version",
]
):
if clean_line:
Expand Down Expand Up @@ -168,6 +195,13 @@ def create_plan(job_name, log_path, errors):
):
is_flake = True
flake_reason = "Transient runner host disk / Darwin sandbox I/O error (OSError: [Errno 5] Input/output error). This is an infrastructure flake, not a codebase defect."
elif any("unable to find bazelisk version" in e.lower() for e in errors):
is_flake = True
flake_reason = (
"Transient network error fetching Bazelisk release metadata during "
"GitHub Actions `setup-bazel` step (`Unable to find Bazelisk version`). "
"This is an infrastructure network flake, not a codebase bug."
)

classification = (
"⚡ **Classification**: **Infrastructure / Flake Issue** (Not a codebase bug)"
Expand Down Expand Up @@ -215,7 +249,8 @@ def main():
os.makedirs(scratch_dir, exist_ok=True)

safe_jname = re.sub(r"[^a-zA-Z0-9]", "_", args.job_name)
log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{args.job_id}.log")
safe_jid = re.sub(r"[^a-zA-Z0-9_-]", "_", args.job_id)
log_path = os.path.join(scratch_dir, f"ci_{safe_jname}_{safe_jid}.log")

fetch_log(args.job_name, args.build_id, args.job_id, log_path)

Expand Down
38 changes: 27 additions & 11 deletions .agents/skills/buildkite-retry-job/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,21 +1,32 @@
---
name: buildkite-retry-job
description: Retry failed Buildkite jobs, unblock manual approval steps, or
rebuild full builds
description: Retry failed Buildkite or GitHub Actions jobs, unblock manual
approval steps, or rebuild full builds
---

Use `scripts/retry_buildkite_jobs.py` to retry individual jobs, unblock manual
approval steps (such as `.bazelci/presubmit.yml` changes), or rebuild full
Buildkite builds.
Use `scripts/retry_buildkite_jobs.py` to retry individual Buildkite or GitHub
Actions jobs, unblock manual approval steps (such as `.bazelci/presubmit.yml`
changes), or rebuild full builds.

When a GitHub Actions job fails early (for example, `Setup Bazel` failing with
`Error: Unable to find Bazelisk version ...` due to a transient network error)
while sibling matrix jobs in the same workflow run are still running, GitHub
rejects rerun requests until the parent workflow run completes. The script
automatically waits for an in-progress workflow run to finish before triggering
`gh run rerun`.

### Retrying & Unblocking Modes

1. **Default Retry & Unblock**:
Unblocks any blocked manual approval steps (`bk job unblock <job_id>
--no-input`) and retries specific failing jobs (`bk job retry <job_id>`):
1. **Default Retry & Unblock (by PR, Buildkite URL, or GitHub Actions URL)**:
Unblocks any blocked Buildkite manual approval steps (`bk job unblock
<job_id> --no-input`), retries specific failing Buildkite jobs (`bk job
retry <job_id>`), and reruns failing GitHub Actions jobs/runs (`gh run
rerun`):
```bash
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
<pr_number_or_url>
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
"https://github.com/<owner>/<repo>/actions/runs/<run_id>/job/<job_id>"
```

2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**:
Expand All @@ -27,16 +38,21 @@ Buildkite builds.
```

3. **Retry Specific Jobs by Name or ID**:
Retries matching Buildkite or GitHub Actions jobs by name/regex pattern,
Buildkite job UUID, or numeric GitHub Actions job ID:
```bash
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
<pr_number> --jobs "compile_pip_requirements"
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
--job-id <job_uuid>
<pr_number> --jobs "Build linux"
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
--job-id <job_uuid_or_github_job_id>
```

4. **Rebuild Full Build**:
Re-runs the entire pipeline build from scratch via `bk build rebuild`:
Re-runs the entire pipeline/workflow build from scratch via `bk build
rebuild` or `gh run rerun`:
```bash
./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \
<pr_number> --rebuild
<pr_number_or_url> --rebuild
```
Loading
Loading