From 1aa168cd7527bec84e30122d4602ea13c3440034 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 12 Aug 2026 14:54:23 +0200 Subject: [PATCH 1/2] ci: fix the Slack alert for failed scheduled tests --- .github/workflows/on_schedule_tests.yaml | 48 ++++++++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/.github/workflows/on_schedule_tests.yaml b/.github/workflows/on_schedule_tests.yaml index 19412a4703..51084cb958 100644 --- a/.github/workflows/on_schedule_tests.yaml +++ b/.github/workflows/on_schedule_tests.yaml @@ -3,6 +3,11 @@ name: Scheduled tests on: # Runs when manually triggered from the GitHub UI. workflow_dispatch: + inputs: + slack_test: + description: 'Skip the tests and only send a Slack notification, to check the alerting path.' + type: boolean + default: false # Runs on weekdays at 01:00 UTC. schedule: @@ -32,6 +37,7 @@ env: jobs: end_to_end_tests: name: End-to-end tests + if: github.event_name != 'workflow_dispatch' || !inputs.slack_test strategy: fail-fast: false max-parallel: 12 @@ -81,11 +87,13 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} # Send a Slack notification to the team alerting channel when scheduled e2e tests fail. - # Skipped on workflow_dispatch (manual runs) so that ad-hoc triggers don't spam the channel. + # Only scheduled runs notify, so that pull requests and ad-hoc triggers don't spam the channel. + # The `slack_test` dispatch input skips the tests and notifies anyway, which is the only way to + # exercise this path without waiting for a real failure. notify_on_failure: name: Notify Slack on failure needs: end_to_end_tests - if: failure() && github.event_name == 'schedule' + if: always() && ((github.event_name == 'schedule' && needs.end_to_end_tests.result == 'failure') || (github.event_name == 'workflow_dispatch' && inputs.slack_test)) runs-on: ubuntu-latest permissions: contents: read @@ -99,15 +107,16 @@ jobs: RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - HEADING: ':red_circle: Scheduled e2e tests failed' + HEADING: "${{ inputs.slack_test && ':white_circle: Slack alerting check, no tests were run' || ':red_circle: Scheduled e2e tests failed' }}" run: | - # Retry the API call to tolerate transient 5xx from GitHub. + # Retry the API call to tolerate transient 5xx from GitHub. A body that is not usable + # counts as a failed attempt as well, so that it falls back instead of aborting the step. max_attempts=5 fetched=0 for attempt in $(seq 1 "${max_attempts}"); do - if failed_jobs=$(gh api \ - "repos/${REPO}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs?per_page=100" \ - --jq '[.jobs[] | select(.conclusion == "failure") | "• \(.name)"] | join("\n")'); then + if gh api \ + "repos/${REPO}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs?per_page=100" > jobs.json \ + && jq -e '.jobs | type == "array"' jobs.json > /dev/null; then fetched=1 break fi @@ -115,10 +124,30 @@ jobs: sleep "$((attempt * 5))" fi done + if [[ "${fetched}" -eq 0 ]]; then echo "Failed to fetch job list after ${max_attempts} attempts; sending notification without it." >&2 - failed_jobs="(unable to fetch job list — see workflow run)" + failed_jobs="(unable to fetch job list - see workflow run)" + else + # This job is still running and so carries no conclusion, which keeps it out of both + # counts without having to match on its own name. + succeeded=$(jq '[.jobs[] | select(.conclusion == "success")] | length' jobs.json) + failed=$(jq '[.jobs[] | select(.conclusion == "failure")] | length' jobs.json) + bullets=$(jq -r '[.jobs[] | select(.conclusion == "failure") | "- \(.name)"] | join("\n")' jobs.json) + + # Slack rejects the whole message with `invalid_blocks` once a section's text passes + # 3000 characters, which a full matrix failure comfortably does. Trim the list at a line + # boundary and count off the remainder instead, leaving room for the count line and the + # trailing summary. + max_list_chars=2500 + if [[ "${#bullets}" -gt "${max_list_chars}" ]]; then + bullets=$(printf '%s' "${bullets}" | head -c "${max_list_chars}" | sed '$d') + shown=$(printf '%s\n' "${bullets}" | wc -l) + bullets="${bullets}"$'\n'"... and $((failed - shown)) more" + fi + failed_jobs="${failed} failed, ${succeeded} succeeded"$'\n'"${bullets}" fi + jq -n \ --arg repo "${REPO}" \ --arg url "${WORKFLOW_URL}" \ @@ -148,6 +177,9 @@ jobs: - name: Send Slack notification uses: slackapi/slack-github-action@v4.0.0 with: + # Without this the action swallows every delivery error into a hidden debug log line and + # reports success, so a rejected or undeliverable alert looks exactly like a sent one. + errors: true webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook payload-file-path: slack-payload.json From 648053f3c821584586a7e49f02524975c67484b5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 12 Aug 2026 15:33:36 +0200 Subject: [PATCH 2/2] ci: paginate the job list and harden the Slack payload trim --- .github/workflows/on_schedule_tests.yaml | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/on_schedule_tests.yaml b/.github/workflows/on_schedule_tests.yaml index 51084cb958..fa27a50100 100644 --- a/.github/workflows/on_schedule_tests.yaml +++ b/.github/workflows/on_schedule_tests.yaml @@ -109,14 +109,18 @@ jobs: WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} HEADING: "${{ inputs.slack_test && ':white_circle: Slack alerting check, no tests were run' || ':red_circle: Scheduled e2e tests failed' }}" run: | - # Retry the API call to tolerate transient 5xx from GitHub. A body that is not usable - # counts as a failed attempt as well, so that it falls back instead of aborting the step. + # Retry the API call to tolerate transient 5xx from GitHub. The matrix nearly fills a + # single 100-item page, so paginate rather than silently count only the first one, and + # flatten the pages into the same shape a single page has. A response that is empty or + # not shaped as expected counts as a failed attempt as well, so that it falls back + # instead of aborting the step. max_attempts=5 fetched=0 for attempt in $(seq 1 "${max_attempts}"); do - if gh api \ - "repos/${REPO}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs?per_page=100" > jobs.json \ - && jq -e '.jobs | type == "array"' jobs.json > /dev/null; then + if gh api --paginate --slurp \ + "repos/${REPO}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs?per_page=100" > pages.json \ + && jq '{jobs: [.[].jobs[]]}' pages.json > jobs.json \ + && jq -e '.jobs | length > 0 and all(.[]; type == "object")' jobs.json > /dev/null; then fetched=1 break fi @@ -136,16 +140,20 @@ jobs: bullets=$(jq -r '[.jobs[] | select(.conclusion == "failure") | "- \(.name)"] | join("\n")' jobs.json) # Slack rejects the whole message with `invalid_blocks` once a section's text passes - # 3000 characters, which a full matrix failure comfortably does. Trim the list at a line - # boundary and count off the remainder instead, leaving room for the count line and the - # trailing summary. + # 3000 characters, which a full matrix failure comfortably does. Cut the list to fit, + # drop the partial line the cut leaves behind, and count off the remainder instead, + # leaving room for the count line and the trailing summary. max_list_chars=2500 if [[ "${#bullets}" -gt "${max_list_chars}" ]]; then - bullets=$(printf '%s' "${bullets}" | head -c "${max_list_chars}" | sed '$d') - shown=$(printf '%s\n' "${bullets}" | wc -l) + bullets="${bullets:0:${max_list_chars}}" + bullets="${bullets%$'\n'*}" + # The list carries one newline fewer than it has lines, so stripping everything but + # the newlines counts what is left. + newlines="${bullets//[!$'\n']/}" + shown=$((${#newlines} + 1)) bullets="${bullets}"$'\n'"... and $((failed - shown)) more" fi - failed_jobs="${failed} failed, ${succeeded} succeeded"$'\n'"${bullets}" + failed_jobs="${failed} failed, ${succeeded} succeeded${bullets:+$'\n'}${bullets}" fi jq -n \