From 732705036e137da0524e205e2cfa9bcbfc2d7fcd Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Fri, 26 Jun 2026 15:38:54 -0700 Subject: [PATCH 01/12] Initial implementation of pulse listener --- services/hackbot-pulse-listener/Dockerfile | 32 + services/hackbot-pulse-listener/README.md | 44 + .../hackbot-pulse-listener/app/__init__.py | 0 .../hackbot-pulse-listener/app/__main__.py | 44 + services/hackbot-pulse-listener/app/client.py | 32 + services/hackbot-pulse-listener/app/config.py | 51 + .../hackbot-pulse-listener/app/consumer.py | 118 +++ services/hackbot-pulse-listener/app/notify.py | 74 ++ .../hackbot-pulse-listener/app/taskcluster.py | 26 + services/hackbot-pulse-listener/app/worker.py | 47 + services/hackbot-pulse-listener/deploy.sh | 85 ++ .../docker-compose.dev.yml | 11 + .../hackbot-pulse-listener/pyproject.toml | 27 + .../hackbot-pulse-listener/tests/__init__.py | 0 .../tests/fixtures/pulse_messages.json | 892 ++++++++++++++++++ .../tests/test_consumer.py | 100 ++ .../tests/test_notify.py | 62 ++ .../tests/test_worker.py | 31 + uv.lock | 33 + 19 files changed, 1709 insertions(+) create mode 100644 services/hackbot-pulse-listener/Dockerfile create mode 100644 services/hackbot-pulse-listener/README.md create mode 100644 services/hackbot-pulse-listener/app/__init__.py create mode 100644 services/hackbot-pulse-listener/app/__main__.py create mode 100644 services/hackbot-pulse-listener/app/client.py create mode 100644 services/hackbot-pulse-listener/app/config.py create mode 100644 services/hackbot-pulse-listener/app/consumer.py create mode 100644 services/hackbot-pulse-listener/app/notify.py create mode 100644 services/hackbot-pulse-listener/app/taskcluster.py create mode 100644 services/hackbot-pulse-listener/app/worker.py create mode 100755 services/hackbot-pulse-listener/deploy.sh create mode 100644 services/hackbot-pulse-listener/docker-compose.dev.yml create mode 100644 services/hackbot-pulse-listener/pyproject.toml create mode 100644 services/hackbot-pulse-listener/tests/__init__.py create mode 100644 services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json create mode 100644 services/hackbot-pulse-listener/tests/test_consumer.py create mode 100644 services/hackbot-pulse-listener/tests/test_notify.py create mode 100644 services/hackbot-pulse-listener/tests/test_worker.py diff --git a/services/hackbot-pulse-listener/Dockerfile b/services/hackbot-pulse-listener/Dockerfile new file mode 100644 index 0000000000..2376966800 --- /dev/null +++ b/services/hackbot-pulse-listener/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.14-slim AS builder + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /app + +# Install external deps without building workspace members. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=VERSION,target=VERSION \ + uv sync --frozen --no-dev --no-install-workspace --package hackbot-pulse-listener + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,target=/app,rw \ + uv sync --locked --no-dev --no-editable --package hackbot-pulse-listener + +FROM python:3.14-slim AS base + +COPY --from=builder /opt/venv /opt/venv +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PATH="/opt/venv/bin:$PATH" + +RUN useradd --create-home --shell /bin/bash app +USER app + +CMD ["python", "-m", "app"] diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md new file mode 100644 index 0000000000..57984bd5e7 --- /dev/null +++ b/services/hackbot-pulse-listener/README.md @@ -0,0 +1,44 @@ +# Hackbot Pulse Listener + +Listens to Taskcluster build-failure pulse messages, and for failed **Firefox build +tasks** triggers the `build-repair` hackbot agent through the hackbot-api. When a run +finishes (minutes later) it emails the developer who pushed the change a link to the +hackbot UI and a summary of the analysis and fix. + +## How it works + +1. Consume `task-failed` messages from `pulse.mozilla.org`. +2. Keep only **build-kind** tasks (`tags.kind == "build"`) on a watched `project` + (`WATCHED_REPOS`, default `try`). Build tasks don't run tests, so a failure is a + compilation/link error. +3. Fetch the task definition to read `GECKO_HEAD_REV` (the revision is not in the message). +4. Dedupe by revision with an in-memory TTL cache, so only one agent run is triggered per + revision even when many build tasks fail for the same push. +5. `POST /agents/build-repair/runs`, then poll `GET /runs/{run_id}` until terminal and send + the report email. + +The dedupe cache and pending-run tracking are in-memory (reset on restart). + +## Run locally + +```bash +export PULSE_USER=... PULSE_PASSWORD=... # https://pulseguardian.mozilla.org +export HACKBOT_API_URL=https://hackbot-api.../ HACKBOT_API_KEY=... +export HACKBOT_UI_URL=https://hackbot-ui.../ +export WATCHED_REPOS=try +export DRY_RUN=true # log intended calls, don't POST +uv run --package hackbot-pulse-listener python -m app +``` + +Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it +is logged and skipped. + +## Test + +```bash +uv run --package hackbot-pulse-listener pytest services/hackbot-pulse-listener/tests +``` + +## Deploy + +Cloud Run worker pool (no HTTP). See `deploy.sh`. diff --git a/services/hackbot-pulse-listener/app/__init__.py b/services/hackbot-pulse-listener/app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/app/__main__.py b/services/hackbot-pulse-listener/app/__main__.py new file mode 100644 index 0000000000..80c99d76e7 --- /dev/null +++ b/services/hackbot-pulse-listener/app/__main__.py @@ -0,0 +1,44 @@ +import logging +import signal +from concurrent.futures import ThreadPoolExecutor + +from app import consumer +from app.config import settings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main() -> None: + if settings.sentry_dsn: + import sentry_sdk + + sentry_sdk.init(dsn=settings.sentry_dsn, environment=settings.environment) + + if not (settings.pulse_user and settings.pulse_password): + logger.warning("PULSE_USER/PULSE_PASSWORD not set; listener will not start") + return + + executor = ThreadPoolExecutor(max_workers=settings.poll_max_workers) + consumer_obj = consumer.build_consumer(executor) + + def shutdown(signum, _frame): + logger.info("Received signal %s; shutting down", signum) + consumer_obj.should_stop = True + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + + logger.info( + "Listening for build failures on %s; watched repos: %s", + ", ".join(consumer.EXCHANGES), + sorted(settings.watched_repos_set), + ) + try: + consumer_obj.run() + finally: + executor.shutdown(wait=False) + + +if __name__ == "__main__": + main() diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py new file mode 100644 index 0000000000..a83e6ce974 --- /dev/null +++ b/services/hackbot-pulse-listener/app/client.py @@ -0,0 +1,32 @@ +import logging + +import httpx + +from app.config import settings + +logger = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0) + + +def _headers() -> dict[str, str]: + return {"X-API-Key": settings.hackbot_api_key} + + +def trigger_run(inputs: dict) -> str | None: + """Create a build-repair run. Returns the run id, or None in dry-run mode.""" + if settings.dry_run: + logger.info("[dry-run] would trigger %s run: %s", settings.agent_name, inputs) + return None + + url = f"{settings.hackbot_api_url}/agents/{settings.agent_name}/runs" + resp = httpx.post(url, json=inputs, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json()["run_id"] + + +def get_run(run_id: str) -> dict: + url = f"{settings.hackbot_api_url}/runs/{run_id}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json() diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py new file mode 100644 index 0000000000..49666308c8 --- /dev/null +++ b/services/hackbot-pulse-listener/app/config.py @@ -0,0 +1,51 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + # Pulse (https://pulseguardian.mozilla.org) + pulse_user: str = "" + pulse_password: str = "" + taskcluster_root_url: str = "https://firefox-ci-tc.services.mozilla.com" + + # hackbot-api + hackbot_api_url: str = "" + hackbot_api_key: str = "" + hackbot_ui_url: str = "" + agent_name: str = "build-repair" + + # Failure filtering and agent inputs. + # ``watched_repos`` is a comma-separated list of Taskcluster ``project`` tags. + watched_repos: str = "try,autoland" + run_try_push: bool = False + model: str | None = None + max_turns: int | None = None + + # Dedupe (in-memory, by git revision) + dedupe_ttl_seconds: int = 6 * 60 * 60 + dedupe_max_size: int = 4096 + + # Polling the API for run completion + poll_interval_seconds: int = 60 + run_max_age_minutes: int = 12 * 60 + poll_max_workers: int = 8 + + # Email notifications (SendGrid) + sendgrid_api_key: str | None = None + notification_sender: str | None = None + + dry_run: bool = False + environment: str = "development" + sentry_dsn: str | None = None + + model_config = { + "env_file": ".env", + "env_file_encoding": "utf-8", + "extra": "ignore", + } + + @property + def watched_repos_set(self) -> set[str]: + return {r.strip() for r in self.watched_repos.split(",") if r.strip()} + + +settings = Settings() diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py new file mode 100644 index 0000000000..a44599768f --- /dev/null +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -0,0 +1,118 @@ +import logging +from concurrent.futures import Executor + +from cachetools import TTLCache +from kombu import Connection, Exchange, Queue +from kombu.mixins import ConsumerMixin + +from app import client, taskcluster, worker +from app.config import settings + +logger = logging.getLogger(__name__) + +CONNECTION_URL = "amqp://{}:{}@pulse.mozilla.org:5671/?ssl=1" + +EXCHANGES = ("exchange/taskcluster-queue/v1/task-failed",) + +# In-memory dedupe of git revisions already handed to the agent. Only the +# single consumer thread touches it, so no lock is needed. +_seen: TTLCache = TTLCache( + maxsize=settings.dedupe_max_size, ttl=settings.dedupe_ttl_seconds +) + + +def process(body: dict, executor: Executor) -> str | None: + """Handle one Taskcluster failure message. Returns the triggered run id.""" + tags = (body.get("task") or {}).get("tags") or {} + + if tags.get("kind") != "build": + return None + + project = tags.get("project") + if project not in settings.watched_repos_set: + return None + + task_id = body["status"]["taskId"] + task_name = tags.get("label") or task_id + developer_email = tags.get("createdForUser") + + revision = taskcluster.get_revision(task_id) + if not revision: + logger.warning("No GECKO_HEAD_REV for task %s; skipping", task_id) + return None + + if revision in _seen: + logger.info("Revision %s already processed; skipping", revision) + return None + _seen[revision] = True + + inputs: dict = { + "git_commit": revision, + "failure_tasks": {task_name: task_id}, + "run_try_push": settings.run_try_push, + } + if settings.model: + inputs["model"] = settings.model + if settings.max_turns is not None: + inputs["max_turns"] = settings.max_turns + + try: + run_id = client.trigger_run(inputs) + except Exception: + logger.exception("Failed to trigger build-repair run for %s", revision) + _seen.pop(revision, None) + return None + + logger.info("Triggered build-repair run %s for %s@%s", run_id, project, revision) + if run_id is not None: + executor.submit( + worker.poll_and_notify, run_id, revision, project, developer_email + ) + return run_id + + +def make_handler(executor: Executor): + def on_message(body, message): + try: + process(body, executor) + except Exception: + logger.exception("Error handling pulse message") + finally: + message.ack() + + return on_message + + +def _build_queues(user: str) -> list[Queue]: + queues = [] + for exchange in EXCHANGES: + suffix = exchange.rsplit("/", 1)[-1] + queues.append( + Queue( + name=f"queue/{user}/build-repair-{suffix}", + exchange=Exchange(exchange, type="topic", no_declare=True), + routing_key="#", + durable=True, + auto_delete=True, + ) + ) + return queues + + +class BuildFailureConsumer(ConsumerMixin): + def __init__(self, connection, queues, on_message): + self.connection = connection + self.queues = queues + self.on_message = on_message + + def get_consumers(self, Consumer, channel): + return [Consumer(queues=self.queues, callbacks=[self.on_message])] + + +def build_consumer(executor: Executor) -> BuildFailureConsumer: + connection = Connection( + CONNECTION_URL.format(settings.pulse_user, settings.pulse_password) + ) + return BuildFailureConsumer( + connection, _build_queues(settings.pulse_user), make_handler(executor) + ) diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py new file mode 100644 index 0000000000..459fae3d95 --- /dev/null +++ b/services/hackbot-pulse-listener/app/notify.py @@ -0,0 +1,74 @@ +import logging + +from app.config import settings + +logger = logging.getLogger(__name__) + + +def send_email( + developer_email: str | None, + revision: str, + repo: str, + run_id: str, + run_doc: dict, +) -> None: + """Email the developer the run outcome. No-op (logged) if not configured.""" + if not developer_email: + logger.info("No developer email for run %s; skipping notification", run_id) + return + if not (settings.sendgrid_api_key and settings.notification_sender): + logger.info("SendGrid not configured; skipping email for run %s", run_id) + return + + import sendgrid + from sendgrid.helpers.mail import Content, From, Mail, Subject, To + + status = run_doc.get("status", "unknown") + subject = f"[build-repair] {status} for {repo}@{revision[:12]}" + + sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) + message = Mail( + From(settings.notification_sender), + To(developer_email), + Subject(subject), + Content("text/plain", _build_body(revision, repo, run_id, run_doc)), + ) + response = sg.send(message=message) + logger.info( + "Sent build-repair notification to %s (status %s)", + developer_email, + response.status_code, + ) + + +def _build_body(revision: str, repo: str, run_id: str, run_doc: dict) -> str: + status = run_doc.get("status", "unknown") + lines = [ + f"The build-repair agent finished with status: {status}.", + "", + f"Repository: {repo}", + f"Revision: {revision}", + ] + + if settings.hackbot_ui_url: + lines += [ + "", + f"Run details: {settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}", + ] + + summary = run_doc.get("summary") or {} + findings = summary.get("findings") or {} + if findings.get("summary"): + lines += ["", "Summary:", findings["summary"]] + if findings.get("analysis"): + lines += ["", "Analysis:", findings["analysis"]] + if findings.get("local_build_verified") is not None: + lines += ["", f"Local build verified: {findings['local_build_verified']}"] + if findings.get("treeherder_url"): + lines += [f"Try push: {findings['treeherder_url']}"] + + error = run_doc.get("error") or summary.get("error") + if status != "succeeded" and error: + lines += ["", f"Error: {error}"] + + return "\n".join(lines) diff --git a/services/hackbot-pulse-listener/app/taskcluster.py b/services/hackbot-pulse-listener/app/taskcluster.py new file mode 100644 index 0000000000..13c5827962 --- /dev/null +++ b/services/hackbot-pulse-listener/app/taskcluster.py @@ -0,0 +1,26 @@ +import logging + +import taskcluster + +from app.config import settings + +logger = logging.getLogger(__name__) + +_queue: taskcluster.Queue | None = None + + +def _get_queue() -> taskcluster.Queue: + global _queue + if _queue is None: + _queue = taskcluster.Queue({"rootUrl": settings.taskcluster_root_url}) + return _queue + + +def get_revision(task_id: str) -> str | None: + """Return the GECKO_HEAD_REV (git SHA) for a task, or None if unavailable. + + The revision is not in the pulse message, so we fetch the full task + definition. Task definitions are public, so no credentials are needed. + """ + task = _get_queue().task(task_id) + return task.get("payload", {}).get("env", {}).get("GECKO_HEAD_REV") diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py new file mode 100644 index 0000000000..4f733968b8 --- /dev/null +++ b/services/hackbot-pulse-listener/app/worker.py @@ -0,0 +1,47 @@ +import logging +import time + +from app import client, notify +from app.config import settings + +logger = logging.getLogger(__name__) + +TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} + + +def poll_and_notify( + run_id: str, revision: str, repo: str, developer_email: str | None +) -> None: + """Poll the run until terminal, then notify the developer. + + Runs on a background executor thread; never lets an exception escape. + """ + try: + run_doc = _poll_until_terminal(run_id) + except Exception: + logger.exception("Polling failed for run %s", run_id) + return + + if run_doc is None: + logger.warning( + "Run %s did not finish within %s minutes; giving up", + run_id, + settings.run_max_age_minutes, + ) + return + + try: + notify.send_email(developer_email, revision, repo, run_id, run_doc) + except Exception: + logger.exception("Failed to send notification for run %s", run_id) + + +def _poll_until_terminal(run_id: str) -> dict | None: + deadline = time.monotonic() + settings.run_max_age_minutes * 60 + while True: + run_doc = client.get_run(run_id) + if run_doc.get("status") in TERMINAL_STATUSES: + return run_doc + if time.monotonic() >= deadline: + return None + time.sleep(settings.poll_interval_seconds) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh new file mode 100755 index 0000000000..4ac3a237a7 --- /dev/null +++ b/services/hackbot-pulse-listener/deploy.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Deploy the build-failure Pulse listener to Cloud Run as a worker pool. +# +# A worker pool runs an always-on, non-request workload (no HTTP port). This +# service consumes Taskcluster build-failure pulse messages and triggers the +# build-repair hackbot agent via the hackbot-api. +# +# Prereqs (one-time): +# gcloud auth login +# gcloud config set project +# gcloud services enable run.googleapis.com artifactregistry.googleapis.com \ +# cloudbuild.googleapis.com secretmanager.googleapis.com +# +# Secrets (one-time) — store in Secret Manager: +# printf '%s' '' | gcloud secrets create pulse-password --data-file=- +# printf '%s' '' | gcloud secrets create sendgrid-api-key --data-file=- +# # HACKBOT_API_KEY reuses the existing shared `external-api-key` secret. +# +# Usage: +# PROJECT=my-proj REGION=us-central1 \ +# HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ +# HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ +# PULSE_USER=my-pulse-user NOTIFICATION_SENDER=build-repair@mozilla.com \ +# ./deploy.sh +set -euo pipefail + +PROJECT="${PROJECT:?set PROJECT to your GCP project id}" +REGION="${REGION:-us-central1}" +SERVICE="${SERVICE:-hackbot-pulse-listener}" +REPO="${REPO:-hackbot}" +HACKBOT_API_URL="${HACKBOT_API_URL:?set HACKBOT_API_URL to the hackbot-api base URL}" +HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL}" +PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" +WATCHED_REPOS="${WATCHED_REPOS:-try}" +NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (from address)}" + +SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" +SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" + +PULSE_SECRET="${PULSE_SECRET:-pulse-password}" +API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" +SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" + +IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" +# Build context is the repo root (the Dockerfile needs the workspace lock files). +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +echo "==> Ensuring runtime service account '${SA_EMAIL}' exists" +gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ + gcloud iam service-accounts create "${SA_NAME}" \ + --display-name="Hackbot Pulse Listener (Cloud Run runtime)" + +echo "==> Granting the SA read access to its secrets" +for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do + gcloud secrets add-iam-policy-binding "$s" \ + --member="serviceAccount:${SA_EMAIL}" \ + --role=roles/secretmanager.secretAccessor >/dev/null +done + +echo "==> Ensuring Artifact Registry repo '${REPO}' exists in ${REGION}" +gcloud artifacts repositories describe "${REPO}" --location="${REGION}" >/dev/null 2>&1 || \ + gcloud artifacts repositories create "${REPO}" \ + --repository-format=docker --location="${REGION}" \ + --description="Hackbot container images" + +echo "==> Building & pushing image with Cloud Build: ${IMAGE}" +gcloud builds submit "${ROOT_DIR}" \ + --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") + +echo "==> Deploying worker pool" +ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" +ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" +ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" + +gcloud run worker-pools deploy "${SERVICE}" \ + --image "${IMAGE}" \ + --region "${REGION}" \ + --min-instances 1 \ + --max-instances 1 \ + --service-account "${SA_EMAIL}" \ + --set-env-vars "${ENV_VARS}" \ + --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" + +echo "==> Deployed worker pool '${SERVICE}'" diff --git a/services/hackbot-pulse-listener/docker-compose.dev.yml b/services/hackbot-pulse-listener/docker-compose.dev.yml new file mode 100644 index 0000000000..441d03e7b1 --- /dev/null +++ b/services/hackbot-pulse-listener/docker-compose.dev.yml @@ -0,0 +1,11 @@ +services: + hackbot-pulse-listener: + build: + context: ../.. + dockerfile: services/hackbot-pulse-listener/Dockerfile + # Inject config into the container (relative to this compose file = repo root .env). + env_file: ../../.env + # Live-edit the source without rebuilding (cwd takes precedence on sys.path). + volumes: + - ./app:/app/app + command: python -m app diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml new file mode 100644 index 0000000000..15b114e329 --- /dev/null +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "hackbot-pulse-listener" +version = "0.1.0" +description = "Listens to Taskcluster build failures and triggers the build-repair hackbot agent" +requires-python = ">=3.12" +dependencies = [ + "kombu>=5.6,<6", + "taskcluster>=97.1,<100.4", + "httpx>=0.26.0", + "pydantic-settings>=2.1.0", + "sendgrid>=6.12.5", + "cachetools>=5.3.0", + "sentry-sdk>=2.51.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/hackbot-pulse-listener/tests/__init__.py b/services/hackbot-pulse-listener/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json new file mode 100644 index 0000000000..3813b8025a --- /dev/null +++ b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json @@ -0,0 +1,892 @@ +[ + { + "payload": { + "status": { + "taskId": "QKbYKE7STVy3_DM6J9NKXw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1400-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1400-r8", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "DJziYijSR9qv-cGTDdS92w", + "priority": "very-low", + "deadline": "2026-06-26T21:22:49.913Z", + "expires": "2026-07-23T21:22:49.913Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "takenUntil": "2026-06-25T22:47:29.995Z", + "scheduled": "2026-06-25T21:36:27.697Z", + "started": "2026-06-25T21:36:28.598Z", + "resolved": "2026-06-25T22:35:05.200Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "web-platform-tests", + "label": "test-macosx1470-64/opt-web-platform-tests-webgpu-backlog-long-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "trust-domain": "gecko", + "test-platform": "macosx1470-64/opt", + "tests_grouped": "1", + "createdForUser": "aleiserson@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.QKbYKE7STVy3_DM6J9NKXw.0.mdc1.macmini-r8-381.releng-hardware.gecko-t-osx-1400-r8.gecko-level-1.DJziYijSR9qv-cGTDdS92w._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TISFEWXiRD-Fg4ifDcLm5A", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.330Z", + "expires": "2026-07-23T13:31:23.330Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "takenUntil": "2026-06-25T23:27:40.611Z", + "scheduled": "2026-06-25T13:39:15.145Z", + "started": "2026-06-25T23:07:40.613Z", + "resolved": "2026-06-25T23:17:25.349Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/debug-mochitest-thunderbird-2", + "project": "try-comm-central", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome-thunderbird", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/debug", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TISFEWXiRD-Fg4ifDcLm5A.0.mdc1.macmini-r8-209.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TWIXsqxDTame750yV3a_0A", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.935Z", + "expires": "2026-07-23T22:29:42.935Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "takenUntil": "2026-06-25T23:21:38.581Z", + "scheduled": "2026-06-25T22:43:40.477Z", + "started": "2026-06-25T22:44:38.441Z", + "resolved": "2026-06-25T23:17:28.610Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/opt-web-platform-tests-async-9", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/opt", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TWIXsqxDTame750yV3a_0A.0.us-east1-c.4411950482911664357.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "CLwxQmXLQFunTzlzrzshOg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "takenUntil": "2026-06-25T23:30:14.189Z", + "scheduled": "2026-06-25T22:18:46.436Z", + "started": "2026-06-25T22:19:13.295Z", + "resolved": "2026-06-25T23:19:31.591Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-8", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.CLwxQmXLQFunTzlzrzshOg.0.southindia.vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "d4YZWZQXR-SdCVwlrVTTUA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "takenUntil": "2026-06-25T23:30:22.934Z", + "scheduled": "2026-06-25T22:18:46.462Z", + "started": "2026-06-25T22:19:22.411Z", + "resolved": "2026-06-25T23:19:37.633Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-2", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.d4YZWZQXR-SdCVwlrVTTUA.0.southindia.vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "EJXnn6xsQ4OeDMK3W1aksg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "takenUntil": "2026-06-25T23:30:32.889Z", + "scheduled": "2026-06-25T22:18:46.487Z", + "started": "2026-06-25T22:19:32.554Z", + "resolved": "2026-06-25T23:19:38.966Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.EJXnn6xsQ4OeDMK3W1aksg.0.westus3.vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "cnTkiN1YROGfMWecDxyMEQ", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.413Z", + "expires": "2026-07-23T22:29:42.413Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "takenUntil": "2026-06-25T23:39:39.063Z", + "scheduled": "2026-06-25T22:44:50.986Z", + "started": "2026-06-25T22:45:38.942Z", + "resolved": "2026-06-25T23:19:39.842Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-12", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.cnTkiN1YROGfMWecDxyMEQ.0.us-west1-b.525836151908471198.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DPltHwhPTI29TinNU_XBbw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.472Z", + "expires": "2027-06-25T20:22:15.472Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "takenUntil": "2026-06-25T23:31:17.350Z", + "scheduled": "2026-06-25T22:19:36.851Z", + "started": "2026-06-25T22:20:16.923Z", + "resolved": "2026-06-25T23:20:25.989Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DPltHwhPTI29TinNU_XBbw.0.canadacentral.vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DEt854d0QAOgI-OZsHoMmw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.468Z", + "expires": "2027-06-25T20:22:15.468Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "takenUntil": "2026-06-25T23:31:13.812Z", + "scheduled": "2026-06-25T22:19:36.839Z", + "started": "2026-06-25T22:20:12.929Z", + "resolved": "2026-06-25T23:20:28.897Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-13", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DEt854d0QAOgI-OZsHoMmw.0.southindia.vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "AlWS0HmFQiKPjBS1HcFAzw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.054Z", + "expires": "2026-07-23T13:31:23.054Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "takenUntil": "2026-06-25T23:28:29.702Z", + "scheduled": "2026-06-25T13:42:42.448Z", + "started": "2026-06-25T23:08:29.704Z", + "resolved": "2026-06-25T23:20:29.089Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/opt-xpcshell-4", + "project": "try-comm-central", + "retrigger": "true", + "test-suite": "xpcshell", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/opt", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.AlWS0HmFQiKPjBS1HcFAzw.0.mdc1.macmini-r8-77.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YO-gvnzTTGS7INXYIN8Nbg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "takenUntil": "2026-06-25T23:31:12.293Z", + "scheduled": "2026-06-25T22:18:46.606Z", + "started": "2026-06-25T22:20:11.674Z", + "resolved": "2026-06-25T23:20:29.185Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-15", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YO-gvnzTTGS7INXYIN8Nbg.0.centralindia.vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YV2f3tnLSCyXUIG4EDq5Xw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.153Z", + "expires": "2027-06-25T20:21:36.153Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "takenUntil": "2026-06-25T23:31:12.727Z", + "scheduled": "2026-06-25T22:18:46.629Z", + "started": "2026-06-25T22:20:12.243Z", + "resolved": "2026-06-25T23:20:30.629Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-4", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YV2f3tnLSCyXUIG4EDq5Xw.0.centralindia.vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "dtOy4UGdTHm2s5XCVuNkqA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.473Z", + "expires": "2027-06-25T20:22:15.473Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "takenUntil": "2026-06-25T23:31:33.280Z", + "scheduled": "2026-06-25T22:19:36.866Z", + "started": "2026-06-25T22:20:32.966Z", + "resolved": "2026-06-25T23:20:41.142Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-11", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.dtOy4UGdTHm2s5XCVuNkqA.0.northcentralus.vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "fRj5qgRVS4adFANUGOexwA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.474Z", + "expires": "2027-06-25T20:22:15.474Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "takenUntil": "2026-06-25T23:31:38.392Z", + "scheduled": "2026-06-25T22:19:36.878Z", + "started": "2026-06-25T22:20:38.071Z", + "resolved": "2026-06-25T23:20:45.381Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-5", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.fRj5qgRVS4adFANUGOexwA.0.westus.vm-slqhmhydqni6evb1vuglbaat8icl5saifla.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "ZFZw8IBSSWKrZmgZcdpUKA", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.408Z", + "expires": "2026-07-23T22:29:42.408Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "takenUntil": "2026-06-25T23:39:44.587Z", + "scheduled": "2026-06-25T22:44:51.205Z", + "started": "2026-06-25T22:45:44.383Z", + "resolved": "2026-06-25T23:21:01.122Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.ZFZw8IBSSWKrZmgZcdpUKA.0.us-central1-b.1690275899809730974.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "f_BW7zM6RCi8p_Ah5i57Qg", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-amd", + "taskQueueId": "gecko-t/t-linux-docker-amd", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "cXOzgHGNRt25Mv8Aq5TvLw", + "priority": "low", + "deadline": "2026-06-26T22:51:20.160Z", + "expires": "2027-06-25T22:51:20.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "takenUntil": "2026-06-25T23:36:36.096Z", + "scheduled": "2026-06-25T23:16:35.979Z", + "started": "2026-06-25T23:16:36.098Z", + "resolved": "2026-06-25T23:21:06.146Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "source-test", + "label": "source-test-node-newtab-unit-tests", + "project": "autoland", + "retrigger": "true", + "trust-domain": "gecko", + "createdForUser": "dwhisman@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.f_BW7zM6RCi8p_Ah5i57Qg.0.us-central1-b.9170129125611932268.gecko-t.t-linux-docker-amd.gecko-level-3.cXOzgHGNRt25Mv8Aq5TvLw._", + "redelivered": false, + "cc": [] + } +] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py new file mode 100644 index 0000000000..6a68968ec9 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -0,0 +1,100 @@ +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from app import consumer + +FIXTURES = Path(__file__).parent / "fixtures" + + +def setup_function(): + consumer._seen.clear() + + +def _sample_bodies(): + data = json.loads((FIXTURES / "pulse_messages.json").read_text()) + # The inspector wraps the real AMQP body under "payload". + return [m["payload"] for m in data] + + +def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): + return { + "status": {"taskId": task_id}, + "runId": 0, + "task": { + "tags": { + "kind": "build", + "project": project, + "label": label, + "createdForUser": "dev@mozilla.com", + } + }, + } + + +def test_sample_messages_are_all_tests_and_skipped(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + for body in _sample_bodies(): + assert consumer.process(body, executor) is None + get_rev.assert_not_called() + trigger.assert_not_called() + executor.submit.assert_not_called() + + +def test_build_failure_triggers_run_and_submits_poll(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + run_id = consumer.process(_build_msg(), executor) + + assert run_id == "run-1" + trigger.assert_called_once() + inputs = trigger.call_args.args[0] + assert inputs["git_commit"] == "deadbeef" + assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} + executor.submit.assert_called_once() + + +def test_same_revision_triggers_once(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + consumer.process(_build_msg(task_id="T1"), executor) + consumer.process(_build_msg(task_id="T2"), executor) + + trigger.assert_called_once() + + +def test_unwatched_project_skipped_before_api_call(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + assert consumer.process(_build_msg(project="mozilla-central"), executor) is None + + get_rev.assert_not_called() + trigger.assert_not_called() + + +def test_trigger_failure_releases_revision_for_retry(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object( + consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] + ) as trigger, + ): + assert consumer.process(_build_msg(task_id="T1"), executor) is None + # Same revision can be retried because the failed claim was released. + assert consumer.process(_build_msg(task_id="T2"), executor) == "run-2" + + assert trigger.call_count == 2 diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py new file mode 100644 index 0000000000..32cf831011 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -0,0 +1,62 @@ +from unittest.mock import MagicMock, patch + +from app import notify + + +def test_skips_without_developer_email(): + # Must not raise even with no SendGrid config. + notify.send_email(None, "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_skips_without_sendgrid_config(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) + monkeypatch.setattr(notify.settings, "notification_sender", None) + notify.send_email("dev@mozilla.com", "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_body_contains_ui_link_and_summary(monkeypatch): + monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") + body = notify._build_body( + "deadbeefcafe1234", + "try", + "run-1", + { + "status": "succeeded", + "summary": { + "findings": { + "summary": "Fixed a missing include", + "analysis": "The commit removed a needed header", + "local_build_verified": True, + } + }, + }, + ) + assert "https://ui.example/runs/run-1" in body + assert "Fixed a missing include" in body + assert "The commit removed a needed header" in body + assert "Local build verified: True" in body + + +def test_failure_body_includes_error(): + body = notify._build_body( + "deadbeef", "try", "run-1", {"status": "failed", "error": "build still broken"} + ) + assert "build still broken" in body + + +def test_sends_email_when_configured(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email( + "dev@mozilla.com", + "rev", + "try", + "run-1", + {"status": "succeeded", "summary": {}}, + ) + + fake_client.send.assert_called_once() diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py new file mode 100644 index 0000000000..1ef39ae448 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_worker.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +from app import worker + + +def test_terminal_run_notifies_once(): + run_doc = {"status": "succeeded", "summary": {}} + with ( + patch.object(worker.client, "get_run", return_value=run_doc) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_called_once() + args = notify.send_email.call_args.args + assert args == ("dev@mozilla.com", "rev", "try", "run-1", run_doc) + + +def test_gives_up_after_max_age(monkeypatch): + monkeypatch.setattr(worker.settings, "run_max_age_minutes", 0) + with ( + patch.object( + worker.client, "get_run", return_value={"status": "running"} + ) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_not_called() diff --git a/uv.lock b/uv.lock index 0de875449c..63ef8e9109 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,7 @@ members = [ "hackbot-agent-frontend-triage", "hackbot-agent-test-plan-generator", "hackbot-api", + "hackbot-pulse-listener", "hackbot-runtime", "reviewhelper-api", ] @@ -2584,6 +2585,38 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "hackbot-pulse-listener" +version = "0.1.0" +source = { editable = "services/hackbot-pulse-listener" } +dependencies = [ + { name = "cachetools" }, + { name = "httpx" }, + { name = "kombu" }, + { name = "pydantic-settings" }, + { name = "sendgrid" }, + { name = "sentry-sdk" }, + { name = "taskcluster" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "cachetools", specifier = ">=5.3.0" }, + { name = "httpx", specifier = ">=0.26.0" }, + { name = "kombu", specifier = ">=5.6,<6" }, + { name = "pydantic-settings", specifier = ">=2.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "sendgrid", specifier = ">=6.12.5" }, + { name = "sentry-sdk", specifier = ">=2.51.0" }, + { name = "taskcluster", specifier = ">=97.1,<100.4" }, +] +provides-extras = ["dev"] + [[package]] name = "hackbot-runtime" version = "0.1.0" From c405f099c51637cf48935d0708386c1e01d2913f Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Tue, 30 Jun 2026 11:29:46 -0700 Subject: [PATCH 02/12] Remove email notifications from pulse listener Notification about run outcomes is out of scope for the listener (#6261); it will live in the API or a separate service. Drop the SendGrid email sender, the run-polling worker, the get_run API client, and the ThreadPoolExecutor that fed them, along with their settings and deploy config. Co-Authored-By: Claude Opus 4.8 (1M context) --- services/hackbot-pulse-listener/README.md | 14 +--- .../hackbot-pulse-listener/app/__main__.py | 9 +-- services/hackbot-pulse-listener/app/client.py | 7 -- services/hackbot-pulse-listener/app/config.py | 10 --- .../hackbot-pulse-listener/app/consumer.py | 18 ++--- services/hackbot-pulse-listener/app/notify.py | 74 ------------------- services/hackbot-pulse-listener/app/worker.py | 47 ------------ services/hackbot-pulse-listener/deploy.sh | 14 +--- .../hackbot-pulse-listener/pyproject.toml | 1 - .../tests/test_consumer.py | 25 +++---- .../tests/test_notify.py | 62 ---------------- .../tests/test_worker.py | 31 -------- uv.lock | 2 - 13 files changed, 25 insertions(+), 289 deletions(-) delete mode 100644 services/hackbot-pulse-listener/app/notify.py delete mode 100644 services/hackbot-pulse-listener/app/worker.py delete mode 100644 services/hackbot-pulse-listener/tests/test_notify.py delete mode 100644 services/hackbot-pulse-listener/tests/test_worker.py diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md index 57984bd5e7..5a356246b2 100644 --- a/services/hackbot-pulse-listener/README.md +++ b/services/hackbot-pulse-listener/README.md @@ -1,9 +1,8 @@ # Hackbot Pulse Listener Listens to Taskcluster build-failure pulse messages, and for failed **Firefox build -tasks** triggers the `build-repair` hackbot agent through the hackbot-api. When a run -finishes (minutes later) it emails the developer who pushed the change a link to the -hackbot UI and a summary of the analysis and fix. +tasks** triggers the `build-repair` hackbot agent through the hackbot-api. Notification +about the run outcome is handled separately (in the API or another service). ## How it works @@ -14,25 +13,20 @@ hackbot UI and a summary of the analysis and fix. 3. Fetch the task definition to read `GECKO_HEAD_REV` (the revision is not in the message). 4. Dedupe by revision with an in-memory TTL cache, so only one agent run is triggered per revision even when many build tasks fail for the same push. -5. `POST /agents/build-repair/runs`, then poll `GET /runs/{run_id}` until terminal and send - the report email. +5. `POST /agents/build-repair/runs`. -The dedupe cache and pending-run tracking are in-memory (reset on restart). +The dedupe cache is in-memory (reset on restart). ## Run locally ```bash export PULSE_USER=... PULSE_PASSWORD=... # https://pulseguardian.mozilla.org export HACKBOT_API_URL=https://hackbot-api.../ HACKBOT_API_KEY=... -export HACKBOT_UI_URL=https://hackbot-ui.../ export WATCHED_REPOS=try export DRY_RUN=true # log intended calls, don't POST uv run --package hackbot-pulse-listener python -m app ``` -Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it -is logged and skipped. - ## Test ```bash diff --git a/services/hackbot-pulse-listener/app/__main__.py b/services/hackbot-pulse-listener/app/__main__.py index 80c99d76e7..6a3b1b5832 100644 --- a/services/hackbot-pulse-listener/app/__main__.py +++ b/services/hackbot-pulse-listener/app/__main__.py @@ -1,6 +1,5 @@ import logging import signal -from concurrent.futures import ThreadPoolExecutor from app import consumer from app.config import settings @@ -19,8 +18,7 @@ def main() -> None: logger.warning("PULSE_USER/PULSE_PASSWORD not set; listener will not start") return - executor = ThreadPoolExecutor(max_workers=settings.poll_max_workers) - consumer_obj = consumer.build_consumer(executor) + consumer_obj = consumer.build_consumer() def shutdown(signum, _frame): logger.info("Received signal %s; shutting down", signum) @@ -34,10 +32,7 @@ def shutdown(signum, _frame): ", ".join(consumer.EXCHANGES), sorted(settings.watched_repos_set), ) - try: - consumer_obj.run() - finally: - executor.shutdown(wait=False) + consumer_obj.run() if __name__ == "__main__": diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py index a83e6ce974..498660fa08 100644 --- a/services/hackbot-pulse-listener/app/client.py +++ b/services/hackbot-pulse-listener/app/client.py @@ -23,10 +23,3 @@ def trigger_run(inputs: dict) -> str | None: resp = httpx.post(url, json=inputs, headers=_headers(), timeout=_TIMEOUT) resp.raise_for_status() return resp.json()["run_id"] - - -def get_run(run_id: str) -> dict: - url = f"{settings.hackbot_api_url}/runs/{run_id}" - resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) - resp.raise_for_status() - return resp.json() diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py index 49666308c8..102c8fa555 100644 --- a/services/hackbot-pulse-listener/app/config.py +++ b/services/hackbot-pulse-listener/app/config.py @@ -10,7 +10,6 @@ class Settings(BaseSettings): # hackbot-api hackbot_api_url: str = "" hackbot_api_key: str = "" - hackbot_ui_url: str = "" agent_name: str = "build-repair" # Failure filtering and agent inputs. @@ -24,15 +23,6 @@ class Settings(BaseSettings): dedupe_ttl_seconds: int = 6 * 60 * 60 dedupe_max_size: int = 4096 - # Polling the API for run completion - poll_interval_seconds: int = 60 - run_max_age_minutes: int = 12 * 60 - poll_max_workers: int = 8 - - # Email notifications (SendGrid) - sendgrid_api_key: str | None = None - notification_sender: str | None = None - dry_run: bool = False environment: str = "development" sentry_dsn: str | None = None diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py index a44599768f..817e65cec3 100644 --- a/services/hackbot-pulse-listener/app/consumer.py +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -1,11 +1,10 @@ import logging -from concurrent.futures import Executor from cachetools import TTLCache from kombu import Connection, Exchange, Queue from kombu.mixins import ConsumerMixin -from app import client, taskcluster, worker +from app import client, taskcluster from app.config import settings logger = logging.getLogger(__name__) @@ -21,7 +20,7 @@ ) -def process(body: dict, executor: Executor) -> str | None: +def process(body: dict) -> str | None: """Handle one Taskcluster failure message. Returns the triggered run id.""" tags = (body.get("task") or {}).get("tags") or {} @@ -34,7 +33,6 @@ def process(body: dict, executor: Executor) -> str | None: task_id = body["status"]["taskId"] task_name = tags.get("label") or task_id - developer_email = tags.get("createdForUser") revision = taskcluster.get_revision(task_id) if not revision: @@ -64,17 +62,13 @@ def process(body: dict, executor: Executor) -> str | None: return None logger.info("Triggered build-repair run %s for %s@%s", run_id, project, revision) - if run_id is not None: - executor.submit( - worker.poll_and_notify, run_id, revision, project, developer_email - ) return run_id -def make_handler(executor: Executor): +def make_handler(): def on_message(body, message): try: - process(body, executor) + process(body) except Exception: logger.exception("Error handling pulse message") finally: @@ -109,10 +103,10 @@ def get_consumers(self, Consumer, channel): return [Consumer(queues=self.queues, callbacks=[self.on_message])] -def build_consumer(executor: Executor) -> BuildFailureConsumer: +def build_consumer() -> BuildFailureConsumer: connection = Connection( CONNECTION_URL.format(settings.pulse_user, settings.pulse_password) ) return BuildFailureConsumer( - connection, _build_queues(settings.pulse_user), make_handler(executor) + connection, _build_queues(settings.pulse_user), make_handler() ) diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py deleted file mode 100644 index 459fae3d95..0000000000 --- a/services/hackbot-pulse-listener/app/notify.py +++ /dev/null @@ -1,74 +0,0 @@ -import logging - -from app.config import settings - -logger = logging.getLogger(__name__) - - -def send_email( - developer_email: str | None, - revision: str, - repo: str, - run_id: str, - run_doc: dict, -) -> None: - """Email the developer the run outcome. No-op (logged) if not configured.""" - if not developer_email: - logger.info("No developer email for run %s; skipping notification", run_id) - return - if not (settings.sendgrid_api_key and settings.notification_sender): - logger.info("SendGrid not configured; skipping email for run %s", run_id) - return - - import sendgrid - from sendgrid.helpers.mail import Content, From, Mail, Subject, To - - status = run_doc.get("status", "unknown") - subject = f"[build-repair] {status} for {repo}@{revision[:12]}" - - sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) - message = Mail( - From(settings.notification_sender), - To(developer_email), - Subject(subject), - Content("text/plain", _build_body(revision, repo, run_id, run_doc)), - ) - response = sg.send(message=message) - logger.info( - "Sent build-repair notification to %s (status %s)", - developer_email, - response.status_code, - ) - - -def _build_body(revision: str, repo: str, run_id: str, run_doc: dict) -> str: - status = run_doc.get("status", "unknown") - lines = [ - f"The build-repair agent finished with status: {status}.", - "", - f"Repository: {repo}", - f"Revision: {revision}", - ] - - if settings.hackbot_ui_url: - lines += [ - "", - f"Run details: {settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}", - ] - - summary = run_doc.get("summary") or {} - findings = summary.get("findings") or {} - if findings.get("summary"): - lines += ["", "Summary:", findings["summary"]] - if findings.get("analysis"): - lines += ["", "Analysis:", findings["analysis"]] - if findings.get("local_build_verified") is not None: - lines += ["", f"Local build verified: {findings['local_build_verified']}"] - if findings.get("treeherder_url"): - lines += [f"Try push: {findings['treeherder_url']}"] - - error = run_doc.get("error") or summary.get("error") - if status != "succeeded" and error: - lines += ["", f"Error: {error}"] - - return "\n".join(lines) diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py deleted file mode 100644 index 4f733968b8..0000000000 --- a/services/hackbot-pulse-listener/app/worker.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import time - -from app import client, notify -from app.config import settings - -logger = logging.getLogger(__name__) - -TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} - - -def poll_and_notify( - run_id: str, revision: str, repo: str, developer_email: str | None -) -> None: - """Poll the run until terminal, then notify the developer. - - Runs on a background executor thread; never lets an exception escape. - """ - try: - run_doc = _poll_until_terminal(run_id) - except Exception: - logger.exception("Polling failed for run %s", run_id) - return - - if run_doc is None: - logger.warning( - "Run %s did not finish within %s minutes; giving up", - run_id, - settings.run_max_age_minutes, - ) - return - - try: - notify.send_email(developer_email, revision, repo, run_id, run_doc) - except Exception: - logger.exception("Failed to send notification for run %s", run_id) - - -def _poll_until_terminal(run_id: str) -> dict | None: - deadline = time.monotonic() + settings.run_max_age_minutes * 60 - while True: - run_doc = client.get_run(run_id) - if run_doc.get("status") in TERMINAL_STATUSES: - return run_doc - if time.monotonic() >= deadline: - return None - time.sleep(settings.poll_interval_seconds) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh index 4ac3a237a7..ee2c129b37 100755 --- a/services/hackbot-pulse-listener/deploy.sh +++ b/services/hackbot-pulse-listener/deploy.sh @@ -14,14 +14,12 @@ # # Secrets (one-time) — store in Secret Manager: # printf '%s' '' | gcloud secrets create pulse-password --data-file=- -# printf '%s' '' | gcloud secrets create sendgrid-api-key --data-file=- # # HACKBOT_API_KEY reuses the existing shared `external-api-key` secret. # # Usage: # PROJECT=my-proj REGION=us-central1 \ # HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ -# HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ -# PULSE_USER=my-pulse-user NOTIFICATION_SENDER=build-repair@mozilla.com \ +# PULSE_USER=my-pulse-user \ # ./deploy.sh set -euo pipefail @@ -30,17 +28,14 @@ REGION="${REGION:-us-central1}" SERVICE="${SERVICE:-hackbot-pulse-listener}" REPO="${REPO:-hackbot}" HACKBOT_API_URL="${HACKBOT_API_URL:?set HACKBOT_API_URL to the hackbot-api base URL}" -HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL}" PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" WATCHED_REPOS="${WATCHED_REPOS:-try}" -NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (from address)}" SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" PULSE_SECRET="${PULSE_SECRET:-pulse-password}" API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" -SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" # Build context is the repo root (the Dockerfile needs the workspace lock files). @@ -52,7 +47,7 @@ gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ --display-name="Hackbot Pulse Listener (Cloud Run runtime)" echo "==> Granting the SA read access to its secrets" -for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do +for s in "${PULSE_SECRET}" "${API_KEY_SECRET}"; do gcloud secrets add-iam-policy-binding "$s" \ --member="serviceAccount:${SA_EMAIL}" \ --role=roles/secretmanager.secretAccessor >/dev/null @@ -69,9 +64,8 @@ gcloud builds submit "${ROOT_DIR}" \ --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") echo "==> Deploying worker pool" -ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" +ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL}" ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" -ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" gcloud run worker-pools deploy "${SERVICE}" \ --image "${IMAGE}" \ @@ -80,6 +74,6 @@ gcloud run worker-pools deploy "${SERVICE}" \ --max-instances 1 \ --service-account "${SA_EMAIL}" \ --set-env-vars "${ENV_VARS}" \ - --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" + --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest" echo "==> Deployed worker pool '${SERVICE}'" diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml index 15b114e329..bba65ac4b3 100644 --- a/services/hackbot-pulse-listener/pyproject.toml +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -8,7 +8,6 @@ dependencies = [ "taskcluster>=97.1,<100.4", "httpx>=0.26.0", "pydantic-settings>=2.1.0", - "sendgrid>=6.12.5", "cachetools>=5.3.0", "sentry-sdk>=2.51.0", ] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index 6a68968ec9..a222e74865 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch from app import consumer @@ -33,68 +33,61 @@ def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): def test_sample_messages_are_all_tests_and_skipped(): - executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): for body in _sample_bodies(): - assert consumer.process(body, executor) is None + assert consumer.process(body) is None get_rev.assert_not_called() trigger.assert_not_called() - executor.submit.assert_not_called() -def test_build_failure_triggers_run_and_submits_poll(): - executor = MagicMock() +def test_build_failure_triggers_run(): with ( patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): - run_id = consumer.process(_build_msg(), executor) + run_id = consumer.process(_build_msg()) assert run_id == "run-1" trigger.assert_called_once() inputs = trigger.call_args.args[0] assert inputs["git_commit"] == "deadbeef" assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} - executor.submit.assert_called_once() def test_same_revision_triggers_once(): - executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): - consumer.process(_build_msg(task_id="T1"), executor) - consumer.process(_build_msg(task_id="T2"), executor) + consumer.process(_build_msg(task_id="T1")) + consumer.process(_build_msg(task_id="T2")) trigger.assert_called_once() def test_unwatched_project_skipped_before_api_call(): - executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): - assert consumer.process(_build_msg(project="mozilla-central"), executor) is None + assert consumer.process(_build_msg(project="mozilla-central")) is None get_rev.assert_not_called() trigger.assert_not_called() def test_trigger_failure_releases_revision_for_retry(): - executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), patch.object( consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] ) as trigger, ): - assert consumer.process(_build_msg(task_id="T1"), executor) is None + assert consumer.process(_build_msg(task_id="T1")) is None # Same revision can be retried because the failed claim was released. - assert consumer.process(_build_msg(task_id="T2"), executor) == "run-2" + assert consumer.process(_build_msg(task_id="T2")) == "run-2" assert trigger.call_count == 2 diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py deleted file mode 100644 index 32cf831011..0000000000 --- a/services/hackbot-pulse-listener/tests/test_notify.py +++ /dev/null @@ -1,62 +0,0 @@ -from unittest.mock import MagicMock, patch - -from app import notify - - -def test_skips_without_developer_email(): - # Must not raise even with no SendGrid config. - notify.send_email(None, "rev", "try", "run-1", {"status": "succeeded"}) - - -def test_skips_without_sendgrid_config(monkeypatch): - monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) - monkeypatch.setattr(notify.settings, "notification_sender", None) - notify.send_email("dev@mozilla.com", "rev", "try", "run-1", {"status": "succeeded"}) - - -def test_body_contains_ui_link_and_summary(monkeypatch): - monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") - body = notify._build_body( - "deadbeefcafe1234", - "try", - "run-1", - { - "status": "succeeded", - "summary": { - "findings": { - "summary": "Fixed a missing include", - "analysis": "The commit removed a needed header", - "local_build_verified": True, - } - }, - }, - ) - assert "https://ui.example/runs/run-1" in body - assert "Fixed a missing include" in body - assert "The commit removed a needed header" in body - assert "Local build verified: True" in body - - -def test_failure_body_includes_error(): - body = notify._build_body( - "deadbeef", "try", "run-1", {"status": "failed", "error": "build still broken"} - ) - assert "build still broken" in body - - -def test_sends_email_when_configured(monkeypatch): - monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") - monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") - - fake_client = MagicMock() - fake_client.send.return_value = MagicMock(status_code=202) - with patch("sendgrid.SendGridAPIClient", return_value=fake_client): - notify.send_email( - "dev@mozilla.com", - "rev", - "try", - "run-1", - {"status": "succeeded", "summary": {}}, - ) - - fake_client.send.assert_called_once() diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py deleted file mode 100644 index 1ef39ae448..0000000000 --- a/services/hackbot-pulse-listener/tests/test_worker.py +++ /dev/null @@ -1,31 +0,0 @@ -from unittest.mock import patch - -from app import worker - - -def test_terminal_run_notifies_once(): - run_doc = {"status": "succeeded", "summary": {}} - with ( - patch.object(worker.client, "get_run", return_value=run_doc) as get_run, - patch.object(worker, "notify") as notify, - ): - worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") - - get_run.assert_called_once() - notify.send_email.assert_called_once() - args = notify.send_email.call_args.args - assert args == ("dev@mozilla.com", "rev", "try", "run-1", run_doc) - - -def test_gives_up_after_max_age(monkeypatch): - monkeypatch.setattr(worker.settings, "run_max_age_minutes", 0) - with ( - patch.object( - worker.client, "get_run", return_value={"status": "running"} - ) as get_run, - patch.object(worker, "notify") as notify, - ): - worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") - - get_run.assert_called_once() - notify.send_email.assert_not_called() diff --git a/uv.lock b/uv.lock index 63ef8e9109..92e61ba5ac 100644 --- a/uv.lock +++ b/uv.lock @@ -2594,7 +2594,6 @@ dependencies = [ { name = "httpx" }, { name = "kombu" }, { name = "pydantic-settings" }, - { name = "sendgrid" }, { name = "sentry-sdk" }, { name = "taskcluster" }, ] @@ -2611,7 +2610,6 @@ requires-dist = [ { name = "kombu", specifier = ">=5.6,<6" }, { name = "pydantic-settings", specifier = ">=2.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, - { name = "sendgrid", specifier = ">=6.12.5" }, { name = "sentry-sdk", specifier = ">=2.51.0" }, { name = "taskcluster", specifier = ">=97.1,<100.4" }, ] From 8646bbc530c82218e41e51011cf0935381c9ee11 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Thu, 2 Jul 2026 10:30:12 -0700 Subject: [PATCH 03/12] Add hg commit mapping to git --- .../hackbot-pulse-listener/app/consumer.py | 39 +++++++++++++------ services/hackbot-pulse-listener/app/lando.py | 19 +++++++++ .../hackbot-pulse-listener/app/taskcluster.py | 6 ++- .../tests/test_consumer.py | 24 +++++++++--- 4 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 services/hackbot-pulse-listener/app/lando.py diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py index 817e65cec3..a468040490 100644 --- a/services/hackbot-pulse-listener/app/consumer.py +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -4,7 +4,7 @@ from kombu import Connection, Exchange, Queue from kombu.mixins import ConsumerMixin -from app import client, taskcluster +from app import client, lando, taskcluster from app.config import settings logger = logging.getLogger(__name__) @@ -13,7 +13,7 @@ EXCHANGES = ("exchange/taskcluster-queue/v1/task-failed",) -# In-memory dedupe of git revisions already handed to the agent. Only the +# In-memory dedupe of hg revisions already handed to the agent. Only the # single consumer thread touches it, so no lock is needed. _seen: TTLCache = TTLCache( maxsize=settings.dedupe_max_size, ttl=settings.dedupe_ttl_seconds @@ -34,18 +34,29 @@ def process(body: dict) -> str | None: task_id = body["status"]["taskId"] task_name = tags.get("label") or task_id - revision = taskcluster.get_revision(task_id) - if not revision: + hg_revision = taskcluster.get_hg_revision(task_id) + if not hg_revision: logger.warning("No GECKO_HEAD_REV for task %s; skipping", task_id) return None - if revision in _seen: - logger.info("Revision %s already processed; skipping", revision) + if hg_revision in _seen: + logger.info("Revision %s already processed; skipping", hg_revision) + return None + _seen[hg_revision] = True + + git_commit = lando.hg_to_git(hg_revision) + if not git_commit: + logger.warning( + "Could not map hg revision %s to git for task %s (%s); skipping", + hg_revision, + task_id, + project, + ) + _seen.pop(hg_revision, None) return None - _seen[revision] = True inputs: dict = { - "git_commit": revision, + "git_commit": git_commit, "failure_tasks": {task_name: task_id}, "run_try_push": settings.run_try_push, } @@ -57,11 +68,17 @@ def process(body: dict) -> str | None: try: run_id = client.trigger_run(inputs) except Exception: - logger.exception("Failed to trigger build-repair run for %s", revision) - _seen.pop(revision, None) + logger.exception("Failed to trigger build-repair run for %s", hg_revision) + _seen.pop(hg_revision, None) return None - logger.info("Triggered build-repair run %s for %s@%s", run_id, project, revision) + logger.info( + "Triggered build-repair run %s for %s@%s (git %s)", + run_id, + project, + hg_revision, + git_commit, + ) return run_id diff --git a/services/hackbot-pulse-listener/app/lando.py b/services/hackbot-pulse-listener/app/lando.py new file mode 100644 index 0000000000..b8a336ee45 --- /dev/null +++ b/services/hackbot-pulse-listener/app/lando.py @@ -0,0 +1,19 @@ +import logging + +import httpx + +logger = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0) + + +def hg_to_git(hg_rev: str) -> str | None: + """Resolve a Mercurial revision to the matching firefox git SHA, or None.""" + url = f"https://lando.moz.tools/api/hg2git/firefox/{hg_rev}" + try: + resp = httpx.get(url, timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json()["git_hash"] + except (httpx.HTTPError, KeyError) as exc: + logger.debug("Failed to map hg revision %s to git: %s", hg_rev, exc) + return None diff --git a/services/hackbot-pulse-listener/app/taskcluster.py b/services/hackbot-pulse-listener/app/taskcluster.py index 13c5827962..f4076e6d0a 100644 --- a/services/hackbot-pulse-listener/app/taskcluster.py +++ b/services/hackbot-pulse-listener/app/taskcluster.py @@ -16,11 +16,13 @@ def _get_queue() -> taskcluster.Queue: return _queue -def get_revision(task_id: str) -> str | None: - """Return the GECKO_HEAD_REV (git SHA) for a task, or None if unavailable. +def get_hg_revision(task_id: str) -> str | None: + """Return the GECKO_HEAD_REV (Mercurial revision) for a task, or None. The revision is not in the pulse message, so we fetch the full task definition. Task definitions are public, so no credentials are needed. + GECKO_HEAD_REV is an hg revision; the build-repair agent needs a git SHA, + so callers must convert it (see app.lando.hg_to_git). """ task = _get_queue().task(task_id) return task.get("payload", {}).get("env", {}).get("GECKO_HEAD_REV") diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index a222e74865..fdb760dee2 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -34,7 +34,7 @@ def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): def test_sample_messages_are_all_tests_and_skipped(): with ( - patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): for body in _sample_bodies(): @@ -45,7 +45,8 @@ def test_sample_messages_are_all_tests_and_skipped(): def test_build_failure_triggers_run(): with ( - patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): run_id = consumer.process(_build_msg()) @@ -59,7 +60,8 @@ def test_build_failure_triggers_run(): def test_same_revision_triggers_once(): with ( - patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): consumer.process(_build_msg(task_id="T1")) @@ -70,7 +72,7 @@ def test_same_revision_triggers_once(): def test_unwatched_project_skipped_before_api_call(): with ( - patch.object(consumer.taskcluster, "get_revision") as get_rev, + patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): assert consumer.process(_build_msg(project="mozilla-central")) is None @@ -79,9 +81,21 @@ def test_unwatched_project_skipped_before_api_call(): trigger.assert_not_called() +def test_unmappable_revision_skipped(): + with ( + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value=None), + patch.object(consumer.client, "trigger_run") as trigger, + ): + assert consumer.process(_build_msg()) is None + + trigger.assert_not_called() + + def test_trigger_failure_releases_revision_for_retry(): with ( - patch.object(consumer.taskcluster, "get_revision", return_value="deadbeef"), + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), patch.object( consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] ) as trigger, From 075c3824bd3f57bdea2a4c142e0ea88f9b31c4c7 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Thu, 2 Jul 2026 11:06:59 -0700 Subject: [PATCH 04/12] Fix CI prettier --- ui/changes/snowpack.config.js | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ui/changes/snowpack.config.js b/ui/changes/snowpack.config.js index 8454c2b72f..8d4b325d6f 100644 --- a/ui/changes/snowpack.config.js +++ b/ui/changes/snowpack.config.js @@ -1,13 +1,7 @@ module.exports = { - plugins: [ - /* ... */ - ], - packageOptions: { - /* ... */ - }, - devOptions: { - /* ... */ - }, + plugins: [/* ... */], + packageOptions: {/* ... */}, + devOptions: {/* ... */}, buildOptions: { out: "dist", /* ... */ @@ -16,7 +10,5 @@ module.exports = { src: "/", /* ... */ }, - alias: { - /* ... */ - }, + alias: {/* ... */}, }; From 9a6657b3072062bb24b041dc0db3a609fa774db4 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Tue, 7 Jul 2026 15:51:41 -0700 Subject: [PATCH 05/12] Add emails back --- services/hackbot-pulse-listener/README.md | 15 +++- .../hackbot-pulse-listener/app/__main__.py | 9 ++- services/hackbot-pulse-listener/app/client.py | 7 ++ services/hackbot-pulse-listener/app/config.py | 16 +++- .../hackbot-pulse-listener/app/consumer.py | 18 +++-- services/hackbot-pulse-listener/app/notify.py | 75 ++++++++++++++++++ services/hackbot-pulse-listener/app/worker.py | 47 +++++++++++ services/hackbot-pulse-listener/deploy.sh | 16 ++-- .../hackbot-pulse-listener/pyproject.toml | 1 + .../tests/test_consumer.py | 38 ++++++--- .../tests/test_notify.py | 77 +++++++++++++++++++ .../tests/test_worker.py | 31 ++++++++ uv.lock | 2 + 13 files changed, 322 insertions(+), 30 deletions(-) create mode 100644 services/hackbot-pulse-listener/app/notify.py create mode 100644 services/hackbot-pulse-listener/app/worker.py create mode 100644 services/hackbot-pulse-listener/tests/test_notify.py create mode 100644 services/hackbot-pulse-listener/tests/test_worker.py diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md index 5a356246b2..b7cb139cf7 100644 --- a/services/hackbot-pulse-listener/README.md +++ b/services/hackbot-pulse-listener/README.md @@ -1,8 +1,9 @@ # Hackbot Pulse Listener Listens to Taskcluster build-failure pulse messages, and for failed **Firefox build -tasks** triggers the `build-repair` hackbot agent through the hackbot-api. Notification -about the run outcome is handled separately (in the API or another service). +tasks** triggers the `build-repair` hackbot agent through the hackbot-api. When a run +finishes (minutes later) it emails the developer who pushed the change a link to the +hackbot UI and a summary of the analysis and fix. ## How it works @@ -13,20 +14,26 @@ about the run outcome is handled separately (in the API or another service). 3. Fetch the task definition to read `GECKO_HEAD_REV` (the revision is not in the message). 4. Dedupe by revision with an in-memory TTL cache, so only one agent run is triggered per revision even when many build tasks fail for the same push. -5. `POST /agents/build-repair/runs`. +5. `POST /agents/build-repair/runs`, then poll `GET /runs/{run_id}` until terminal and send + the report email. -The dedupe cache is in-memory (reset on restart). +The dedupe cache and pending-run tracking are in-memory (reset on restart). ## Run locally ```bash export PULSE_USER=... PULSE_PASSWORD=... # https://pulseguardian.mozilla.org export HACKBOT_API_URL=https://hackbot-api.../ HACKBOT_API_KEY=... +export HACKBOT_UI_URL=https://hackbot-ui.../ export WATCHED_REPOS=try export DRY_RUN=true # log intended calls, don't POST uv run --package hackbot-pulse-listener python -m app ``` +Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it +is logged and skipped. Set `NOTIFICATION_OVERRIDE_EMAIL` to route every notification to a +single address (useful for local testing). + ## Test ```bash diff --git a/services/hackbot-pulse-listener/app/__main__.py b/services/hackbot-pulse-listener/app/__main__.py index 6a3b1b5832..80c99d76e7 100644 --- a/services/hackbot-pulse-listener/app/__main__.py +++ b/services/hackbot-pulse-listener/app/__main__.py @@ -1,5 +1,6 @@ import logging import signal +from concurrent.futures import ThreadPoolExecutor from app import consumer from app.config import settings @@ -18,7 +19,8 @@ def main() -> None: logger.warning("PULSE_USER/PULSE_PASSWORD not set; listener will not start") return - consumer_obj = consumer.build_consumer() + executor = ThreadPoolExecutor(max_workers=settings.poll_max_workers) + consumer_obj = consumer.build_consumer(executor) def shutdown(signum, _frame): logger.info("Received signal %s; shutting down", signum) @@ -32,7 +34,10 @@ def shutdown(signum, _frame): ", ".join(consumer.EXCHANGES), sorted(settings.watched_repos_set), ) - consumer_obj.run() + try: + consumer_obj.run() + finally: + executor.shutdown(wait=False) if __name__ == "__main__": diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py index 498660fa08..a83e6ce974 100644 --- a/services/hackbot-pulse-listener/app/client.py +++ b/services/hackbot-pulse-listener/app/client.py @@ -23,3 +23,10 @@ def trigger_run(inputs: dict) -> str | None: resp = httpx.post(url, json=inputs, headers=_headers(), timeout=_TIMEOUT) resp.raise_for_status() return resp.json()["run_id"] + + +def get_run(run_id: str) -> dict: + url = f"{settings.hackbot_api_url}/runs/{run_id}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json() diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py index 102c8fa555..a1cbd61ac1 100644 --- a/services/hackbot-pulse-listener/app/config.py +++ b/services/hackbot-pulse-listener/app/config.py @@ -10,19 +10,31 @@ class Settings(BaseSettings): # hackbot-api hackbot_api_url: str = "" hackbot_api_key: str = "" + hackbot_ui_url: str = "" agent_name: str = "build-repair" # Failure filtering and agent inputs. # ``watched_repos`` is a comma-separated list of Taskcluster ``project`` tags. - watched_repos: str = "try,autoland" + watched_repos: str = "autoland" run_try_push: bool = False model: str | None = None max_turns: int | None = None - # Dedupe (in-memory, by git revision) + # Dedupe (in-memory, by hg revision) dedupe_ttl_seconds: int = 6 * 60 * 60 dedupe_max_size: int = 4096 + # Polling the API for run completion + poll_interval_seconds: int = 60 + run_max_age_minutes: int = 12 * 60 + poll_max_workers: int = 8 + + # Email notifications (SendGrid) + sendgrid_api_key: str | None = None + notification_sender: str | None = None + # Send all notifications to this address instead of the developer (local testing). + notification_override_email: str | None = None + dry_run: bool = False environment: str = "development" sentry_dsn: str | None = None diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py index a468040490..66304cee69 100644 --- a/services/hackbot-pulse-listener/app/consumer.py +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -1,10 +1,11 @@ import logging +from concurrent.futures import Executor from cachetools import TTLCache from kombu import Connection, Exchange, Queue from kombu.mixins import ConsumerMixin -from app import client, lando, taskcluster +from app import client, lando, taskcluster, worker from app.config import settings logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ ) -def process(body: dict) -> str | None: +def process(body: dict, executor: Executor) -> str | None: """Handle one Taskcluster failure message. Returns the triggered run id.""" tags = (body.get("task") or {}).get("tags") or {} @@ -33,6 +34,7 @@ def process(body: dict) -> str | None: task_id = body["status"]["taskId"] task_name = tags.get("label") or task_id + developer_email = tags.get("createdForUser") hg_revision = taskcluster.get_hg_revision(task_id) if not hg_revision: @@ -79,13 +81,17 @@ def process(body: dict) -> str | None: hg_revision, git_commit, ) + if run_id is not None: + executor.submit( + worker.poll_and_notify, run_id, git_commit, project, developer_email + ) return run_id -def make_handler(): +def make_handler(executor: Executor): def on_message(body, message): try: - process(body) + process(body, executor) except Exception: logger.exception("Error handling pulse message") finally: @@ -120,10 +126,10 @@ def get_consumers(self, Consumer, channel): return [Consumer(queues=self.queues, callbacks=[self.on_message])] -def build_consumer() -> BuildFailureConsumer: +def build_consumer(executor: Executor) -> BuildFailureConsumer: connection = Connection( CONNECTION_URL.format(settings.pulse_user, settings.pulse_password) ) return BuildFailureConsumer( - connection, _build_queues(settings.pulse_user), make_handler() + connection, _build_queues(settings.pulse_user), make_handler(executor) ) diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py new file mode 100644 index 0000000000..658d55f19b --- /dev/null +++ b/services/hackbot-pulse-listener/app/notify.py @@ -0,0 +1,75 @@ +import logging + +from app.config import settings + +logger = logging.getLogger(__name__) + + +def send_email( + developer_email: str | None, + revision: str, + repo: str, + run_id: str, + run_doc: dict, +) -> None: + """Email the developer the run outcome. No-op (logged) if not configured.""" + recipient = settings.notification_override_email or developer_email + if not recipient: + logger.info("No developer email for run %s; skipping notification", run_id) + return + if not (settings.sendgrid_api_key and settings.notification_sender): + logger.info("SendGrid not configured; skipping email for run %s", run_id) + return + + import sendgrid + from sendgrid.helpers.mail import Content, From, Mail, Subject, To + + status = run_doc.get("status", "unknown") + subject = f"[build-repair] {status} for {repo}@{revision[:12]}" + + sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) + message = Mail( + From(settings.notification_sender), + To(recipient), + Subject(subject), + Content("text/plain", _build_body(revision, repo, run_id, run_doc)), + ) + response = sg.send(message=message) + logger.info( + "Sent build-repair notification to %s (status %s)", + recipient, + response.status_code, + ) + + +def _build_body(revision: str, repo: str, run_id: str, run_doc: dict) -> str: + status = run_doc.get("status", "unknown") + lines = [ + f"The build-repair agent finished with status: {status}.", + "", + f"Repository: {repo}", + f"Revision: {revision}", + ] + + if settings.hackbot_ui_url: + lines += [ + "", + f"Run details: {settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}", + ] + + summary = run_doc.get("summary") or {} + findings = summary.get("findings") or {} + if findings.get("summary"): + lines += ["", "Summary:", findings["summary"]] + if findings.get("analysis"): + lines += ["", "Analysis:", findings["analysis"]] + if findings.get("local_build_verified") is not None: + lines += ["", f"Local build verified: {findings['local_build_verified']}"] + if findings.get("treeherder_url"): + lines += [f"Try push: {findings['treeherder_url']}"] + + error = run_doc.get("error") or summary.get("error") + if status != "succeeded" and error: + lines += ["", f"Error: {error}"] + + return "\n".join(lines) diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py new file mode 100644 index 0000000000..4f733968b8 --- /dev/null +++ b/services/hackbot-pulse-listener/app/worker.py @@ -0,0 +1,47 @@ +import logging +import time + +from app import client, notify +from app.config import settings + +logger = logging.getLogger(__name__) + +TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} + + +def poll_and_notify( + run_id: str, revision: str, repo: str, developer_email: str | None +) -> None: + """Poll the run until terminal, then notify the developer. + + Runs on a background executor thread; never lets an exception escape. + """ + try: + run_doc = _poll_until_terminal(run_id) + except Exception: + logger.exception("Polling failed for run %s", run_id) + return + + if run_doc is None: + logger.warning( + "Run %s did not finish within %s minutes; giving up", + run_id, + settings.run_max_age_minutes, + ) + return + + try: + notify.send_email(developer_email, revision, repo, run_id, run_doc) + except Exception: + logger.exception("Failed to send notification for run %s", run_id) + + +def _poll_until_terminal(run_id: str) -> dict | None: + deadline = time.monotonic() + settings.run_max_age_minutes * 60 + while True: + run_doc = client.get_run(run_id) + if run_doc.get("status") in TERMINAL_STATUSES: + return run_doc + if time.monotonic() >= deadline: + return None + time.sleep(settings.poll_interval_seconds) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh index ee2c129b37..7307b7a43f 100755 --- a/services/hackbot-pulse-listener/deploy.sh +++ b/services/hackbot-pulse-listener/deploy.sh @@ -14,12 +14,14 @@ # # Secrets (one-time) — store in Secret Manager: # printf '%s' '' | gcloud secrets create pulse-password --data-file=- +# printf '%s' '' | gcloud secrets create sendgrid-api-key --data-file=- # # HACKBOT_API_KEY reuses the existing shared `external-api-key` secret. # # Usage: # PROJECT=my-proj REGION=us-central1 \ # HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ -# PULSE_USER=my-pulse-user \ +# HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ +# PULSE_USER=my-pulse-user NOTIFICATION_SENDER= \ # ./deploy.sh set -euo pipefail @@ -28,14 +30,17 @@ REGION="${REGION:-us-central1}" SERVICE="${SERVICE:-hackbot-pulse-listener}" REPO="${REPO:-hackbot}" HACKBOT_API_URL="${HACKBOT_API_URL:?set HACKBOT_API_URL to the hackbot-api base URL}" +HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL}" PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" -WATCHED_REPOS="${WATCHED_REPOS:-try}" +WATCHED_REPOS="${WATCHED_REPOS:-autoland}" +NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (verified SendGrid sender)}" SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" PULSE_SECRET="${PULSE_SECRET:-pulse-password}" API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" +SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" # Build context is the repo root (the Dockerfile needs the workspace lock files). @@ -47,7 +52,7 @@ gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ --display-name="Hackbot Pulse Listener (Cloud Run runtime)" echo "==> Granting the SA read access to its secrets" -for s in "${PULSE_SECRET}" "${API_KEY_SECRET}"; do +for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do gcloud secrets add-iam-policy-binding "$s" \ --member="serviceAccount:${SA_EMAIL}" \ --role=roles/secretmanager.secretAccessor >/dev/null @@ -64,8 +69,9 @@ gcloud builds submit "${ROOT_DIR}" \ --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") echo "==> Deploying worker pool" -ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL}" +ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" +ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" gcloud run worker-pools deploy "${SERVICE}" \ --image "${IMAGE}" \ @@ -74,6 +80,6 @@ gcloud run worker-pools deploy "${SERVICE}" \ --max-instances 1 \ --service-account "${SA_EMAIL}" \ --set-env-vars "${ENV_VARS}" \ - --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest" + --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" echo "==> Deployed worker pool '${SERVICE}'" diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml index bba65ac4b3..15b114e329 100644 --- a/services/hackbot-pulse-listener/pyproject.toml +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "taskcluster>=97.1,<100.4", "httpx>=0.26.0", "pydantic-settings>=2.1.0", + "sendgrid>=6.12.5", "cachetools>=5.3.0", "sentry-sdk>=2.51.0", ] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index fdb760dee2..5789e7d2a6 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch from app import consumer @@ -17,7 +17,7 @@ def _sample_bodies(): return [m["payload"] for m in data] -def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): +def _build_msg(task_id="ABC", project="autoland", label="build-linux64/opt"): return { "status": {"taskId": task_id}, "runId": 0, @@ -33,66 +33,82 @@ def _build_msg(task_id="ABC", project="try", label="build-linux64/opt"): def test_sample_messages_are_all_tests_and_skipped(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): for body in _sample_bodies(): - assert consumer.process(body) is None + assert consumer.process(body, executor) is None get_rev.assert_not_called() trigger.assert_not_called() + executor.submit.assert_not_called() -def test_build_failure_triggers_run(): +def test_build_failure_triggers_run_and_submits_poll(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): - run_id = consumer.process(_build_msg()) + run_id = consumer.process(_build_msg(), executor) assert run_id == "run-1" trigger.assert_called_once() inputs = trigger.call_args.args[0] assert inputs["git_commit"] == "deadbeef" assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} + executor.submit.assert_called_once() + assert executor.submit.call_args.args == ( + consumer.worker.poll_and_notify, + "run-1", + "deadbeef", + "autoland", + "dev@mozilla.com", + ) def test_same_revision_triggers_once(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, ): - consumer.process(_build_msg(task_id="T1")) - consumer.process(_build_msg(task_id="T2")) + consumer.process(_build_msg(task_id="T1"), executor) + consumer.process(_build_msg(task_id="T2"), executor) trigger.assert_called_once() def test_unwatched_project_skipped_before_api_call(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): - assert consumer.process(_build_msg(project="mozilla-central")) is None + assert consumer.process(_build_msg(project="mozilla-central"), executor) is None get_rev.assert_not_called() trigger.assert_not_called() def test_unmappable_revision_skipped(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), patch.object(consumer.lando, "hg_to_git", return_value=None), patch.object(consumer.client, "trigger_run") as trigger, ): - assert consumer.process(_build_msg()) is None + assert consumer.process(_build_msg(), executor) is None trigger.assert_not_called() + executor.submit.assert_not_called() def test_trigger_failure_releases_revision_for_retry(): + executor = MagicMock() with ( patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), @@ -100,8 +116,8 @@ def test_trigger_failure_releases_revision_for_retry(): consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] ) as trigger, ): - assert consumer.process(_build_msg(task_id="T1")) is None + assert consumer.process(_build_msg(task_id="T1"), executor) is None # Same revision can be retried because the failed claim was released. - assert consumer.process(_build_msg(task_id="T2")) == "run-2" + assert consumer.process(_build_msg(task_id="T2"), executor) == "run-2" assert trigger.call_count == 2 diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py new file mode 100644 index 0000000000..5b73c73c6b --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -0,0 +1,77 @@ +from unittest.mock import MagicMock, patch + +from app import notify + + +def test_skips_without_developer_email(): + # Must not raise even with no SendGrid config. + notify.send_email(None, "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_skips_without_sendgrid_config(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) + monkeypatch.setattr(notify.settings, "notification_sender", None) + notify.send_email("dev@mozilla.com", "rev", "try", "run-1", {"status": "succeeded"}) + + +def test_body_contains_ui_link_and_summary(monkeypatch): + monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") + body = notify._build_body( + "deadbeefcafe1234", + "try", + "run-1", + { + "status": "succeeded", + "summary": { + "findings": { + "summary": "Fixed a missing include", + "analysis": "The commit removed a needed header", + "local_build_verified": True, + } + }, + }, + ) + assert "https://ui.example/runs/run-1" in body + assert "Fixed a missing include" in body + assert "The commit removed a needed header" in body + assert "Local build verified: True" in body + + +def test_failure_body_includes_error(): + body = notify._build_body( + "deadbeef", "try", "run-1", {"status": "failed", "error": "build still broken"} + ) + assert "build still broken" in body + + +def test_sends_email_when_configured(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email( + "dev@mozilla.com", + "rev", + "try", + "run-1", + {"status": "succeeded", "summary": {}}, + ) + + fake_client.send.assert_called_once() + + +def test_override_sends_even_without_developer_email(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr( + notify.settings, "notification_override_email", "me@mozilla.com" + ) + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email( + None, "rev", "try", "run-1", {"status": "succeeded", "summary": {}} + ) diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py new file mode 100644 index 0000000000..1ef39ae448 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_worker.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +from app import worker + + +def test_terminal_run_notifies_once(): + run_doc = {"status": "succeeded", "summary": {}} + with ( + patch.object(worker.client, "get_run", return_value=run_doc) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_called_once() + args = notify.send_email.call_args.args + assert args == ("dev@mozilla.com", "rev", "try", "run-1", run_doc) + + +def test_gives_up_after_max_age(monkeypatch): + monkeypatch.setattr(worker.settings, "run_max_age_minutes", 0) + with ( + patch.object( + worker.client, "get_run", return_value={"status": "running"} + ) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + + get_run.assert_called_once() + notify.send_email.assert_not_called() diff --git a/uv.lock b/uv.lock index 92e61ba5ac..63ef8e9109 100644 --- a/uv.lock +++ b/uv.lock @@ -2594,6 +2594,7 @@ dependencies = [ { name = "httpx" }, { name = "kombu" }, { name = "pydantic-settings" }, + { name = "sendgrid" }, { name = "sentry-sdk" }, { name = "taskcluster" }, ] @@ -2610,6 +2611,7 @@ requires-dist = [ { name = "kombu", specifier = ">=5.6,<6" }, { name = "pydantic-settings", specifier = ">=2.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "sendgrid", specifier = ">=6.12.5" }, { name = "sentry-sdk", specifier = ">=2.51.0" }, { name = "taskcluster", specifier = ">=97.1,<100.4" }, ] From 53497a7a8e92df44585e5b16d579423210377711 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 8 Jul 2026 11:51:45 -0700 Subject: [PATCH 06/12] Improve email --- services/hackbot-pulse-listener/README.md | 2 +- services/hackbot-pulse-listener/app/client.py | 12 ++ services/hackbot-pulse-listener/app/config.py | 2 + services/hackbot-pulse-listener/app/notify.py | 145 ++++++++++++++---- services/hackbot-pulse-listener/deploy.sh | 3 + .../hackbot-pulse-listener/pyproject.toml | 1 + .../tests/test_consumer.py | 2 +- .../tests/test_notify.py | 82 +++++++++- uv.lock | 2 + 9 files changed, 212 insertions(+), 39 deletions(-) diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md index b7cb139cf7..537acf08e4 100644 --- a/services/hackbot-pulse-listener/README.md +++ b/services/hackbot-pulse-listener/README.md @@ -31,7 +31,7 @@ uv run --package hackbot-pulse-listener python -m app ``` Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it -is logged and skipped. Set `NOTIFICATION_OVERRIDE_EMAIL` to route every notification to a +is logged and skipped. Notifications go to the revision author and, if set, the `NOTIFICATION_TEAM_EMAIL` team address. Set `NOTIFICATION_OVERRIDE_EMAIL` to route every notification to a single address (useful for local testing). ## Test diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py index a83e6ce974..a4f98ba518 100644 --- a/services/hackbot-pulse-listener/app/client.py +++ b/services/hackbot-pulse-listener/app/client.py @@ -30,3 +30,15 @@ def get_run(run_id: str) -> dict: resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) resp.raise_for_status() return resp.json() + + +def get_artifact(run_id: str, name: str) -> str | None: + """Download a run artifact's text content, or None if it is missing.""" + url = f"{settings.hackbot_api_url}/runs/{run_id}/artifacts/{name}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + if resp.status_code == 404: + return None + resp.raise_for_status() + download = httpx.get(resp.json()["url"], timeout=_TIMEOUT) + download.raise_for_status() + return download.text diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py index a1cbd61ac1..6406cd6403 100644 --- a/services/hackbot-pulse-listener/app/config.py +++ b/services/hackbot-pulse-listener/app/config.py @@ -32,6 +32,8 @@ class Settings(BaseSettings): # Email notifications (SendGrid) sendgrid_api_key: str | None = None notification_sender: str | None = None + # Team address CC'd on every notification alongside the revision author. + notification_team_email: str | None = None # Send all notifications to this address instead of the developer (local testing). notification_override_email: str | None = None diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py index 658d55f19b..74629aaa7e 100644 --- a/services/hackbot-pulse-listener/app/notify.py +++ b/services/hackbot-pulse-listener/app/notify.py @@ -1,9 +1,14 @@ import logging +import re +from app import client from app.config import settings logger = logging.getLogger(__name__) +PATCH_ARTIFACT = "changes/changes.patch" +MAX_PATCH_LINES = 400 + def send_email( developer_email: str | None, @@ -12,64 +17,144 @@ def send_email( run_id: str, run_doc: dict, ) -> None: - """Email the developer the run outcome. No-op (logged) if not configured.""" - recipient = settings.notification_override_email or developer_email - if not recipient: - logger.info("No developer email for run %s; skipping notification", run_id) + """Email the developer the fix. Only succeeded runs are notified.""" + if run_doc.get("status") != "succeeded": + logger.info("Run %s did not succeed; skipping notification", run_id) + return + + recipients = _recipients(developer_email) + if not recipients: + logger.info("No recipients for run %s; skipping notification", run_id) return if not (settings.sendgrid_api_key and settings.notification_sender): logger.info("SendGrid not configured; skipping email for run %s", run_id) return + import markdown2 import sendgrid - from sendgrid.helpers.mail import Content, From, Mail, Subject, To + from sendgrid.helpers.mail import ( + Cc, + Content, + From, + HtmlContent, + Mail, + Subject, + To, + ) - status = run_doc.get("status", "unknown") - subject = f"[build-repair] {status} for {repo}@{revision[:12]}" + subject = f"[build-repair] fix for {repo}@{revision[:12]}" + + patch = _fetch_patch(run_id, run_doc) + body_md = _build_body(revision, repo, run_id, run_doc, patch) + html = markdown2.markdown(body_md, extras=["fenced-code-blocks", "tables"]) sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) + to_emails = [To(recipients[0])] + [Cc(addr) for addr in recipients[1:]] message = Mail( From(settings.notification_sender), - To(recipient), + to_emails, Subject(subject), - Content("text/plain", _build_body(revision, repo, run_id, run_doc)), + Content("text/plain", body_md), + HtmlContent(html), ) response = sg.send(message=message) logger.info( "Sent build-repair notification to %s (status %s)", - recipient, + ", ".join(recipients), response.status_code, ) -def _build_body(revision: str, repo: str, run_id: str, run_doc: dict) -> str: - status = run_doc.get("status", "unknown") +def _recipients(developer_email: str | None) -> list[str]: + """Recipients for a run: the revision author plus the team address, deduped. + + ``notification_override_email`` short-circuits to a single address so local + testing never mails real developers or the team. + """ + if settings.notification_override_email: + return [settings.notification_override_email] + recipients: list[str] = [] + for addr in (developer_email, settings.notification_team_email): + if addr and addr not in recipients: + recipients.append(addr) + return recipients + + +def _fetch_patch(run_id: str, run_doc: dict) -> str | None: + """Download the proposed-fix patch artifact, if the run produced one.""" + artifacts = run_doc.get("artifacts") or [] + if not any(a.get("name") == PATCH_ARTIFACT for a in artifacts): + return None + try: + return client.get_artifact(run_id, PATCH_ARTIFACT) + except Exception: + logger.exception("Failed to fetch patch for run %s", run_id) + return None + + +def _build_body( + revision: str, repo: str, run_id: str, run_doc: dict, patch: str | None = None +) -> str: + summary = run_doc.get("summary") or {} + findings = summary.get("findings") or {} + lines = [ - f"The build-repair agent finished with status: {status}.", + "# Build-repair fix ready", "", - f"Repository: {repo}", - f"Revision: {revision}", + f"- **Repository:** {repo}", + f"- **Revision:** `{revision}`", ] - if settings.hackbot_ui_url: - lines += [ - "", - f"Run details: {settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}", - ] + run_url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}" + lines.append(f"- **Run details:** {run_url}") - summary = run_doc.get("summary") or {} - findings = summary.get("findings") or {} if findings.get("summary"): - lines += ["", "Summary:", findings["summary"]] + lines += ["", "## Summary", "", _demote_headings(findings["summary"])] if findings.get("analysis"): - lines += ["", "Analysis:", findings["analysis"]] + lines += ["", "## Analysis", "", _demote_headings(findings["analysis"])] + if findings.get("local_build_verified") is not None: - lines += ["", f"Local build verified: {findings['local_build_verified']}"] - if findings.get("treeherder_url"): - lines += [f"Try push: {findings['treeherder_url']}"] + lines += [ + "", + "## Verification", + "", + f"- Local build verified: {findings['local_build_verified']}", + ] - error = run_doc.get("error") or summary.get("error") - if status != "succeeded" and error: - lines += ["", f"Error: {error}"] + if patch: + lines += ["", "## Proposed patch", "", _patch_block(patch)] return "\n".join(lines) + + +def _demote_headings(md: str, by: int = 2) -> str: + """Shift ATX headings down ``by`` levels so agent docs nest under our own. + + Lines inside code fences (and ``#include`` and the like, which lack the + required space after ``#``) are left untouched. + """ + out = [] + in_fence = False + for line in md.splitlines(): + if line.lstrip().startswith(("```", "~~~")): + in_fence = not in_fence + out.append(line) + continue + match = re.match(r"(#{1,6}) ", line) if not in_fence else None + if match: + level = min(len(match.group(1)) + by, 6) + line = "#" * level + line[len(match.group(1)) :] + out.append(line) + return "\n".join(out) + + +def _patch_block(patch: str) -> str: + patch_lines = patch.splitlines() + shown = patch_lines[:MAX_PATCH_LINES] + block = ["```diff", *shown, "```"] + if len(patch_lines) > MAX_PATCH_LINES: + block.append( + f"\n_Patch truncated to {MAX_PATCH_LINES} lines; " + "see run details for the full diff._" + ) + return "\n".join(block) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh index 7307b7a43f..7bfc134f06 100755 --- a/services/hackbot-pulse-listener/deploy.sh +++ b/services/hackbot-pulse-listener/deploy.sh @@ -22,6 +22,7 @@ # HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ # HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ # PULSE_USER=my-pulse-user NOTIFICATION_SENDER= \ +# NOTIFICATION_TEAM_EMAIL=hackbot-developers@mozilla.com \ # ./deploy.sh set -euo pipefail @@ -34,6 +35,7 @@ HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL} PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" WATCHED_REPOS="${WATCHED_REPOS:-autoland}" NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (verified SendGrid sender)}" +NOTIFICATION_TEAM_EMAIL="${NOTIFICATION_TEAM_EMAIL:-}" SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" @@ -72,6 +74,7 @@ echo "==> Deploying worker pool" ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" +ENV_VARS="${ENV_VARS},NOTIFICATION_TEAM_EMAIL=${NOTIFICATION_TEAM_EMAIL}" gcloud run worker-pools deploy "${SERVICE}" \ --image "${IMAGE}" \ diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml index 15b114e329..7940bc4940 100644 --- a/services/hackbot-pulse-listener/pyproject.toml +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "httpx>=0.26.0", "pydantic-settings>=2.1.0", "sendgrid>=6.12.5", + "markdown2>=2.4.0", "cachetools>=5.3.0", "sentry-sdk>=2.51.0", ] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index 5789e7d2a6..7d0351ba24 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -88,7 +88,7 @@ def test_unwatched_project_skipped_before_api_call(): patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, patch.object(consumer.client, "trigger_run") as trigger, ): - assert consumer.process(_build_msg(project="mozilla-central"), executor) is None + assert consumer.process(_build_msg(project="try"), executor) is None get_rev.assert_not_called() trigger.assert_not_called() diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py index 5b73c73c6b..476dbc0be4 100644 --- a/services/hackbot-pulse-listener/tests/test_notify.py +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -37,13 +37,6 @@ def test_body_contains_ui_link_and_summary(monkeypatch): assert "Local build verified: True" in body -def test_failure_body_includes_error(): - body = notify._build_body( - "deadbeef", "try", "run-1", {"status": "failed", "error": "build still broken"} - ) - assert "build still broken" in body - - def test_sends_email_when_configured(monkeypatch): monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") @@ -75,3 +68,78 @@ def test_override_sends_even_without_developer_email(monkeypatch): notify.send_email( None, "rev", "try", "run-1", {"status": "succeeded", "summary": {}} ) + + +def test_body_includes_patch(): + body = notify._build_body( + "deadbeef", + "autoland", + "run-1", + {"status": "succeeded", "summary": {}}, + patch="--- a/f\n+++ b/f\n@@ -1 +1 @@\n-old\n+new\n", + ) + assert "## Proposed patch" in body + assert "```diff" in body + assert "+new" in body + + +def test_fetch_patch_returns_none_without_artifact(): + assert notify._fetch_patch("run-1", {"artifacts": []}) is None + + +def test_fetch_patch_downloads_listed_artifact(): + run_doc = {"artifacts": [{"name": notify.PATCH_ARTIFACT}]} + with patch.object(notify.client, "get_artifact", return_value="THE PATCH") as ga: + assert notify._fetch_patch("run-1", run_doc) == "THE PATCH" + ga.assert_called_once_with("run-1", notify.PATCH_ARTIFACT) + + +def test_skips_when_not_succeeded(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + with patch("sendgrid.SendGridAPIClient") as sg: + notify.send_email( + "dev@mozilla.com", "rev", "autoland", "run-1", {"status": "failed"} + ) + sg.assert_not_called() + + +def test_analysis_headings_demoted_under_section(): + run_doc = { + "status": "succeeded", + "summary": {"findings": {"analysis": "# Root cause\n\n## Details\ntext"}}, + } + body = notify._build_body("rev", "autoland", "run-1", run_doc) + assert "## Analysis" in body + assert "### Root cause" in body + assert "#### Details" in body + + +def test_demote_headings_leaves_code_fences_and_includes_alone(): + md = "```cpp\n#include \n```" + assert notify._demote_headings(md) == md + + +def test_recipients_author_and_team(monkeypatch): + monkeypatch.setattr(notify.settings, "notification_override_email", None) + monkeypatch.setattr(notify.settings, "notification_team_email", "team@mozilla.com") + assert notify._recipients("dev@mozilla.com") == [ + "dev@mozilla.com", + "team@mozilla.com", + ] + + +def test_recipients_override_wins(monkeypatch): + monkeypatch.setattr( + notify.settings, "notification_override_email", "me@mozilla.com" + ) + monkeypatch.setattr(notify.settings, "notification_team_email", "team@mozilla.com") + assert notify._recipients("dev@mozilla.com") == ["me@mozilla.com"] + + +def test_recipients_dedupes_and_skips_empty(monkeypatch): + monkeypatch.setattr(notify.settings, "notification_override_email", None) + monkeypatch.setattr(notify.settings, "notification_team_email", "dev@mozilla.com") + assert notify._recipients("dev@mozilla.com") == ["dev@mozilla.com"] + monkeypatch.setattr(notify.settings, "notification_team_email", None) + assert notify._recipients(None) == [] diff --git a/uv.lock b/uv.lock index 63ef8e9109..2ce6678e7e 100644 --- a/uv.lock +++ b/uv.lock @@ -2593,6 +2593,7 @@ dependencies = [ { name = "cachetools" }, { name = "httpx" }, { name = "kombu" }, + { name = "markdown2" }, { name = "pydantic-settings" }, { name = "sendgrid" }, { name = "sentry-sdk" }, @@ -2609,6 +2610,7 @@ requires-dist = [ { name = "cachetools", specifier = ">=5.3.0" }, { name = "httpx", specifier = ">=0.26.0" }, { name = "kombu", specifier = ">=5.6,<6" }, + { name = "markdown2", specifier = ">=2.4.0" }, { name = "pydantic-settings", specifier = ">=2.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "sendgrid", specifier = ">=6.12.5" }, From e700bb7677d941147ed72dea0e49bdea5ad24086 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 8 Jul 2026 12:38:20 -0700 Subject: [PATCH 07/12] Add repo links --- services/hackbot-pulse-listener/app/config.py | 5 + .../hackbot-pulse-listener/app/consumer.py | 11 +- services/hackbot-pulse-listener/app/models.py | 13 ++ services/hackbot-pulse-listener/app/notify.py | 58 +++++--- services/hackbot-pulse-listener/app/worker.py | 17 ++- .../tests/test_consumer.py | 15 ++- .../tests/test_notify.py | 126 ++++++++++-------- .../tests/test_worker.py | 18 ++- 8 files changed, 166 insertions(+), 97 deletions(-) create mode 100644 services/hackbot-pulse-listener/app/models.py diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py index 6406cd6403..a25c8ed193 100644 --- a/services/hackbot-pulse-listener/app/config.py +++ b/services/hackbot-pulse-listener/app/config.py @@ -13,6 +13,11 @@ class Settings(BaseSettings): hackbot_ui_url: str = "" agent_name: str = "build-repair" + # Source links shown in notifications. + firefox_git_url: str = "https://github.com/mozilla-firefox/firefox" + firefox_hg_url: str = "https://hg.mozilla.org/mozilla-unified" + bugzilla_url: str = "https://bugzilla.mozilla.org" + # Failure filtering and agent inputs. # ``watched_repos`` is a comma-separated list of Taskcluster ``project`` tags. watched_repos: str = "autoland" diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py index 66304cee69..0280cb6135 100644 --- a/services/hackbot-pulse-listener/app/consumer.py +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -7,6 +7,7 @@ from app import client, lando, taskcluster, worker from app.config import settings +from app.models import RunContext logger = logging.getLogger(__name__) @@ -82,9 +83,15 @@ def process(body: dict, executor: Executor) -> str | None: git_commit, ) if run_id is not None: - executor.submit( - worker.poll_and_notify, run_id, git_commit, project, developer_email + ctx = RunContext( + run_id=run_id, + repo=project, + git_commit=git_commit, + hg_revision=hg_revision, + task_id=task_id, + developer_email=developer_email, ) + executor.submit(worker.poll_and_notify, ctx) return run_id diff --git a/services/hackbot-pulse-listener/app/models.py b/services/hackbot-pulse-listener/app/models.py new file mode 100644 index 0000000000..2bcaf33cfa --- /dev/null +++ b/services/hackbot-pulse-listener/app/models.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + + +@dataclass +class RunContext: + """What the notifier needs about a triggered build-repair run.""" + + run_id: str + repo: str + git_commit: str + hg_revision: str + task_id: str + developer_email: str | None diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py index 74629aaa7e..b2cfe587fe 100644 --- a/services/hackbot-pulse-listener/app/notify.py +++ b/services/hackbot-pulse-listener/app/notify.py @@ -3,6 +3,7 @@ from app import client from app.config import settings +from app.models import RunContext logger = logging.getLogger(__name__) @@ -10,24 +11,18 @@ MAX_PATCH_LINES = 400 -def send_email( - developer_email: str | None, - revision: str, - repo: str, - run_id: str, - run_doc: dict, -) -> None: - """Email the developer the fix. Only succeeded runs are notified.""" +def send_email(ctx: RunContext, run_doc: dict) -> None: + """Email the developer and team the fix. Only succeeded runs are notified.""" if run_doc.get("status") != "succeeded": - logger.info("Run %s did not succeed; skipping notification", run_id) + logger.info("Run %s did not succeed; skipping notification", ctx.run_id) return - recipients = _recipients(developer_email) + recipients = _recipients(ctx.developer_email) if not recipients: - logger.info("No recipients for run %s; skipping notification", run_id) + logger.info("No recipients for run %s; skipping notification", ctx.run_id) return if not (settings.sendgrid_api_key and settings.notification_sender): - logger.info("SendGrid not configured; skipping email for run %s", run_id) + logger.info("SendGrid not configured; skipping email for run %s", ctx.run_id) return import markdown2 @@ -42,10 +37,10 @@ def send_email( To, ) - subject = f"[build-repair] fix for {repo}@{revision[:12]}" + subject = f"[build-repair] fix for {ctx.repo}@{ctx.git_commit[:12]}" - patch = _fetch_patch(run_id, run_doc) - body_md = _build_body(revision, repo, run_id, run_doc, patch) + patch = _fetch_patch(ctx.run_id, run_doc) + body_md = _build_body(ctx, run_doc, patch) html = markdown2.markdown(body_md, extras=["fenced-code-blocks", "tables"]) sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) @@ -92,20 +87,41 @@ def _fetch_patch(run_id: str, run_doc: dict) -> str | None: return None -def _build_body( - revision: str, repo: str, run_id: str, run_doc: dict, patch: str | None = None -) -> str: +def _git_url(git_commit: str) -> str: + return f"{settings.firefox_git_url.rstrip('/')}/commit/{git_commit}" + + +def _hg_url(hg_revision: str) -> str: + return f"{settings.firefox_hg_url.rstrip('/')}/rev/{hg_revision}" + + +def _task_url(task_id: str) -> str: + return f"{settings.taskcluster_root_url.rstrip('/')}/tasks/{task_id}" + + +def _bug_url(bug_id: object) -> str: + return f"{settings.bugzilla_url.rstrip('/')}/show_bug.cgi?id={bug_id}" + + +def _build_body(ctx: RunContext, run_doc: dict, patch: str | None = None) -> str: summary = run_doc.get("summary") or {} findings = summary.get("findings") or {} lines = [ "# Build-repair fix ready", "", - f"- **Repository:** {repo}", - f"- **Revision:** `{revision}`", + f"- **Repository:** {ctx.repo}", + f"- **Revision (git):** [`{ctx.git_commit[:12]}`]({_git_url(ctx.git_commit)})", + f"- **Revision (hg):** [`{ctx.hg_revision[:12]}`]({_hg_url(ctx.hg_revision)})", + f"- **Failed task:** [`{ctx.task_id}`]({_task_url(ctx.task_id)})", ] + + bug_id = findings.get("bug_id") or (run_doc.get("inputs") or {}).get("bug_id") + if bug_id: + lines.append(f"- **Bug:** [{bug_id}]({_bug_url(bug_id)})") + if settings.hackbot_ui_url: - run_url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{run_id}" + run_url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{ctx.run_id}" lines.append(f"- **Run details:** {run_url}") if findings.get("summary"): diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py index 4f733968b8..188ea81376 100644 --- a/services/hackbot-pulse-listener/app/worker.py +++ b/services/hackbot-pulse-listener/app/worker.py @@ -3,37 +3,36 @@ from app import client, notify from app.config import settings +from app.models import RunContext logger = logging.getLogger(__name__) TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} -def poll_and_notify( - run_id: str, revision: str, repo: str, developer_email: str | None -) -> None: - """Poll the run until terminal, then notify the developer. +def poll_and_notify(ctx: RunContext) -> None: + """Poll the run until terminal, then notify. Runs on a background executor thread; never lets an exception escape. """ try: - run_doc = _poll_until_terminal(run_id) + run_doc = _poll_until_terminal(ctx.run_id) except Exception: - logger.exception("Polling failed for run %s", run_id) + logger.exception("Polling failed for run %s", ctx.run_id) return if run_doc is None: logger.warning( "Run %s did not finish within %s minutes; giving up", - run_id, + ctx.run_id, settings.run_max_age_minutes, ) return try: - notify.send_email(developer_email, revision, repo, run_id, run_doc) + notify.send_email(ctx, run_doc) except Exception: - logger.exception("Failed to send notification for run %s", run_id) + logger.exception("Failed to send notification for run %s", ctx.run_id) def _poll_until_terminal(run_id: str) -> dict | None: diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index 7d0351ba24..5fdecf37a2 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -60,13 +60,14 @@ def test_build_failure_triggers_run_and_submits_poll(): assert inputs["git_commit"] == "deadbeef" assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} executor.submit.assert_called_once() - assert executor.submit.call_args.args == ( - consumer.worker.poll_and_notify, - "run-1", - "deadbeef", - "autoland", - "dev@mozilla.com", - ) + fn, ctx = executor.submit.call_args.args + assert fn is consumer.worker.poll_and_notify + assert ctx.run_id == "run-1" + assert ctx.git_commit == "deadbeef" + assert ctx.hg_revision == "hgrev" + assert ctx.task_id == "ABC" + assert ctx.repo == "autoland" + assert ctx.developer_email == "dev@mozilla.com" def test_same_revision_triggers_once(): diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py index 476dbc0be4..cfac423b0f 100644 --- a/services/hackbot-pulse-listener/tests/test_notify.py +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -1,25 +1,61 @@ from unittest.mock import MagicMock, patch from app import notify +from app.models import RunContext -def test_skips_without_developer_email(): - # Must not raise even with no SendGrid config. - notify.send_email(None, "rev", "try", "run-1", {"status": "succeeded"}) +def _ctx(**over): + base = dict( + run_id="run-1", + repo="autoland", + git_commit="deadbeefcafe", + hg_revision="0123456789ab", + task_id="TASK123", + developer_email="dev@mozilla.com", + ) + base.update(over) + return RunContext(**base) + + +def test_skips_without_recipient(): + # No developer, no team, no override -> nothing to send, must not raise. + notify.send_email(_ctx(developer_email=None), {"status": "succeeded"}) def test_skips_without_sendgrid_config(monkeypatch): monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) monkeypatch.setattr(notify.settings, "notification_sender", None) - notify.send_email("dev@mozilla.com", "rev", "try", "run-1", {"status": "succeeded"}) + notify.send_email(_ctx(), {"status": "succeeded"}) + + +def test_skips_when_not_succeeded(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + with patch("sendgrid.SendGridAPIClient") as sg: + notify.send_email(_ctx(), {"status": "failed"}) + sg.assert_not_called() + + +def test_body_contains_source_links(): + body = notify._build_body(_ctx(), {"status": "succeeded", "summary": {}}) + assert "https://github.com/mozilla-firefox/firefox/commit/deadbeefcafe" in body + assert "https://hg.mozilla.org/mozilla-unified/rev/0123456789ab" in body + assert "https://firefox-ci-tc.services.mozilla.com/tasks/TASK123" in body + + +def test_body_contains_bug_link_when_present(): + run_doc = {"status": "succeeded", "summary": {"findings": {"bug_id": 12345}}} + body = notify._build_body(_ctx(), run_doc) + assert "https://bugzilla.mozilla.org/show_bug.cgi?id=12345" in body + + no_bug = notify._build_body(_ctx(), {"status": "succeeded", "summary": {}}) + assert "show_bug.cgi" not in no_bug def test_body_contains_ui_link_and_summary(monkeypatch): monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") body = notify._build_body( - "deadbeefcafe1234", - "try", - "run-1", + _ctx(), { "status": "succeeded", "summary": { @@ -37,6 +73,33 @@ def test_body_contains_ui_link_and_summary(monkeypatch): assert "Local build verified: True" in body +def test_body_includes_patch(): + body = notify._build_body( + _ctx(), + {"status": "succeeded", "summary": {}}, + patch="--- a/f\n+++ b/f\n@@ -1 +1 @@\n-old\n+new\n", + ) + assert "## Proposed patch" in body + assert "```diff" in body + assert "+new" in body + + +def test_analysis_headings_demoted_under_section(): + run_doc = { + "status": "succeeded", + "summary": {"findings": {"analysis": "# Root cause\n\n## Details\ntext"}}, + } + body = notify._build_body(_ctx(), run_doc) + assert "## Analysis" in body + assert "### Root cause" in body + assert "#### Details" in body + + +def test_demote_headings_leaves_code_fences_and_includes_alone(): + md = "```cpp\n#include \n```" + assert notify._demote_headings(md) == md + + def test_sends_email_when_configured(monkeypatch): monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") @@ -44,13 +107,7 @@ def test_sends_email_when_configured(monkeypatch): fake_client = MagicMock() fake_client.send.return_value = MagicMock(status_code=202) with patch("sendgrid.SendGridAPIClient", return_value=fake_client): - notify.send_email( - "dev@mozilla.com", - "rev", - "try", - "run-1", - {"status": "succeeded", "summary": {}}, - ) + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) fake_client.send.assert_called_once() @@ -66,21 +123,10 @@ def test_override_sends_even_without_developer_email(monkeypatch): fake_client.send.return_value = MagicMock(status_code=202) with patch("sendgrid.SendGridAPIClient", return_value=fake_client): notify.send_email( - None, "rev", "try", "run-1", {"status": "succeeded", "summary": {}} + _ctx(developer_email=None), {"status": "succeeded", "summary": {}} ) - -def test_body_includes_patch(): - body = notify._build_body( - "deadbeef", - "autoland", - "run-1", - {"status": "succeeded", "summary": {}}, - patch="--- a/f\n+++ b/f\n@@ -1 +1 @@\n-old\n+new\n", - ) - assert "## Proposed patch" in body - assert "```diff" in body - assert "+new" in body + fake_client.send.assert_called_once() def test_fetch_patch_returns_none_without_artifact(): @@ -94,32 +140,6 @@ def test_fetch_patch_downloads_listed_artifact(): ga.assert_called_once_with("run-1", notify.PATCH_ARTIFACT) -def test_skips_when_not_succeeded(monkeypatch): - monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") - monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") - with patch("sendgrid.SendGridAPIClient") as sg: - notify.send_email( - "dev@mozilla.com", "rev", "autoland", "run-1", {"status": "failed"} - ) - sg.assert_not_called() - - -def test_analysis_headings_demoted_under_section(): - run_doc = { - "status": "succeeded", - "summary": {"findings": {"analysis": "# Root cause\n\n## Details\ntext"}}, - } - body = notify._build_body("rev", "autoland", "run-1", run_doc) - assert "## Analysis" in body - assert "### Root cause" in body - assert "#### Details" in body - - -def test_demote_headings_leaves_code_fences_and_includes_alone(): - md = "```cpp\n#include \n```" - assert notify._demote_headings(md) == md - - def test_recipients_author_and_team(monkeypatch): monkeypatch.setattr(notify.settings, "notification_override_email", None) monkeypatch.setattr(notify.settings, "notification_team_email", "team@mozilla.com") diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py index 1ef39ae448..d1533be3d6 100644 --- a/services/hackbot-pulse-listener/tests/test_worker.py +++ b/services/hackbot-pulse-listener/tests/test_worker.py @@ -1,6 +1,16 @@ from unittest.mock import patch from app import worker +from app.models import RunContext + +CTX = RunContext( + run_id="run-1", + repo="autoland", + git_commit="deadbeef", + hg_revision="hg123", + task_id="T1", + developer_email="dev@mozilla.com", +) def test_terminal_run_notifies_once(): @@ -9,12 +19,10 @@ def test_terminal_run_notifies_once(): patch.object(worker.client, "get_run", return_value=run_doc) as get_run, patch.object(worker, "notify") as notify, ): - worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + worker.poll_and_notify(CTX) get_run.assert_called_once() - notify.send_email.assert_called_once() - args = notify.send_email.call_args.args - assert args == ("dev@mozilla.com", "rev", "try", "run-1", run_doc) + notify.send_email.assert_called_once_with(CTX, run_doc) def test_gives_up_after_max_age(monkeypatch): @@ -25,7 +33,7 @@ def test_gives_up_after_max_age(monkeypatch): ) as get_run, patch.object(worker, "notify") as notify, ): - worker.poll_and_notify("run-1", "rev", "try", "dev@mozilla.com") + worker.poll_and_notify(CTX) get_run.assert_called_once() notify.send_email.assert_not_called() From ea268dd060c7e6bcb301d1aeec0489647e49a491 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 8 Jul 2026 13:25:34 -0700 Subject: [PATCH 08/12] Fix deployment --- services/hackbot-pulse-listener/deploy.sh | 43 ++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh index 7bfc134f06..5f52c373b6 100755 --- a/services/hackbot-pulse-listener/deploy.sh +++ b/services/hackbot-pulse-listener/deploy.sh @@ -9,15 +9,20 @@ # Prereqs (one-time): # gcloud auth login # gcloud config set project +# gcloud components install beta # worker pools live under the beta track # gcloud services enable run.googleapis.com artifactregistry.googleapis.com \ # cloudbuild.googleapis.com secretmanager.googleapis.com # -# Secrets (one-time) — store in Secret Manager: -# printf '%s' '' | gcloud secrets create pulse-password --data-file=- -# printf '%s' '' | gcloud secrets create sendgrid-api-key --data-file=- -# # HACKBOT_API_KEY reuses the existing shared `external-api-key` secret. +# Secrets — the values use the same env var names as the app's .env, so +# `source .env` populates them. Any secret missing from Secret Manager is +# created from its value; existing secrets are never overwritten (rotate with +# `gcloud secrets versions add`): +# PULSE_PASSWORD -> secret `pulse-password` +# SENDGRID_API_KEY -> secret `sendgrid-api-key` +# HACKBOT_API_KEY -> secret `external-api-key` (shared with hackbot-api) # # Usage: +# source .env # provides PULSE_PASSWORD, HACKBOT_API_KEY, SENDGRID_API_KEY, etc. # PROJECT=my-proj REGION=us-central1 \ # HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ # HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ @@ -40,10 +45,17 @@ NOTIFICATION_TEAM_EMAIL="${NOTIFICATION_TEAM_EMAIL:-}" SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" +# Secret Manager secret names (where the values live). PULSE_SECRET="${PULSE_SECRET:-pulse-password}" API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" +# Secret values, using the same names as the app's .env so `source .env` works. +# Used only to seed a secret that does not exist yet (never overwrites). +PULSE_PASSWORD="${PULSE_PASSWORD:-}" +HACKBOT_API_KEY="${HACKBOT_API_KEY:-}" +SENDGRID_API_KEY="${SENDGRID_API_KEY:-}" + IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" # Build context is the repo root (the Dockerfile needs the workspace lock files). ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -53,6 +65,22 @@ gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ gcloud iam service-accounts create "${SA_NAME}" \ --display-name="Hackbot Pulse Listener (Cloud Run runtime)" +echo "==> Ensuring secrets exist (seeding from env when missing)" +ensure_secret() { # secret_name value + local name="$1" value="${2:-}" + if gcloud secrets describe "$name" >/dev/null 2>&1; then + return 0 + fi + if [ -z "$value" ]; then + echo "ERROR: secret '$name' is missing and no value was provided to create it" >&2 + exit 1 + fi + printf '%s' "$value" | gcloud secrets create "$name" --data-file=- +} +ensure_secret "${PULSE_SECRET}" "${PULSE_PASSWORD}" +ensure_secret "${API_KEY_SECRET}" "${HACKBOT_API_KEY}" +ensure_secret "${SENDGRID_SECRET}" "${SENDGRID_API_KEY}" + echo "==> Granting the SA read access to its secrets" for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do gcloud secrets add-iam-policy-binding "$s" \ @@ -68,7 +96,7 @@ gcloud artifacts repositories describe "${REPO}" --location="${REGION}" >/dev/nu echo "==> Building & pushing image with Cloud Build: ${IMAGE}" gcloud builds submit "${ROOT_DIR}" \ - --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") + --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n env: ["DOCKER_BUILDKIT=1"]\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") echo "==> Deploying worker pool" ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" @@ -76,11 +104,10 @@ ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" ENV_VARS="${ENV_VARS},NOTIFICATION_TEAM_EMAIL=${NOTIFICATION_TEAM_EMAIL}" -gcloud run worker-pools deploy "${SERVICE}" \ +gcloud beta run worker-pools deploy "${SERVICE}" \ --image "${IMAGE}" \ --region "${REGION}" \ - --min-instances 1 \ - --max-instances 1 \ + --scaling 1 \ --service-account "${SA_EMAIL}" \ --set-env-vars "${ENV_VARS}" \ --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" From ac86d0366380248083a75eb3acf5c0b05a3b4712 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 8 Jul 2026 14:47:59 -0700 Subject: [PATCH 09/12] Attach patch file --- services/hackbot-pulse-listener/app/notify.py | 15 +++++++++++- .../tests/test_notify.py | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py index b2cfe587fe..ad790b28e6 100644 --- a/services/hackbot-pulse-listener/app/notify.py +++ b/services/hackbot-pulse-listener/app/notify.py @@ -1,3 +1,4 @@ +import base64 import logging import re @@ -28,8 +29,13 @@ def send_email(ctx: RunContext, run_doc: dict) -> None: import markdown2 import sendgrid from sendgrid.helpers.mail import ( + Attachment, Cc, Content, + Disposition, + FileContent, + FileName, + FileType, From, HtmlContent, Mail, @@ -52,6 +58,13 @@ def send_email(ctx: RunContext, run_doc: dict) -> None: Content("text/plain", body_md), HtmlContent(html), ) + if patch: + message.attachment = Attachment( + FileContent(base64.b64encode(patch.encode()).decode()), + FileName("changes.patch"), + FileType("text/x-patch"), + Disposition("attachment"), + ) response = sg.send(message=message) logger.info( "Sent build-repair notification to %s (status %s)", @@ -171,6 +184,6 @@ def _patch_block(patch: str) -> str: if len(patch_lines) > MAX_PATCH_LINES: block.append( f"\n_Patch truncated to {MAX_PATCH_LINES} lines; " - "see run details for the full diff._" + "see the attached changes.patch for the full diff._" ) return "\n".join(block) diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py index cfac423b0f..249d610e9b 100644 --- a/services/hackbot-pulse-listener/tests/test_notify.py +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -1,3 +1,4 @@ +import base64 from unittest.mock import MagicMock, patch from app import notify @@ -163,3 +164,25 @@ def test_recipients_dedupes_and_skips_empty(monkeypatch): assert notify._recipients("dev@mozilla.com") == ["dev@mozilla.com"] monkeypatch.setattr(notify.settings, "notification_team_email", None) assert notify._recipients(None) == [] + + +def test_attaches_patch_file(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + run_doc = { + "status": "succeeded", + "artifacts": [{"name": notify.PATCH_ARTIFACT}], + "summary": {"findings": {}}, + } + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with ( + patch("sendgrid.SendGridAPIClient", return_value=fake_client), + patch.object(notify.client, "get_artifact", return_value="DIFF-CONTENT"), + ): + notify.send_email(_ctx(), run_doc) + + attachments = fake_client.send.call_args.kwargs["message"].get()["attachments"] + assert len(attachments) == 1 + assert attachments[0]["filename"] == "changes.patch" + assert base64.b64decode(attachments[0]["content"]).decode() == "DIFF-CONTENT" From 40704d8f78318a16bbe438775174c3be4436f6a1 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 8 Jul 2026 15:31:18 -0700 Subject: [PATCH 10/12] Relax pulse filter --- services/hackbot-pulse-listener/app/consumer.py | 3 ++- services/hackbot-pulse-listener/tests/test_consumer.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py index 0280cb6135..427dc403bf 100644 --- a/services/hackbot-pulse-listener/app/consumer.py +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -26,7 +26,8 @@ def process(body: dict, executor: Executor) -> str | None: """Handle one Taskcluster failure message. Returns the triggered run id.""" tags = (body.get("task") or {}).get("tags") or {} - if tags.get("kind") != "build": + task_label = tags.get("label") or "" + if "build" not in task_label or "test" in task_label: return None project = tags.get("project") diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py index 5fdecf37a2..08e5981b7c 100644 --- a/services/hackbot-pulse-listener/tests/test_consumer.py +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -45,6 +45,14 @@ def test_sample_messages_are_all_tests_and_skipped(): executor.submit.assert_not_called() +def test_missing_label_is_skipped_not_crashed(): + executor = MagicMock() + body = {"status": {"taskId": "XYZ"}, "task": {"tags": {"project": "autoland"}}} + with patch.object(consumer.client, "trigger_run") as trigger: + assert consumer.process(body, executor) is None + trigger.assert_not_called() + + def test_build_failure_triggers_run_and_submits_poll(): executor = MagicMock() with ( From 5388e844a7f01570db3f591980c1a6310160b035 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Thu, 9 Jul 2026 09:16:06 -0700 Subject: [PATCH 11/12] Disable notifications if there's no patch --- services/hackbot-pulse-listener/README.md | 3 ++- services/hackbot-pulse-listener/app/config.py | 2 ++ services/hackbot-pulse-listener/app/notify.py | 17 ++++++++++--- .../tests/test_notify.py | 25 +++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md index 537acf08e4..d098b4fd77 100644 --- a/services/hackbot-pulse-listener/README.md +++ b/services/hackbot-pulse-listener/README.md @@ -32,7 +32,8 @@ uv run --package hackbot-pulse-listener python -m app Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it is logged and skipped. Notifications go to the revision author and, if set, the `NOTIFICATION_TEAM_EMAIL` team address. Set `NOTIFICATION_OVERRIDE_EMAIL` to route every notification to a -single address (useful for local testing). +single address (useful for local testing). By default only runs that produced a patch are +emailed; set `NOTIFY_ONLY_WITH_PATCH=false` to also notify on transient / not-to-blame runs. ## Test diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py index a25c8ed193..969cff3e8c 100644 --- a/services/hackbot-pulse-listener/app/config.py +++ b/services/hackbot-pulse-listener/app/config.py @@ -41,6 +41,8 @@ class Settings(BaseSettings): notification_team_email: str | None = None # Send all notifications to this address instead of the developer (local testing). notification_override_email: str | None = None + # Only notify when the run produced a patch (skip transient / not-to-blame runs). + notify_only_with_patch: bool = True dry_run: bool = False environment: str = "development" diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py index ad790b28e6..9769878616 100644 --- a/services/hackbot-pulse-listener/app/notify.py +++ b/services/hackbot-pulse-listener/app/notify.py @@ -13,11 +13,19 @@ def send_email(ctx: RunContext, run_doc: dict) -> None: - """Email the developer and team the fix. Only succeeded runs are notified.""" + """Email the developer and team the build failure analysis. + + Only succeeded runs are notified. + """ if run_doc.get("status") != "succeeded": logger.info("Run %s did not succeed; skipping notification", ctx.run_id) return + patch = _fetch_patch(ctx.run_id, run_doc) + if settings.notify_only_with_patch and not patch: + logger.info("Run %s produced no patch; skipping notification", ctx.run_id) + return + recipients = _recipients(ctx.developer_email) if not recipients: logger.info("No recipients for run %s; skipping notification", ctx.run_id) @@ -43,9 +51,10 @@ def send_email(ctx: RunContext, run_doc: dict) -> None: To, ) - subject = f"[build-repair] fix for {ctx.repo}@{ctx.git_commit[:12]}" + subject = ( + f"[build-repair] Build failure analysis for {ctx.repo}@{ctx.git_commit[:12]}" + ) - patch = _fetch_patch(ctx.run_id, run_doc) body_md = _build_body(ctx, run_doc, patch) html = markdown2.markdown(body_md, extras=["fenced-code-blocks", "tables"]) @@ -121,7 +130,7 @@ def _build_body(ctx: RunContext, run_doc: dict, patch: str | None = None) -> str findings = summary.get("findings") or {} lines = [ - "# Build-repair fix ready", + "# Build failure analysis", "", f"- **Repository:** {ctx.repo}", f"- **Revision (git):** [`{ctx.git_commit[:12]}`]({_git_url(ctx.git_commit)})", diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py index 249d610e9b..42f83e6331 100644 --- a/services/hackbot-pulse-listener/tests/test_notify.py +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -104,6 +104,7 @@ def test_demote_headings_leaves_code_fences_and_includes_alone(): def test_sends_email_when_configured(monkeypatch): monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) fake_client = MagicMock() fake_client.send.return_value = MagicMock(status_code=202) @@ -119,6 +120,7 @@ def test_override_sends_even_without_developer_email(monkeypatch): monkeypatch.setattr( notify.settings, "notification_override_email", "me@mozilla.com" ) + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) fake_client = MagicMock() fake_client.send.return_value = MagicMock(status_code=202) @@ -130,6 +132,29 @@ def test_override_sends_even_without_developer_email(monkeypatch): fake_client.send.assert_called_once() +def test_skips_when_no_patch_and_notify_only_with_patch(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", True) + + with patch("sendgrid.SendGridAPIClient") as sg: + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) + sg.assert_not_called() + + +def test_sends_without_patch_when_gate_disabled(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) + + fake_client.send.assert_called_once() + + def test_fetch_patch_returns_none_without_artifact(): assert notify._fetch_patch("run-1", {"artifacts": []}) is None From 8ec83514e7376ed74b3dd43e181c74318da9423d Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Thu, 9 Jul 2026 12:28:11 -0700 Subject: [PATCH 12/12] Revert "Fix CI prettier" This reverts commit 075c3824bd3f57bdea2a4c142e0ea88f9b31c4c7. --- ui/changes/snowpack.config.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ui/changes/snowpack.config.js b/ui/changes/snowpack.config.js index 8d4b325d6f..8454c2b72f 100644 --- a/ui/changes/snowpack.config.js +++ b/ui/changes/snowpack.config.js @@ -1,7 +1,13 @@ module.exports = { - plugins: [/* ... */], - packageOptions: {/* ... */}, - devOptions: {/* ... */}, + plugins: [ + /* ... */ + ], + packageOptions: { + /* ... */ + }, + devOptions: { + /* ... */ + }, buildOptions: { out: "dist", /* ... */ @@ -10,5 +16,7 @@ module.exports = { src: "/", /* ... */ }, - alias: {/* ... */}, + alias: { + /* ... */ + }, };