diff --git a/services/hackbot-pulse-listener/Dockerfile b/services/hackbot-pulse-listener/Dockerfile new file mode 100644 index 0000000000..2376966800 --- /dev/null +++ b/services/hackbot-pulse-listener/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.14-slim AS builder + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /app + +# Install external deps without building workspace members. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=VERSION,target=VERSION \ + uv sync --frozen --no-dev --no-install-workspace --package hackbot-pulse-listener + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,target=/app,rw \ + uv sync --locked --no-dev --no-editable --package hackbot-pulse-listener + +FROM python:3.14-slim AS base + +COPY --from=builder /opt/venv /opt/venv +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PATH="/opt/venv/bin:$PATH" + +RUN useradd --create-home --shell /bin/bash app +USER app + +CMD ["python", "-m", "app"] diff --git a/services/hackbot-pulse-listener/README.md b/services/hackbot-pulse-listener/README.md new file mode 100644 index 0000000000..d098b4fd77 --- /dev/null +++ b/services/hackbot-pulse-listener/README.md @@ -0,0 +1,46 @@ +# Hackbot Pulse Listener + +Listens to Taskcluster build-failure pulse messages, and for failed **Firefox build +tasks** triggers the `build-repair` hackbot agent through the hackbot-api. When a run +finishes (minutes later) it emails the developer who pushed the change a link to the +hackbot UI and a summary of the analysis and fix. + +## How it works + +1. Consume `task-failed` messages from `pulse.mozilla.org`. +2. Keep only **build-kind** tasks (`tags.kind == "build"`) on a watched `project` + (`WATCHED_REPOS`, default `try`). Build tasks don't run tests, so a failure is a + compilation/link error. +3. Fetch the task definition to read `GECKO_HEAD_REV` (the revision is not in the message). +4. Dedupe by revision with an in-memory TTL cache, so only one agent run is triggered per + revision even when many build tasks fail for the same push. +5. `POST /agents/build-repair/runs`, then poll `GET /runs/{run_id}` until terminal and send + the report email. + +The dedupe cache and pending-run tracking are in-memory (reset on restart). + +## Run locally + +```bash +export PULSE_USER=... PULSE_PASSWORD=... # https://pulseguardian.mozilla.org +export HACKBOT_API_URL=https://hackbot-api.../ HACKBOT_API_KEY=... +export HACKBOT_UI_URL=https://hackbot-ui.../ +export WATCHED_REPOS=try +export DRY_RUN=true # log intended calls, don't POST +uv run --package hackbot-pulse-listener python -m app +``` + +Email is sent only when `SENDGRID_API_KEY` and `NOTIFICATION_SENDER` are set; otherwise it +is logged and skipped. Notifications go to the revision author and, if set, the `NOTIFICATION_TEAM_EMAIL` team address. Set `NOTIFICATION_OVERRIDE_EMAIL` to route every notification to a +single address (useful for local testing). By default only runs that produced a patch are +emailed; set `NOTIFY_ONLY_WITH_PATCH=false` to also notify on transient / not-to-blame runs. + +## Test + +```bash +uv run --package hackbot-pulse-listener pytest services/hackbot-pulse-listener/tests +``` + +## Deploy + +Cloud Run worker pool (no HTTP). See `deploy.sh`. diff --git a/services/hackbot-pulse-listener/app/__init__.py b/services/hackbot-pulse-listener/app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/app/__main__.py b/services/hackbot-pulse-listener/app/__main__.py new file mode 100644 index 0000000000..80c99d76e7 --- /dev/null +++ b/services/hackbot-pulse-listener/app/__main__.py @@ -0,0 +1,44 @@ +import logging +import signal +from concurrent.futures import ThreadPoolExecutor + +from app import consumer +from app.config import settings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main() -> None: + if settings.sentry_dsn: + import sentry_sdk + + sentry_sdk.init(dsn=settings.sentry_dsn, environment=settings.environment) + + if not (settings.pulse_user and settings.pulse_password): + logger.warning("PULSE_USER/PULSE_PASSWORD not set; listener will not start") + return + + executor = ThreadPoolExecutor(max_workers=settings.poll_max_workers) + consumer_obj = consumer.build_consumer(executor) + + def shutdown(signum, _frame): + logger.info("Received signal %s; shutting down", signum) + consumer_obj.should_stop = True + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + + logger.info( + "Listening for build failures on %s; watched repos: %s", + ", ".join(consumer.EXCHANGES), + sorted(settings.watched_repos_set), + ) + try: + consumer_obj.run() + finally: + executor.shutdown(wait=False) + + +if __name__ == "__main__": + main() diff --git a/services/hackbot-pulse-listener/app/client.py b/services/hackbot-pulse-listener/app/client.py new file mode 100644 index 0000000000..a4f98ba518 --- /dev/null +++ b/services/hackbot-pulse-listener/app/client.py @@ -0,0 +1,44 @@ +import logging + +import httpx + +from app.config import settings + +logger = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0) + + +def _headers() -> dict[str, str]: + return {"X-API-Key": settings.hackbot_api_key} + + +def trigger_run(inputs: dict) -> str | None: + """Create a build-repair run. Returns the run id, or None in dry-run mode.""" + if settings.dry_run: + logger.info("[dry-run] would trigger %s run: %s", settings.agent_name, inputs) + return None + + url = f"{settings.hackbot_api_url}/agents/{settings.agent_name}/runs" + resp = httpx.post(url, json=inputs, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json()["run_id"] + + +def get_run(run_id: str) -> dict: + url = f"{settings.hackbot_api_url}/runs/{run_id}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json() + + +def get_artifact(run_id: str, name: str) -> str | None: + """Download a run artifact's text content, or None if it is missing.""" + url = f"{settings.hackbot_api_url}/runs/{run_id}/artifacts/{name}" + resp = httpx.get(url, headers=_headers(), timeout=_TIMEOUT) + if resp.status_code == 404: + return None + resp.raise_for_status() + download = httpx.get(resp.json()["url"], timeout=_TIMEOUT) + download.raise_for_status() + return download.text diff --git a/services/hackbot-pulse-listener/app/config.py b/services/hackbot-pulse-listener/app/config.py new file mode 100644 index 0000000000..969cff3e8c --- /dev/null +++ b/services/hackbot-pulse-listener/app/config.py @@ -0,0 +1,62 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + # Pulse (https://pulseguardian.mozilla.org) + pulse_user: str = "" + pulse_password: str = "" + taskcluster_root_url: str = "https://firefox-ci-tc.services.mozilla.com" + + # hackbot-api + hackbot_api_url: str = "" + hackbot_api_key: str = "" + hackbot_ui_url: str = "" + agent_name: str = "build-repair" + + # Source links shown in notifications. + firefox_git_url: str = "https://github.com/mozilla-firefox/firefox" + firefox_hg_url: str = "https://hg.mozilla.org/mozilla-unified" + bugzilla_url: str = "https://bugzilla.mozilla.org" + + # Failure filtering and agent inputs. + # ``watched_repos`` is a comma-separated list of Taskcluster ``project`` tags. + watched_repos: str = "autoland" + run_try_push: bool = False + model: str | None = None + max_turns: int | None = None + + # Dedupe (in-memory, by hg revision) + dedupe_ttl_seconds: int = 6 * 60 * 60 + dedupe_max_size: int = 4096 + + # Polling the API for run completion + poll_interval_seconds: int = 60 + run_max_age_minutes: int = 12 * 60 + poll_max_workers: int = 8 + + # Email notifications (SendGrid) + sendgrid_api_key: str | None = None + notification_sender: str | None = None + # Team address CC'd on every notification alongside the revision author. + notification_team_email: str | None = None + # Send all notifications to this address instead of the developer (local testing). + notification_override_email: str | None = None + # Only notify when the run produced a patch (skip transient / not-to-blame runs). + notify_only_with_patch: bool = True + + dry_run: bool = False + environment: str = "development" + sentry_dsn: str | None = None + + model_config = { + "env_file": ".env", + "env_file_encoding": "utf-8", + "extra": "ignore", + } + + @property + def watched_repos_set(self) -> set[str]: + return {r.strip() for r in self.watched_repos.split(",") if r.strip()} + + +settings = Settings() diff --git a/services/hackbot-pulse-listener/app/consumer.py b/services/hackbot-pulse-listener/app/consumer.py new file mode 100644 index 0000000000..427dc403bf --- /dev/null +++ b/services/hackbot-pulse-listener/app/consumer.py @@ -0,0 +1,143 @@ +import logging +from concurrent.futures import Executor + +from cachetools import TTLCache +from kombu import Connection, Exchange, Queue +from kombu.mixins import ConsumerMixin + +from app import client, lando, taskcluster, worker +from app.config import settings +from app.models import RunContext + +logger = logging.getLogger(__name__) + +CONNECTION_URL = "amqp://{}:{}@pulse.mozilla.org:5671/?ssl=1" + +EXCHANGES = ("exchange/taskcluster-queue/v1/task-failed",) + +# In-memory dedupe of hg revisions already handed to the agent. Only the +# single consumer thread touches it, so no lock is needed. +_seen: TTLCache = TTLCache( + maxsize=settings.dedupe_max_size, ttl=settings.dedupe_ttl_seconds +) + + +def process(body: dict, executor: Executor) -> str | None: + """Handle one Taskcluster failure message. Returns the triggered run id.""" + tags = (body.get("task") or {}).get("tags") or {} + + task_label = tags.get("label") or "" + if "build" not in task_label or "test" in task_label: + return None + + project = tags.get("project") + if project not in settings.watched_repos_set: + return None + + task_id = body["status"]["taskId"] + task_name = tags.get("label") or task_id + developer_email = tags.get("createdForUser") + + hg_revision = taskcluster.get_hg_revision(task_id) + if not hg_revision: + logger.warning("No GECKO_HEAD_REV for task %s; skipping", task_id) + return None + + if hg_revision in _seen: + logger.info("Revision %s already processed; skipping", hg_revision) + return None + _seen[hg_revision] = True + + git_commit = lando.hg_to_git(hg_revision) + if not git_commit: + logger.warning( + "Could not map hg revision %s to git for task %s (%s); skipping", + hg_revision, + task_id, + project, + ) + _seen.pop(hg_revision, None) + return None + + inputs: dict = { + "git_commit": git_commit, + "failure_tasks": {task_name: task_id}, + "run_try_push": settings.run_try_push, + } + if settings.model: + inputs["model"] = settings.model + if settings.max_turns is not None: + inputs["max_turns"] = settings.max_turns + + try: + run_id = client.trigger_run(inputs) + except Exception: + logger.exception("Failed to trigger build-repair run for %s", hg_revision) + _seen.pop(hg_revision, None) + return None + + logger.info( + "Triggered build-repair run %s for %s@%s (git %s)", + run_id, + project, + hg_revision, + git_commit, + ) + if run_id is not None: + ctx = RunContext( + run_id=run_id, + repo=project, + git_commit=git_commit, + hg_revision=hg_revision, + task_id=task_id, + developer_email=developer_email, + ) + executor.submit(worker.poll_and_notify, ctx) + return run_id + + +def make_handler(executor: Executor): + def on_message(body, message): + try: + process(body, executor) + except Exception: + logger.exception("Error handling pulse message") + finally: + message.ack() + + return on_message + + +def _build_queues(user: str) -> list[Queue]: + queues = [] + for exchange in EXCHANGES: + suffix = exchange.rsplit("/", 1)[-1] + queues.append( + Queue( + name=f"queue/{user}/build-repair-{suffix}", + exchange=Exchange(exchange, type="topic", no_declare=True), + routing_key="#", + durable=True, + auto_delete=True, + ) + ) + return queues + + +class BuildFailureConsumer(ConsumerMixin): + def __init__(self, connection, queues, on_message): + self.connection = connection + self.queues = queues + self.on_message = on_message + + def get_consumers(self, Consumer, channel): + return [Consumer(queues=self.queues, callbacks=[self.on_message])] + + +def build_consumer(executor: Executor) -> BuildFailureConsumer: + connection = Connection( + CONNECTION_URL.format(settings.pulse_user, settings.pulse_password) + ) + return BuildFailureConsumer( + connection, _build_queues(settings.pulse_user), make_handler(executor) + ) diff --git a/services/hackbot-pulse-listener/app/lando.py b/services/hackbot-pulse-listener/app/lando.py new file mode 100644 index 0000000000..b8a336ee45 --- /dev/null +++ b/services/hackbot-pulse-listener/app/lando.py @@ -0,0 +1,19 @@ +import logging + +import httpx + +logger = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0) + + +def hg_to_git(hg_rev: str) -> str | None: + """Resolve a Mercurial revision to the matching firefox git SHA, or None.""" + url = f"https://lando.moz.tools/api/hg2git/firefox/{hg_rev}" + try: + resp = httpx.get(url, timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json()["git_hash"] + except (httpx.HTTPError, KeyError) as exc: + logger.debug("Failed to map hg revision %s to git: %s", hg_rev, exc) + return None diff --git a/services/hackbot-pulse-listener/app/models.py b/services/hackbot-pulse-listener/app/models.py new file mode 100644 index 0000000000..2bcaf33cfa --- /dev/null +++ b/services/hackbot-pulse-listener/app/models.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + + +@dataclass +class RunContext: + """What the notifier needs about a triggered build-repair run.""" + + run_id: str + repo: str + git_commit: str + hg_revision: str + task_id: str + developer_email: str | None diff --git a/services/hackbot-pulse-listener/app/notify.py b/services/hackbot-pulse-listener/app/notify.py new file mode 100644 index 0000000000..9769878616 --- /dev/null +++ b/services/hackbot-pulse-listener/app/notify.py @@ -0,0 +1,198 @@ +import base64 +import logging +import re + +from app import client +from app.config import settings +from app.models import RunContext + +logger = logging.getLogger(__name__) + +PATCH_ARTIFACT = "changes/changes.patch" +MAX_PATCH_LINES = 400 + + +def send_email(ctx: RunContext, run_doc: dict) -> None: + """Email the developer and team the build failure analysis. + + Only succeeded runs are notified. + """ + if run_doc.get("status") != "succeeded": + logger.info("Run %s did not succeed; skipping notification", ctx.run_id) + return + + patch = _fetch_patch(ctx.run_id, run_doc) + if settings.notify_only_with_patch and not patch: + logger.info("Run %s produced no patch; skipping notification", ctx.run_id) + return + + recipients = _recipients(ctx.developer_email) + if not recipients: + logger.info("No recipients for run %s; skipping notification", ctx.run_id) + return + if not (settings.sendgrid_api_key and settings.notification_sender): + logger.info("SendGrid not configured; skipping email for run %s", ctx.run_id) + return + + import markdown2 + import sendgrid + from sendgrid.helpers.mail import ( + Attachment, + Cc, + Content, + Disposition, + FileContent, + FileName, + FileType, + From, + HtmlContent, + Mail, + Subject, + To, + ) + + subject = ( + f"[build-repair] Build failure analysis for {ctx.repo}@{ctx.git_commit[:12]}" + ) + + body_md = _build_body(ctx, run_doc, patch) + html = markdown2.markdown(body_md, extras=["fenced-code-blocks", "tables"]) + + sg = sendgrid.SendGridAPIClient(api_key=settings.sendgrid_api_key) + to_emails = [To(recipients[0])] + [Cc(addr) for addr in recipients[1:]] + message = Mail( + From(settings.notification_sender), + to_emails, + Subject(subject), + Content("text/plain", body_md), + HtmlContent(html), + ) + if patch: + message.attachment = Attachment( + FileContent(base64.b64encode(patch.encode()).decode()), + FileName("changes.patch"), + FileType("text/x-patch"), + Disposition("attachment"), + ) + response = sg.send(message=message) + logger.info( + "Sent build-repair notification to %s (status %s)", + ", ".join(recipients), + response.status_code, + ) + + +def _recipients(developer_email: str | None) -> list[str]: + """Recipients for a run: the revision author plus the team address, deduped. + + ``notification_override_email`` short-circuits to a single address so local + testing never mails real developers or the team. + """ + if settings.notification_override_email: + return [settings.notification_override_email] + recipients: list[str] = [] + for addr in (developer_email, settings.notification_team_email): + if addr and addr not in recipients: + recipients.append(addr) + return recipients + + +def _fetch_patch(run_id: str, run_doc: dict) -> str | None: + """Download the proposed-fix patch artifact, if the run produced one.""" + artifacts = run_doc.get("artifacts") or [] + if not any(a.get("name") == PATCH_ARTIFACT for a in artifacts): + return None + try: + return client.get_artifact(run_id, PATCH_ARTIFACT) + except Exception: + logger.exception("Failed to fetch patch for run %s", run_id) + return None + + +def _git_url(git_commit: str) -> str: + return f"{settings.firefox_git_url.rstrip('/')}/commit/{git_commit}" + + +def _hg_url(hg_revision: str) -> str: + return f"{settings.firefox_hg_url.rstrip('/')}/rev/{hg_revision}" + + +def _task_url(task_id: str) -> str: + return f"{settings.taskcluster_root_url.rstrip('/')}/tasks/{task_id}" + + +def _bug_url(bug_id: object) -> str: + return f"{settings.bugzilla_url.rstrip('/')}/show_bug.cgi?id={bug_id}" + + +def _build_body(ctx: RunContext, run_doc: dict, patch: str | None = None) -> str: + summary = run_doc.get("summary") or {} + findings = summary.get("findings") or {} + + lines = [ + "# Build failure analysis", + "", + f"- **Repository:** {ctx.repo}", + f"- **Revision (git):** [`{ctx.git_commit[:12]}`]({_git_url(ctx.git_commit)})", + f"- **Revision (hg):** [`{ctx.hg_revision[:12]}`]({_hg_url(ctx.hg_revision)})", + f"- **Failed task:** [`{ctx.task_id}`]({_task_url(ctx.task_id)})", + ] + + bug_id = findings.get("bug_id") or (run_doc.get("inputs") or {}).get("bug_id") + if bug_id: + lines.append(f"- **Bug:** [{bug_id}]({_bug_url(bug_id)})") + + if settings.hackbot_ui_url: + run_url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{ctx.run_id}" + lines.append(f"- **Run details:** {run_url}") + + if findings.get("summary"): + lines += ["", "## Summary", "", _demote_headings(findings["summary"])] + if findings.get("analysis"): + lines += ["", "## Analysis", "", _demote_headings(findings["analysis"])] + + if findings.get("local_build_verified") is not None: + lines += [ + "", + "## Verification", + "", + f"- Local build verified: {findings['local_build_verified']}", + ] + + if patch: + lines += ["", "## Proposed patch", "", _patch_block(patch)] + + return "\n".join(lines) + + +def _demote_headings(md: str, by: int = 2) -> str: + """Shift ATX headings down ``by`` levels so agent docs nest under our own. + + Lines inside code fences (and ``#include`` and the like, which lack the + required space after ``#``) are left untouched. + """ + out = [] + in_fence = False + for line in md.splitlines(): + if line.lstrip().startswith(("```", "~~~")): + in_fence = not in_fence + out.append(line) + continue + match = re.match(r"(#{1,6}) ", line) if not in_fence else None + if match: + level = min(len(match.group(1)) + by, 6) + line = "#" * level + line[len(match.group(1)) :] + out.append(line) + return "\n".join(out) + + +def _patch_block(patch: str) -> str: + patch_lines = patch.splitlines() + shown = patch_lines[:MAX_PATCH_LINES] + block = ["```diff", *shown, "```"] + if len(patch_lines) > MAX_PATCH_LINES: + block.append( + f"\n_Patch truncated to {MAX_PATCH_LINES} lines; " + "see the attached changes.patch for the full diff._" + ) + return "\n".join(block) diff --git a/services/hackbot-pulse-listener/app/taskcluster.py b/services/hackbot-pulse-listener/app/taskcluster.py new file mode 100644 index 0000000000..f4076e6d0a --- /dev/null +++ b/services/hackbot-pulse-listener/app/taskcluster.py @@ -0,0 +1,28 @@ +import logging + +import taskcluster + +from app.config import settings + +logger = logging.getLogger(__name__) + +_queue: taskcluster.Queue | None = None + + +def _get_queue() -> taskcluster.Queue: + global _queue + if _queue is None: + _queue = taskcluster.Queue({"rootUrl": settings.taskcluster_root_url}) + return _queue + + +def get_hg_revision(task_id: str) -> str | None: + """Return the GECKO_HEAD_REV (Mercurial revision) for a task, or None. + + The revision is not in the pulse message, so we fetch the full task + definition. Task definitions are public, so no credentials are needed. + GECKO_HEAD_REV is an hg revision; the build-repair agent needs a git SHA, + so callers must convert it (see app.lando.hg_to_git). + """ + task = _get_queue().task(task_id) + return task.get("payload", {}).get("env", {}).get("GECKO_HEAD_REV") diff --git a/services/hackbot-pulse-listener/app/worker.py b/services/hackbot-pulse-listener/app/worker.py new file mode 100644 index 0000000000..188ea81376 --- /dev/null +++ b/services/hackbot-pulse-listener/app/worker.py @@ -0,0 +1,46 @@ +import logging +import time + +from app import client, notify +from app.config import settings +from app.models import RunContext + +logger = logging.getLogger(__name__) + +TERMINAL_STATUSES = {"succeeded", "failed", "timed_out"} + + +def poll_and_notify(ctx: RunContext) -> None: + """Poll the run until terminal, then notify. + + Runs on a background executor thread; never lets an exception escape. + """ + try: + run_doc = _poll_until_terminal(ctx.run_id) + except Exception: + logger.exception("Polling failed for run %s", ctx.run_id) + return + + if run_doc is None: + logger.warning( + "Run %s did not finish within %s minutes; giving up", + ctx.run_id, + settings.run_max_age_minutes, + ) + return + + try: + notify.send_email(ctx, run_doc) + except Exception: + logger.exception("Failed to send notification for run %s", ctx.run_id) + + +def _poll_until_terminal(run_id: str) -> dict | None: + deadline = time.monotonic() + settings.run_max_age_minutes * 60 + while True: + run_doc = client.get_run(run_id) + if run_doc.get("status") in TERMINAL_STATUSES: + return run_doc + if time.monotonic() >= deadline: + return None + time.sleep(settings.poll_interval_seconds) diff --git a/services/hackbot-pulse-listener/deploy.sh b/services/hackbot-pulse-listener/deploy.sh new file mode 100755 index 0000000000..5f52c373b6 --- /dev/null +++ b/services/hackbot-pulse-listener/deploy.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Deploy the build-failure Pulse listener to Cloud Run as a worker pool. +# +# A worker pool runs an always-on, non-request workload (no HTTP port). This +# service consumes Taskcluster build-failure pulse messages and triggers the +# build-repair hackbot agent via the hackbot-api. +# +# Prereqs (one-time): +# gcloud auth login +# gcloud config set project +# gcloud components install beta # worker pools live under the beta track +# gcloud services enable run.googleapis.com artifactregistry.googleapis.com \ +# cloudbuild.googleapis.com secretmanager.googleapis.com +# +# Secrets — the values use the same env var names as the app's .env, so +# `source .env` populates them. Any secret missing from Secret Manager is +# created from its value; existing secrets are never overwritten (rotate with +# `gcloud secrets versions add`): +# PULSE_PASSWORD -> secret `pulse-password` +# SENDGRID_API_KEY -> secret `sendgrid-api-key` +# HACKBOT_API_KEY -> secret `external-api-key` (shared with hackbot-api) +# +# Usage: +# source .env # provides PULSE_PASSWORD, HACKBOT_API_KEY, SENDGRID_API_KEY, etc. +# PROJECT=my-proj REGION=us-central1 \ +# HACKBOT_API_URL=https://hackbot-api-xxxx.run.app \ +# HACKBOT_UI_URL=https://hackbot-ui-xxxx.run.app \ +# PULSE_USER=my-pulse-user NOTIFICATION_SENDER= \ +# NOTIFICATION_TEAM_EMAIL=hackbot-developers@mozilla.com \ +# ./deploy.sh +set -euo pipefail + +PROJECT="${PROJECT:?set PROJECT to your GCP project id}" +REGION="${REGION:-us-central1}" +SERVICE="${SERVICE:-hackbot-pulse-listener}" +REPO="${REPO:-hackbot}" +HACKBOT_API_URL="${HACKBOT_API_URL:?set HACKBOT_API_URL to the hackbot-api base URL}" +HACKBOT_UI_URL="${HACKBOT_UI_URL:?set HACKBOT_UI_URL to the hackbot-ui base URL}" +PULSE_USER="${PULSE_USER:?set PULSE_USER (https://pulseguardian.mozilla.org)}" +WATCHED_REPOS="${WATCHED_REPOS:-autoland}" +NOTIFICATION_SENDER="${NOTIFICATION_SENDER:?set NOTIFICATION_SENDER (verified SendGrid sender)}" +NOTIFICATION_TEAM_EMAIL="${NOTIFICATION_TEAM_EMAIL:-}" + +SA_NAME="${SA_NAME:-hackbot-pulse-listener-run}" +SA_EMAIL="${SA_EMAIL:-${SA_NAME}@${PROJECT}.iam.gserviceaccount.com}" + +# Secret Manager secret names (where the values live). +PULSE_SECRET="${PULSE_SECRET:-pulse-password}" +API_KEY_SECRET="${API_KEY_SECRET:-external-api-key}" +SENDGRID_SECRET="${SENDGRID_SECRET:-sendgrid-api-key}" + +# Secret values, using the same names as the app's .env so `source .env` works. +# Used only to seed a secret that does not exist yet (never overwrites). +PULSE_PASSWORD="${PULSE_PASSWORD:-}" +HACKBOT_API_KEY="${HACKBOT_API_KEY:-}" +SENDGRID_API_KEY="${SENDGRID_API_KEY:-}" + +IMAGE="${REGION}-docker.pkg.dev/${PROJECT}/${REPO}/${SERVICE}:latest" +# Build context is the repo root (the Dockerfile needs the workspace lock files). +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +echo "==> Ensuring runtime service account '${SA_EMAIL}' exists" +gcloud iam service-accounts describe "${SA_EMAIL}" >/dev/null 2>&1 || \ + gcloud iam service-accounts create "${SA_NAME}" \ + --display-name="Hackbot Pulse Listener (Cloud Run runtime)" + +echo "==> Ensuring secrets exist (seeding from env when missing)" +ensure_secret() { # secret_name value + local name="$1" value="${2:-}" + if gcloud secrets describe "$name" >/dev/null 2>&1; then + return 0 + fi + if [ -z "$value" ]; then + echo "ERROR: secret '$name' is missing and no value was provided to create it" >&2 + exit 1 + fi + printf '%s' "$value" | gcloud secrets create "$name" --data-file=- +} +ensure_secret "${PULSE_SECRET}" "${PULSE_PASSWORD}" +ensure_secret "${API_KEY_SECRET}" "${HACKBOT_API_KEY}" +ensure_secret "${SENDGRID_SECRET}" "${SENDGRID_API_KEY}" + +echo "==> Granting the SA read access to its secrets" +for s in "${PULSE_SECRET}" "${API_KEY_SECRET}" "${SENDGRID_SECRET}"; do + gcloud secrets add-iam-policy-binding "$s" \ + --member="serviceAccount:${SA_EMAIL}" \ + --role=roles/secretmanager.secretAccessor >/dev/null +done + +echo "==> Ensuring Artifact Registry repo '${REPO}' exists in ${REGION}" +gcloud artifacts repositories describe "${REPO}" --location="${REGION}" >/dev/null 2>&1 || \ + gcloud artifacts repositories create "${REPO}" \ + --repository-format=docker --location="${REGION}" \ + --description="Hackbot container images" + +echo "==> Building & pushing image with Cloud Build: ${IMAGE}" +gcloud builds submit "${ROOT_DIR}" \ + --config <(printf 'steps:\n- name: gcr.io/cloud-builders/docker\n env: ["DOCKER_BUILDKIT=1"]\n args: ["build","-t","%s","-f","services/%s/Dockerfile","."]\nimages: ["%s"]\n' "${IMAGE}" "${SERVICE}" "${IMAGE}") + +echo "==> Deploying worker pool" +ENV_VARS="HACKBOT_API_URL=${HACKBOT_API_URL},HACKBOT_UI_URL=${HACKBOT_UI_URL}" +ENV_VARS="${ENV_VARS},PULSE_USER=${PULSE_USER},WATCHED_REPOS=${WATCHED_REPOS}" +ENV_VARS="${ENV_VARS},NOTIFICATION_SENDER=${NOTIFICATION_SENDER}" +ENV_VARS="${ENV_VARS},NOTIFICATION_TEAM_EMAIL=${NOTIFICATION_TEAM_EMAIL}" + +gcloud beta run worker-pools deploy "${SERVICE}" \ + --image "${IMAGE}" \ + --region "${REGION}" \ + --scaling 1 \ + --service-account "${SA_EMAIL}" \ + --set-env-vars "${ENV_VARS}" \ + --set-secrets "PULSE_PASSWORD=${PULSE_SECRET}:latest,HACKBOT_API_KEY=${API_KEY_SECRET}:latest,SENDGRID_API_KEY=${SENDGRID_SECRET}:latest" + +echo "==> Deployed worker pool '${SERVICE}'" diff --git a/services/hackbot-pulse-listener/docker-compose.dev.yml b/services/hackbot-pulse-listener/docker-compose.dev.yml new file mode 100644 index 0000000000..441d03e7b1 --- /dev/null +++ b/services/hackbot-pulse-listener/docker-compose.dev.yml @@ -0,0 +1,11 @@ +services: + hackbot-pulse-listener: + build: + context: ../.. + dockerfile: services/hackbot-pulse-listener/Dockerfile + # Inject config into the container (relative to this compose file = repo root .env). + env_file: ../../.env + # Live-edit the source without rebuilding (cwd takes precedence on sys.path). + volumes: + - ./app:/app/app + command: python -m app diff --git a/services/hackbot-pulse-listener/pyproject.toml b/services/hackbot-pulse-listener/pyproject.toml new file mode 100644 index 0000000000..7940bc4940 --- /dev/null +++ b/services/hackbot-pulse-listener/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "hackbot-pulse-listener" +version = "0.1.0" +description = "Listens to Taskcluster build failures and triggers the build-repair hackbot agent" +requires-python = ">=3.12" +dependencies = [ + "kombu>=5.6,<6", + "taskcluster>=97.1,<100.4", + "httpx>=0.26.0", + "pydantic-settings>=2.1.0", + "sendgrid>=6.12.5", + "markdown2>=2.4.0", + "cachetools>=5.3.0", + "sentry-sdk>=2.51.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/hackbot-pulse-listener/tests/__init__.py b/services/hackbot-pulse-listener/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json new file mode 100644 index 0000000000..3813b8025a --- /dev/null +++ b/services/hackbot-pulse-listener/tests/fixtures/pulse_messages.json @@ -0,0 +1,892 @@ +[ + { + "payload": { + "status": { + "taskId": "QKbYKE7STVy3_DM6J9NKXw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1400-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1400-r8", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "DJziYijSR9qv-cGTDdS92w", + "priority": "very-low", + "deadline": "2026-06-26T21:22:49.913Z", + "expires": "2026-07-23T21:22:49.913Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "takenUntil": "2026-06-25T22:47:29.995Z", + "scheduled": "2026-06-25T21:36:27.697Z", + "started": "2026-06-25T21:36:28.598Z", + "resolved": "2026-06-25T22:35:05.200Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "web-platform-tests", + "label": "test-macosx1470-64/opt-web-platform-tests-webgpu-backlog-long-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "trust-domain": "gecko", + "test-platform": "macosx1470-64/opt", + "tests_grouped": "1", + "createdForUser": "aleiserson@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-381", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.QKbYKE7STVy3_DM6J9NKXw.0.mdc1.macmini-r8-381.releng-hardware.gecko-t-osx-1400-r8.gecko-level-1.DJziYijSR9qv-cGTDdS92w._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TISFEWXiRD-Fg4ifDcLm5A", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.330Z", + "expires": "2026-07-23T13:31:23.330Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "takenUntil": "2026-06-25T23:27:40.611Z", + "scheduled": "2026-06-25T13:39:15.145Z", + "started": "2026-06-25T23:07:40.613Z", + "resolved": "2026-06-25T23:17:25.349Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/debug-mochitest-thunderbird-2", + "project": "try-comm-central", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome-thunderbird", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/debug", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-209", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TISFEWXiRD-Fg4ifDcLm5A.0.mdc1.macmini-r8-209.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "TWIXsqxDTame750yV3a_0A", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.935Z", + "expires": "2026-07-23T22:29:42.935Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "takenUntil": "2026-06-25T23:21:38.581Z", + "scheduled": "2026-06-25T22:43:40.477Z", + "started": "2026-06-25T22:44:38.441Z", + "resolved": "2026-06-25T23:17:28.610Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/opt-web-platform-tests-async-9", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/opt", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-east1-c", + "workerId": "4411950482911664357", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.TWIXsqxDTame750yV3a_0A.0.us-east1-c.4411950482911664357.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "CLwxQmXLQFunTzlzrzshOg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "takenUntil": "2026-06-25T23:30:14.189Z", + "scheduled": "2026-06-25T22:18:46.436Z", + "started": "2026-06-25T22:19:13.295Z", + "resolved": "2026-06-25T23:19:31.591Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-8", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.CLwxQmXLQFunTzlzrzshOg.0.southindia.vm-c53ljhlksec57kwlo3atvgxoxvt5wctd2ma.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "d4YZWZQXR-SdCVwlrVTTUA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "takenUntil": "2026-06-25T23:30:22.934Z", + "scheduled": "2026-06-25T22:18:46.462Z", + "started": "2026-06-25T22:19:22.411Z", + "resolved": "2026-06-25T23:19:37.633Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-2", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.d4YZWZQXR-SdCVwlrVTTUA.0.southindia.vm-t8mqfiqktf2mzzqnjzm4iaek5dqgelspc09.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "EJXnn6xsQ4OeDMK3W1aksg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.160Z", + "expires": "2027-06-25T20:21:36.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "takenUntil": "2026-06-25T23:30:32.889Z", + "scheduled": "2026-06-25T22:18:46.487Z", + "started": "2026-06-25T22:19:32.554Z", + "resolved": "2026-06-25T23:19:38.966Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus3", + "workerId": "vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.EJXnn6xsQ4OeDMK3W1aksg.0.westus3.vm-gm0hjns2r2q6os6d6n0jngdliu65qqsbmlf.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "cnTkiN1YROGfMWecDxyMEQ", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.413Z", + "expires": "2026-07-23T22:29:42.413Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "takenUntil": "2026-06-25T23:39:39.063Z", + "scheduled": "2026-06-25T22:44:50.986Z", + "started": "2026-06-25T22:45:38.942Z", + "resolved": "2026-06-25T23:19:39.842Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-12", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-west1-b", + "workerId": "525836151908471198", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.cnTkiN1YROGfMWecDxyMEQ.0.us-west1-b.525836151908471198.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DPltHwhPTI29TinNU_XBbw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.472Z", + "expires": "2027-06-25T20:22:15.472Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "takenUntil": "2026-06-25T23:31:17.350Z", + "scheduled": "2026-06-25T22:19:36.851Z", + "started": "2026-06-25T22:20:16.923Z", + "resolved": "2026-06-25T23:20:25.989Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-7", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "canadacentral", + "workerId": "vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DPltHwhPTI29TinNU_XBbw.0.canadacentral.vm-oxyh6fj4tz0aaicetdxwaqmjphnymqusaz6.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "DEt854d0QAOgI-OZsHoMmw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.468Z", + "expires": "2027-06-25T20:22:15.468Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "takenUntil": "2026-06-25T23:31:13.812Z", + "scheduled": "2026-06-25T22:19:36.839Z", + "started": "2026-06-25T22:20:12.929Z", + "resolved": "2026-06-25T23:20:28.897Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-13", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "southindia", + "workerId": "vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.DEt854d0QAOgI-OZsHoMmw.0.southindia.vm-ytkzofbcsxkitaq19prrnqpg6vq5jq3ailg.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "AlWS0HmFQiKPjBS1HcFAzw", + "provisionerId": "releng-hardware", + "workerType": "gecko-t-osx-1015-r8", + "taskQueueId": "releng-hardware/gecko-t-osx-1015-r8", + "schedulerId": "comm-level-1", + "projectId": "none", + "taskGroupId": "VuZl_WA9QKq3m4U7zIVcUA", + "priority": "low", + "deadline": "2026-06-26T13:31:23.054Z", + "expires": "2026-07-23T13:31:23.054Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "takenUntil": "2026-06-25T23:28:29.702Z", + "scheduled": "2026-06-25T13:42:42.448Z", + "started": "2026-06-25T23:08:29.704Z", + "resolved": "2026-06-25T23:20:29.089Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "macosx", + "kind": "test", + "label": "test-macosx1015-64-qr/opt-xpcshell-4", + "project": "try-comm-central", + "retrigger": "true", + "test-suite": "xpcshell", + "trust-domain": "comm", + "test-platform": "macosx1015-64-qr/opt", + "tests_grouped": "1", + "createdForUser": "h.w.forms@arcor.de", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "mdc1", + "workerId": "macmini-r8-77", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.AlWS0HmFQiKPjBS1HcFAzw.0.mdc1.macmini-r8-77.releng-hardware.gecko-t-osx-1015-r8.comm-level-1.VuZl_WA9QKq3m4U7zIVcUA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YO-gvnzTTGS7INXYIN8Nbg", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.156Z", + "expires": "2027-06-25T20:21:36.156Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "takenUntil": "2026-06-25T23:31:12.293Z", + "scheduled": "2026-06-25T22:18:46.606Z", + "started": "2026-06-25T22:20:11.674Z", + "resolved": "2026-06-25T23:20:29.185Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-15", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YO-gvnzTTGS7INXYIN8Nbg.0.centralindia.vm-qf6plze9qdes12vlwojguayb66pk1vqrdz0.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "YV2f3tnLSCyXUIG4EDq5Xw", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "b49IIC4mRAGhUGzjpvP0jQ", + "priority": "low", + "deadline": "2026-06-26T20:21:36.153Z", + "expires": "2027-06-25T20:21:36.153Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "takenUntil": "2026-06-25T23:31:12.727Z", + "scheduled": "2026-06-25T22:18:46.629Z", + "started": "2026-06-25T22:20:12.243Z", + "resolved": "2026-06-25T23:20:30.629Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-4", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "mozilla@kaply.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "centralindia", + "workerId": "vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.YV2f3tnLSCyXUIG4EDq5Xw.0.centralindia.vm-uussuzjr5yzwxdeekeggdet9hyushkx30tm.gecko-t.win11-64-25h2.gecko-level-3.b49IIC4mRAGhUGzjpvP0jQ._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "dtOy4UGdTHm2s5XCVuNkqA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.473Z", + "expires": "2027-06-25T20:22:15.473Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "takenUntil": "2026-06-25T23:31:33.280Z", + "scheduled": "2026-06-25T22:19:36.866Z", + "started": "2026-06-25T22:20:32.966Z", + "resolved": "2026-06-25T23:20:41.142Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-11", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "northcentralus", + "workerId": "vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.dtOy4UGdTHm2s5XCVuNkqA.0.northcentralus.vm-fpfcr7qcafdv8bktbhzwmz07xcsbq2ohk4h.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "fRj5qgRVS4adFANUGOexwA", + "provisionerId": "gecko-t", + "workerType": "win11-64-25h2", + "taskQueueId": "gecko-t/win11-64-25h2", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "K3dxP_ueQcueqrIFkLqwAA", + "priority": "low", + "deadline": "2026-06-26T20:22:15.474Z", + "expires": "2027-06-25T20:22:15.474Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "takenUntil": "2026-06-25T23:31:38.392Z", + "scheduled": "2026-06-25T22:19:36.878Z", + "started": "2026-06-25T22:20:38.071Z", + "resolved": "2026-06-25T23:20:45.381Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "windows", + "kind": "mochitest", + "label": "test-windows11-32-25h2-shippable/opt-mochitest-browser-chrome-5", + "project": "autoland", + "retrigger": "true", + "test-type": "mochitest", + "test-suite": "mochitest-browser-chrome", + "trust-domain": "gecko", + "test-platform": "windows11-32-25h2-shippable/opt", + "tests_grouped": "1", + "createdForUser": "agoloman@mozilla.com", + "worker-implementation": "generic-worker" + } + }, + "workerGroup": "westus", + "workerId": "vm-slqhmhydqni6evb1vuglbaat8icl5saifla", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.fRj5qgRVS4adFANUGOexwA.0.westus.vm-slqhmhydqni6evb1vuglbaat8icl5saifla.gecko-t.win11-64-25h2.gecko-level-3.K3dxP_ueQcueqrIFkLqwAA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "ZFZw8IBSSWKrZmgZcdpUKA", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-noscratch-amd", + "taskQueueId": "gecko-t/t-linux-docker-noscratch-amd", + "schedulerId": "gecko-level-1", + "projectId": "none", + "taskGroupId": "ea3Ml2aSTM2XtJSn5x9OgA", + "priority": "very-low", + "deadline": "2026-06-26T22:29:42.408Z", + "expires": "2026-07-23T22:29:42.408Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "takenUntil": "2026-06-25T23:39:44.587Z", + "scheduled": "2026-06-25T22:44:51.205Z", + "started": "2026-06-25T22:45:44.383Z", + "resolved": "2026-06-25T23:21:01.122Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "web-platform-tests", + "label": "test-linux2404-64/debug-web-platform-tests-async-1", + "project": "try", + "retrigger": "true", + "test-suite": "web-platform-tests", + "test-variant": "async-event-dispatching", + "trust-domain": "gecko", + "test-platform": "linux2404-64/debug", + "tests_grouped": "1", + "createdForUser": "hikezoe.birchill@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "1690275899809730974", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.ZFZw8IBSSWKrZmgZcdpUKA.0.us-central1-b.1690275899809730974.gecko-t.t-linux-docker-noscratch-amd.gecko-level-1.ea3Ml2aSTM2XtJSn5x9OgA._", + "redelivered": false, + "cc": [] + }, + { + "payload": { + "status": { + "taskId": "f_BW7zM6RCi8p_Ah5i57Qg", + "provisionerId": "gecko-t", + "workerType": "t-linux-docker-amd", + "taskQueueId": "gecko-t/t-linux-docker-amd", + "schedulerId": "gecko-level-3", + "projectId": "none", + "taskGroupId": "cXOzgHGNRt25Mv8Aq5TvLw", + "priority": "low", + "deadline": "2026-06-26T22:51:20.160Z", + "expires": "2027-06-25T22:51:20.160Z", + "retriesLeft": 5, + "state": "failed", + "runs": [ + { + "runId": 0, + "state": "failed", + "reasonCreated": "scheduled", + "reasonResolved": "failed", + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "takenUntil": "2026-06-25T23:36:36.096Z", + "scheduled": "2026-06-25T23:16:35.979Z", + "started": "2026-06-25T23:16:36.098Z", + "resolved": "2026-06-25T23:21:06.146Z" + } + ] + }, + "runId": 0, + "task": { + "tags": { + "os": "linux", + "kind": "source-test", + "label": "source-test-node-newtab-unit-tests", + "project": "autoland", + "retrigger": "true", + "trust-domain": "gecko", + "createdForUser": "dwhisman@mozilla.com", + "worker-implementation": "docker-worker" + } + }, + "workerGroup": "us-central1-b", + "workerId": "9170129125611932268", + "version": 1 + }, + "exchange": "exchange/taskcluster-queue/v1/task-failed", + "routingKey": "primary.f_BW7zM6RCi8p_Ah5i57Qg.0.us-central1-b.9170129125611932268.gecko-t.t-linux-docker-amd.gecko-level-3.cXOzgHGNRt25Mv8Aq5TvLw._", + "redelivered": false, + "cc": [] + } +] diff --git a/services/hackbot-pulse-listener/tests/test_consumer.py b/services/hackbot-pulse-listener/tests/test_consumer.py new file mode 100644 index 0000000000..08e5981b7c --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_consumer.py @@ -0,0 +1,132 @@ +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from app import consumer + +FIXTURES = Path(__file__).parent / "fixtures" + + +def setup_function(): + consumer._seen.clear() + + +def _sample_bodies(): + data = json.loads((FIXTURES / "pulse_messages.json").read_text()) + # The inspector wraps the real AMQP body under "payload". + return [m["payload"] for m in data] + + +def _build_msg(task_id="ABC", project="autoland", label="build-linux64/opt"): + return { + "status": {"taskId": task_id}, + "runId": 0, + "task": { + "tags": { + "kind": "build", + "project": project, + "label": label, + "createdForUser": "dev@mozilla.com", + } + }, + } + + +def test_sample_messages_are_all_tests_and_skipped(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + for body in _sample_bodies(): + assert consumer.process(body, executor) is None + get_rev.assert_not_called() + trigger.assert_not_called() + executor.submit.assert_not_called() + + +def test_missing_label_is_skipped_not_crashed(): + executor = MagicMock() + body = {"status": {"taskId": "XYZ"}, "task": {"tags": {"project": "autoland"}}} + with patch.object(consumer.client, "trigger_run") as trigger: + assert consumer.process(body, executor) is None + trigger.assert_not_called() + + +def test_build_failure_triggers_run_and_submits_poll(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + run_id = consumer.process(_build_msg(), executor) + + assert run_id == "run-1" + trigger.assert_called_once() + inputs = trigger.call_args.args[0] + assert inputs["git_commit"] == "deadbeef" + assert inputs["failure_tasks"] == {"build-linux64/opt": "ABC"} + executor.submit.assert_called_once() + fn, ctx = executor.submit.call_args.args + assert fn is consumer.worker.poll_and_notify + assert ctx.run_id == "run-1" + assert ctx.git_commit == "deadbeef" + assert ctx.hg_revision == "hgrev" + assert ctx.task_id == "ABC" + assert ctx.repo == "autoland" + assert ctx.developer_email == "dev@mozilla.com" + + +def test_same_revision_triggers_once(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), + patch.object(consumer.client, "trigger_run", return_value="run-1") as trigger, + ): + consumer.process(_build_msg(task_id="T1"), executor) + consumer.process(_build_msg(task_id="T2"), executor) + + trigger.assert_called_once() + + +def test_unwatched_project_skipped_before_api_call(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision") as get_rev, + patch.object(consumer.client, "trigger_run") as trigger, + ): + assert consumer.process(_build_msg(project="try"), executor) is None + + get_rev.assert_not_called() + trigger.assert_not_called() + + +def test_unmappable_revision_skipped(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value=None), + patch.object(consumer.client, "trigger_run") as trigger, + ): + assert consumer.process(_build_msg(), executor) is None + + trigger.assert_not_called() + executor.submit.assert_not_called() + + +def test_trigger_failure_releases_revision_for_retry(): + executor = MagicMock() + with ( + patch.object(consumer.taskcluster, "get_hg_revision", return_value="hgrev"), + patch.object(consumer.lando, "hg_to_git", return_value="deadbeef"), + patch.object( + consumer.client, "trigger_run", side_effect=[RuntimeError("boom"), "run-2"] + ) as trigger, + ): + assert consumer.process(_build_msg(task_id="T1"), executor) is None + # Same revision can be retried because the failed claim was released. + assert consumer.process(_build_msg(task_id="T2"), executor) == "run-2" + + assert trigger.call_count == 2 diff --git a/services/hackbot-pulse-listener/tests/test_notify.py b/services/hackbot-pulse-listener/tests/test_notify.py new file mode 100644 index 0000000000..42f83e6331 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_notify.py @@ -0,0 +1,213 @@ +import base64 +from unittest.mock import MagicMock, patch + +from app import notify +from app.models import RunContext + + +def _ctx(**over): + base = dict( + run_id="run-1", + repo="autoland", + git_commit="deadbeefcafe", + hg_revision="0123456789ab", + task_id="TASK123", + developer_email="dev@mozilla.com", + ) + base.update(over) + return RunContext(**base) + + +def test_skips_without_recipient(): + # No developer, no team, no override -> nothing to send, must not raise. + notify.send_email(_ctx(developer_email=None), {"status": "succeeded"}) + + +def test_skips_without_sendgrid_config(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", None) + monkeypatch.setattr(notify.settings, "notification_sender", None) + notify.send_email(_ctx(), {"status": "succeeded"}) + + +def test_skips_when_not_succeeded(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + with patch("sendgrid.SendGridAPIClient") as sg: + notify.send_email(_ctx(), {"status": "failed"}) + sg.assert_not_called() + + +def test_body_contains_source_links(): + body = notify._build_body(_ctx(), {"status": "succeeded", "summary": {}}) + assert "https://github.com/mozilla-firefox/firefox/commit/deadbeefcafe" in body + assert "https://hg.mozilla.org/mozilla-unified/rev/0123456789ab" in body + assert "https://firefox-ci-tc.services.mozilla.com/tasks/TASK123" in body + + +def test_body_contains_bug_link_when_present(): + run_doc = {"status": "succeeded", "summary": {"findings": {"bug_id": 12345}}} + body = notify._build_body(_ctx(), run_doc) + assert "https://bugzilla.mozilla.org/show_bug.cgi?id=12345" in body + + no_bug = notify._build_body(_ctx(), {"status": "succeeded", "summary": {}}) + assert "show_bug.cgi" not in no_bug + + +def test_body_contains_ui_link_and_summary(monkeypatch): + monkeypatch.setattr(notify.settings, "hackbot_ui_url", "https://ui.example/") + body = notify._build_body( + _ctx(), + { + "status": "succeeded", + "summary": { + "findings": { + "summary": "Fixed a missing include", + "analysis": "The commit removed a needed header", + "local_build_verified": True, + } + }, + }, + ) + assert "https://ui.example/runs/run-1" in body + assert "Fixed a missing include" in body + assert "The commit removed a needed header" in body + assert "Local build verified: True" in body + + +def test_body_includes_patch(): + body = notify._build_body( + _ctx(), + {"status": "succeeded", "summary": {}}, + patch="--- a/f\n+++ b/f\n@@ -1 +1 @@\n-old\n+new\n", + ) + assert "## Proposed patch" in body + assert "```diff" in body + assert "+new" in body + + +def test_analysis_headings_demoted_under_section(): + run_doc = { + "status": "succeeded", + "summary": {"findings": {"analysis": "# Root cause\n\n## Details\ntext"}}, + } + body = notify._build_body(_ctx(), run_doc) + assert "## Analysis" in body + assert "### Root cause" in body + assert "#### Details" in body + + +def test_demote_headings_leaves_code_fences_and_includes_alone(): + md = "```cpp\n#include \n```" + assert notify._demote_headings(md) == md + + +def test_sends_email_when_configured(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) + + fake_client.send.assert_called_once() + + +def test_override_sends_even_without_developer_email(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr( + notify.settings, "notification_override_email", "me@mozilla.com" + ) + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email( + _ctx(developer_email=None), {"status": "succeeded", "summary": {}} + ) + + fake_client.send.assert_called_once() + + +def test_skips_when_no_patch_and_notify_only_with_patch(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", True) + + with patch("sendgrid.SendGridAPIClient") as sg: + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) + sg.assert_not_called() + + +def test_sends_without_patch_when_gate_disabled(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + monkeypatch.setattr(notify.settings, "notify_only_with_patch", False) + + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with patch("sendgrid.SendGridAPIClient", return_value=fake_client): + notify.send_email(_ctx(), {"status": "succeeded", "summary": {}}) + + fake_client.send.assert_called_once() + + +def test_fetch_patch_returns_none_without_artifact(): + assert notify._fetch_patch("run-1", {"artifacts": []}) is None + + +def test_fetch_patch_downloads_listed_artifact(): + run_doc = {"artifacts": [{"name": notify.PATCH_ARTIFACT}]} + with patch.object(notify.client, "get_artifact", return_value="THE PATCH") as ga: + assert notify._fetch_patch("run-1", run_doc) == "THE PATCH" + ga.assert_called_once_with("run-1", notify.PATCH_ARTIFACT) + + +def test_recipients_author_and_team(monkeypatch): + monkeypatch.setattr(notify.settings, "notification_override_email", None) + monkeypatch.setattr(notify.settings, "notification_team_email", "team@mozilla.com") + assert notify._recipients("dev@mozilla.com") == [ + "dev@mozilla.com", + "team@mozilla.com", + ] + + +def test_recipients_override_wins(monkeypatch): + monkeypatch.setattr( + notify.settings, "notification_override_email", "me@mozilla.com" + ) + monkeypatch.setattr(notify.settings, "notification_team_email", "team@mozilla.com") + assert notify._recipients("dev@mozilla.com") == ["me@mozilla.com"] + + +def test_recipients_dedupes_and_skips_empty(monkeypatch): + monkeypatch.setattr(notify.settings, "notification_override_email", None) + monkeypatch.setattr(notify.settings, "notification_team_email", "dev@mozilla.com") + assert notify._recipients("dev@mozilla.com") == ["dev@mozilla.com"] + monkeypatch.setattr(notify.settings, "notification_team_email", None) + assert notify._recipients(None) == [] + + +def test_attaches_patch_file(monkeypatch): + monkeypatch.setattr(notify.settings, "sendgrid_api_key", "key") + monkeypatch.setattr(notify.settings, "notification_sender", "from@mozilla.com") + run_doc = { + "status": "succeeded", + "artifacts": [{"name": notify.PATCH_ARTIFACT}], + "summary": {"findings": {}}, + } + fake_client = MagicMock() + fake_client.send.return_value = MagicMock(status_code=202) + with ( + patch("sendgrid.SendGridAPIClient", return_value=fake_client), + patch.object(notify.client, "get_artifact", return_value="DIFF-CONTENT"), + ): + notify.send_email(_ctx(), run_doc) + + attachments = fake_client.send.call_args.kwargs["message"].get()["attachments"] + assert len(attachments) == 1 + assert attachments[0]["filename"] == "changes.patch" + assert base64.b64decode(attachments[0]["content"]).decode() == "DIFF-CONTENT" diff --git a/services/hackbot-pulse-listener/tests/test_worker.py b/services/hackbot-pulse-listener/tests/test_worker.py new file mode 100644 index 0000000000..d1533be3d6 --- /dev/null +++ b/services/hackbot-pulse-listener/tests/test_worker.py @@ -0,0 +1,39 @@ +from unittest.mock import patch + +from app import worker +from app.models import RunContext + +CTX = RunContext( + run_id="run-1", + repo="autoland", + git_commit="deadbeef", + hg_revision="hg123", + task_id="T1", + developer_email="dev@mozilla.com", +) + + +def test_terminal_run_notifies_once(): + run_doc = {"status": "succeeded", "summary": {}} + with ( + patch.object(worker.client, "get_run", return_value=run_doc) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify(CTX) + + get_run.assert_called_once() + notify.send_email.assert_called_once_with(CTX, run_doc) + + +def test_gives_up_after_max_age(monkeypatch): + monkeypatch.setattr(worker.settings, "run_max_age_minutes", 0) + with ( + patch.object( + worker.client, "get_run", return_value={"status": "running"} + ) as get_run, + patch.object(worker, "notify") as notify, + ): + worker.poll_and_notify(CTX) + + get_run.assert_called_once() + notify.send_email.assert_not_called() diff --git a/uv.lock b/uv.lock index 0de875449c..2ce6678e7e 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,7 @@ members = [ "hackbot-agent-frontend-triage", "hackbot-agent-test-plan-generator", "hackbot-api", + "hackbot-pulse-listener", "hackbot-runtime", "reviewhelper-api", ] @@ -2584,6 +2585,40 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "hackbot-pulse-listener" +version = "0.1.0" +source = { editable = "services/hackbot-pulse-listener" } +dependencies = [ + { name = "cachetools" }, + { name = "httpx" }, + { name = "kombu" }, + { name = "markdown2" }, + { name = "pydantic-settings" }, + { name = "sendgrid" }, + { name = "sentry-sdk" }, + { name = "taskcluster" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "cachetools", specifier = ">=5.3.0" }, + { name = "httpx", specifier = ">=0.26.0" }, + { name = "kombu", specifier = ">=5.6,<6" }, + { name = "markdown2", specifier = ">=2.4.0" }, + { name = "pydantic-settings", specifier = ">=2.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "sendgrid", specifier = ">=6.12.5" }, + { name = "sentry-sdk", specifier = ">=2.51.0" }, + { name = "taskcluster", specifier = ">=97.1,<100.4" }, +] +provides-extras = ["dev"] + [[package]] name = "hackbot-runtime" version = "0.1.0"