refactor: each observability service's config sits in a root folder named after it, out of backend/

.github/labeler.yml

@@ -65,9 +65,12 @@
           - backend/app/core/tracing/**
           - backend/app/core/logging.py
           - backend/app/core/middlewares/metrics.py
-          - backend/grafana/**
-          - backend/loki/**
-          - backend/otel-collector-config.yaml
+          - grafana/**
+          - host-metrics/**
+          - loki/**
+          - opensearch/**
+          - otel-collector/**
+          - victoria-metrics/**
           - backend/tests/contract/**
 
 "area: deploy":

.github/workflows/ci.yml

@@ -32,6 +32,7 @@ jobs:
       backend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.backend == 'true' }}
       frontend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.frontend == 'true' }}
       infra: ${{ github.event_name != 'pull_request' || steps.filter.outputs.infra == 'true' }}
+      observability: ${{ github.event_name != 'pull_request' || steps.filter.outputs.observability == 'true' }}
       api: ${{ github.event_name != 'pull_request' || steps.filter.outputs.api == 'true' }}
       docs: ${{ github.event_name != 'pull_request' || steps.filter.outputs.docs == 'true' }}
     steps:
@@ -54,6 +55,15 @@ jobs:
               - 'docker-compose.yaml'
               - 'deploy.sh'
               - '.github/actions/**'
+            # The configs of the observability services, each in the folder named after its compose service
+            observability: &observability
+              - *workflow
+              - 'grafana/**'
+              - 'host-metrics/**'
+              - 'loki/**'
+              - 'opensearch/**'
+              - 'otel-collector/**'
+              - 'victoria-metrics/**'
             api:
               - *workflow
               - 'backend/**'
@@ -63,9 +73,10 @@ jobs:
               - 'frontend/openapi-ts.config.ts'
               - 'frontend/package.json'
               - 'frontend/package-lock.json'
-            # The docs include snippets of backend and frontend files and build the spec from the backend
+            # The docs include snippets of backend, frontend and observability files and build the spec from the backend
             docs:
               - *workflow
+              - *observability
               - 'docs/**'
               - 'mkdocs.yml'
               - 'backend/**'
@@ -346,12 +357,6 @@ jobs:
             --cov=app \
             --cov-report=xml --cov-report=term
 
-      - name: Run Grafana contract tests
-        timeout-minutes: 2
-        run: |
-          cd backend
-          uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract
-
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5  # v7.1.1
         if: always()
@@ -363,6 +368,33 @@ jobs:
           fail_ci_if_error: false
           verbose: true
 
+  # Every metric a dashboard or alert rule reads exists, and every metric the collector makes is read
+  grafana-contract:
+    name: Grafana Contract Tests
+    needs: [changes]
+    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.observability == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884  # v10.3.0
+        with:
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
+
+      - name: Install Python dependencies
+        run: |
+          cd backend
+          uv python install 3.14
+          uv sync --frozen --group test --no-dev --no-install-project --no-build
+
+      - name: Run Grafana contract tests
+        timeout-minutes: 2
+        run: |
+          cd backend
+          uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract
+
   frontend-unit:
     name: Frontend Unit Tests
     needs: [changes]
@@ -686,6 +718,7 @@ jobs:
       - generated-api
       - docs
       - backend-unit
+      - grafana-contract
       - frontend-unit
       - build-images
       - backend-e2e
@@ -696,5 +729,5 @@ jobs:
         with:
           allowed-skips: >-
             ruff, mypy, grimp, security, frontend, generated-api, docs,
-            backend-unit, frontend-unit, build-images, backend-e2e, frontend-e2e
+            backend-unit, grafana-contract, frontend-unit, build-images, backend-e2e, frontend-e2e
           jobs: ${{ toJSON(needs) }}

.github/workflows/docker.yml

@@ -58,7 +58,7 @@ jobs:
           ignore-unfixed: true
           severity: 'CRITICAL,HIGH'
           timeout: '5m0s'
-          trivyignores: 'backend/.trivyignore'
+          trivyignores: '.trivyignore'
           version: 'v0.75.0'
           exit-code: '1'
           # without it the action drops the severity filter for SARIF, and the gate would also fail on LOW and MEDIUM

.github/workflows/release-deploy.yml

@@ -225,12 +225,12 @@ jobs:
 
             # These read their mounted config only at start, and compose recreates a container only when its service definition changes
             RESTART=""
-            git diff --quiet "$PREVIOUS" HEAD -- backend/otel-collector-config.yaml || RESTART="$RESTART otel-collector"
-            git diff --quiet "$PREVIOUS" HEAD -- backend/otel-host-metrics.yaml || RESTART="$RESTART host-metrics"
-            git diff --quiet "$PREVIOUS" HEAD -- backend/victoria-metrics || RESTART="$RESTART victoria-metrics"
-            git diff --quiet "$PREVIOUS" HEAD -- backend/loki || RESTART="$RESTART loki"
-            git diff --quiet "$PREVIOUS" HEAD -- backend/opensearch || RESTART="$RESTART opensearch"
-            git diff --quiet "$PREVIOUS" HEAD -- backend/grafana ':(exclude)backend/grafana/provisioning/dashboards' || RESTART="$RESTART grafana"
+            git diff --quiet "$PREVIOUS" HEAD -- otel-collector || RESTART="$RESTART otel-collector"
+            git diff --quiet "$PREVIOUS" HEAD -- host-metrics || RESTART="$RESTART host-metrics"
+            git diff --quiet "$PREVIOUS" HEAD -- victoria-metrics || RESTART="$RESTART victoria-metrics"
+            git diff --quiet "$PREVIOUS" HEAD -- loki || RESTART="$RESTART loki"
+            git diff --quiet "$PREVIOUS" HEAD -- opensearch || RESTART="$RESTART opensearch"
+            git diff --quiet "$PREVIOUS" HEAD -- grafana ':(exclude)grafana/provisioning/dashboards' || RESTART="$RESTART grafana"
             if [ -n "$RESTART" ]; then
               echo "Config changed, restarting:$RESTART"
               docker compose restart $RESTART

AGENTS.md

@@ -51,8 +51,17 @@ frontend/
     ├── lib/notifications/   # Notification SSE stream
     ├── lib/query-client.ts  # the one TanStack Query client, provided by the root layout
     └── styles/              # Global CSS
+
+grafana/                     # grafana.ini and provisioning/: datasources, dashboards, alert rules
+host-metrics/                # otel-host-metrics.yaml, the host's CPU, memory, disk and network
+loki/                        # loki-config.yaml
+opensearch/                  # start.sh (the entrypoint), the logs index template and retention policy
+otel-collector/              # otel-collector-config.yaml: every pipeline, span metrics (signaltometrics), exporters
+victoria-metrics/            # scrape.yaml
 ```
 
+backend/ and frontend/ are the build contexts of the two images. Each other folder holds the config of the compose service it's named after, mounted read-only.
+
 ## Development Commands
 
 ```bash
@@ -465,7 +474,7 @@ class ExecutionService:
 
 - Location: `app/services/`
 - Constructor injection: repo, producer, logger, settings, and `trace.Tracer` when the service makes spans
-- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `backend/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer
+- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `otel-collector/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer
 - Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`. An event sent after its work is done, which no caller waits on (auth and settings events, pod progress, the result processor's announcements), goes through `produce_or_log`: it still raises a MongoDB error but only logs a Kafka one, so a Kafka outage doesn't fail a login or leave a finished pod behind
 - Services orchestrate domain logic; no direct DB or HTTP calls
 - Services build new documents with the document's constructor and hand them to the repository; they never import `app.schemas_pydantic`

backend/tests/contract/test_grafana_metrics.py

@@ -17,11 +17,11 @@
 
 from app.core.metrics.base import PROCESS_METRICS
 
-BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent
-GRAFANA_DIR = BACKEND_ROOT / "grafana" / "provisioning"
+REPO_ROOT = Path(__file__).resolve().parents[3]
+GRAFANA_DIR = REPO_ROOT / "grafana" / "provisioning"
 DASHBOARDS_DIR = GRAFANA_DIR / "dashboards"
 ALERTING_DIR = GRAFANA_DIR / "alerting"
-COLLECTOR_CONFIG = BACKEND_ROOT / "otel-collector-config.yaml"
+COLLECTOR_CONFIG = REPO_ROOT / "otel-collector" / "otel-collector-config.yaml"
 
 # Series written by otel-collector receivers rather than by any Python process, keyed by name prefix.
 COLLECTOR_PREFIXES: dict[str, str] = {

docker-compose.yaml

@@ -171,8 +171,8 @@ services:
     ports:
       - "127.0.0.1:3000:3000"
     volumes:
-      - ./backend/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
-      - ./backend/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
       - grafana_data:/var/lib/grafana
     networks:
       - app-network
@@ -345,7 +345,7 @@ services:
     ports:
       - "127.0.0.1:3100:3100"
     volumes:
-      - ./backend/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
       - loki_data:/loki
     command: -config.file=/etc/loki/local-config.yaml
     networks:
@@ -373,7 +373,7 @@ services:
     ports:
       - "127.0.0.1:9200:9200"
     volumes:
-      - ./backend/opensearch:/etc/opensearch-setup:ro
+      - ./opensearch:/etc/opensearch-setup:ro
       - opensearch_data:/usr/share/opensearch/data
     networks:
       - app-network
@@ -398,7 +398,7 @@ services:
       - "127.0.0.1:8428:8428"
     volumes:
       - victoria_metrics_data:/victoria-metrics-data
-      - ./backend/victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro
+      - ./victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--httpListenAddr=0.0.0.0:8428"
@@ -428,7 +428,7 @@ services:
     mem_limit: 192m
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
-      - ./backend/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      - ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
     environment:
       # The backend creates this user on start; until then the mongodb receiver's scrapes fail and retry
       - MONGO_MONITOR_USER=${MONGO_MONITOR_USER:-otel-monitor}
@@ -480,7 +480,7 @@ services:
     mem_limit: 128m
     command: ["--config=/etc/otel-host-metrics.yaml"]
     volumes:
-      - ./backend/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro
+      - ./host-metrics/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro
       - /proc:/hostfs/proc:ro
       - /sys:/hostfs/sys:ro
     depends_on:

docs/operations/cicd.md

@@ -5,10 +5,11 @@ CI is eight GitHub Actions workflows, two composite actions and a Dependabot con
 ```mermaid
 graph LR
     subgraph ST["CI"]
-        UB["backend unit + Grafana contract"] --> Build["build and push 2 images"]
+        UB["backend unit"] --> Build["build and push 2 images"]
         UF["frontend unit"] --> Build
         Build --> EB["backend E2E"]
         Build --> EF["frontend E2E, 2 shards"]
+        GC["Grafana contract"]
     end
     subgraph DSP["Docker Scan & Promote"]
         Scan["Trivy, 2 images"] --> Promote["crane: sha tag to latest"]
@@ -49,11 +50,12 @@ CI is the only workflow a PR has to pass, and it runs only what the PR can affec
 
 | Changed                                                         | Jobs                                                                 |
 |-----------------------------------------------------------------|----------------------------------------------------------------------|
-| `backend/`                                                      | ruff, mypy, grimp, bandit, backend unit tests                        |
+| `backend/`                                                      | ruff, mypy, grimp, bandit, backend unit tests, Grafana contract tests |
+| `grafana/`, `host-metrics/`, `loki/`, `opensearch/`, `otel-collector/`, `victoria-metrics/` | Grafana contract tests, Docs Build |
 | `frontend/`                                                     | frontend lint and type check, frontend unit tests                    |
 | `backend/`, `frontend/`, `.env.example`, compose, deploy.sh or the composite actions | the images and frontend E2E; backend E2E unless only `frontend/` changed |
 | `backend/`, the spec, the generated client or its generator config | Spec and client match the code                                    |
-| `docs/`, mkdocs.yml, or `backend/` and `frontend/`, since the docs include snippets of them | Docs Build                                  |
+| `docs/`, mkdocs.yml, or `backend/`, `frontend/` and the service config folders, since the docs include snippets of them | Docs Build |
 
 A change to ci.yml itself runs every job, and so does every push to `main`. The last job, All checks pass, runs `re-actors/alls-green` over the others: a job skipped because its files didn't change counts as passed, and a failed or cancelled one fails it. FastAPI, pydantic and CPython gate their PRs the same way. All checks pass is the one required status check, so a PR that only touches the docs waits for the docs build and nothing else, and a new job only needs adding to its `needs`.
 
@@ -63,7 +65,7 @@ Stacked PRs come from `gh stack`. CI runs for every PR in a stack, each against
 
 ## Tests and images
 
-The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. The same job then runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract`. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`.
+The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. A job of its own runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract` when `backend/` or a service config folder changed, so a dashboard edit doesn't build the images or run E2E. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`.
 
 Once both pass, or are skipped because only one side changed, one job builds the two images and pushes them to GHCR as `sha-<7 chars>`:
 
@@ -86,7 +88,7 @@ The backend E2E job runs `pytest tests/e2e` inside the backend container after `
 
 ## Docker Scan & Promote
 
-The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `backend/.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22.
+The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22.
 
 ## Release & Deploy
 

docs/operations/deployment.md

@@ -106,10 +106,10 @@ How long each store keeps its data, and how much of it gets there, is set in the
 | What | Where | Value |
 |---|---|---|
 | Metrics | `--retentionPeriod` of `victoria-metrics` in docker-compose.yaml | 30 days |
-| Logs | `retention_period` in backend/loki/loki-config.yaml | 720h, 30 days |
-| Searchable logs | the `logs-30d` policy in backend/opensearch/logs-retention-policy.json | 30 days, one index a day |
+| Logs | `retention_period` in loki/loki-config.yaml | 720h, 30 days |
+| Searchable logs | the `logs-30d` policy in opensearch/logs-retention-policy.json | 30 days, one index a day |
 | Traces | `max_traces` in the `--set` flag of `jaeger` in docker-compose.yaml | 50 000, about 4 days at 12 600 a day; in memory, so a restart empties it |
-| Trace sampling | `tail_sampling` in backend/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest |
+| Trace sampling | `tail_sampling` in otel-collector/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest |
 
 Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki`, `opensearch` and `otel-collector` when their config changes.