From 62c779b6709440dce1291ac3dcd7e0a162763097 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 23 Oct 2026 19:33:10 +0200 Subject: [PATCH] refactor: each observability service's config sits in a root folder named after it, out of backend/ Grafana, Loki, OpenSearch, VictoriaMetrics and both collectors read their config through compose mounts; none of it is in the backend image. In backend/ a dashboard edit counted as a backend change and ran every CI job. The Grafana contract tests get their own job for backend and service config changes, and .trivyignore moves to the root since it covers both images. --- .github/labeler.yml | 9 ++-- .github/workflows/ci.yml | 49 ++++++++++++++++--- .github/workflows/docker.yml | 2 +- .github/workflows/release-deploy.yml | 12 ++--- backend/.trivyignore => .trivyignore | 0 AGENTS.md | 11 ++++- .../tests/contract/test_grafana_metrics.py | 6 +-- docker-compose.yaml | 14 +++--- docs/operations/cicd.md | 12 +++-- docs/operations/deployment.md | 6 +-- docs/operations/grafana-integration.md | 8 +-- docs/operations/loki.md | 4 +- docs/operations/metrics-reference.md | 4 +- docs/operations/opensearch.md | 8 +-- docs/security/supply-chain.md | 2 +- docs/testing/contract-testing.md | 2 +- {backend/grafana => grafana}/grafana.ini | 0 .../provisioning/alerting/alerting.yml | 0 .../dashboards/coordinator-execution.json | 0 .../provisioning/dashboards/dashboard.yml | 0 .../dashboards/dlq-monitoring.json | 0 .../provisioning/dashboards/errors.json | 0 .../provisioning/dashboards/event-replay.json | 0 .../dashboards/event-stream-monitoring.json | 0 .../dashboards/http-middleware.json | 0 .../provisioning/dashboards/integr8scode.json | 0 .../dashboards/kafka-events-monitoring.json | 0 .../dashboards/kubernetes-pods.json | 0 .../dashboards/mongodb-monitoring.json | 0 .../dashboards/notifications.json | 0 .../dashboards/rate-limiting-dashboard.json | 0 .../dashboards/security-auth.json | 0 .../provisioning/datasources/datasources.yml | 0 .../provisioning/plugins/plugins.yml | 0 .../otel-host-metrics.yaml | 0 {backend/loki => loki}/loki-config.yaml | 0 .../logs-retention-policy.json | 0 .../logs-template.json | 0 {backend/opensearch => opensearch}/start.sh | 0 .../otel-collector-config.yaml | 4 +- .../scrape.yaml | 0 41 files changed, 100 insertions(+), 53 deletions(-) rename backend/.trivyignore => .trivyignore (100%) rename {backend/grafana => grafana}/grafana.ini (100%) rename {backend/grafana => grafana}/provisioning/alerting/alerting.yml (100%) rename {backend/grafana => grafana}/provisioning/dashboards/coordinator-execution.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/dashboard.yml (100%) rename {backend/grafana => grafana}/provisioning/dashboards/dlq-monitoring.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/errors.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/event-replay.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/event-stream-monitoring.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/http-middleware.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/integr8scode.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/kafka-events-monitoring.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/kubernetes-pods.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/mongodb-monitoring.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/notifications.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/rate-limiting-dashboard.json (100%) rename {backend/grafana => grafana}/provisioning/dashboards/security-auth.json (100%) rename {backend/grafana => grafana}/provisioning/datasources/datasources.yml (100%) rename {backend/grafana => grafana}/provisioning/plugins/plugins.yml (100%) rename {backend => host-metrics}/otel-host-metrics.yaml (100%) rename {backend/loki => loki}/loki-config.yaml (100%) rename {backend/opensearch => opensearch}/logs-retention-policy.json (100%) rename {backend/opensearch => opensearch}/logs-template.json (100%) rename {backend/opensearch => opensearch}/start.sh (100%) rename {backend => otel-collector}/otel-collector-config.yaml (99%) rename {backend/victoria-metrics => victoria-metrics}/scrape.yaml (100%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 628037589..46ac01cb2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -65,9 +65,12 @@ - backend/app/core/tracing/** - backend/app/core/logging.py - backend/app/core/middlewares/metrics.py - - backend/grafana/** - - backend/loki/** - - backend/otel-collector-config.yaml + - grafana/** + - host-metrics/** + - loki/** + - opensearch/** + - otel-collector/** + - victoria-metrics/** - backend/tests/contract/** "area: deploy": diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dd5de3b7..cdd4d56fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: backend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.backend == 'true' }} frontend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.frontend == 'true' }} infra: ${{ github.event_name != 'pull_request' || steps.filter.outputs.infra == 'true' }} + observability: ${{ github.event_name != 'pull_request' || steps.filter.outputs.observability == 'true' }} api: ${{ github.event_name != 'pull_request' || steps.filter.outputs.api == 'true' }} docs: ${{ github.event_name != 'pull_request' || steps.filter.outputs.docs == 'true' }} steps: @@ -54,6 +55,15 @@ jobs: - 'docker-compose.yaml' - 'deploy.sh' - '.github/actions/**' + # The configs of the observability services, each in the folder named after its compose service + observability: &observability + - *workflow + - 'grafana/**' + - 'host-metrics/**' + - 'loki/**' + - 'opensearch/**' + - 'otel-collector/**' + - 'victoria-metrics/**' api: - *workflow - 'backend/**' @@ -63,9 +73,10 @@ jobs: - 'frontend/openapi-ts.config.ts' - 'frontend/package.json' - 'frontend/package-lock.json' - # The docs include snippets of backend and frontend files and build the spec from the backend + # The docs include snippets of backend, frontend and observability files and build the spec from the backend docs: - *workflow + - *observability - 'docs/**' - 'mkdocs.yml' - 'backend/**' @@ -346,12 +357,6 @@ jobs: --cov=app \ --cov-report=xml --cov-report=term - - name: Run Grafana contract tests - timeout-minutes: 2 - run: | - cd backend - uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract - - name: Upload coverage to Codecov uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 if: always() @@ -363,6 +368,33 @@ jobs: fail_ci_if_error: false verbose: true + # Every metric a dashboard or alert rule reads exists, and every metric the collector makes is read + grafana-contract: + name: Grafana Contract Tests + needs: [changes] + if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.observability == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up uv + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + enable-cache: true + cache-dependency-glob: "backend/uv.lock" + + - name: Install Python dependencies + run: | + cd backend + uv python install 3.14 + uv sync --frozen --group test --no-dev --no-install-project --no-build + + - name: Run Grafana contract tests + timeout-minutes: 2 + run: | + cd backend + uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract + frontend-unit: name: Frontend Unit Tests needs: [changes] @@ -686,6 +718,7 @@ jobs: - generated-api - docs - backend-unit + - grafana-contract - frontend-unit - build-images - backend-e2e @@ -696,5 +729,5 @@ jobs: with: allowed-skips: >- ruff, mypy, grimp, security, frontend, generated-api, docs, - backend-unit, frontend-unit, build-images, backend-e2e, frontend-e2e + backend-unit, grafana-contract, frontend-unit, build-images, backend-e2e, frontend-e2e jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 39b938851..010ae4eb2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -58,7 +58,7 @@ jobs: ignore-unfixed: true severity: 'CRITICAL,HIGH' timeout: '5m0s' - trivyignores: 'backend/.trivyignore' + trivyignores: '.trivyignore' version: 'v0.75.0' exit-code: '1' # without it the action drops the severity filter for SARIF, and the gate would also fail on LOW and MEDIUM diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml index fb5fe2cbc..45437a8f5 100644 --- a/.github/workflows/release-deploy.yml +++ b/.github/workflows/release-deploy.yml @@ -225,12 +225,12 @@ jobs: # These read their mounted config only at start, and compose recreates a container only when its service definition changes RESTART="" - git diff --quiet "$PREVIOUS" HEAD -- backend/otel-collector-config.yaml || RESTART="$RESTART otel-collector" - git diff --quiet "$PREVIOUS" HEAD -- backend/otel-host-metrics.yaml || RESTART="$RESTART host-metrics" - git diff --quiet "$PREVIOUS" HEAD -- backend/victoria-metrics || RESTART="$RESTART victoria-metrics" - git diff --quiet "$PREVIOUS" HEAD -- backend/loki || RESTART="$RESTART loki" - git diff --quiet "$PREVIOUS" HEAD -- backend/opensearch || RESTART="$RESTART opensearch" - git diff --quiet "$PREVIOUS" HEAD -- backend/grafana ':(exclude)backend/grafana/provisioning/dashboards' || RESTART="$RESTART grafana" + git diff --quiet "$PREVIOUS" HEAD -- otel-collector || RESTART="$RESTART otel-collector" + git diff --quiet "$PREVIOUS" HEAD -- host-metrics || RESTART="$RESTART host-metrics" + git diff --quiet "$PREVIOUS" HEAD -- victoria-metrics || RESTART="$RESTART victoria-metrics" + git diff --quiet "$PREVIOUS" HEAD -- loki || RESTART="$RESTART loki" + git diff --quiet "$PREVIOUS" HEAD -- opensearch || RESTART="$RESTART opensearch" + git diff --quiet "$PREVIOUS" HEAD -- grafana ':(exclude)grafana/provisioning/dashboards' || RESTART="$RESTART grafana" if [ -n "$RESTART" ]; then echo "Config changed, restarting:$RESTART" docker compose restart $RESTART diff --git a/backend/.trivyignore b/.trivyignore similarity index 100% rename from backend/.trivyignore rename to .trivyignore diff --git a/AGENTS.md b/AGENTS.md index 99b1615ac..0c1a03edf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,8 +51,17 @@ frontend/ ├── lib/notifications/ # Notification SSE stream ├── lib/query-client.ts # the one TanStack Query client, provided by the root layout └── styles/ # Global CSS + +grafana/ # grafana.ini and provisioning/: datasources, dashboards, alert rules +host-metrics/ # otel-host-metrics.yaml, the host's CPU, memory, disk and network +loki/ # loki-config.yaml +opensearch/ # start.sh (the entrypoint), the logs index template and retention policy +otel-collector/ # otel-collector-config.yaml: every pipeline, span metrics (signaltometrics), exporters +victoria-metrics/ # scrape.yaml ``` +backend/ and frontend/ are the build contexts of the two images. Each other folder holds the config of the compose service it's named after, mounted read-only. + ## Development Commands ```bash @@ -465,7 +474,7 @@ class ExecutionService: - Location: `app/services/` - Constructor injection: repo, producer, logger, settings, and `trace.Tracer` when the service makes spans -- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `backend/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer +- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `otel-collector/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer - Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`. An event sent after its work is done, which no caller waits on (auth and settings events, pod progress, the result processor's announcements), goes through `produce_or_log`: it still raises a MongoDB error but only logs a Kafka one, so a Kafka outage doesn't fail a login or leave a finished pod behind - Services orchestrate domain logic; no direct DB or HTTP calls - Services build new documents with the document's constructor and hand them to the repository; they never import `app.schemas_pydantic` diff --git a/backend/tests/contract/test_grafana_metrics.py b/backend/tests/contract/test_grafana_metrics.py index d64526c12..823e9af0c 100644 --- a/backend/tests/contract/test_grafana_metrics.py +++ b/backend/tests/contract/test_grafana_metrics.py @@ -17,11 +17,11 @@ from app.core.metrics.base import PROCESS_METRICS -BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent -GRAFANA_DIR = BACKEND_ROOT / "grafana" / "provisioning" +REPO_ROOT = Path(__file__).resolve().parents[3] +GRAFANA_DIR = REPO_ROOT / "grafana" / "provisioning" DASHBOARDS_DIR = GRAFANA_DIR / "dashboards" ALERTING_DIR = GRAFANA_DIR / "alerting" -COLLECTOR_CONFIG = BACKEND_ROOT / "otel-collector-config.yaml" +COLLECTOR_CONFIG = REPO_ROOT / "otel-collector" / "otel-collector-config.yaml" # Series written by otel-collector receivers rather than by any Python process, keyed by name prefix. COLLECTOR_PREFIXES: dict[str, str] = { diff --git a/docker-compose.yaml b/docker-compose.yaml index 0220d21eb..3dd0b2946 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -171,8 +171,8 @@ services: ports: - "127.0.0.1:3000:3000" volumes: - - ./backend/grafana/grafana.ini:/etc/grafana/grafana.ini:ro - - ./backend/grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./grafana/provisioning:/etc/grafana/provisioning:ro - grafana_data:/var/lib/grafana networks: - app-network @@ -345,7 +345,7 @@ services: ports: - "127.0.0.1:3100:3100" volumes: - - ./backend/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro + - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro - loki_data:/loki command: -config.file=/etc/loki/local-config.yaml networks: @@ -373,7 +373,7 @@ services: ports: - "127.0.0.1:9200:9200" volumes: - - ./backend/opensearch:/etc/opensearch-setup:ro + - ./opensearch:/etc/opensearch-setup:ro - opensearch_data:/usr/share/opensearch/data networks: - app-network @@ -398,7 +398,7 @@ services: - "127.0.0.1:8428:8428" volumes: - victoria_metrics_data:/victoria-metrics-data - - ./backend/victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro + - ./victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro command: - "--storageDataPath=/victoria-metrics-data" - "--httpListenAddr=0.0.0.0:8428" @@ -428,7 +428,7 @@ services: mem_limit: 192m command: ["--config=/etc/otel-collector-config.yaml"] volumes: - - ./backend/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + - ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro environment: # The backend creates this user on start; until then the mongodb receiver's scrapes fail and retry - MONGO_MONITOR_USER=${MONGO_MONITOR_USER:-otel-monitor} @@ -480,7 +480,7 @@ services: mem_limit: 128m command: ["--config=/etc/otel-host-metrics.yaml"] volumes: - - ./backend/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro + - ./host-metrics/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro - /proc:/hostfs/proc:ro - /sys:/hostfs/sys:ro depends_on: diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index f4dbe0475..4ea5ab163 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -5,10 +5,11 @@ CI is eight GitHub Actions workflows, two composite actions and a Dependabot con ```mermaid graph LR subgraph ST["CI"] - UB["backend unit + Grafana contract"] --> Build["build and push 2 images"] + UB["backend unit"] --> Build["build and push 2 images"] UF["frontend unit"] --> Build Build --> EB["backend E2E"] Build --> EF["frontend E2E, 2 shards"] + GC["Grafana contract"] end subgraph DSP["Docker Scan & Promote"] Scan["Trivy, 2 images"] --> Promote["crane: sha tag to latest"] @@ -49,11 +50,12 @@ CI is the only workflow a PR has to pass, and it runs only what the PR can affec | Changed | Jobs | |-----------------------------------------------------------------|----------------------------------------------------------------------| -| `backend/` | ruff, mypy, grimp, bandit, backend unit tests | +| `backend/` | ruff, mypy, grimp, bandit, backend unit tests, Grafana contract tests | +| `grafana/`, `host-metrics/`, `loki/`, `opensearch/`, `otel-collector/`, `victoria-metrics/` | Grafana contract tests, Docs Build | | `frontend/` | frontend lint and type check, frontend unit tests | | `backend/`, `frontend/`, `.env.example`, compose, deploy.sh or the composite actions | the images and frontend E2E; backend E2E unless only `frontend/` changed | | `backend/`, the spec, the generated client or its generator config | Spec and client match the code | -| `docs/`, mkdocs.yml, or `backend/` and `frontend/`, since the docs include snippets of them | Docs Build | +| `docs/`, mkdocs.yml, or `backend/`, `frontend/` and the service config folders, since the docs include snippets of them | Docs Build | A change to ci.yml itself runs every job, and so does every push to `main`. The last job, All checks pass, runs `re-actors/alls-green` over the others: a job skipped because its files didn't change counts as passed, and a failed or cancelled one fails it. FastAPI, pydantic and CPython gate their PRs the same way. All checks pass is the one required status check, so a PR that only touches the docs waits for the docs build and nothing else, and a new job only needs adding to its `needs`. @@ -63,7 +65,7 @@ Stacked PRs come from `gh stack`. CI runs for every PR in a stack, each against ## Tests and images -The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. The same job then runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract`. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`. +The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. A job of its own runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract` when `backend/` or a service config folder changed, so a dashboard edit doesn't build the images or run E2E. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`. Once both pass, or are skipped because only one side changed, one job builds the two images and pushes them to GHCR as `sha-<7 chars>`: @@ -86,7 +88,7 @@ The backend E2E job runs `pytest tests/e2e` inside the backend container after ` ## Docker Scan & Promote -The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `backend/.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22. +The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22. ## Release & Deploy diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index c5da15265..7506ccd88 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -106,10 +106,10 @@ How long each store keeps its data, and how much of it gets there, is set in the | What | Where | Value | |---|---|---| | Metrics | `--retentionPeriod` of `victoria-metrics` in docker-compose.yaml | 30 days | -| Logs | `retention_period` in backend/loki/loki-config.yaml | 720h, 30 days | -| Searchable logs | the `logs-30d` policy in backend/opensearch/logs-retention-policy.json | 30 days, one index a day | +| Logs | `retention_period` in loki/loki-config.yaml | 720h, 30 days | +| Searchable logs | the `logs-30d` policy in opensearch/logs-retention-policy.json | 30 days, one index a day | | Traces | `max_traces` in the `--set` flag of `jaeger` in docker-compose.yaml | 50 000, about 4 days at 12 600 a day; in memory, so a restart empties it | -| Trace sampling | `tail_sampling` in backend/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest | +| Trace sampling | `tail_sampling` in otel-collector/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest | Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki`, `opensearch` and `otel-collector` when their config changes. diff --git a/docs/operations/grafana-integration.md b/docs/operations/grafana-integration.md index ab87780ca..c21122d62 100644 --- a/docs/operations/grafana-integration.md +++ b/docs/operations/grafana-integration.md @@ -1,6 +1,6 @@ # Grafana integration -Grafana runs under the `observability` compose profile. Everything it shows is provisioned from [`backend/grafana/`](https://github.com/HardMax71/Integr8sCode/tree/main/backend/grafana): three datasources, thirteen dashboards, thirteen alert rules, an email contact point and a heartbeat webhook. Locally it's at `http://localhost:3000`; in production nginx serves it on `grafana.integr8scode.cc` (see [Nginx configuration](nginx-configuration.md#grafana-server)). Anonymous access is off. The admin login comes from `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` (default `admin` / `admin123`), and the entrypoint resets the password to that value on every start. +Grafana runs under the `observability` compose profile. Everything it shows is provisioned from [`grafana/`](https://github.com/HardMax71/Integr8sCode/tree/main/grafana): three datasources, thirteen dashboards, thirteen alert rules, an email contact point and a heartbeat webhook. Locally it's at `http://localhost:3000`; in production nginx serves it on `grafana.integr8scode.cc` (see [Nginx configuration](nginx-configuration.md#grafana-server)). Anonymous access is off. The admin login comes from `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` (default `admin` / `admin123`), and the entrypoint resets the password to that value on every start. ## Datasources @@ -35,10 +35,10 @@ All dashboards land in one folder, `Integr8sCode`, with the fixed UID `integr8sc ## Alerting -[`provisioning/alerting/alerting.yml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/grafana/provisioning/alerting/alerting.yml) holds the rules, the contact point and the policy: +[`provisioning/alerting/alerting.yml`](https://github.com/HardMax71/Integr8sCode/blob/main/grafana/provisioning/alerting/alerting.yml) holds the rules, the contact point and the policy: ```yaml ---8<-- "backend/grafana/provisioning/alerting/alerting.yml" +--8<-- "grafana/provisioning/alerting/alerting.yml" ``` Every rule is an instant query evaluated every minute, and each summary names the service or mount and the value. Rules that point at a panel carry its `dashboardUid` and `panelId`, so the mail links to it. @@ -63,7 +63,7 @@ A Grafana volume from before that fix may still have the stray folder holding th ## grafana.ini ```ini ---8<-- "backend/grafana/grafana.ini:grafana_ini" +--8<-- "grafana/grafana.ini:grafana_ini" ``` `[alerting]` is the legacy engine and stays off. `[unified_alerting]` runs the provisioned rules. There's no `root_url` here; compose sets `GF_SERVER_ROOT_URL` from `GRAFANA_ROOT_URL`. diff --git a/docs/operations/loki.md b/docs/operations/loki.md index 6e532067c..cc77710e9 100644 --- a/docs/operations/loki.md +++ b/docs/operations/loki.md @@ -21,7 +21,7 @@ The OTel resource on every record has `service.name` from `SERVICE_NAME` (`integ ## Collector -The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml). +The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml). The browser's records come in on a pipeline of their own, `logs/browser`. Its receiver, `otlp/browser`, takes OTLP over HTTP on port 4319 with bodies up to 64 KB, and only nginx's `/otlp/v1/logs` reaches it ([Frontend logging](logging.md#frontend)). The browser sends every level, and `filter/browser` is where lines get dropped: below `warning` when the collector's `ENVIRONMENT` is `production`, none anywhere else. Anyone can post to that path, so `transform/browser` drops whatever resource a record arrives with and sets `service.name` to `browser`: a posted record can't pass as the backend or any other service. The Errors dashboard and the error-lines alert group by `service_name`, so `browser` shows up there like any container. @@ -40,7 +40,7 @@ Loki 3 accepts OTLP natively. Resource attributes such as `service.name` and `se Single instance with filesystem storage, TSDB index and schema v13. Structured metadata is on, and retention is 30 days, the same as Victoria Metrics. ```yaml ---8<-- "backend/loki/loki-config.yaml" +--8<-- "loki/loki-config.yaml" ``` The container has a 512m memory limit (a 30-day search took it to 207 MiB), keeps data in the `loki_data` volume and listens on `127.0.0.1:3100`. Its image is distroless, so there's no healthcheck; Grafana and the collector start once the container has started. diff --git a/docs/operations/metrics-reference.md b/docs/operations/metrics-reference.md index 45f5dec0b..88bceacde 100644 --- a/docs/operations/metrics-reference.md +++ b/docs/operations/metrics-reference.md @@ -17,11 +17,11 @@ flowchart LR `create_app` and the worker's `main()` each call `setup_metrics` (`app/core/metrics/base.py`) once. It installs a global `MeterProvider` whose `PeriodicExportingMetricReader` exports every 60 000 ms through an OTLP gRPC exporter. The exporter targets `OTEL_EXPORTER_OTLP_ENDPOINT`, which `config.toml` sets to `http://otel-collector:4317`. It also starts the system metrics instrumentation for the process's own CPU and memory. Without an endpoint it returns early and the no-op provider stays. The OpenTelemetry instrumentations take their meters from that provider. The app's own metrics have no meter at all: they come from spans, described next. -The collector's metrics pipeline (`backend/otel-collector-config.yaml`) reads the `otlp`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. Host metrics come from a second collector, the `host-metrics` service (`backend/otel-host-metrics.yaml`). It runs only the `host_metrics` receiver in the host's network namespace, since in a container the network scraper sees only that container's `eth0` and `lo`, and sends to the main collector's OTLP port on `127.0.0.1:4317`. Both collectors, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next. +The collector's metrics pipeline (`otel-collector/otel-collector-config.yaml`) reads the `otlp`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. Host metrics come from a second collector, the `host-metrics` service (`host-metrics/otel-host-metrics.yaml`). It runs only the `host_metrics` receiver in the host's network namespace, since in a container the network scraper sees only that container's `eth0` and `lo`, and sends to the main collector's OTLP port on `127.0.0.1:4317`. Both collectors, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next. ## Metrics made from spans -Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `backend/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`backend/victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. Every app metric works this way since [#389](https://github.com/HardMax71/Integr8sCode/issues/389). +Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `otel-collector/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. Every app metric works this way since [#389](https://github.com/HardMax71/Integr8sCode/issues/389). ## Names and labels in VictoriaMetrics diff --git a/docs/operations/opensearch.md b/docs/operations/opensearch.md index 1235b3e3c..8a9d2eb32 100644 --- a/docs/operations/opensearch.md +++ b/docs/operations/opensearch.md @@ -11,7 +11,7 @@ flowchart LR ## Collector -Both log pipelines in [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml) send their lines to Loki and to the `forward/search` connector, so OpenSearch gets exactly what Loki gets, after the same filters. The `logs/search` pipeline runs `transform/search`, which parses the JSON body of the backend's and worker's records into attributes (`execution_id`, `user_id`, `event` and the rest of structlog's keys), and the `opensearch` exporter writes one index a day, `logs-2026.10.22`, in the exporter's default SS4O document shape: +Both log pipelines in [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml) send their lines to Loki and to the `forward/search` connector, so OpenSearch gets exactly what Loki gets, after the same filters. The `logs/search` pipeline runs `transform/search`, which parses the JSON body of the backend's and worker's records into attributes (`execution_id`, `user_id`, `event` and the rest of structlog's keys), and the `opensearch` exporter writes one index a day, `logs-2026.10.22`, in the exporter's default SS4O document shape: ```json { @@ -31,16 +31,16 @@ The exporter treats a refused connection as a permanent error and doesn't retry Single node, OpenSearch 3.9, security plugin off, listening on `127.0.0.1:9200` and on the compose network, like Loki. `node.store.allow_mmap=false` means the host's `vm.max_map_count` stays as it is. The heap is 512m and the container limit 1536m: 30 days at prod's volume sit near 1.1 GiB. Data lives in the `opensearch_data` volume, about 1.1 GB for 30 days. -The container's entrypoint is [`backend/opensearch/start.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/opensearch/start.sh). It starts OpenSearch, waits for the cluster, and then applies two things through the API: the index template and the retention policy. The healthcheck passes only once the template exists. +The container's entrypoint is [`opensearch/start.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/opensearch/start.sh). It starts OpenSearch, waits for the cluster, and then applies two things through the API: the index template and the retention policy. The healthcheck passes only once the template exists. ```json ---8<-- "backend/opensearch/logs-template.json" +--8<-- "opensearch/logs-template.json" ``` `attributes` and `resource` are `flat_object` fields. Every key under them is searchable by its dotted path, but they add no fields to the mapping, so a key that's a number in one line and a string in another (`status: 500`, `status: "running"`) can't make OpenSearch reject a line. ```json ---8<-- "backend/opensearch/logs-retention-policy.json" +--8<-- "opensearch/logs-retention-policy.json" ``` The policy deletes each daily index 30 days after it was created, the same retention as Loki. A stored policy is never overwritten, so a changed policy needs a new id. diff --git a/docs/security/supply-chain.md b/docs/security/supply-chain.md index 025e93495..d82973cba 100644 --- a/docs/security/supply-chain.md +++ b/docs/security/supply-chain.md @@ -10,7 +10,7 @@ Findings show up in the repository's Security tab under code scanning, as `backe ## Image and code scans -Docker Scan & Promote runs Trivy v0.75.0 on the `backend` and `frontend` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `backend/.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages in a step CI never caches (the backend's last stage, the frontend's `production` stage), so a fix released after a cached build still gets in, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file lists no CVE today. See [CI/CD](../operations/cicd.md#docker-scan-promote). +Docker Scan & Promote runs Trivy v0.75.0 on the `backend` and `frontend` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages in a step CI never caches (the backend's last stage, the frontend's `production` stage), so a fix released after a cached build still gets in, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file lists no CVE today. See [CI/CD](../operations/cicd.md#docker-scan-promote). CI's Security Scanning job runs `uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/ -ll` on pushes to `main` and on PRs that change `backend/`. `-ll` reports medium severity and up, and any finding fails the workflow. diff --git a/docs/testing/contract-testing.md b/docs/testing/contract-testing.md index 98bcfc91b..b9fe0961c 100644 --- a/docs/testing/contract-testing.md +++ b/docs/testing/contract-testing.md @@ -22,7 +22,7 @@ All three tests carry the `grafana_contract` marker. ## Running -CI runs them in the backend unit job, right after the unit tests: +CI runs them in the Grafana Contract Tests job whenever `backend/` or one of the service config folders (`grafana/`, `otel-collector/` and the rest) changes: ```bash cd backend diff --git a/backend/grafana/grafana.ini b/grafana/grafana.ini similarity index 100% rename from backend/grafana/grafana.ini rename to grafana/grafana.ini diff --git a/backend/grafana/provisioning/alerting/alerting.yml b/grafana/provisioning/alerting/alerting.yml similarity index 100% rename from backend/grafana/provisioning/alerting/alerting.yml rename to grafana/provisioning/alerting/alerting.yml diff --git a/backend/grafana/provisioning/dashboards/coordinator-execution.json b/grafana/provisioning/dashboards/coordinator-execution.json similarity index 100% rename from backend/grafana/provisioning/dashboards/coordinator-execution.json rename to grafana/provisioning/dashboards/coordinator-execution.json diff --git a/backend/grafana/provisioning/dashboards/dashboard.yml b/grafana/provisioning/dashboards/dashboard.yml similarity index 100% rename from backend/grafana/provisioning/dashboards/dashboard.yml rename to grafana/provisioning/dashboards/dashboard.yml diff --git a/backend/grafana/provisioning/dashboards/dlq-monitoring.json b/grafana/provisioning/dashboards/dlq-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/dlq-monitoring.json rename to grafana/provisioning/dashboards/dlq-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/errors.json b/grafana/provisioning/dashboards/errors.json similarity index 100% rename from backend/grafana/provisioning/dashboards/errors.json rename to grafana/provisioning/dashboards/errors.json diff --git a/backend/grafana/provisioning/dashboards/event-replay.json b/grafana/provisioning/dashboards/event-replay.json similarity index 100% rename from backend/grafana/provisioning/dashboards/event-replay.json rename to grafana/provisioning/dashboards/event-replay.json diff --git a/backend/grafana/provisioning/dashboards/event-stream-monitoring.json b/grafana/provisioning/dashboards/event-stream-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/event-stream-monitoring.json rename to grafana/provisioning/dashboards/event-stream-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/http-middleware.json b/grafana/provisioning/dashboards/http-middleware.json similarity index 100% rename from backend/grafana/provisioning/dashboards/http-middleware.json rename to grafana/provisioning/dashboards/http-middleware.json diff --git a/backend/grafana/provisioning/dashboards/integr8scode.json b/grafana/provisioning/dashboards/integr8scode.json similarity index 100% rename from backend/grafana/provisioning/dashboards/integr8scode.json rename to grafana/provisioning/dashboards/integr8scode.json diff --git a/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json b/grafana/provisioning/dashboards/kafka-events-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/kafka-events-monitoring.json rename to grafana/provisioning/dashboards/kafka-events-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/kubernetes-pods.json b/grafana/provisioning/dashboards/kubernetes-pods.json similarity index 100% rename from backend/grafana/provisioning/dashboards/kubernetes-pods.json rename to grafana/provisioning/dashboards/kubernetes-pods.json diff --git a/backend/grafana/provisioning/dashboards/mongodb-monitoring.json b/grafana/provisioning/dashboards/mongodb-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/mongodb-monitoring.json rename to grafana/provisioning/dashboards/mongodb-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/notifications.json b/grafana/provisioning/dashboards/notifications.json similarity index 100% rename from backend/grafana/provisioning/dashboards/notifications.json rename to grafana/provisioning/dashboards/notifications.json diff --git a/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json b/grafana/provisioning/dashboards/rate-limiting-dashboard.json similarity index 100% rename from backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json rename to grafana/provisioning/dashboards/rate-limiting-dashboard.json diff --git a/backend/grafana/provisioning/dashboards/security-auth.json b/grafana/provisioning/dashboards/security-auth.json similarity index 100% rename from backend/grafana/provisioning/dashboards/security-auth.json rename to grafana/provisioning/dashboards/security-auth.json diff --git a/backend/grafana/provisioning/datasources/datasources.yml b/grafana/provisioning/datasources/datasources.yml similarity index 100% rename from backend/grafana/provisioning/datasources/datasources.yml rename to grafana/provisioning/datasources/datasources.yml diff --git a/backend/grafana/provisioning/plugins/plugins.yml b/grafana/provisioning/plugins/plugins.yml similarity index 100% rename from backend/grafana/provisioning/plugins/plugins.yml rename to grafana/provisioning/plugins/plugins.yml diff --git a/backend/otel-host-metrics.yaml b/host-metrics/otel-host-metrics.yaml similarity index 100% rename from backend/otel-host-metrics.yaml rename to host-metrics/otel-host-metrics.yaml diff --git a/backend/loki/loki-config.yaml b/loki/loki-config.yaml similarity index 100% rename from backend/loki/loki-config.yaml rename to loki/loki-config.yaml diff --git a/backend/opensearch/logs-retention-policy.json b/opensearch/logs-retention-policy.json similarity index 100% rename from backend/opensearch/logs-retention-policy.json rename to opensearch/logs-retention-policy.json diff --git a/backend/opensearch/logs-template.json b/opensearch/logs-template.json similarity index 100% rename from backend/opensearch/logs-template.json rename to opensearch/logs-template.json diff --git a/backend/opensearch/start.sh b/opensearch/start.sh similarity index 100% rename from backend/opensearch/start.sh rename to opensearch/start.sh diff --git a/backend/otel-collector-config.yaml b/otel-collector/otel-collector-config.yaml similarity index 99% rename from backend/otel-collector-config.yaml rename to otel-collector/otel-collector-config.yaml index 5713ac89f..cecd4a400 100644 --- a/backend/otel-collector-config.yaml +++ b/otel-collector/otel-collector-config.yaml @@ -939,7 +939,7 @@ exporters: tls: insecure: true - # One index a day, so the 30-day retention policy in backend/opensearch deletes whole days + # One index a day, so the 30-day retention policy in opensearch/logs-retention-policy.json deletes whole days opensearch: http: endpoint: http://opensearch:9200 @@ -982,7 +982,7 @@ service: processors: [memory_limiter, batch, resource] exporters: [prometheus_remote_write, debug] - # VictoriaMetrics scrapes these every 15 s, see backend/victoria-metrics/scrape.yaml + # VictoriaMetrics scrapes these every 15 s, see victoria-metrics/scrape.yaml metrics/span_counts: receivers: [signaltometrics] processors: [memory_limiter, filter/no_gauges, deltatocumulative, batch, resource] diff --git a/backend/victoria-metrics/scrape.yaml b/victoria-metrics/scrape.yaml similarity index 100% rename from backend/victoria-metrics/scrape.yaml rename to victoria-metrics/scrape.yaml