diff --git a/.github/labeler.yml b/.github/labeler.yml index 628037589..46ac01cb2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -65,9 +65,12 @@ - backend/app/core/tracing/** - backend/app/core/logging.py - backend/app/core/middlewares/metrics.py - - backend/grafana/** - - backend/loki/** - - backend/otel-collector-config.yaml + - grafana/** + - host-metrics/** + - loki/** + - opensearch/** + - otel-collector/** + - victoria-metrics/** - backend/tests/contract/** "area: deploy": diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dd5de3b7..cdd4d56fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: backend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.backend == 'true' }} frontend: ${{ github.event_name != 'pull_request' || steps.filter.outputs.frontend == 'true' }} infra: ${{ github.event_name != 'pull_request' || steps.filter.outputs.infra == 'true' }} + observability: ${{ github.event_name != 'pull_request' || steps.filter.outputs.observability == 'true' }} api: ${{ github.event_name != 'pull_request' || steps.filter.outputs.api == 'true' }} docs: ${{ github.event_name != 'pull_request' || steps.filter.outputs.docs == 'true' }} steps: @@ -54,6 +55,15 @@ jobs: - 'docker-compose.yaml' - 'deploy.sh' - '.github/actions/**' + # The configs of the observability services, each in the folder named after its compose service + observability: &observability + - *workflow + - 'grafana/**' + - 'host-metrics/**' + - 'loki/**' + - 'opensearch/**' + - 'otel-collector/**' + - 'victoria-metrics/**' api: - *workflow - 'backend/**' @@ -63,9 +73,10 @@ jobs: - 'frontend/openapi-ts.config.ts' - 'frontend/package.json' - 'frontend/package-lock.json' - # The docs include snippets of backend and frontend files and build the spec from the backend + # The docs include snippets of backend, frontend and observability files and build the spec from the backend docs: - *workflow + - *observability - 'docs/**' - 'mkdocs.yml' - 'backend/**' @@ -346,12 +357,6 @@ jobs: --cov=app \ --cov-report=xml --cov-report=term - - name: Run Grafana contract tests - timeout-minutes: 2 - run: | - cd backend - uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract - - name: Upload coverage to Codecov uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 if: always() @@ -363,6 +368,33 @@ jobs: fail_ci_if_error: false verbose: true + # Every metric a dashboard or alert rule reads exists, and every metric the collector makes is read + grafana-contract: + name: Grafana Contract Tests + needs: [changes] + if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.observability == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up uv + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + enable-cache: true + cache-dependency-glob: "backend/uv.lock" + + - name: Install Python dependencies + run: | + cd backend + uv python install 3.14 + uv sync --frozen --group test --no-dev --no-install-project --no-build + + - name: Run Grafana contract tests + timeout-minutes: 2 + run: | + cd backend + uv run --no-sync --no-build pytest tests/contract -v -rs -m grafana_contract + frontend-unit: name: Frontend Unit Tests needs: [changes] @@ -686,6 +718,7 @@ jobs: - generated-api - docs - backend-unit + - grafana-contract - frontend-unit - build-images - backend-e2e @@ -696,5 +729,5 @@ jobs: with: allowed-skips: >- ruff, mypy, grimp, security, frontend, generated-api, docs, - backend-unit, frontend-unit, build-images, backend-e2e, frontend-e2e + backend-unit, grafana-contract, frontend-unit, build-images, backend-e2e, frontend-e2e jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 39b938851..010ae4eb2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -58,7 +58,7 @@ jobs: ignore-unfixed: true severity: 'CRITICAL,HIGH' timeout: '5m0s' - trivyignores: 'backend/.trivyignore' + trivyignores: '.trivyignore' version: 'v0.75.0' exit-code: '1' # without it the action drops the severity filter for SARIF, and the gate would also fail on LOW and MEDIUM diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml index fb5fe2cbc..45437a8f5 100644 --- a/.github/workflows/release-deploy.yml +++ b/.github/workflows/release-deploy.yml @@ -225,12 +225,12 @@ jobs: # These read their mounted config only at start, and compose recreates a container only when its service definition changes RESTART="" - git diff --quiet "$PREVIOUS" HEAD -- backend/otel-collector-config.yaml || RESTART="$RESTART otel-collector" - git diff --quiet "$PREVIOUS" HEAD -- backend/otel-host-metrics.yaml || RESTART="$RESTART host-metrics" - git diff --quiet "$PREVIOUS" HEAD -- backend/victoria-metrics || RESTART="$RESTART victoria-metrics" - git diff --quiet "$PREVIOUS" HEAD -- backend/loki || RESTART="$RESTART loki" - git diff --quiet "$PREVIOUS" HEAD -- backend/opensearch || RESTART="$RESTART opensearch" - git diff --quiet "$PREVIOUS" HEAD -- backend/grafana ':(exclude)backend/grafana/provisioning/dashboards' || RESTART="$RESTART grafana" + git diff --quiet "$PREVIOUS" HEAD -- otel-collector || RESTART="$RESTART otel-collector" + git diff --quiet "$PREVIOUS" HEAD -- host-metrics || RESTART="$RESTART host-metrics" + git diff --quiet "$PREVIOUS" HEAD -- victoria-metrics || RESTART="$RESTART victoria-metrics" + git diff --quiet "$PREVIOUS" HEAD -- loki || RESTART="$RESTART loki" + git diff --quiet "$PREVIOUS" HEAD -- opensearch || RESTART="$RESTART opensearch" + git diff --quiet "$PREVIOUS" HEAD -- grafana ':(exclude)grafana/provisioning/dashboards' || RESTART="$RESTART grafana" if [ -n "$RESTART" ]; then echo "Config changed, restarting:$RESTART" docker compose restart $RESTART diff --git a/backend/.trivyignore b/.trivyignore similarity index 100% rename from backend/.trivyignore rename to .trivyignore diff --git a/AGENTS.md b/AGENTS.md index 99b1615ac..0c1a03edf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,8 +51,17 @@ frontend/ ├── lib/notifications/ # Notification SSE stream ├── lib/query-client.ts # the one TanStack Query client, provided by the root layout └── styles/ # Global CSS + +grafana/ # grafana.ini and provisioning/: datasources, dashboards, alert rules +host-metrics/ # otel-host-metrics.yaml, the host's CPU, memory, disk and network +loki/ # loki-config.yaml +opensearch/ # start.sh (the entrypoint), the logs index template and retention policy +otel-collector/ # otel-collector-config.yaml: every pipeline, span metrics (signaltometrics), exporters +victoria-metrics/ # scrape.yaml ``` +backend/ and frontend/ are the build contexts of the two images. Each other folder holds the config of the compose service it's named after, mounted read-only. + ## Development Commands ```bash @@ -465,7 +474,7 @@ class ExecutionService: - Location: `app/services/` - Constructor injection: repo, producer, logger, settings, and `trace.Tracer` when the service makes spans -- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `backend/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer +- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `otel-collector/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer - Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`. An event sent after its work is done, which no caller waits on (auth and settings events, pod progress, the result processor's announcements), goes through `produce_or_log`: it still raises a MongoDB error but only logs a Kafka one, so a Kafka outage doesn't fail a login or leave a finished pod behind - Services orchestrate domain logic; no direct DB or HTTP calls - Services build new documents with the document's constructor and hand them to the repository; they never import `app.schemas_pydantic` diff --git a/backend/tests/contract/test_grafana_metrics.py b/backend/tests/contract/test_grafana_metrics.py index d64526c12..823e9af0c 100644 --- a/backend/tests/contract/test_grafana_metrics.py +++ b/backend/tests/contract/test_grafana_metrics.py @@ -17,11 +17,11 @@ from app.core.metrics.base import PROCESS_METRICS -BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent -GRAFANA_DIR = BACKEND_ROOT / "grafana" / "provisioning" +REPO_ROOT = Path(__file__).resolve().parents[3] +GRAFANA_DIR = REPO_ROOT / "grafana" / "provisioning" DASHBOARDS_DIR = GRAFANA_DIR / "dashboards" ALERTING_DIR = GRAFANA_DIR / "alerting" -COLLECTOR_CONFIG = BACKEND_ROOT / "otel-collector-config.yaml" +COLLECTOR_CONFIG = REPO_ROOT / "otel-collector" / "otel-collector-config.yaml" # Series written by otel-collector receivers rather than by any Python process, keyed by name prefix. COLLECTOR_PREFIXES: dict[str, str] = { diff --git a/docker-compose.yaml b/docker-compose.yaml index 0220d21eb..3dd0b2946 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -171,8 +171,8 @@ services: ports: - "127.0.0.1:3000:3000" volumes: - - ./backend/grafana/grafana.ini:/etc/grafana/grafana.ini:ro - - ./backend/grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./grafana/provisioning:/etc/grafana/provisioning:ro - grafana_data:/var/lib/grafana networks: - app-network @@ -345,7 +345,7 @@ services: ports: - "127.0.0.1:3100:3100" volumes: - - ./backend/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro + - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro - loki_data:/loki command: -config.file=/etc/loki/local-config.yaml networks: @@ -373,7 +373,7 @@ services: ports: - "127.0.0.1:9200:9200" volumes: - - ./backend/opensearch:/etc/opensearch-setup:ro + - ./opensearch:/etc/opensearch-setup:ro - opensearch_data:/usr/share/opensearch/data networks: - app-network @@ -398,7 +398,7 @@ services: - "127.0.0.1:8428:8428" volumes: - victoria_metrics_data:/victoria-metrics-data - - ./backend/victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro + - ./victoria-metrics/scrape.yaml:/etc/victoria-metrics/scrape.yaml:ro command: - "--storageDataPath=/victoria-metrics-data" - "--httpListenAddr=0.0.0.0:8428" @@ -428,7 +428,7 @@ services: mem_limit: 192m command: ["--config=/etc/otel-collector-config.yaml"] volumes: - - ./backend/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + - ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro environment: # The backend creates this user on start; until then the mongodb receiver's scrapes fail and retry - MONGO_MONITOR_USER=${MONGO_MONITOR_USER:-otel-monitor} @@ -480,7 +480,7 @@ services: mem_limit: 128m command: ["--config=/etc/otel-host-metrics.yaml"] volumes: - - ./backend/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro + - ./host-metrics/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro - /proc:/hostfs/proc:ro - /sys:/hostfs/sys:ro depends_on: diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index f4dbe0475..4ea5ab163 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -5,10 +5,11 @@ CI is eight GitHub Actions workflows, two composite actions and a Dependabot con ```mermaid graph LR subgraph ST["CI"] - UB["backend unit + Grafana contract"] --> Build["build and push 2 images"] + UB["backend unit"] --> Build["build and push 2 images"] UF["frontend unit"] --> Build Build --> EB["backend E2E"] Build --> EF["frontend E2E, 2 shards"] + GC["Grafana contract"] end subgraph DSP["Docker Scan & Promote"] Scan["Trivy, 2 images"] --> Promote["crane: sha tag to latest"] @@ -49,11 +50,12 @@ CI is the only workflow a PR has to pass, and it runs only what the PR can affec | Changed | Jobs | |-----------------------------------------------------------------|----------------------------------------------------------------------| -| `backend/` | ruff, mypy, grimp, bandit, backend unit tests | +| `backend/` | ruff, mypy, grimp, bandit, backend unit tests, Grafana contract tests | +| `grafana/`, `host-metrics/`, `loki/`, `opensearch/`, `otel-collector/`, `victoria-metrics/` | Grafana contract tests, Docs Build | | `frontend/` | frontend lint and type check, frontend unit tests | | `backend/`, `frontend/`, `.env.example`, compose, deploy.sh or the composite actions | the images and frontend E2E; backend E2E unless only `frontend/` changed | | `backend/`, the spec, the generated client or its generator config | Spec and client match the code | -| `docs/`, mkdocs.yml, or `backend/` and `frontend/`, since the docs include snippets of them | Docs Build | +| `docs/`, mkdocs.yml, or `backend/`, `frontend/` and the service config folders, since the docs include snippets of them | Docs Build | A change to ci.yml itself runs every job, and so does every push to `main`. The last job, All checks pass, runs `re-actors/alls-green` over the others: a job skipped because its files didn't change counts as passed, and a failed or cancelled one fails it. FastAPI, pydantic and CPython gate their PRs the same way. All checks pass is the one required status check, so a PR that only touches the docs waits for the docs build and nothing else, and a new job only needs adding to its `needs`. @@ -63,7 +65,7 @@ Stacked PRs come from `gh stack`. CI runs for every PR in a stack, each against ## Tests and images -The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. The same job then runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract`. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`. +The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. A job of its own runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract` when `backend/` or a service config folder changed, so a dashboard edit doesn't build the images or run E2E. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`. Once both pass, or are skipped because only one side changed, one job builds the two images and pushes them to GHCR as `sha-<7 chars>`: @@ -86,7 +88,7 @@ The backend E2E job runs `pytest tests/e2e` inside the backend container after ` ## Docker Scan & Promote -The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `backend/.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22. +The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. CI runs finish in any order, so promotes run one at a time (`concurrency: promote-latest`), and each first checks the commits on `main` after its own: when `latest` already holds the image of one of them, it leaves `latest` alone and the Summary job is skipped. Without that check, an older merge whose CI finished last moved `latest` back a commit on 2026-10-22. ## Release & Deploy diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index c5da15265..7506ccd88 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -106,10 +106,10 @@ How long each store keeps its data, and how much of it gets there, is set in the | What | Where | Value | |---|---|---| | Metrics | `--retentionPeriod` of `victoria-metrics` in docker-compose.yaml | 30 days | -| Logs | `retention_period` in backend/loki/loki-config.yaml | 720h, 30 days | -| Searchable logs | the `logs-30d` policy in backend/opensearch/logs-retention-policy.json | 30 days, one index a day | +| Logs | `retention_period` in loki/loki-config.yaml | 720h, 30 days | +| Searchable logs | the `logs-30d` policy in opensearch/logs-retention-policy.json | 30 days, one index a day | | Traces | `max_traces` in the `--set` flag of `jaeger` in docker-compose.yaml | 50 000, about 4 days at 12 600 a day; in memory, so a restart empties it | -| Trace sampling | `tail_sampling` in backend/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest | +| Trace sampling | `tail_sampling` in otel-collector/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest | Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki`, `opensearch` and `otel-collector` when their config changes. diff --git a/docs/operations/grafana-integration.md b/docs/operations/grafana-integration.md index ab87780ca..c21122d62 100644 --- a/docs/operations/grafana-integration.md +++ b/docs/operations/grafana-integration.md @@ -1,6 +1,6 @@ # Grafana integration -Grafana runs under the `observability` compose profile. Everything it shows is provisioned from [`backend/grafana/`](https://github.com/HardMax71/Integr8sCode/tree/main/backend/grafana): three datasources, thirteen dashboards, thirteen alert rules, an email contact point and a heartbeat webhook. Locally it's at `http://localhost:3000`; in production nginx serves it on `grafana.integr8scode.cc` (see [Nginx configuration](nginx-configuration.md#grafana-server)). Anonymous access is off. The admin login comes from `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` (default `admin` / `admin123`), and the entrypoint resets the password to that value on every start. +Grafana runs under the `observability` compose profile. Everything it shows is provisioned from [`grafana/`](https://github.com/HardMax71/Integr8sCode/tree/main/grafana): three datasources, thirteen dashboards, thirteen alert rules, an email contact point and a heartbeat webhook. Locally it's at `http://localhost:3000`; in production nginx serves it on `grafana.integr8scode.cc` (see [Nginx configuration](nginx-configuration.md#grafana-server)). Anonymous access is off. The admin login comes from `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` (default `admin` / `admin123`), and the entrypoint resets the password to that value on every start. ## Datasources @@ -35,10 +35,10 @@ All dashboards land in one folder, `Integr8sCode`, with the fixed UID `integr8sc ## Alerting -[`provisioning/alerting/alerting.yml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/grafana/provisioning/alerting/alerting.yml) holds the rules, the contact point and the policy: +[`provisioning/alerting/alerting.yml`](https://github.com/HardMax71/Integr8sCode/blob/main/grafana/provisioning/alerting/alerting.yml) holds the rules, the contact point and the policy: ```yaml ---8<-- "backend/grafana/provisioning/alerting/alerting.yml" +--8<-- "grafana/provisioning/alerting/alerting.yml" ``` Every rule is an instant query evaluated every minute, and each summary names the service or mount and the value. Rules that point at a panel carry its `dashboardUid` and `panelId`, so the mail links to it. @@ -63,7 +63,7 @@ A Grafana volume from before that fix may still have the stray folder holding th ## grafana.ini ```ini ---8<-- "backend/grafana/grafana.ini:grafana_ini" +--8<-- "grafana/grafana.ini:grafana_ini" ``` `[alerting]` is the legacy engine and stays off. `[unified_alerting]` runs the provisioned rules. There's no `root_url` here; compose sets `GF_SERVER_ROOT_URL` from `GRAFANA_ROOT_URL`. diff --git a/docs/operations/loki.md b/docs/operations/loki.md index 6e532067c..cc77710e9 100644 --- a/docs/operations/loki.md +++ b/docs/operations/loki.md @@ -21,7 +21,7 @@ The OTel resource on every record has `service.name` from `SERVICE_NAME` (`integ ## Collector -The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml). +The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml). The browser's records come in on a pipeline of their own, `logs/browser`. Its receiver, `otlp/browser`, takes OTLP over HTTP on port 4319 with bodies up to 64 KB, and only nginx's `/otlp/v1/logs` reaches it ([Frontend logging](logging.md#frontend)). The browser sends every level, and `filter/browser` is where lines get dropped: below `warning` when the collector's `ENVIRONMENT` is `production`, none anywhere else. Anyone can post to that path, so `transform/browser` drops whatever resource a record arrives with and sets `service.name` to `browser`: a posted record can't pass as the backend or any other service. The Errors dashboard and the error-lines alert group by `service_name`, so `browser` shows up there like any container. @@ -40,7 +40,7 @@ Loki 3 accepts OTLP natively. Resource attributes such as `service.name` and `se Single instance with filesystem storage, TSDB index and schema v13. Structured metadata is on, and retention is 30 days, the same as Victoria Metrics. ```yaml ---8<-- "backend/loki/loki-config.yaml" +--8<-- "loki/loki-config.yaml" ``` The container has a 512m memory limit (a 30-day search took it to 207 MiB), keeps data in the `loki_data` volume and listens on `127.0.0.1:3100`. Its image is distroless, so there's no healthcheck; Grafana and the collector start once the container has started. diff --git a/docs/operations/metrics-reference.md b/docs/operations/metrics-reference.md index 45f5dec0b..88bceacde 100644 --- a/docs/operations/metrics-reference.md +++ b/docs/operations/metrics-reference.md @@ -17,11 +17,11 @@ flowchart LR `create_app` and the worker's `main()` each call `setup_metrics` (`app/core/metrics/base.py`) once. It installs a global `MeterProvider` whose `PeriodicExportingMetricReader` exports every 60 000 ms through an OTLP gRPC exporter. The exporter targets `OTEL_EXPORTER_OTLP_ENDPOINT`, which `config.toml` sets to `http://otel-collector:4317`. It also starts the system metrics instrumentation for the process's own CPU and memory. Without an endpoint it returns early and the no-op provider stays. The OpenTelemetry instrumentations take their meters from that provider. The app's own metrics have no meter at all: they come from spans, described next. -The collector's metrics pipeline (`backend/otel-collector-config.yaml`) reads the `otlp`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. Host metrics come from a second collector, the `host-metrics` service (`backend/otel-host-metrics.yaml`). It runs only the `host_metrics` receiver in the host's network namespace, since in a container the network scraper sees only that container's `eth0` and `lo`, and sends to the main collector's OTLP port on `127.0.0.1:4317`. Both collectors, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next. +The collector's metrics pipeline (`otel-collector/otel-collector-config.yaml`) reads the `otlp`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. Host metrics come from a second collector, the `host-metrics` service (`host-metrics/otel-host-metrics.yaml`). It runs only the `host_metrics` receiver in the host's network namespace, since in a container the network scraper sees only that container's `eth0` and `lo`, and sends to the main collector's OTLP port on `127.0.0.1:4317`. Both collectors, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next. ## Metrics made from spans -Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `backend/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`backend/victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. Every app metric works this way since [#389](https://github.com/HardMax71/Integr8sCode/issues/389). +Some metrics have no instrument in the code. A service puts the facts on a span, and the collector's `signaltometrics` connector (alpha in collector v0.162.0) turns them into counters, histograms and gauges, defined under `connectors` in `otel-collector/otel-collector-config.yaml`. The SDK sends every span (`TRACING_SAMPLING_RATE = 1.0`), and the traces pipeline gives each span to the connector before `tail_sampling` decides what Jaeger keeps, so the counts don't depend on sampling. The connector emits a point only when a matching span arrives. Sent straight to VictoriaMetrics, a counter that sees one event every half hour would get one point every half hour, and `increase` and `rate` stop finding the previous point once two points are more than about 8 to 10 minutes apart (measured on v1.153.0): `increase` then counts the whole total again, and `rate` returns nothing. So counters and histograms go another way. `deltatocumulative` adds up the connector's deltas and keeps each total through 24 hours without spans (its default is 5 minutes, after which an idle counter would start again from zero). The collector's `prometheus` exporter serves the totals on `otel-collector:8889`, and VictoriaMetrics scrapes it every 15 seconds (`victoria-metrics/scrape.yaml`, with `honor_labels` so each service's `job` and `instance` stay). A series the exporter hasn't updated for 24 hours drops out until its next span. Gauges still go by remote write, so a gauge goes stale when the job that reports it on a timer stops. Panels wrap counter rates in `or vector(0)` so a series that hasn't started yet reads 0. Every app metric works this way since [#389](https://github.com/HardMax71/Integr8sCode/issues/389). ## Names and labels in VictoriaMetrics diff --git a/docs/operations/opensearch.md b/docs/operations/opensearch.md index 1235b3e3c..8a9d2eb32 100644 --- a/docs/operations/opensearch.md +++ b/docs/operations/opensearch.md @@ -11,7 +11,7 @@ flowchart LR ## Collector -Both log pipelines in [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml) send their lines to Loki and to the `forward/search` connector, so OpenSearch gets exactly what Loki gets, after the same filters. The `logs/search` pipeline runs `transform/search`, which parses the JSON body of the backend's and worker's records into attributes (`execution_id`, `user_id`, `event` and the rest of structlog's keys), and the `opensearch` exporter writes one index a day, `logs-2026.10.22`, in the exporter's default SS4O document shape: +Both log pipelines in [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml) send their lines to Loki and to the `forward/search` connector, so OpenSearch gets exactly what Loki gets, after the same filters. The `logs/search` pipeline runs `transform/search`, which parses the JSON body of the backend's and worker's records into attributes (`execution_id`, `user_id`, `event` and the rest of structlog's keys), and the `opensearch` exporter writes one index a day, `logs-2026.10.22`, in the exporter's default SS4O document shape: ```json { @@ -31,16 +31,16 @@ The exporter treats a refused connection as a permanent error and doesn't retry Single node, OpenSearch 3.9, security plugin off, listening on `127.0.0.1:9200` and on the compose network, like Loki. `node.store.allow_mmap=false` means the host's `vm.max_map_count` stays as it is. The heap is 512m and the container limit 1536m: 30 days at prod's volume sit near 1.1 GiB. Data lives in the `opensearch_data` volume, about 1.1 GB for 30 days. -The container's entrypoint is [`backend/opensearch/start.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/opensearch/start.sh). It starts OpenSearch, waits for the cluster, and then applies two things through the API: the index template and the retention policy. The healthcheck passes only once the template exists. +The container's entrypoint is [`opensearch/start.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/opensearch/start.sh). It starts OpenSearch, waits for the cluster, and then applies two things through the API: the index template and the retention policy. The healthcheck passes only once the template exists. ```json ---8<-- "backend/opensearch/logs-template.json" +--8<-- "opensearch/logs-template.json" ``` `attributes` and `resource` are `flat_object` fields. Every key under them is searchable by its dotted path, but they add no fields to the mapping, so a key that's a number in one line and a string in another (`status: 500`, `status: "running"`) can't make OpenSearch reject a line. ```json ---8<-- "backend/opensearch/logs-retention-policy.json" +--8<-- "opensearch/logs-retention-policy.json" ``` The policy deletes each daily index 30 days after it was created, the same retention as Loki. A stored policy is never overwritten, so a changed policy needs a new id. diff --git a/docs/security/supply-chain.md b/docs/security/supply-chain.md index 025e93495..d82973cba 100644 --- a/docs/security/supply-chain.md +++ b/docs/security/supply-chain.md @@ -10,7 +10,7 @@ Findings show up in the repository's Security tab under code scanning, as `backe ## Image and code scans -Docker Scan & Promote runs Trivy v0.75.0 on the `backend` and `frontend` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `backend/.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages in a step CI never caches (the backend's last stage, the frontend's `production` stage), so a fix released after a cached build still gets in, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file lists no CVE today. See [CI/CD](../operations/cicd.md#docker-scan-promote). +Docker Scan & Promote runs Trivy v0.75.0 on the `backend` and `frontend` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages in a step CI never caches (the backend's last stage, the frontend's `production` stage), so a fix released after a cached build still gets in, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file lists no CVE today. See [CI/CD](../operations/cicd.md#docker-scan-promote). CI's Security Scanning job runs `uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/ -ll` on pushes to `main` and on PRs that change `backend/`. `-ll` reports medium severity and up, and any finding fails the workflow. diff --git a/docs/testing/contract-testing.md b/docs/testing/contract-testing.md index 98bcfc91b..b9fe0961c 100644 --- a/docs/testing/contract-testing.md +++ b/docs/testing/contract-testing.md @@ -22,7 +22,7 @@ All three tests carry the `grafana_contract` marker. ## Running -CI runs them in the backend unit job, right after the unit tests: +CI runs them in the Grafana Contract Tests job whenever `backend/` or one of the service config folders (`grafana/`, `otel-collector/` and the rest) changes: ```bash cd backend diff --git a/backend/grafana/grafana.ini b/grafana/grafana.ini similarity index 100% rename from backend/grafana/grafana.ini rename to grafana/grafana.ini diff --git a/backend/grafana/provisioning/alerting/alerting.yml b/grafana/provisioning/alerting/alerting.yml similarity index 100% rename from backend/grafana/provisioning/alerting/alerting.yml rename to grafana/provisioning/alerting/alerting.yml diff --git a/backend/grafana/provisioning/dashboards/coordinator-execution.json b/grafana/provisioning/dashboards/coordinator-execution.json similarity index 100% rename from backend/grafana/provisioning/dashboards/coordinator-execution.json rename to grafana/provisioning/dashboards/coordinator-execution.json diff --git a/backend/grafana/provisioning/dashboards/dashboard.yml b/grafana/provisioning/dashboards/dashboard.yml similarity index 100% rename from backend/grafana/provisioning/dashboards/dashboard.yml rename to grafana/provisioning/dashboards/dashboard.yml diff --git a/backend/grafana/provisioning/dashboards/dlq-monitoring.json b/grafana/provisioning/dashboards/dlq-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/dlq-monitoring.json rename to grafana/provisioning/dashboards/dlq-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/errors.json b/grafana/provisioning/dashboards/errors.json similarity index 100% rename from backend/grafana/provisioning/dashboards/errors.json rename to grafana/provisioning/dashboards/errors.json diff --git a/backend/grafana/provisioning/dashboards/event-replay.json b/grafana/provisioning/dashboards/event-replay.json similarity index 100% rename from backend/grafana/provisioning/dashboards/event-replay.json rename to grafana/provisioning/dashboards/event-replay.json diff --git a/backend/grafana/provisioning/dashboards/event-stream-monitoring.json b/grafana/provisioning/dashboards/event-stream-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/event-stream-monitoring.json rename to grafana/provisioning/dashboards/event-stream-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/http-middleware.json b/grafana/provisioning/dashboards/http-middleware.json similarity index 100% rename from backend/grafana/provisioning/dashboards/http-middleware.json rename to grafana/provisioning/dashboards/http-middleware.json diff --git a/backend/grafana/provisioning/dashboards/integr8scode.json b/grafana/provisioning/dashboards/integr8scode.json similarity index 100% rename from backend/grafana/provisioning/dashboards/integr8scode.json rename to grafana/provisioning/dashboards/integr8scode.json diff --git a/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json b/grafana/provisioning/dashboards/kafka-events-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/kafka-events-monitoring.json rename to grafana/provisioning/dashboards/kafka-events-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/kubernetes-pods.json b/grafana/provisioning/dashboards/kubernetes-pods.json similarity index 100% rename from backend/grafana/provisioning/dashboards/kubernetes-pods.json rename to grafana/provisioning/dashboards/kubernetes-pods.json diff --git a/backend/grafana/provisioning/dashboards/mongodb-monitoring.json b/grafana/provisioning/dashboards/mongodb-monitoring.json similarity index 100% rename from backend/grafana/provisioning/dashboards/mongodb-monitoring.json rename to grafana/provisioning/dashboards/mongodb-monitoring.json diff --git a/backend/grafana/provisioning/dashboards/notifications.json b/grafana/provisioning/dashboards/notifications.json similarity index 100% rename from backend/grafana/provisioning/dashboards/notifications.json rename to grafana/provisioning/dashboards/notifications.json diff --git a/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json b/grafana/provisioning/dashboards/rate-limiting-dashboard.json similarity index 100% rename from backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json rename to grafana/provisioning/dashboards/rate-limiting-dashboard.json diff --git a/backend/grafana/provisioning/dashboards/security-auth.json b/grafana/provisioning/dashboards/security-auth.json similarity index 100% rename from backend/grafana/provisioning/dashboards/security-auth.json rename to grafana/provisioning/dashboards/security-auth.json diff --git a/backend/grafana/provisioning/datasources/datasources.yml b/grafana/provisioning/datasources/datasources.yml similarity index 100% rename from backend/grafana/provisioning/datasources/datasources.yml rename to grafana/provisioning/datasources/datasources.yml diff --git a/backend/grafana/provisioning/plugins/plugins.yml b/grafana/provisioning/plugins/plugins.yml similarity index 100% rename from backend/grafana/provisioning/plugins/plugins.yml rename to grafana/provisioning/plugins/plugins.yml diff --git a/backend/otel-host-metrics.yaml b/host-metrics/otel-host-metrics.yaml similarity index 100% rename from backend/otel-host-metrics.yaml rename to host-metrics/otel-host-metrics.yaml diff --git a/backend/loki/loki-config.yaml b/loki/loki-config.yaml similarity index 100% rename from backend/loki/loki-config.yaml rename to loki/loki-config.yaml diff --git a/backend/opensearch/logs-retention-policy.json b/opensearch/logs-retention-policy.json similarity index 100% rename from backend/opensearch/logs-retention-policy.json rename to opensearch/logs-retention-policy.json diff --git a/backend/opensearch/logs-template.json b/opensearch/logs-template.json similarity index 100% rename from backend/opensearch/logs-template.json rename to opensearch/logs-template.json diff --git a/backend/opensearch/start.sh b/opensearch/start.sh similarity index 100% rename from backend/opensearch/start.sh rename to opensearch/start.sh diff --git a/backend/otel-collector-config.yaml b/otel-collector/otel-collector-config.yaml similarity index 99% rename from backend/otel-collector-config.yaml rename to otel-collector/otel-collector-config.yaml index 5713ac89f..cecd4a400 100644 --- a/backend/otel-collector-config.yaml +++ b/otel-collector/otel-collector-config.yaml @@ -939,7 +939,7 @@ exporters: tls: insecure: true - # One index a day, so the 30-day retention policy in backend/opensearch deletes whole days + # One index a day, so the 30-day retention policy in opensearch/logs-retention-policy.json deletes whole days opensearch: http: endpoint: http://opensearch:9200 @@ -982,7 +982,7 @@ service: processors: [memory_limiter, batch, resource] exporters: [prometheus_remote_write, debug] - # VictoriaMetrics scrapes these every 15 s, see backend/victoria-metrics/scrape.yaml + # VictoriaMetrics scrapes these every 15 s, see victoria-metrics/scrape.yaml metrics/span_counts: receivers: [signaltometrics] processors: [memory_limiter, filter/no_gauges, deltatocumulative, batch, resource] diff --git a/backend/victoria-metrics/scrape.yaml b/victoria-metrics/scrape.yaml similarity index 100% rename from backend/victoria-metrics/scrape.yaml rename to victoria-metrics/scrape.yaml