refactor: deploy.sh goes, each task runs through its own tool, and the Mongo backup is a compose service

.env.example

@@ -1,4 +1,4 @@
-# Local development values. ./deploy.sh dev copies this file to .env, which compose reads.
+# Local development values. Copy this file to .env once (cp .env.example .env); compose reads .env.
 # Production never sees this file: its deploy writes .env from the repository secrets.
 SECRET_KEY=CHANGE_ME_min_32_chars_long_!!!!
 MONGO_USER=root

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -49,7 +49,7 @@ body:
         - Kafka events (consumers, DLQ, replay)
         - Live updates and notifications
         - Metrics, dashboards and logs
-        - Deployment (compose, deploy.sh, production)
+        - Deployment (compose, production)
         - CI and tooling
         - Documentation
         - Not sure
@@ -62,7 +62,7 @@ body:
       label: Environment
       description: Whatever applies.
       placeholder: |
-        - Where: local dev (./deploy.sh dev) or production
+        - Where: local dev (docker compose up) or production
         - Version: release tag or commit, e.g. 2026.10.3 or 7b9f060
         - Script language and version, e.g. Go 1.27
         - Browser, e.g. Firefox 145

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -45,7 +45,7 @@ body:
         - Kafka events (consumers, DLQ, replay)
         - Live updates and notifications
         - Metrics, dashboards and logs
-        - Deployment (compose, deploy.sh, production)
+        - Deployment (compose, production)
         - CI and tooling
         - Documentation
         - Not sure

.github/PULL_REQUEST_TEMPLATE.md

@@ -31,7 +31,7 @@ Fixes #
 - [ ] Frontend unit tests pass (`cd frontend && npm run test`)
 - [ ] Backend E2E tests pass (`docker compose exec -T backend sh -c 'uv sync --group test --no-dev --frozen --no-install-project --no-build && pytest tests/e2e'`)
 - [ ] Frontend E2E tests pass (`cd frontend && npm run test:e2e`)
-- [ ] API spec and client regenerated if routes or schemas changed (`./deploy.sh openapi`, then `./deploy.sh types`), never edited by hand
+- [ ] API spec and client regenerated if routes or schemas changed (`uv run python -m scripts.export_openapi` in backend/, then `npm run generate:api` in frontend/), never edited by hand
 - [ ] Manual testing performed
 
 **Manual test steps:**

.github/actions/e2e-ready/action.yml

@@ -36,7 +36,8 @@ runs:
       shell: bash
       run: |
         set -e
-        cp backend/config.test.toml backend/config.toml
+        # compose reads .env for every later command, so the backend and the worker both run with the test settings
+        echo BACKEND_CONFIG=config.test.toml >> .env
 
         timeout 120 tail --pid="$(cat /tmp/infra-pull.pid)" -f /dev/null 2>/dev/null || true
         cat /tmp/infra-pull.log 2>/dev/null || true

.github/labeler.yml

@@ -77,7 +77,6 @@
   - changed-files:
       - any-glob-to-any-file:
           - docker-compose.yaml
-          - deploy.sh
           - "**/Dockerfile*"
           - .env.example
           - .github/workflows/docker.yml

.github/labels.yml

@@ -9,7 +9,7 @@
   description: "GitHub workflows, test infrastructure, repo tooling"
 - name: "area: deploy"
   color: "#c5def5"
-  description: "Compose, deploy.sh, images, release workflow, production host"
+  description: "Compose, images, release workflow, production host"
 - name: "area: events"
   color: "#c5def5"
   description: "Kafka consumers and middlewares, producer, idempotency, DLQ, replay"

.github/workflows/ci.yml

@@ -53,7 +53,6 @@ jobs:
               - *workflow
               - '.env.example'
               - 'docker-compose.yaml'
-              - 'deploy.sh'
               - '.github/actions/**'
             # The configs of the observability services, each in the folder named after its compose service
             observability: &observability
@@ -67,7 +66,6 @@ jobs:
             api:
               - *workflow
               - 'backend/**'
-              - 'deploy.sh'
               - 'docs/reference/openapi.json'
               - 'frontend/src/lib/api/**'
               - 'frontend/openapi-ts.config.ts'
@@ -82,7 +80,6 @@ jobs:
               - 'backend/**'
               - 'frontend/**'
               - 'docker-compose.yaml'
-              - 'deploy.sh'
 
   ruff:
     name: Ruff Linting
@@ -176,7 +173,7 @@ jobs:
           cache-dependency-glob: "backend/uv.lock"
 
       - name: Run bandit
-        run: uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/ -ll
+        run: uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/,backend/.venv/ -ll
 
   frontend:
     name: Frontend Lint & Type Check
@@ -250,14 +247,14 @@ jobs:
 
       - name: Regenerate the spec and the client
         run: |
-          ./deploy.sh openapi
+          (cd backend && uv run --no-sync --no-build python -m scripts.export_openapi)
           cd frontend && npm run generate:api
 
       - name: Fail on any difference
         run: |
           changes=$(git status --porcelain -- docs/reference/openapi.json frontend/src/lib/api)
           if [ -n "$changes" ]; then
-            echo "::error::docs/reference/openapi.json and frontend/src/lib/api must come from the generators, never from hand edits. Run ./deploy.sh openapi, then ./deploy.sh types, and commit the result."
+            echo "::error::docs/reference/openapi.json and frontend/src/lib/api must come from the generators, never from hand edits. Run uv run python -m scripts.export_openapi in backend/, then npm run generate:api in frontend/, and commit the result."
             echo "$changes"
             git diff --stat -- docs/reference/openapi.json frontend/src/lib/api
             exit 1
@@ -281,7 +278,7 @@ jobs:
         run: cd backend && uv sync --frozen --no-dev --no-install-project --no-build
 
       - name: Generate OpenAPI spec
-        run: ./deploy.sh openapi
+        run: cd backend && uv run --no-sync --no-build python -m scripts.export_openapi
 
       - name: Build documentation
         run: >-

.pre-commit-config.yaml

@@ -38,6 +38,14 @@ repos:
         files: ^backend/.*\.py$
         pass_filenames: false
 
+      # Bandit - matches CI: uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/,backend/.venv/ -ll
+      - id: bandit-backend
+        name: bandit (backend)
+        entry: uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/,backend/.venv/ -ll
+        language: system
+        files: ^backend/.*\.py$
+        pass_filenames: false
+
       # ESLint - matches CI: cd frontend && npm run lint
       - id: eslint-frontend
         name: eslint (frontend)

AGENTS.md

@@ -10,7 +10,7 @@ Platform for running scripts online in isolated Kubernetes pods. Users write Pyt
 | Backend | FastAPI, Dishka DI, Beanie ODM (MongoDB), Python 3.14 (backend/.python-version), async throughout, httpx2 for HTTP |
 | Events | Kafka (KRaft, no ZooKeeper) + FastStream (Pydantic JSON) |
 | Worker | one process (`app/worker.py`): execution scheduler, k8s worker, pod monitor, result processor, DLQ retries |
-| Infra | MongoDB 9, Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector plus its host-metrics collector (`--observability`) and Kafdrop (`--debug`) |
+| Infra | MongoDB 9 with a daily mongodump (`mongo-backup`), Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector plus its host-metrics collector (compose profile `observability`) and Kafdrop (profile `debug`) |
 | Runtimes | Python 3.11 to 3.14, Node 22/24/26, Ruby 3.3/3.4/4.0, Bash 5.3, Go 1.26/1.27 (app/runtime_registry.py) |
 
 ## Key Directory Map
@@ -87,17 +87,20 @@ npm run test:e2e:browsers # once: Playwright Chromium
 npm run test:e2e          # Playwright E2E, needs the running stack
 
 # Full stack (from project root)
-./deploy.sh dev                  # start
-./deploy.sh dev --build          # rebuild and start
-./deploy.sh dev --wait           # start and wait for healthy
-./deploy.sh dev --observability  # plus Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector, host-metrics
-./deploy.sh dev --debug          # observability plus Kafdrop
-./deploy.sh down                 # stop
-./deploy.sh openapi              # regenerate docs/reference/openapi.json
-./deploy.sh types                # regenerate frontend/src/lib/api/ from that spec
+cp .env.example .env                           # once: the local secrets
+docker compose up -d                           # start
+docker compose up -d --build                   # rebuild and start
+docker compose up -d --wait                    # start and wait for healthy
+docker compose --profile observability up -d   # plus Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector, host-metrics
+docker compose --profile debug up -d           # plus Kafdrop
+docker compose down                            # stop
+
+# Generated API client
+cd backend && uv run python -m scripts.export_openapi   # regenerate docs/reference/openapi.json
+cd frontend && npm run generate:api                     # regenerate frontend/src/lib/api/ from that spec
 ```
 
-`npm run dev` serves HTTPS when ./certs/server.key and server.crt exist and proxies /api to `https://backend:443` with the certificate checked, so it needs `NODE_EXTRA_CA_CERTS` pointing at the backend's `server.crt` (the `backend_tls_cert` volume). `backend` only resolves inside the compose network, so on the host the API calls fail. The compose `frontend` service already serves a production build on :5001; rebuild it with `./deploy.sh dev --build` after frontend changes.
+`npm run dev` serves HTTPS when ./certs/server.key and server.crt exist and proxies /api to `https://backend:443` with the certificate checked, so it needs `NODE_EXTRA_CA_CERTS` pointing at the backend's `server.crt` (the `backend_tls_cert` volume). `backend` only resolves inside the compose network, so on the host the API calls fail. The compose `frontend` service already serves a production build on :5001; rebuild it with `docker compose up -d --build frontend` after frontend changes.
 
 Plain `uv run pytest` also collects tests/e2e, which needs the full stack, so always pass a path. E2E tests run inside the backend container against a running stack, the way CI does it (CI copies config.test.toml over config.toml before starting the stack):
 
@@ -504,7 +507,7 @@ settings: FromDishka[Settings]
 Config load order (each overrides the previous):
 1. `config.toml`: base settings (committed)
 2. `config.worker.toml`: the worker's override (optional)
-3. Secrets: `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD`. Each one is read from the env var first, then the TOML value, then a dev default, so a missing secret never raises in `Settings`. Compose has no defaults for them: `./deploy.sh dev` copies `.env.example` to `.env`, the deploy writes `.env` from the GitHub secrets, and compose refuses to start while one is empty. `MONGODB_URL` is built from the Mongo values
+3. Secrets: `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD`. Each one is read from the env var first, then the TOML value, then a dev default, so a missing secret never raises in `Settings`. Compose has no defaults for them: locally `.env` is a copy of `.env.example`, the deploy writes `.env` from the GitHub secrets, and compose refuses to start while one is empty. `MONGODB_URL` is built from the Mongo values
 
 ```python
 Settings()                                               # config.toml + secrets
@@ -656,11 +659,10 @@ Use `interface` for object shapes, `type` for unions/intersections/aliases. Defi
 
 Hard rule: `docs/reference/openapi.json` and everything under `frontend/src/lib/api/` come from the generators only. Never edit them by hand, not even a one-line fix; change the backend (routes, schemas) and regenerate. CI's `Spec and client match the code` job regenerates both on every PR that touches the backend, the spec or the client, and fails on any difference, so a hand edit can't merge.
 
-`npm run generate:api` reads `../docs/reference/openapi.json`, and `./deploy.sh types` regenerates the spec only when that file is missing. After any backend API change, regenerate the spec first:
+`npm run generate:api` reads `../docs/reference/openapi.json`. After any backend API change, regenerate the spec first:
 ```bash
-# From project root
-./deploy.sh openapi   # writes docs/reference/openapi.json from create_app()
-./deploy.sh types     # runs npm run generate:api into frontend/src/lib/api/
+cd backend && uv run python -m scripts.export_openapi   # writes docs/reference/openapi.json from create_app()
+cd frontend && npm run generate:api                     # writes frontend/src/lib/api/ from that spec
 ```
 
 Function names are derived from URL paths: `getUserRateLimitsApiV1AdminRateLimitsUserIdGet`.
@@ -881,7 +883,7 @@ npm run build
 npm run test:coverage   # Vitest with coverage thresholds
 
 # Generated API (from the project root): must leave no diff
-./deploy.sh openapi && (cd frontend && npm run generate:api)
+(cd backend && uv run python -m scripts.export_openapi) && (cd frontend && npm run generate:api)
 git status --porcelain -- docs/reference/openapi.json frontend/src/lib/api
 ```
 

CONTRIBUTING.md

@@ -25,7 +25,8 @@ uv tool install pre-commit
 pre-commit install
 
 # Start the stack
-./deploy.sh dev
+cp .env.example .env
+docker compose up -d
 ```
 
 ### Pre-commit Hooks
@@ -73,7 +74,7 @@ uv run pytest tests/unit
 Always pass a path to pytest. A plain `uv run pytest` also collects tests/e2e, which needs the running stack and Kubernetes. The E2E suite runs inside the backend container, the same way CI runs it:
 
 ```bash
-./deploy.sh dev --wait
+docker compose up -d --wait
 docker compose exec -T backend sh -c 'uv sync --group test --no-dev --frozen --no-install-project && pytest tests/e2e'
 ```
 
@@ -90,13 +91,13 @@ npm run test
 npm run check
 npm run lint
 
-# E2E tests (Playwright, needs the stack from ./deploy.sh dev)
+# E2E tests (Playwright, needs the running stack)
 npm run test:e2e:deps      # once, installs system deps for Chromium
 npm run test:e2e:browsers  # once, installs Chromium
 npm run test:e2e
 ```
 
-The stack serves a production build of the frontend on https://localhost:5001, so rebuild it with `./deploy.sh dev --build` after frontend changes. `npm run dev` only works inside the compose network, since it proxies to the backend container and needs the shared CA.
+The stack serves a production build of the frontend on https://localhost:5001, so rebuild it with `docker compose up -d --build frontend` after frontend changes. `npm run dev` only works inside the compose network, since it proxies to the backend container and needs the shared CA.
 
 ## Code Style
 

README.md

@@ -66,10 +66,11 @@
 ```bash
 git clone https://github.com/HardMax71/Integr8sCode.git
 cd Integr8sCode
-./deploy.sh dev
+cp .env.example .env
+docker compose up -d
 ```
 
-`./deploy.sh dev` copies `.env.example` to `.env` on its first run, so local development needs no config. Compose has no defaults for secrets: anywhere else, write a `.env` with your own values, and compose refuses to start while one is missing.
+`.env.example` holds the local secrets, so local development needs no other config. Compose has no defaults for secrets: anywhere else, write a `.env` with your own values, and compose refuses to start while one is missing.
 
 ### Verify
 
@@ -79,15 +80,15 @@ curl -k https://localhost/api/v1/health/live
 
 ### Access
 
-| Service            | URL                                              | Notes                                         |
-|--------------------|--------------------------------------------------|-----------------------------------------------|
-| Frontend           | [https://localhost:5001](https://localhost:5001) | `user` / `user123` or `admin` / `admin123`    |
-| Backend API        | [https://localhost:443](https://localhost:443)   |                                               |
-| Grafana            | [http://localhost:3000](http://localhost:3000)   | needs `--observability`; `admin` / `admin123` |
-| Jaeger (Tracing)   | [http://localhost:16686](http://localhost:16686) | needs `--observability`                       |
-| Kafdrop (Kafka UI) | [http://localhost:9000](http://localhost:9000)   | needs `--debug`                               |
+| Service            | URL                                              | Notes                                                   |
+|--------------------|--------------------------------------------------|---------------------------------------------------------|
+| Frontend           | [https://localhost:5001](https://localhost:5001) | `user` / `user123` or `admin` / `admin123`              |
+| Backend API        | [https://localhost:443](https://localhost:443)   |                                                         |
+| Grafana            | [http://localhost:3000](http://localhost:3000)   | needs the `observability` profile; `admin` / `admin123` |
+| Jaeger (Tracing)   | [http://localhost:16686](http://localhost:16686) | needs the `observability` profile                       |
+| Kafdrop (Kafka UI) | [http://localhost:9000](http://localhost:9000)   | needs the `debug` profile                               |
 
-`./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch and the OTel Collector. `--debug` adds all of those plus Kafdrop.
+`docker compose --profile observability up -d` adds Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch and the OTel Collector, and `--profile debug` adds Kafdrop.
 
 nginx and the backend each make a self-signed certificate when they first start, so accept the browser warning. nginx verifies the backend's certificate on every proxied request. A certificate mounted at `/etc/nginx/certs` in the frontend container is used as is.
 
@@ -105,7 +106,7 @@ The E2E suites need the full stack; [CONTRIBUTING.md](CONTRIBUTING.md) has the c
 ### Stop
 
 ```bash
-./deploy.sh down
+docker compose down
 ```
 
 See the [full deployment guide](https://hardmax71.github.io/Integr8sCode/operations/deployment/) for Docker build strategy, troubleshooting, pre-built images, and more.

backend/scripts/export_openapi.py

@@ -0,0 +1,18 @@
+import json
+from pathlib import Path
+
+from app.main import create_app
+from app.settings import Settings
+
+SPEC_PATH = Path(__file__).resolve().parents[2] / "docs" / "reference" / "openapi.json"
+
+
+def export_openapi() -> None:
+    """Write the API's OpenAPI spec to docs/reference/openapi.json, where the frontend's client generator reads it."""
+    # config.toml's collector address resolves only inside compose; exporting there adds 11 s of failed retries
+    app = create_app(settings=Settings().model_copy(update={"OTEL_EXPORTER_OTLP_ENDPOINT": None}))
+    SPEC_PATH.write_text(json.dumps(app.openapi(), indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    export_openapi()