From dc175b0038e38b41b7a0a14c03ad83b17792f162 Mon Sep 17 00:00:00 2001 From: Shae Feltz Date: Tue, 15 Sep 2026 15:53:43 -0500 Subject: [PATCH] feat: run scheduled maintenance endpoints in self-hosted deployments --- .github/workflows/test-self-hosting.yml | 21 ++++++++++++ apps/web/content/docs/self-hosting.mdx | 24 ++++++++++++-- docker-compose.coolify.env.example | 4 +++ docker-compose.coolify.yml | 43 +++++++++++++++++++++++++ docker-compose.yml | 43 +++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-self-hosting.yml b/.github/workflows/test-self-hosting.yml index 3f5255fda97..3b8db27f3be 100644 --- a/.github/workflows/test-self-hosting.yml +++ b/.github/workflows/test-self-hosting.yml @@ -95,6 +95,27 @@ jobs: exit 1 fi + - name: Test scheduled maintenance endpoints + run: | + echo "Waiting for cap-cron..." + timeout 150 bash -c 'until docker inspect cap-cron --format="{{.State.Health.Status}}" 2>/dev/null | grep -q "healthy"; do sleep 2; done' + for route in recover-failed-video-processing finalize-stale-desktop-segments cleanup-agent-api; do + code=$(docker exec cap-cron sh -c 'curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $CRON_SECRET" "$CAP_WEB_INTERNAL_URL/api/cron/'"$route"'"') + if [ "$code" = "200" ]; then + echo "✓ /api/cron/$route (HTTP $code)" + else + echo "✗ /api/cron/$route failed (HTTP $code)" + exit 1 + fi + done + code=$(docker exec cap-cron sh -c 'curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong" "$CAP_WEB_INTERNAL_URL/api/cron/cleanup-agent-api"') + if [ "$code" = "401" ]; then + echo "✓ Wrong CRON_SECRET is rejected" + else + echo "✗ Wrong CRON_SECRET returned HTTP $code" + exit 1 + fi + - name: Test database has tables run: | echo "Testing database..." diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index 45219a81bd0..43af2d4d654 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -42,6 +42,7 @@ Best for VPS, home servers, or any Docker-capable host. - Media server (FFmpeg processing) - MySQL database - MinIO (S3-compatible storage) +- Scheduled maintenance (`cap-cron`) **Steps:** 1. Clone the repository @@ -89,6 +90,23 @@ For Coolify users, use `docker-compose.coolify.yml` which includes environment v > **Note:** The Coolify compose file uses slightly different environment variable names: `WEB_URL` instead of `CAP_URL`, and `S3_PUBLIC_ENDPOINT` instead of `S3_PUBLIC_URL`. +## Scheduled Maintenance + +Cap Web relies on a few scheduled endpoints to recover work that stalls. On Cap.so these run as Vercel Cron Jobs; in the Docker Compose and Coolify deployments the `cap-cron` service calls them on the same UTC schedule: + +| Endpoint | Schedule | Purpose | +|---|---|---| +| `/api/cron/recover-failed-video-processing` | :07, :22, :37, :52 | Retries uploads that failed with transient media server errors and restarts processing, transcription and AI generation stalled for over an hour | +| `/api/cron/finalize-stale-desktop-segments` | Every 15 minutes | Finalizes segmented desktop recordings whose upload never completed | +| `/api/cron/cleanup-agent-api` | 03:17 daily | Deletes expired Agent API records | + +The endpoints require `Authorization: Bearer $CRON_SECRET`, and the same `CRON_SECRET` must be set on `cap-web` and `cap-cron`. Without the scheduler, videos that hit a transient processing error stay stuck. If you run Cap Web without these compose files, call the endpoints on the schedule above from your own scheduler. + +Check it with: +```bash +docker compose logs cap-cron +``` + ## Connecting Cap Desktop 1. Open Cap Desktop settings @@ -194,6 +212,7 @@ The default `docker-compose.yml` contains **hardcoded placeholder secrets** that - **Forge authentication sessions** (via `NEXTAUTH_SECRET`) - **Decrypt sensitive database fields** (via `DATABASE_ENCRYPTION_KEY`) - **Spoof webhook requests** (via `MEDIA_SERVER_WEBHOOK_SECRET`) +- **Trigger maintenance endpoints** (via `CRON_SECRET`) This is fine for local development or testing on a private network, but **you must generate unique secrets before exposing Cap to the internet**. @@ -203,15 +222,16 @@ This is fine for local development or testing on a private network, but **you mu openssl rand -hex 32 ``` -Run this command three times to generate values for: +Run this command four times to generate values for: - `NEXTAUTH_SECRET` - `DATABASE_ENCRYPTION_KEY` - `MEDIA_SERVER_WEBHOOK_SECRET` +- `CRON_SECRET` **Full production checklist:** - [ ] Set secure passwords: `MYSQL_PASSWORD`, `MINIO_ROOT_PASSWORD` -- [ ] Set secure secrets: `DATABASE_ENCRYPTION_KEY`, `NEXTAUTH_SECRET`, `MEDIA_SERVER_WEBHOOK_SECRET` +- [ ] Set secure secrets: `DATABASE_ENCRYPTION_KEY`, `NEXTAUTH_SECRET`, `MEDIA_SERVER_WEBHOOK_SECRET`, `CRON_SECRET` - [ ] Set `CAP_URL` to your public URL - [ ] Set `S3_PUBLIC_URL` to your MinIO/S3 public URL - [ ] Configure a reverse proxy (nginx, Caddy, Traefik) with SSL diff --git a/docker-compose.coolify.env.example b/docker-compose.coolify.env.example index b57c7441713..4f8d7a0222f 100644 --- a/docker-compose.coolify.env.example +++ b/docker-compose.coolify.env.example @@ -20,6 +20,10 @@ NEXTAUTH_SECRET= # Generate with: openssl rand -hex 32 MEDIA_SERVER_WEBHOOK_SECRET= +# Secret the cap-cron service uses to call Cap Web's scheduled maintenance endpoints +# Generate with: openssl rand -hex 32 +CRON_SECRET= + # =================== # DATABASE # =================== diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index ff11962c90b..0c49fccd6e9 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -31,6 +31,7 @@ services: MEDIA_SERVER_URL: 'http://media-server:3456' MEDIA_SERVER_WEBHOOK_SECRET: '${SERVICE_HEX_32_MEDIASERVER}' MEDIA_SERVER_WEBHOOK_URL: 'http://cap-web:3000' + CRON_SECRET: '${SERVICE_HEX_32_CRONSECRET}' healthcheck: test: - CMD @@ -59,6 +60,48 @@ services: timeout: 10s retries: 3 start_period: 10s + cap-cron: + image: 'curlimages/curl:8.22.0' + restart: unless-stopped + depends_on: + cap-web: + condition: service_healthy + environment: + CRON_SECRET: '${SERVICE_HEX_32_CRONSECRET}' + CAP_WEB_INTERNAL_URL: 'http://cap-web:3000' + entrypoint: + - /bin/sh + - '-c' + - | + trap 'exit 0' TERM INT + run() { + mkdir "/tmp/lock.$$1" 2>/dev/null || return 0 + code=$$(curl -s -o "/tmp/out.$$1" -w '%{http_code}' --max-time 600 -H "Authorization: Bearer $$CRON_SECRET" "$$CAP_WEB_INTERNAL_URL/api/cron/$$1") + echo "$$(date -u +%FT%TZ) $$1 $$code $$(head -c 500 "/tmp/out.$$1")" + rmdir "/tmp/lock.$$1" + } + rmdir /tmp/lock.* 2>/dev/null + while :; do + jobs >/dev/null + touch /tmp/heartbeat + sleep $$((60 - $$(date +%s) % 60)) & + wait $$! + m=$$(date -u +%M); m=$${m#0} + h=$$(date -u +%H); h=$${h#0} + case $$m in 7|22|37|52) run recover-failed-video-processing & ;; esac + if [ $$((m % 15)) -eq 0 ]; then run finalize-stale-desktop-segments & fi + if [ "$$h" = 3 ] && [ "$$m" = 17 ]; then run cleanup-agent-api & fi + done + healthcheck: + test: + - CMD + - sh + - '-c' + - 'find /tmp/heartbeat -mmin -3 | grep -q .' + interval: 60s + timeout: 5s + retries: 3 + start_period: 90s mysql: image: 'mysql:8.0' restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 039cefba12e..3626a5ff835 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: DATABASE_ENCRYPTION_KEY: ${DATABASE_ENCRYPTION_KEY:-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef} NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789} MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:-fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210} + CRON_SECRET: ${CRON_SECRET:-0f1e2d3c4b5a69780f1e2d3c4b5a69780f1e2d3c4b5a69780f1e2d3c4b5a6978} CAP_AWS_ACCESS_KEY: ${MINIO_ROOT_USER:-cap-admin} CAP_AWS_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-cap-minio-pwd-456} CAP_AWS_BUCKET: cap @@ -61,6 +62,48 @@ services: networks: - cap-network + cap-cron: + container_name: cap-cron + image: curlimages/curl:8.22.0 + restart: unless-stopped + depends_on: + cap-web: + condition: service_healthy + environment: + CRON_SECRET: ${CRON_SECRET:-0f1e2d3c4b5a69780f1e2d3c4b5a69780f1e2d3c4b5a69780f1e2d3c4b5a6978} + CAP_WEB_INTERNAL_URL: http://cap-web:3000 + entrypoint: + - /bin/sh + - -c + - | + trap 'exit 0' TERM INT + run() { + mkdir "/tmp/lock.$$1" 2>/dev/null || return 0 + code=$$(curl -s -o "/tmp/out.$$1" -w '%{http_code}' --max-time 600 -H "Authorization: Bearer $$CRON_SECRET" "$$CAP_WEB_INTERNAL_URL/api/cron/$$1") + echo "$$(date -u +%FT%TZ) $$1 $$code $$(head -c 500 "/tmp/out.$$1")" + rmdir "/tmp/lock.$$1" + } + rmdir /tmp/lock.* 2>/dev/null + while :; do + jobs >/dev/null + touch /tmp/heartbeat + sleep $$((60 - $$(date +%s) % 60)) & + wait $$! + m=$$(date -u +%M); m=$${m#0} + h=$$(date -u +%H); h=$${h#0} + case $$m in 7|22|37|52) run recover-failed-video-processing & ;; esac + if [ $$((m % 15)) -eq 0 ]; then run finalize-stale-desktop-segments & fi + if [ "$$h" = 3 ] && [ "$$m" = 17 ]; then run cleanup-agent-api & fi + done + healthcheck: + test: ["CMD", "sh", "-c", "find /tmp/heartbeat -mmin -3 | grep -q ."] + interval: 60s + timeout: 5s + retries: 3 + start_period: 90s + networks: + - cap-network + mysql: container_name: cap-mysql image: mysql:8.0