diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 6d2757fcc..f5388559a 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -17,7 +17,7 @@ cloud_run_set_defaults() { CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}" # Stage 2-qualified runtime shape, pinned explicitly on every deploy — # rationale in docs/migration/cloud-run-operations.md ("Runtime shape"). - CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-4}" + CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" diff --git a/changelog.d/cloud-run-concurrency-6.changed.md b/changelog.d/cloud-run-concurrency-6.changed.md new file mode 100644 index 000000000..3948399d1 --- /dev/null +++ b/changelog.d/cloud-run-concurrency-6.changed.md @@ -0,0 +1 @@ +Raise Cloud Run per-instance request concurrency from 4 to 6 to reduce bot-burst scale-out churn. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index aed16fa13..7f53c5a53 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -61,9 +61,13 @@ Values measured and justified in - **`WEB_CONCURRENCY=2`** — the engine is GIL-bound; two worker processes materially outperform one on the 4-vCPU instance. Set via the deploy script's env vars. -- **`--concurrency 4`** — admission control is what keeps the two-worker instance +- **`--concurrency 6`** — admission control is what keeps the two-worker instance memory-safe in 16Gi; excess load queues at the platform and fails as client timeouts - rather than dying in-instance. **Both knobs are pinned explicitly on every deploy**: + rather than dying in-instance. Raised from the original 4 on 2026-07-21 (re-qualified + by a dedicated 30-min soak — memory peaked 61%, zero OOM/5xx; see the 2026-07-21 + amendment in `history/pr4-stage2-runtime-timing.md`) to stop bot-burst scale-out + churn: the autoscaler targets ~60% of concurrency, so c4 spawned instances at ~2.4 + sustained concurrent requests. **Both knobs are pinned explicitly on every deploy**: gcloud inherits unspecified template fields (a mid-campaign CI deploy once shipped an inherited test concurrency), and `--concurrency default` resolves to the *platform* default (640), not the historical 80. diff --git a/docs/migration/history/pr4-stage2-runtime-timing.md b/docs/migration/history/pr4-stage2-runtime-timing.md index 24f8311cd..77cc07f68 100644 --- a/docs/migration/history/pr4-stage2-runtime-timing.md +++ b/docs/migration/history/pr4-stage2-runtime-timing.md @@ -131,3 +131,37 @@ memory-safe. request mix (likely O(5,000+) with jitter ~50%). 4. Re-baseline after metadata slimming lands; it dominates warm-parity and transfer costs. + +## Amendment 2026-07-21 — concurrency re-qualified at 6 (c6) + +Context: overnight bot-traffic waves (2026-07-21 00:00–06:00Z) churned Cloud +Run scale-outs 1→4→1→4 (18 boots); at c4 the autoscaler's ~60% target spawns +an instance at ~2.4 sustained concurrent requests, and every boot queues +early-bind-routed requests behind the ~3-min import. Decision (user, +2026-07-21): raise `--concurrency` to 6, moving the trigger to ~3.6. + +Qualification soak per this document's discipline (never change concurrency +without its own soak — c8 OOM'd a worker at 92% in the original campaign): + +- No-traffic revision `policyengine-api-c6-qual` (concurrency 6, 4 vCPU/16Gi, + WEB_CONCURRENCY=2), tag URL only, public traffic untouched. +- 30-min soak, 12 closed-loop clients, cache-busted mix + (`scripts/measure_cloud_run_runtime.py --clients 12 --duration-seconds 1800 + --cache-bust always`), window 14:31:35–15:01:35Z; CPU p99 sustained 35–79% + through 15:05Z (fully saturated 6-slot instance with queue pressure). +- **Gates, evaluated server-side: PASS.** Memory p99-per-2min peaked at 61% + (14:55Z) and held 60% steady to soak end — under the 70% bar (vs 92% at c8, + 69% knife-edge at c4's qualifying soak); zero OOM / worker restarts (only + the two initial boots at instance start); zero 5xx — 320/320 requests 200 + in the revision's request log. +- Caveat recorded honestly: the load-generator process was killed at soak end + before writing its client-side JSON (operator session exit), so client-side + latency percentiles were lost. All three gates are server-side observable + and were evaluated from Monitoring + request logs over the full window; the + loss is informational only. +- Cleanup: tag removed; revision undeletable while latest-created (same + precedent as `s2-restore2`) — zero traffic, scaled to zero, superseded by + the next CI deploy. Revision YAML captured at qualification time. + +Shipped as `CLOUD_RUN_CONCURRENCY` default 4→6 in +`.github/scripts/cloud_run_env.sh` with the dry-run invariant updated. diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index eb215817e..32895da97 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -371,7 +371,7 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): assert result.returncode == 0, result.stderr # Stage 2-qualified values, pinned on every deploy — rationale in # docs/migration/cloud-run-operations.md ("Runtime shape and scaling"). - assert "--concurrency 4 " in result.stdout + assert "--concurrency 6 " in result.stdout assert "WEB_CONCURRENCY=2 " in result.stdout assert "--min-instances 0 " in result.stdout