From aa09257e4a56828d97e7f5789889b096980e46cd Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 21 Jul 2026 17:33:41 +0300 Subject: [PATCH 1/2] Raise Cloud Run request concurrency from 4 to 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud Run's autoscaler targets ~60% of containerConcurrency, so c4 spawns a new instance at ~2.4 sustained concurrent requests — overnight bot bursts churned 18 instance boots on 2026-07-21, each queueing requests behind the ~3-min import. c6 moves the trigger to ~3.6 while staying within the memory envelope (c8 OOM'd in Stage 2; c6 re-qualified by soak, evidence in docs/migration/history/pr4-stage2-runtime-timing.md). Co-Authored-By: Claude Fable 5 --- .github/scripts/cloud_run_env.sh | 2 +- changelog.d/cloud-run-concurrency-6.changed.md | 1 + tests/unit/test_cloud_run_deploy_scripts.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/cloud-run-concurrency-6.changed.md diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 6d2757fcc..f5388559a 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -17,7 +17,7 @@ cloud_run_set_defaults() { CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}" # Stage 2-qualified runtime shape, pinned explicitly on every deploy — # rationale in docs/migration/cloud-run-operations.md ("Runtime shape"). - CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-4}" + CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" diff --git a/changelog.d/cloud-run-concurrency-6.changed.md b/changelog.d/cloud-run-concurrency-6.changed.md new file mode 100644 index 000000000..3948399d1 --- /dev/null +++ b/changelog.d/cloud-run-concurrency-6.changed.md @@ -0,0 +1 @@ +Raise Cloud Run per-instance request concurrency from 4 to 6 to reduce bot-burst scale-out churn. diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index eb215817e..32895da97 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -371,7 +371,7 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): assert result.returncode == 0, result.stderr # Stage 2-qualified values, pinned on every deploy — rationale in # docs/migration/cloud-run-operations.md ("Runtime shape and scaling"). - assert "--concurrency 4 " in result.stdout + assert "--concurrency 6 " in result.stdout assert "WEB_CONCURRENCY=2 " in result.stdout assert "--min-instances 0 " in result.stdout From 64b3b3a5153ad14271668d3697da636c2cb5a5f4 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 21 Jul 2026 22:05:29 +0300 Subject: [PATCH 2/2] Record c6 qualification soak evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 30-min 12-client cache-busted soak against no-traffic c6-qual revision: memory p99 peaked 61% (bar 70%; c8 failed at 92%), zero OOM/worker restarts, 320/320 requests 200. Gates evaluated server-side (client-side JSON lost to an operator session exit at soak end — informational only). Co-Authored-By: Claude Fable 5 --- docs/migration/cloud-run-operations.md | 8 +++-- .../history/pr4-stage2-runtime-timing.md | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index aed16fa13..7f53c5a53 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -61,9 +61,13 @@ Values measured and justified in - **`WEB_CONCURRENCY=2`** — the engine is GIL-bound; two worker processes materially outperform one on the 4-vCPU instance. Set via the deploy script's env vars. -- **`--concurrency 4`** — admission control is what keeps the two-worker instance +- **`--concurrency 6`** — admission control is what keeps the two-worker instance memory-safe in 16Gi; excess load queues at the platform and fails as client timeouts - rather than dying in-instance. **Both knobs are pinned explicitly on every deploy**: + rather than dying in-instance. Raised from the original 4 on 2026-07-21 (re-qualified + by a dedicated 30-min soak — memory peaked 61%, zero OOM/5xx; see the 2026-07-21 + amendment in `history/pr4-stage2-runtime-timing.md`) to stop bot-burst scale-out + churn: the autoscaler targets ~60% of concurrency, so c4 spawned instances at ~2.4 + sustained concurrent requests. **Both knobs are pinned explicitly on every deploy**: gcloud inherits unspecified template fields (a mid-campaign CI deploy once shipped an inherited test concurrency), and `--concurrency default` resolves to the *platform* default (640), not the historical 80. diff --git a/docs/migration/history/pr4-stage2-runtime-timing.md b/docs/migration/history/pr4-stage2-runtime-timing.md index 24f8311cd..77cc07f68 100644 --- a/docs/migration/history/pr4-stage2-runtime-timing.md +++ b/docs/migration/history/pr4-stage2-runtime-timing.md @@ -131,3 +131,37 @@ memory-safe. request mix (likely O(5,000+) with jitter ~50%). 4. Re-baseline after metadata slimming lands; it dominates warm-parity and transfer costs. + +## Amendment 2026-07-21 — concurrency re-qualified at 6 (c6) + +Context: overnight bot-traffic waves (2026-07-21 00:00–06:00Z) churned Cloud +Run scale-outs 1→4→1→4 (18 boots); at c4 the autoscaler's ~60% target spawns +an instance at ~2.4 sustained concurrent requests, and every boot queues +early-bind-routed requests behind the ~3-min import. Decision (user, +2026-07-21): raise `--concurrency` to 6, moving the trigger to ~3.6. + +Qualification soak per this document's discipline (never change concurrency +without its own soak — c8 OOM'd a worker at 92% in the original campaign): + +- No-traffic revision `policyengine-api-c6-qual` (concurrency 6, 4 vCPU/16Gi, + WEB_CONCURRENCY=2), tag URL only, public traffic untouched. +- 30-min soak, 12 closed-loop clients, cache-busted mix + (`scripts/measure_cloud_run_runtime.py --clients 12 --duration-seconds 1800 + --cache-bust always`), window 14:31:35–15:01:35Z; CPU p99 sustained 35–79% + through 15:05Z (fully saturated 6-slot instance with queue pressure). +- **Gates, evaluated server-side: PASS.** Memory p99-per-2min peaked at 61% + (14:55Z) and held 60% steady to soak end — under the 70% bar (vs 92% at c8, + 69% knife-edge at c4's qualifying soak); zero OOM / worker restarts (only + the two initial boots at instance start); zero 5xx — 320/320 requests 200 + in the revision's request log. +- Caveat recorded honestly: the load-generator process was killed at soak end + before writing its client-side JSON (operator session exit), so client-side + latency percentiles were lost. All three gates are server-side observable + and were evaluated from Monitoring + request logs over the full window; the + loss is informational only. +- Cleanup: tag removed; revision undeletable while latest-created (same + precedent as `s2-restore2`) — zero traffic, scaled to zero, superseded by + the next CI deploy. Revision YAML captured at qualification time. + +Shipped as `CLOUD_RUN_CONCURRENCY` default 4→6 in +`.github/scripts/cloud_run_env.sh` with the dry-run invariant updated.