From c91c92c8a513337bc5cee517017b4042aaa450c2 Mon Sep 17 00:00:00 2001 From: Bo-Cheng Chu Date: Wed, 28 Oct 2026 10:18:19 -0700 Subject: [PATCH] Fix parking demo worker capacity, probe, and load.sh failure reporting WorkerPool/parking omitted container resource limits, so the Kubernetes Downward API reported the node's allocatable capacity in /run/worker-capacity. Two workers held all four 1000m actors at once without triggering request parking. Set per-worker container limits to 1 CPU and 1Gi memory so each worker hosts at most one actor. Add a wakeupProbe on /readyz port 80 to parking-template.yaml.tmpl so golden memory snapshots are captured after the counter server binds port 80. In load.sh, wait for each proxy to start listening before generating load, aggregate only actor request logs (excluding .proxy.log files), and exit non-zero when requests fail with a non-200/503 status or when zero requests return 200 OK. Signed-off-by: Bo-Cheng Chu --- demos/parking/load.sh | 25 ++++++++++++++++++++++-- demos/parking/parking-template.yaml.tmpl | 8 ++++++-- demos/parking/parking.yaml.tmpl | 15 ++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/demos/parking/load.sh b/demos/parking/load.sh index bf8d6d3f95..8a7cbf94e2 100755 --- a/demos/parking/load.sh +++ b/demos/parking/load.sh @@ -111,7 +111,18 @@ for i in "${!ACTORS[@]}"; do kubectl ate proxy actor "${a}" "${port}:80" --atespace "${ATESPACE}" >"${TMP}/${a}.proxy.log" 2>&1 & proxy_pids+=($!) done -sleep 3 +# Wait for each proxy to print its "Proxying ..." ready line before sending load. +for a in "${ACTORS[@]}"; do + for _ in $(seq 1 60); do + grep -q "^Proxying " "${TMP}/${a}.proxy.log" 2>/dev/null && break + sleep 0.2 + done + if ! grep -q "^Proxying " "${TMP}/${a}.proxy.log" 2>/dev/null; then + echo " proxy for ${a} failed to start:" >&2 + cat "${TMP}/${a}.proxy.log" >&2 || true + exit 1 + fi +done # One worker per actor: hammer it with request->suspend until the deadline. worker() { @@ -138,7 +149,9 @@ wait "${pids[@]}" # ---- aggregate ------------------------------------------------------------- echo echo "==> results" -cat "${TMP}"/*.log 2>/dev/null >"${TMP}/all" || true +for a in "${ACTORS[@]}"; do + cat "${TMP}/${a}.log" 2>/dev/null || true +done >"${TMP}/all" total=$(wc -l <"${TMP}/all" | tr -d ' ') if [[ "${total}" -eq 0 ]]; then echo " no responses recorded -- check proxy logs" @@ -157,6 +170,14 @@ printf " 503 unavailable: %s\n" "${busy}" printf " other : %s\n" "${other}" printf " 200 latency : avg %ss, slowest %ss <- parked requests sit here\n" "${avg_ok}" "${slowest_ok:-n/a}" echo +if [[ "${other}" -gt 0 ]]; then + echo " => ${other} requests failed with a non-200/503 status." + exit 1 +fi +if [[ "${ok}" -eq 0 ]]; then + echo " => 0 requests succeeded (${busy} shed with 503)." + exit 1 +fi if [[ "${busy}" -eq 0 ]]; then echo " => 0 failures under saturation: parking absorbed the contention." echo " Re-run with the router started --parked-request-max=0 to see 503s." diff --git a/demos/parking/parking-template.yaml.tmpl b/demos/parking/parking-template.yaml.tmpl index 4753f7a02a..cbe93fb1a2 100644 --- a/demos/parking/parking-template.yaml.tmpl +++ b/demos/parking/parking-template.yaml.tmpl @@ -26,8 +26,12 @@ containers: - name: counter image: ko://go.ate.dev/substrate/demos/counter command: ["/ko-app/counter"] -# Sandbox size. The pool sets no worker limits, so the worker offers the -# node's allocatable and these only need to fit the node. + wakeupProbe: + httpGet: + path: /readyz + port: 80 +# One worker's worth of CPU (see parking.yaml.tmpl), so each worker hosts at +# most one actor at a time. resources: limits: cpuMillicores: 1000 diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index b8a5cb3f58..b2a6ea7d6d 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -40,9 +40,20 @@ metadata: labels: workload: parking spec: - # Intentionally small: 2 workers, so 3+ concurrently-active actors saturate - # the pool and exercise the parking path. + # Intentionally small: 2 workers, each sized for one actor, so 3+ + # concurrently-active actors saturate the pool and exercise the parking path. replicas: 2 workerImage: ko://go.ate.dev/substrate/cmd/ateworker-gvisor sandboxClasses: - name: gvisor + template: + # Each worker advertises its container limits as capacity. Sizing CPU to + # one actor's limit in parking-template.yaml.tmpl caps each worker at one + # running actor. + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 1Gi