From 59131f44866d4d89527391a3d9604991c8f99417 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 02:14:07 +0000 Subject: [PATCH 1/8] Add optional autoscaling and disruption budget for the AI Gateway The gateway Deployment had a fixed replica count and no HPA, while the AWS module's gateway ECS service autoscales 2-6 tasks on CPU and memory. Add aiGateway.autoscaling, a standard autoscaling/v2 HPA on ai-gateway container CPU (70%) and memory (75%) with the same bounds and scale windows, and an optional aiGateway.podDisruptionBudget. Both default to disabled, so existing releases render unchanged. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 25 ++ braintrust/templates/_helpers.tpl | 41 ++ .../templates/ai-gateway-deployment.yaml | 3 + braintrust/templates/ai-gateway-hpa.yaml | 47 +++ braintrust/templates/ai-gateway-pdb.yaml | 26 ++ .../pre-ai-gateway-autoscaling-values.yaml | 9 + .../tests/ai-gateway-autoscaling_test.yaml | 375 ++++++++++++++++++ braintrust/values.yaml | 23 ++ 8 files changed, 549 insertions(+) create mode 100644 braintrust/templates/ai-gateway-hpa.yaml create mode 100644 braintrust/templates/ai-gateway-pdb.yaml create mode 100644 braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml create mode 100644 braintrust/tests/ai-gateway-autoscaling_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 5adb943..a553a03 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -278,6 +278,26 @@ api: When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. +## AI Gateway Autoscaling + +The AI Gateway can autoscale with a standard `autoscaling/v2` Horizontal Pod Autoscaler on CPU and memory, scoped to the `ai-gateway` container. The defaults match the gateway ECS service in the AWS Terraform module: 2-6 replicas, 70% CPU, 75% memory, a 60 second scale-up window, and a 300 second scale-down window. + +The HPA reads the Kubernetes resource metrics API. GKE and AKS provide it. On EKS, install [metrics-server](https://github.com/kubernetes-sigs/metrics-server) first. + +```yaml +aiGateway: + enabled: true + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 6 + podDisruptionBudget: + enabled: true + minAvailable: 1 +``` + +When enabled, `aiGateway.replicas` is ignored and the HPA controls the replica count. Turning autoscaling on for a running gateway removes `replicas` from the Deployment. Kubernetes then resets the Deployment to 1 replica until the HPA raises it to `minReplicas`, usually within one HPA sync period (15 seconds by default). + ## API workload isolation `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest` @@ -425,8 +445,13 @@ api: podDisruptionBudget: enabled: true maxUnavailable: 1 +aiGateway: + podDisruptionBudget: + enabled: true + minAvailable: 1 ``` +The AI Gateway budget defaults to `minAvailable: 1` and renders only when `aiGateway.enabled` is true. The API retains its existing `minAvailable` behavior unless `maxUnavailable` is set. An explicit maximum takes precedence over the inherited minimum. Each role has an independent budget. Evictions can proceed simultaneously across roles. Readiness probes determine healthy replicas for PDBs. Deployment `minReadySeconds` does not delay PDB eviction permission. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 4393355..564c69f 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -146,6 +146,47 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} +{{/* +AI Gateway autoscaling defaults. `helm upgrade --reuse-values` from a chart +that predates gateway autoscaling retains no aiGateway.autoscaling map, and a +partial --set on top of those values would otherwise render empty fields. +*/}} +{{- define "braintrust.aiGateway.autoscaling.defaults" -}} +enabled: false +minReplicas: 2 +maxReplicas: 6 +cpu: + targetAverageUtilization: 70 +memory: + targetAverageUtilization: 75 +behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 +{{- end -}} + +{{/* +Effective AI Gateway autoscaling settings: template defaults merged with any +provided values, validated when enabled. +*/}} +{{- define "braintrust.aiGateway.autoscaling" -}} +{{- $defaults := include "braintrust.aiGateway.autoscaling.defaults" . | fromYaml -}} +{{- $provided := deepCopy (.Values.aiGateway.autoscaling | default dict) -}} +{{- $autoscaling := mergeOverwrite (deepCopy $defaults) $provided -}} +{{- if $autoscaling.enabled -}} +{{- $min := int $autoscaling.minReplicas -}} +{{- $max := int $autoscaling.maxReplicas -}} +{{- if lt $min 1 -}} +{{- fail "aiGateway.autoscaling.minReplicas must be at least 1" -}} +{{- end -}} +{{- if lt $max $min -}} +{{- fail "aiGateway.autoscaling.maxReplicas must be greater than or equal to aiGateway.autoscaling.minReplicas" -}} +{{- end -}} +{{- end -}} +{{- toYaml $autoscaling -}} +{{- end -}} + {{/* Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). */}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index e78b254..7fb8c72 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -1,4 +1,5 @@ {{- if .Values.aiGateway.enabled }} +{{- $autoscaling := include "braintrust.aiGateway.autoscaling" . | fromYaml }} --- apiVersion: apps/v1 kind: Deployment @@ -14,7 +15,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if not $autoscaling.enabled }} replicas: {{ .Values.aiGateway.replicas }} + {{- end }} strategy: type: RollingUpdate rollingUpdate: diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml new file mode 100644 index 0000000..2377c57 --- /dev/null +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -0,0 +1,47 @@ +{{- if .Values.aiGateway.enabled }} +{{- $autoscaling := include "braintrust.aiGateway.autoscaling" . | fromYaml }} +{{- if $autoscaling.enabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.aiGateway.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (merge (deepCopy .Values.aiGateway.labels) .Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with (dig "hpa" dict (.Values.aiGateway.annotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.aiGateway.name }} + minReplicas: {{ $autoscaling.minReplicas }} + maxReplicas: {{ $autoscaling.maxReplicas }} + metrics: + # ContainerResource scopes utilization to the ai-gateway container so + # sidecars / extraContainers do not skew it. + - type: ContainerResource + containerResource: + name: cpu + container: ai-gateway + target: + type: Utilization + averageUtilization: {{ $autoscaling.cpu.targetAverageUtilization }} + - type: ContainerResource + containerResource: + name: memory + container: ai-gateway + target: + type: Utilization + averageUtilization: {{ $autoscaling.memory.targetAverageUtilization }} + {{- with $autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/braintrust/templates/ai-gateway-pdb.yaml b/braintrust/templates/ai-gateway-pdb.yaml new file mode 100644 index 0000000..5011aea --- /dev/null +++ b/braintrust/templates/ai-gateway-pdb.yaml @@ -0,0 +1,26 @@ +{{- if .Values.aiGateway.enabled }} +{{- $pdb := .Values.aiGateway.podDisruptionBudget | default dict }} +{{- if $pdb.enabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Values.aiGateway.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (merge (deepCopy .Values.aiGateway.labels) .Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if ne $pdb.maxUnavailable nil }} + maxUnavailable: {{ $pdb.maxUnavailable | toYaml | trim }} + {{- else if ne $pdb.minAvailable nil }} + minAvailable: {{ $pdb.minAvailable | toYaml | trim }} + {{- else }} + minAvailable: 1 + {{- end }} + selector: + matchLabels: + app: {{ .Values.aiGateway.name }} +{{- end }} +{{- end }} diff --git a/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml new file mode 100644 index 0000000..2d0cba5 --- /dev/null +++ b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml @@ -0,0 +1,9 @@ +# Simulate values retained by `helm upgrade --reuse-values` from a chart release +# that predates AI Gateway autoscaling and disruption budgets. An explicit null +# removes the new chart default during Helm value coalescing. +aiGateway: + enabled: true + annotations: + hpa: null + autoscaling: null + podDisruptionBudget: null diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml new file mode 100644 index 0000000..8c6f1f7 --- /dev/null +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -0,0 +1,375 @@ +suite: AI Gateway autoscaling and disruption budget +templates: + - ai-gateway-hpa.yaml + - ai-gateway-pdb.yaml + - ai-gateway-deployment.yaml + - ai-gateway-configmap.yaml +values: + - __fixtures__/base-values.yaml +release: + namespace: braintrust +tests: + - it: renders no HPA or PDB when the gateway is disabled, even if both are enabled + set: + aiGateway.autoscaling.enabled: true + aiGateway.podDisruptionBudget.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders no PDB when the gateway is disabled + set: + aiGateway.podDisruptionBudget.enabled: true + template: ai-gateway-pdb.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders no HPA by default + set: + aiGateway.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders no PDB by default + set: + aiGateway.enabled: true + template: ai-gateway-pdb.yaml + asserts: + - hasDocuments: + count: 0 + + - it: keeps fixed replicas on the deployment by default + set: + aiGateway.enabled: true + aiGateway.replicas: 3 + template: ai-gateway-deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 3 + + - it: renders an HPA with AWS-parity defaults when enabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HorizontalPodAutoscaler + - isAPIVersion: + of: autoscaling/v2 + - equal: + path: metadata.name + value: braintrust-ai-gateway + - equal: + path: metadata.namespace + value: braintrust + - equal: + path: spec.scaleTargetRef + value: + apiVersion: apps/v1 + kind: Deployment + name: braintrust-ai-gateway + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + - equal: + path: spec.metrics + value: + - type: ContainerResource + containerResource: + name: cpu + container: ai-gateway + target: + type: Utilization + averageUtilization: 70 + - type: ContainerResource + containerResource: + name: memory + container: ai-gateway + target: + type: Utilization + averageUtilization: 75 + - equal: + path: spec.behavior + value: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 + - notExists: + path: metadata.annotations + + - it: omits fixed replicas from the deployment when autoscaling is enabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + template: ai-gateway-deployment.yaml + asserts: + - notExists: + path: spec.replicas + + - it: applies custom bounds, targets, behavior, names, labels, and annotations + set: + aiGateway.enabled: true + aiGateway.name: custom-gateway + aiGateway.labels: + team: gateway + global.labels: + env: test + aiGateway.annotations.hpa: + example.com/owner: gateway + aiGateway.autoscaling: + enabled: true + minReplicas: 4 + maxReplicas: 40 + cpu: + targetAverageUtilization: 60 + memory: + targetAverageUtilization: 80 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: metadata.name + value: custom-gateway + - equal: + path: spec.scaleTargetRef.name + value: custom-gateway + - equal: + path: metadata.labels + value: + team: gateway + env: test + - equal: + path: metadata.annotations + value: + example.com/owner: gateway + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 40 + - equal: + path: spec.metrics[0].containerResource.target.averageUtilization + value: 60 + - equal: + path: spec.metrics[1].containerResource.target.averageUtilization + value: 80 + - equal: + path: spec.behavior.scaleUp.stabilizationWindowSeconds + value: 0 + + - it: uses the managed namespace when the chart creates it + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + global.createNamespace: true + global.namespace: custom-namespace + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: metadata.namespace + value: custom-namespace + + - it: rejects zero minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + aiGateway.autoscaling.minReplicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: "aiGateway.autoscaling.minReplicas must be at least 1" + + - it: rejects maxReplicas below minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + aiGateway.autoscaling.minReplicas: 4 + aiGateway.autoscaling.maxReplicas: 3 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: "aiGateway.autoscaling.maxReplicas must be greater than or equal to aiGateway.autoscaling.minReplicas" + + - it: does not validate bounds while autoscaling is disabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 0 + template: ai-gateway-deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: keeps fixed replicas and no HPA for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: renders no HPA for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders no PDB for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-pdb.yaml + asserts: + - hasDocuments: + count: 0 + + - it: fills defaults for a partial autoscaling --set on reused values + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.autoscaling.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + - equal: + path: spec.metrics[1].containerResource.target.averageUtilization + value: 75 + - equal: + path: spec.behavior.scaleDown.stabilizationWindowSeconds + value: 300 + - notExists: + path: metadata.annotations + + - it: defaults minAvailable for a partial PDB --set on reused values + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.podDisruptionBudget.enabled: true + template: ai-gateway-pdb.yaml + asserts: + - equal: + path: spec.minAvailable + value: 1 + + - it: renders the HPA on aws + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + + - it: renders the HPA on azure + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + + - it: renders the HPA on google without the AutoscalingMetric API + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + + - it: renders a PDB with minAvailable 1 when enabled + set: + aiGateway.enabled: true + aiGateway.podDisruptionBudget.enabled: true + template: ai-gateway-pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: metadata.name + value: braintrust-ai-gateway + - equal: + path: metadata.namespace + value: braintrust + - equal: + path: spec.minAvailable + value: 1 + - notExists: + path: spec.maxUnavailable + - equal: + path: spec.selector.matchLabels.app + value: braintrust-ai-gateway + + - it: preserves a zero minAvailable + set: + aiGateway.enabled: true + aiGateway.podDisruptionBudget.enabled: true + aiGateway.podDisruptionBudget.minAvailable: 0 + template: ai-gateway-pdb.yaml + asserts: + - equal: + path: spec.minAvailable + value: 0 + + - it: prefers an explicit maxUnavailable + set: + aiGateway.enabled: true + aiGateway.podDisruptionBudget.enabled: true + aiGateway.podDisruptionBudget.maxUnavailable: 1 + template: ai-gateway-pdb.yaml + asserts: + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: preserves a zero maxUnavailable + set: + aiGateway.enabled: true + aiGateway.podDisruptionBudget.enabled: true + aiGateway.podDisruptionBudget.maxUnavailable: 0 + template: ai-gateway-pdb.yaml + asserts: + - equal: + path: spec.maxUnavailable + value: 0 + - notExists: + path: spec.minAvailable diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d3b85fa..ca44787 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -324,7 +324,30 @@ aiGateway: deployment: {} service: {} pod: {} + hpa: {} + # Ignored when autoscaling.enabled is true; the HPA controls the replica count. replicas: 2 + # Optional Horizontal Pod Autoscaler on CPU and memory, scoped to the + # ai-gateway container. Defaults mirror the AWS module's gateway ECS service. + # Requires the Kubernetes resource metrics API (metrics-server). GKE and AKS + # provide it; EKS requires installing metrics-server. + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 6 + cpu: + targetAverageUtilization: 70 + memory: + targetAverageUtilization: 75 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 + podDisruptionBudget: + enabled: false + # An explicit maxUnavailable overrides this minimum. + minAvailable: 1 image: repository: public.ecr.aws/braintrust/gateway tag: v2.21.0 From a36d6c1d21ce02006b5e85581fd3b543260abe59 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 02:18:13 +0000 Subject: [PATCH 2/8] Enable AI Gateway autoscaling by default Without the resource metrics API the HPA holds the gateway at minReplicas, which matches the previous fixed count of 2, so a default-on HPA does not regress clusters without metrics-server. Releases that already run the gateway briefly drop to 1 replica on the upgrade that removes the fixed Deployment replica count; the README documents how to avoid that. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 9 +-- braintrust/templates/_helpers.tpl | 2 +- .../tests/ai-gateway-autoscaling_test.yaml | 58 ++++++++++++++----- .../tests/ai-gateway-deployment_test.yaml | 1 + braintrust/values.yaml | 11 ++-- 5 files changed, 58 insertions(+), 23 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index a553a03..9f0bf37 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -280,15 +280,14 @@ When enabled for a pool, that pool's `replicas` setting is ignored and the HPA c ## AI Gateway Autoscaling -The AI Gateway can autoscale with a standard `autoscaling/v2` Horizontal Pod Autoscaler on CPU and memory, scoped to the `ai-gateway` container. The defaults match the gateway ECS service in the AWS Terraform module: 2-6 replicas, 70% CPU, 75% memory, a 60 second scale-up window, and a 300 second scale-down window. +When `aiGateway.enabled` is true, the AI Gateway autoscales by default. It uses a standard `autoscaling/v2` Horizontal Pod Autoscaler on CPU and memory, scoped to the `ai-gateway` container. The defaults match the gateway ECS service in the AWS Terraform module: 2-6 replicas, 70% CPU, 75% memory, a 60 second scale-up window, and a 300 second scale-down window. -The HPA reads the Kubernetes resource metrics API. GKE and AKS provide it. On EKS, install [metrics-server](https://github.com/kubernetes-sigs/metrics-server) first. +The HPA reads the Kubernetes resource metrics API. GKE and AKS provide it. On EKS, install [metrics-server](https://github.com/kubernetes-sigs/metrics-server); without it, the HPA cannot read utilization and holds the gateway at `minReplicas`. ```yaml aiGateway: enabled: true autoscaling: - enabled: true minReplicas: 2 maxReplicas: 6 podDisruptionBudget: @@ -296,7 +295,9 @@ aiGateway: minAvailable: 1 ``` -When enabled, `aiGateway.replicas` is ignored and the HPA controls the replica count. Turning autoscaling on for a running gateway removes `replicas` from the Deployment. Kubernetes then resets the Deployment to 1 replica until the HPA raises it to `minReplicas`, usually within one HPA sync period (15 seconds by default). +While autoscaling is enabled, `aiGateway.replicas` is ignored and the HPA controls the replica count. Set `aiGateway.autoscaling.enabled: false` to keep a fixed `aiGateway.replicas` count. + +Upgrading a release that already runs the gateway with a fixed replica count removes `replicas` from the Deployment. Kubernetes then resets the Deployment to 1 replica until the HPA raises it to `minReplicas`, usually within one HPA sync period (15 seconds by default). To avoid the dip, set `aiGateway.autoscaling.enabled: false` for that upgrade, or schedule it outside peak traffic. ## API workload isolation diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 564c69f..75f66cf 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -152,7 +152,7 @@ that predates gateway autoscaling retains no aiGateway.autoscaling map, and a partial --set on top of those values would otherwise render empty fields. */}} {{- define "braintrust.aiGateway.autoscaling.defaults" -}} -enabled: false +enabled: true minReplicas: 2 maxReplicas: 6 cpu: diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index 8c6f1f7..d1abc71 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -26,25 +26,27 @@ tests: - hasDocuments: count: 0 - - it: renders no HPA by default + - it: renders no PDB by default set: aiGateway.enabled: true - template: ai-gateway-hpa.yaml + template: ai-gateway-pdb.yaml asserts: - hasDocuments: count: 0 - - it: renders no PDB by default + - it: renders no HPA when autoscaling is disabled set: aiGateway.enabled: true - template: ai-gateway-pdb.yaml + aiGateway.autoscaling.enabled: false + template: ai-gateway-hpa.yaml asserts: - hasDocuments: count: 0 - - it: keeps fixed replicas on the deployment by default + - it: keeps fixed replicas on the deployment when autoscaling is disabled set: aiGateway.enabled: true + aiGateway.autoscaling.enabled: false aiGateway.replicas: 3 template: ai-gateway-deployment.yaml asserts: @@ -52,10 +54,9 @@ tests: path: spec.replicas value: 3 - - it: renders an HPA with AWS-parity defaults when enabled + - it: renders an HPA with AWS-parity defaults by default set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true template: ai-gateway-hpa.yaml asserts: - hasDocuments: @@ -109,10 +110,9 @@ tests: - notExists: path: metadata.annotations - - it: omits fixed replicas from the deployment when autoscaling is enabled + - it: omits fixed replicas from the deployment by default set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true template: ai-gateway-deployment.yaml asserts: - notExists: @@ -208,6 +208,7 @@ tests: - it: does not validate bounds while autoscaling is disabled set: aiGateway.enabled: true + aiGateway.autoscaling.enabled: false aiGateway.autoscaling.minReplicas: 0 template: ai-gateway-deployment.yaml asserts: @@ -215,20 +216,51 @@ tests: path: spec.replicas value: 2 - - it: keeps fixed replicas and no HPA for reused values from an older chart + - it: omits fixed replicas for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-deployment.yaml + asserts: + - notExists: + path: spec.replicas + + - it: renders the default HPA for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + - equal: + path: spec.metrics[0].containerResource.target.averageUtilization + value: 70 + - notExists: + path: metadata.annotations + + - it: keeps fixed replicas when reused values disable autoscaling values: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.autoscaling.enabled: false template: ai-gateway-deployment.yaml asserts: - equal: path: spec.replicas value: 2 - - it: renders no HPA for reused values from an older chart + - it: renders no HPA when reused values disable autoscaling values: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.autoscaling.enabled: false template: ai-gateway-hpa.yaml asserts: - hasDocuments: @@ -248,7 +280,7 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml set: - aiGateway.autoscaling.enabled: true + aiGateway.autoscaling.maxReplicas: 20 template: ai-gateway-hpa.yaml asserts: - equal: @@ -256,7 +288,7 @@ tests: value: 2 - equal: path: spec.maxReplicas - value: 6 + value: 20 - equal: path: spec.metrics[1].containerResource.target.averageUtilization value: 75 diff --git a/braintrust/tests/ai-gateway-deployment_test.yaml b/braintrust/tests/ai-gateway-deployment_test.yaml index af84e6f..b6ad304 100644 --- a/braintrust/tests/ai-gateway-deployment_test.yaml +++ b/braintrust/tests/ai-gateway-deployment_test.yaml @@ -19,6 +19,7 @@ tests: - __fixtures__/base-values.yaml set: aiGateway.enabled: true + aiGateway.autoscaling.enabled: false aiGateway.image.repository: test/gateway aiGateway.image.tag: v1.0.0 release: diff --git a/braintrust/values.yaml b/braintrust/values.yaml index ca44787..1714290 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -327,12 +327,13 @@ aiGateway: hpa: {} # Ignored when autoscaling.enabled is true; the HPA controls the replica count. replicas: 2 - # Optional Horizontal Pod Autoscaler on CPU and memory, scoped to the - # ai-gateway container. Defaults mirror the AWS module's gateway ECS service. - # Requires the Kubernetes resource metrics API (metrics-server). GKE and AKS - # provide it; EKS requires installing metrics-server. + # Horizontal Pod Autoscaler on CPU and memory, scoped to the ai-gateway + # container. Defaults mirror the AWS module's gateway ECS service. + # Scaling reads the Kubernetes resource metrics API (metrics-server). GKE and + # AKS provide it; on EKS, install metrics-server. Without it, the HPA holds + # the gateway at minReplicas. autoscaling: - enabled: false + enabled: true minReplicas: 2 maxReplicas: 6 cpu: From a675059a43aab3834364d35a6a97c26868b30381 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 02:20:55 +0000 Subject: [PATCH 3/8] Simplify AI Gateway autoscaling Drop the disruption budget, HPA annotations, configurable scale windows, and the defaults/validation helper. Defaults are read inline with dig so --reuse-values upgrades from older charts still render. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 17 +- braintrust/templates/_helpers.tpl | 41 --- .../templates/ai-gateway-deployment.yaml | 3 +- braintrust/templates/ai-gateway-hpa.yaml | 27 +- braintrust/templates/ai-gateway-pdb.yaml | 26 -- .../pre-ai-gateway-autoscaling-values.yaml | 7 +- .../tests/ai-gateway-autoscaling_test.yaml | 309 +++--------------- braintrust/values.yaml | 20 +- 8 files changed, 59 insertions(+), 391 deletions(-) delete mode 100644 braintrust/templates/ai-gateway-pdb.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 9f0bf37..212e8a0 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -280,24 +280,20 @@ When enabled for a pool, that pool's `replicas` setting is ignored and the HPA c ## AI Gateway Autoscaling -When `aiGateway.enabled` is true, the AI Gateway autoscales by default. It uses a standard `autoscaling/v2` Horizontal Pod Autoscaler on CPU and memory, scoped to the `ai-gateway` container. The defaults match the gateway ECS service in the AWS Terraform module: 2-6 replicas, 70% CPU, 75% memory, a 60 second scale-up window, and a 300 second scale-down window. +When the AI Gateway is enabled, it autoscales by default. An HPA scales the `ai-gateway` container on CPU and memory with the same defaults as the AWS module: 2-6 replicas at 70% CPU and 75% memory. -The HPA reads the Kubernetes resource metrics API. GKE and AKS provide it. On EKS, install [metrics-server](https://github.com/kubernetes-sigs/metrics-server); without it, the HPA cannot read utilization and holds the gateway at `minReplicas`. +The HPA needs metrics-server. GKE and AKS include it; on EKS, [install it](https://github.com/kubernetes-sigs/metrics-server). Without it, the gateway stays at `minReplicas`. ```yaml aiGateway: - enabled: true autoscaling: minReplicas: 2 maxReplicas: 6 - podDisruptionBudget: - enabled: true - minAvailable: 1 ``` -While autoscaling is enabled, `aiGateway.replicas` is ignored and the HPA controls the replica count. Set `aiGateway.autoscaling.enabled: false` to keep a fixed `aiGateway.replicas` count. +Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas` instead. -Upgrading a release that already runs the gateway with a fixed replica count removes `replicas` from the Deployment. Kubernetes then resets the Deployment to 1 replica until the HPA raises it to `minReplicas`, usually within one HPA sync period (15 seconds by default). To avoid the dip, set `aiGateway.autoscaling.enabled: false` for that upgrade, or schedule it outside peak traffic. +On a release that already runs the gateway, the upgrade that turns on autoscaling drops the gateway to 1 replica for about 15 seconds until the HPA scales it back up. ## API workload isolation @@ -446,13 +442,8 @@ api: podDisruptionBudget: enabled: true maxUnavailable: 1 -aiGateway: - podDisruptionBudget: - enabled: true - minAvailable: 1 ``` -The AI Gateway budget defaults to `minAvailable: 1` and renders only when `aiGateway.enabled` is true. The API retains its existing `minAvailable` behavior unless `maxUnavailable` is set. An explicit maximum takes precedence over the inherited minimum. Each role has an independent budget. Evictions can proceed simultaneously across roles. Readiness probes determine healthy replicas for PDBs. Deployment `minReadySeconds` does not delay PDB eviction permission. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 75f66cf..4393355 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -146,47 +146,6 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} -{{/* -AI Gateway autoscaling defaults. `helm upgrade --reuse-values` from a chart -that predates gateway autoscaling retains no aiGateway.autoscaling map, and a -partial --set on top of those values would otherwise render empty fields. -*/}} -{{- define "braintrust.aiGateway.autoscaling.defaults" -}} -enabled: true -minReplicas: 2 -maxReplicas: 6 -cpu: - targetAverageUtilization: 70 -memory: - targetAverageUtilization: 75 -behavior: - scaleDown: - stabilizationWindowSeconds: 300 - scaleUp: - stabilizationWindowSeconds: 60 -{{- end -}} - -{{/* -Effective AI Gateway autoscaling settings: template defaults merged with any -provided values, validated when enabled. -*/}} -{{- define "braintrust.aiGateway.autoscaling" -}} -{{- $defaults := include "braintrust.aiGateway.autoscaling.defaults" . | fromYaml -}} -{{- $provided := deepCopy (.Values.aiGateway.autoscaling | default dict) -}} -{{- $autoscaling := mergeOverwrite (deepCopy $defaults) $provided -}} -{{- if $autoscaling.enabled -}} -{{- $min := int $autoscaling.minReplicas -}} -{{- $max := int $autoscaling.maxReplicas -}} -{{- if lt $min 1 -}} -{{- fail "aiGateway.autoscaling.minReplicas must be at least 1" -}} -{{- end -}} -{{- if lt $max $min -}} -{{- fail "aiGateway.autoscaling.maxReplicas must be greater than or equal to aiGateway.autoscaling.minReplicas" -}} -{{- end -}} -{{- end -}} -{{- toYaml $autoscaling -}} -{{- end -}} - {{/* Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). */}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index 7fb8c72..ef18994 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -1,5 +1,4 @@ {{- if .Values.aiGateway.enabled }} -{{- $autoscaling := include "braintrust.aiGateway.autoscaling" . | fromYaml }} --- apiVersion: apps/v1 kind: Deployment @@ -15,7 +14,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if not $autoscaling.enabled }} + {{- if not (dig "enabled" true (.Values.aiGateway.autoscaling | default dict)) }} replicas: {{ .Values.aiGateway.replicas }} {{- end }} strategy: diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml index 2377c57..fcd72fd 100644 --- a/braintrust/templates/ai-gateway-hpa.yaml +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -1,6 +1,5 @@ -{{- if .Values.aiGateway.enabled }} -{{- $autoscaling := include "braintrust.aiGateway.autoscaling" . | fromYaml }} -{{- if $autoscaling.enabled }} +{{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} +{{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -11,37 +10,31 @@ metadata: labels: {{- toYaml . | nindent 4 }} {{- end }} - {{- with (dig "hpa" dict (.Values.aiGateway.annotations | default dict)) }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: {{ .Values.aiGateway.name }} - minReplicas: {{ $autoscaling.minReplicas }} - maxReplicas: {{ $autoscaling.maxReplicas }} + minReplicas: {{ dig "minReplicas" 2 $autoscaling }} + maxReplicas: {{ dig "maxReplicas" 6 $autoscaling }} metrics: - # ContainerResource scopes utilization to the ai-gateway container so - # sidecars / extraContainers do not skew it. - type: ContainerResource containerResource: name: cpu container: ai-gateway target: type: Utilization - averageUtilization: {{ $autoscaling.cpu.targetAverageUtilization }} + averageUtilization: {{ dig "cpu" "targetAverageUtilization" 70 $autoscaling }} - type: ContainerResource containerResource: name: memory container: ai-gateway target: type: Utilization - averageUtilization: {{ $autoscaling.memory.targetAverageUtilization }} - {{- with $autoscaling.behavior }} + averageUtilization: {{ dig "memory" "targetAverageUtilization" 75 $autoscaling }} behavior: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} + scaleUp: + stabilizationWindowSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 {{- end }} diff --git a/braintrust/templates/ai-gateway-pdb.yaml b/braintrust/templates/ai-gateway-pdb.yaml deleted file mode 100644 index 5011aea..0000000 --- a/braintrust/templates/ai-gateway-pdb.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.aiGateway.enabled }} -{{- $pdb := .Values.aiGateway.podDisruptionBudget | default dict }} -{{- if $pdb.enabled }} ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ .Values.aiGateway.name }} - namespace: {{ include "braintrust.namespace" . }} - {{- with (merge (deepCopy .Values.aiGateway.labels) .Values.global.labels) }} - labels: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if ne $pdb.maxUnavailable nil }} - maxUnavailable: {{ $pdb.maxUnavailable | toYaml | trim }} - {{- else if ne $pdb.minAvailable nil }} - minAvailable: {{ $pdb.minAvailable | toYaml | trim }} - {{- else }} - minAvailable: 1 - {{- end }} - selector: - matchLabels: - app: {{ .Values.aiGateway.name }} -{{- end }} -{{- end }} diff --git a/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml index 2d0cba5..7179721 100644 --- a/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml +++ b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml @@ -1,9 +1,6 @@ # Simulate values retained by `helm upgrade --reuse-values` from a chart release -# that predates AI Gateway autoscaling and disruption budgets. An explicit null -# removes the new chart default during Helm value coalescing. +# that predates AI Gateway autoscaling. An explicit null removes the new chart +# default during Helm value coalescing. aiGateway: enabled: true - annotations: - hpa: null autoscaling: null - podDisruptionBudget: null diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index d1abc71..bbf7176 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -1,7 +1,6 @@ -suite: AI Gateway autoscaling and disruption budget +suite: AI Gateway autoscaling templates: - ai-gateway-hpa.yaml - - ai-gateway-pdb.yaml - ai-gateway-deployment.yaml - ai-gateway-configmap.yaml values: @@ -9,74 +8,22 @@ values: release: namespace: braintrust tests: - - it: renders no HPA or PDB when the gateway is disabled, even if both are enabled - set: - aiGateway.autoscaling.enabled: true - aiGateway.podDisruptionBudget.enabled: true + - it: renders no HPA when the gateway is disabled template: ai-gateway-hpa.yaml asserts: - hasDocuments: count: 0 - - it: renders no PDB when the gateway is disabled - set: - aiGateway.podDisruptionBudget.enabled: true - template: ai-gateway-pdb.yaml - asserts: - - hasDocuments: - count: 0 - - - it: renders no PDB by default + - it: renders an HPA with AWS-parity defaults set: aiGateway.enabled: true - template: ai-gateway-pdb.yaml - asserts: - - hasDocuments: - count: 0 - - - it: renders no HPA when autoscaling is disabled - set: - aiGateway.enabled: true - aiGateway.autoscaling.enabled: false template: ai-gateway-hpa.yaml asserts: - - hasDocuments: - count: 0 - - - it: keeps fixed replicas on the deployment when autoscaling is disabled - set: - aiGateway.enabled: true - aiGateway.autoscaling.enabled: false - aiGateway.replicas: 3 - template: ai-gateway-deployment.yaml - asserts: - - equal: - path: spec.replicas - value: 3 - - - it: renders an HPA with AWS-parity defaults by default - set: - aiGateway.enabled: true - template: ai-gateway-hpa.yaml - asserts: - - hasDocuments: - count: 1 - isKind: of: HorizontalPodAutoscaler - - isAPIVersion: - of: autoscaling/v2 - equal: - path: metadata.name + path: spec.scaleTargetRef.name value: braintrust-ai-gateway - - equal: - path: metadata.namespace - value: braintrust - - equal: - path: spec.scaleTargetRef - value: - apiVersion: apps/v1 - kind: Deployment - name: braintrust-ai-gateway - equal: path: spec.minReplicas value: 2 @@ -84,33 +31,23 @@ tests: path: spec.maxReplicas value: 6 - equal: - path: spec.metrics + path: spec.metrics[0].containerResource value: - - type: ContainerResource - containerResource: - name: cpu - container: ai-gateway - target: - type: Utilization - averageUtilization: 70 - - type: ContainerResource - containerResource: - name: memory - container: ai-gateway - target: - type: Utilization - averageUtilization: 75 + name: cpu + container: ai-gateway + target: + type: Utilization + averageUtilization: 70 - equal: - path: spec.behavior + path: spec.metrics[1].containerResource value: - scaleDown: - stabilizationWindowSeconds: 300 - scaleUp: - stabilizationWindowSeconds: 60 - - notExists: - path: metadata.annotations + name: memory + container: ai-gateway + target: + type: Utilization + averageUtilization: 75 - - it: omits fixed replicas from the deployment by default + - it: lets the HPA own replicas by default set: aiGateway.enabled: true template: ai-gateway-deployment.yaml @@ -118,44 +55,15 @@ tests: - notExists: path: spec.replicas - - it: applies custom bounds, targets, behavior, names, labels, and annotations + - it: applies custom bounds and targets set: aiGateway.enabled: true - aiGateway.name: custom-gateway - aiGateway.labels: - team: gateway - global.labels: - env: test - aiGateway.annotations.hpa: - example.com/owner: gateway - aiGateway.autoscaling: - enabled: true - minReplicas: 4 - maxReplicas: 40 - cpu: - targetAverageUtilization: 60 - memory: - targetAverageUtilization: 80 - behavior: - scaleUp: - stabilizationWindowSeconds: 0 + aiGateway.autoscaling.minReplicas: 4 + aiGateway.autoscaling.maxReplicas: 40 + aiGateway.autoscaling.cpu.targetAverageUtilization: 60 + aiGateway.autoscaling.memory.targetAverageUtilization: 80 template: ai-gateway-hpa.yaml asserts: - - equal: - path: metadata.name - value: custom-gateway - - equal: - path: spec.scaleTargetRef.name - value: custom-gateway - - equal: - path: metadata.labels - value: - team: gateway - env: test - - equal: - path: metadata.annotations - value: - example.com/owner: gateway - equal: path: spec.minReplicas value: 4 @@ -168,64 +76,28 @@ tests: - equal: path: spec.metrics[1].containerResource.target.averageUtilization value: 80 - - equal: - path: spec.behavior.scaleUp.stabilizationWindowSeconds - value: 0 - - - it: uses the managed namespace when the chart creates it - set: - aiGateway.enabled: true - aiGateway.autoscaling.enabled: true - global.createNamespace: true - global.namespace: custom-namespace - template: ai-gateway-hpa.yaml - asserts: - - equal: - path: metadata.namespace - value: custom-namespace - - - it: rejects zero minReplicas - set: - aiGateway.enabled: true - aiGateway.autoscaling.enabled: true - aiGateway.autoscaling.minReplicas: 0 - template: ai-gateway-hpa.yaml - asserts: - - failedTemplate: - errorMessage: "aiGateway.autoscaling.minReplicas must be at least 1" - - it: rejects maxReplicas below minReplicas + - it: renders no HPA when autoscaling is disabled set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true - aiGateway.autoscaling.minReplicas: 4 - aiGateway.autoscaling.maxReplicas: 3 + aiGateway.autoscaling.enabled: false template: ai-gateway-hpa.yaml asserts: - - failedTemplate: - errorMessage: "aiGateway.autoscaling.maxReplicas must be greater than or equal to aiGateway.autoscaling.minReplicas" + - hasDocuments: + count: 0 - - it: does not validate bounds while autoscaling is disabled + - it: keeps fixed replicas when autoscaling is disabled set: aiGateway.enabled: true aiGateway.autoscaling.enabled: false - aiGateway.autoscaling.minReplicas: 0 + aiGateway.replicas: 3 template: ai-gateway-deployment.yaml asserts: - equal: path: spec.replicas - value: 2 - - - it: omits fixed replicas for reused values from an older chart - values: - - __fixtures__/base-values.yaml - - __fixtures__/pre-ai-gateway-autoscaling-values.yaml - template: ai-gateway-deployment.yaml - asserts: - - notExists: - path: spec.replicas + value: 3 - - it: renders the default HPA for reused values from an older chart + - it: uses default bounds for reused values from an older chart values: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml @@ -238,44 +110,10 @@ tests: path: spec.maxReplicas value: 6 - equal: - path: spec.metrics[0].containerResource.target.averageUtilization - value: 70 - - notExists: - path: metadata.annotations - - - it: keeps fixed replicas when reused values disable autoscaling - values: - - __fixtures__/base-values.yaml - - __fixtures__/pre-ai-gateway-autoscaling-values.yaml - set: - aiGateway.autoscaling.enabled: false - template: ai-gateway-deployment.yaml - asserts: - - equal: - path: spec.replicas - value: 2 - - - it: renders no HPA when reused values disable autoscaling - values: - - __fixtures__/base-values.yaml - - __fixtures__/pre-ai-gateway-autoscaling-values.yaml - set: - aiGateway.autoscaling.enabled: false - template: ai-gateway-hpa.yaml - asserts: - - hasDocuments: - count: 0 - - - it: renders no PDB for reused values from an older chart - values: - - __fixtures__/base-values.yaml - - __fixtures__/pre-ai-gateway-autoscaling-values.yaml - template: ai-gateway-pdb.yaml - asserts: - - hasDocuments: - count: 0 + path: spec.metrics[1].containerResource.target.averageUtilization + value: 75 - - it: fills defaults for a partial autoscaling --set on reused values + - it: fills defaults around a partial --set on reused values values: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml @@ -289,26 +127,18 @@ tests: - equal: path: spec.maxReplicas value: 20 - - equal: - path: spec.metrics[1].containerResource.target.averageUtilization - value: 75 - - equal: - path: spec.behavior.scaleDown.stabilizationWindowSeconds - value: 300 - - notExists: - path: metadata.annotations - - it: defaults minAvailable for a partial PDB --set on reused values + - it: keeps fixed replicas when reused values disable autoscaling values: - __fixtures__/base-values.yaml - __fixtures__/pre-ai-gateway-autoscaling-values.yaml set: - aiGateway.podDisruptionBudget.enabled: true - template: ai-gateway-pdb.yaml + aiGateway.autoscaling.enabled: false + template: ai-gateway-deployment.yaml asserts: - equal: - path: spec.minAvailable - value: 1 + path: spec.replicas + value: 2 - it: renders the HPA on aws values: @@ -316,7 +146,6 @@ tests: - __fixtures__/aws-values.yaml set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true template: ai-gateway-hpa.yaml asserts: - isKind: @@ -328,7 +157,6 @@ tests: - __fixtures__/azure-values.yaml set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true template: ai-gateway-hpa.yaml asserts: - isKind: @@ -340,68 +168,7 @@ tests: - __fixtures__/google-values.yaml set: aiGateway.enabled: true - aiGateway.autoscaling.enabled: true template: ai-gateway-hpa.yaml asserts: - isKind: of: HorizontalPodAutoscaler - - - it: renders a PDB with minAvailable 1 when enabled - set: - aiGateway.enabled: true - aiGateway.podDisruptionBudget.enabled: true - template: ai-gateway-pdb.yaml - asserts: - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: braintrust-ai-gateway - - equal: - path: metadata.namespace - value: braintrust - - equal: - path: spec.minAvailable - value: 1 - - notExists: - path: spec.maxUnavailable - - equal: - path: spec.selector.matchLabels.app - value: braintrust-ai-gateway - - - it: preserves a zero minAvailable - set: - aiGateway.enabled: true - aiGateway.podDisruptionBudget.enabled: true - aiGateway.podDisruptionBudget.minAvailable: 0 - template: ai-gateway-pdb.yaml - asserts: - - equal: - path: spec.minAvailable - value: 0 - - - it: prefers an explicit maxUnavailable - set: - aiGateway.enabled: true - aiGateway.podDisruptionBudget.enabled: true - aiGateway.podDisruptionBudget.maxUnavailable: 1 - template: ai-gateway-pdb.yaml - asserts: - - equal: - path: spec.maxUnavailable - value: 1 - - notExists: - path: spec.minAvailable - - - it: preserves a zero maxUnavailable - set: - aiGateway.enabled: true - aiGateway.podDisruptionBudget.enabled: true - aiGateway.podDisruptionBudget.maxUnavailable: 0 - template: ai-gateway-pdb.yaml - asserts: - - equal: - path: spec.maxUnavailable - value: 0 - - notExists: - path: spec.minAvailable diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 1714290..f361f2c 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -324,14 +324,11 @@ aiGateway: deployment: {} service: {} pod: {} - hpa: {} - # Ignored when autoscaling.enabled is true; the HPA controls the replica count. + # Ignored when autoscaling.enabled is true. replicas: 2 - # Horizontal Pod Autoscaler on CPU and memory, scoped to the ai-gateway - # container. Defaults mirror the AWS module's gateway ECS service. - # Scaling reads the Kubernetes resource metrics API (metrics-server). GKE and - # AKS provide it; on EKS, install metrics-server. Without it, the HPA holds - # the gateway at minReplicas. + # HPA on ai-gateway container CPU and memory, matching the AWS module's + # gateway ECS service. Needs metrics-server; without it the HPA holds + # minReplicas. autoscaling: enabled: true minReplicas: 2 @@ -340,15 +337,6 @@ aiGateway: targetAverageUtilization: 70 memory: targetAverageUtilization: 75 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - scaleUp: - stabilizationWindowSeconds: 60 - podDisruptionBudget: - enabled: false - # An explicit maxUnavailable overrides this minimum. - minAvailable: 1 image: repository: public.ecr.aws/braintrust/gateway tag: v2.21.0 From 04436898736ba15117eb768637f7e63824b75f24 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 02:22:49 +0000 Subject: [PATCH 4/8] Test AI Gateway HPA label isolation Cover global/gateway label precedence on the HPA and confirm gateway labels do not reach another rendered component, per the label-merge test rule. Co-Authored-By: Claude Opus 5.5 --- .../tests/labels-merge-autoscaling_test.yaml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/braintrust/tests/labels-merge-autoscaling_test.yaml b/braintrust/tests/labels-merge-autoscaling_test.yaml index a04fb5c..ffd365e 100644 --- a/braintrust/tests/labels-merge-autoscaling_test.yaml +++ b/braintrust/tests/labels-merge-autoscaling_test.yaml @@ -2,6 +2,7 @@ suite: test label merge isolation for API autoscaling resources templates: - api-autoscaling-metric.yaml - api-hpa.yaml + - ai-gateway-hpa.yaml - brainstore-reader-configmap.yaml capabilities: apiVersions: @@ -84,3 +85,55 @@ tests: value: reader-config - isNull: path: metadata.labels["api-specific"] + + - it: should give the AI Gateway HPA only global and gateway-specific labels + template: ai-gateway-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + aiGateway.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + aiGateway.labels.tier: gateway-tier + aiGateway.labels.gateway-specific: gateway-autoscaling + api.labels.api-specific: api-autoscaling + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: gateway-tier + - equal: + path: metadata.labels["gateway-specific"] + value: gateway-autoscaling + - isNull: + path: metadata.labels["api-specific"] + + - it: should not leak AI Gateway labels into another rendered component + template: brainstore-reader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + aiGateway.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + aiGateway.labels.tier: gateway-tier + aiGateway.labels.gateway-specific: gateway-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: global-tier + - equal: + path: metadata.labels["reader-specific"] + value: reader-config + - isNull: + path: metadata.labels["gateway-specific"] From bfc0edb05781a7d55f5aaa1d175326b8baf163cc Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 23:24:27 +0000 Subject: [PATCH 5/8] Reject AI Gateway HPA configs Kubernetes can't use Fail rendering when autoscaling.minReplicas is below 1 or the gateway has no CPU or memory request, since utilization targets need both. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 2 +- braintrust/templates/ai-gateway-hpa.yaml | 10 ++++- .../tests/ai-gateway-autoscaling_test.yaml | 37 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 212e8a0..1d7bb62 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -291,7 +291,7 @@ aiGateway: maxReplicas: 6 ``` -Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas` instead. +The targets are percentages of `aiGateway.resources.requests`, so the chart requires CPU and memory requests while autoscaling is on. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas` instead. On a release that already runs the gateway, the upgrade that turns on autoscaling drops the gateway to 1 replica for about 15 seconds until the HPA scales it back up. diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml index fcd72fd..7f8480c 100644 --- a/braintrust/templates/ai-gateway-hpa.yaml +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -1,5 +1,13 @@ {{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} {{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} +{{- $minReplicas := dig "minReplicas" 2 $autoscaling }} +{{- if lt (int $minReplicas) 1 }} +{{- fail "aiGateway.autoscaling.minReplicas must be at least 1" }} +{{- end }} +{{- $requests := (.Values.aiGateway.resources | default dict).requests | default dict }} +{{- if not (and $requests.cpu $requests.memory) }} +{{- fail "aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false." }} +{{- end }} --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -15,7 +23,7 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ .Values.aiGateway.name }} - minReplicas: {{ dig "minReplicas" 2 $autoscaling }} + minReplicas: {{ $minReplicas }} maxReplicas: {{ dig "maxReplicas" 6 $autoscaling }} metrics: - type: ContainerResource diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index bbf7176..1b7342e 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -77,6 +77,43 @@ tests: path: spec.metrics[1].containerResource.target.averageUtilization value: 80 + - it: rejects zero minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling.minReplicas must be at least 1 + + - it: rejects missing CPU and memory requests + set: + aiGateway.enabled: true + aiGateway.resources.requests: null + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false. + + - it: rejects a missing memory request + set: + aiGateway.enabled: true + aiGateway.resources.requests.memory: null + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false. + + - it: allows missing requests when autoscaling is disabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: false + aiGateway.resources.requests: null + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + - it: renders no HPA when autoscaling is disabled set: aiGateway.enabled: true From 3e059f6f2926e46409a8b3d4c317049ba9cbe380 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 23:28:58 +0000 Subject: [PATCH 6/8] Drop AI Gateway HPA render guards and trim docs The API HPA has neither check, the default values set requests, and Kubernetes rejects minReplicas 0 on its own. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 15 +------- braintrust/templates/ai-gateway-hpa.yaml | 10 +---- .../tests/ai-gateway-autoscaling_test.yaml | 37 ------------------- braintrust/values.yaml | 4 +- 4 files changed, 4 insertions(+), 62 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 1d7bb62..b7fe255 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -280,20 +280,9 @@ When enabled for a pool, that pool's `replicas` setting is ignored and the HPA c ## AI Gateway Autoscaling -When the AI Gateway is enabled, it autoscales by default. An HPA scales the `ai-gateway` container on CPU and memory with the same defaults as the AWS module: 2-6 replicas at 70% CPU and 75% memory. +The AI Gateway autoscales by default. An HPA runs 2-6 replicas at 70% CPU and 75% memory, the same as the AWS module. It needs metrics-server, which GKE and AKS include; without it the gateway stays at 2. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas`. -The HPA needs metrics-server. GKE and AKS include it; on EKS, [install it](https://github.com/kubernetes-sigs/metrics-server). Without it, the gateway stays at `minReplicas`. - -```yaml -aiGateway: - autoscaling: - minReplicas: 2 - maxReplicas: 6 -``` - -The targets are percentages of `aiGateway.resources.requests`, so the chart requires CPU and memory requests while autoscaling is on. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas` instead. - -On a release that already runs the gateway, the upgrade that turns on autoscaling drops the gateway to 1 replica for about 15 seconds until the HPA scales it back up. +Turning on autoscaling for a running gateway briefly drops it to 1 replica until the HPA scales it back up. ## API workload isolation diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml index 7f8480c..fcd72fd 100644 --- a/braintrust/templates/ai-gateway-hpa.yaml +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -1,13 +1,5 @@ {{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} {{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} -{{- $minReplicas := dig "minReplicas" 2 $autoscaling }} -{{- if lt (int $minReplicas) 1 }} -{{- fail "aiGateway.autoscaling.minReplicas must be at least 1" }} -{{- end }} -{{- $requests := (.Values.aiGateway.resources | default dict).requests | default dict }} -{{- if not (and $requests.cpu $requests.memory) }} -{{- fail "aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false." }} -{{- end }} --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -23,7 +15,7 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ .Values.aiGateway.name }} - minReplicas: {{ $minReplicas }} + minReplicas: {{ dig "minReplicas" 2 $autoscaling }} maxReplicas: {{ dig "maxReplicas" 6 $autoscaling }} metrics: - type: ContainerResource diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index 1b7342e..bbf7176 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -77,43 +77,6 @@ tests: path: spec.metrics[1].containerResource.target.averageUtilization value: 80 - - it: rejects zero minReplicas - set: - aiGateway.enabled: true - aiGateway.autoscaling.minReplicas: 0 - template: ai-gateway-hpa.yaml - asserts: - - failedTemplate: - errorMessage: aiGateway.autoscaling.minReplicas must be at least 1 - - - it: rejects missing CPU and memory requests - set: - aiGateway.enabled: true - aiGateway.resources.requests: null - template: ai-gateway-hpa.yaml - asserts: - - failedTemplate: - errorMessage: aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false. - - - it: rejects a missing memory request - set: - aiGateway.enabled: true - aiGateway.resources.requests.memory: null - template: ai-gateway-hpa.yaml - asserts: - - failedTemplate: - errorMessage: aiGateway.autoscaling needs aiGateway.resources.requests.cpu and .memory. Set them, or set aiGateway.autoscaling.enabled to false. - - - it: allows missing requests when autoscaling is disabled - set: - aiGateway.enabled: true - aiGateway.autoscaling.enabled: false - aiGateway.resources.requests: null - template: ai-gateway-hpa.yaml - asserts: - - hasDocuments: - count: 0 - - it: renders no HPA when autoscaling is disabled set: aiGateway.enabled: true diff --git a/braintrust/values.yaml b/braintrust/values.yaml index f361f2c..cb5ba9a 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -326,9 +326,7 @@ aiGateway: pod: {} # Ignored when autoscaling.enabled is true. replicas: 2 - # HPA on ai-gateway container CPU and memory, matching the AWS module's - # gateway ECS service. Needs metrics-server; without it the HPA holds - # minReplicas. + # Needs metrics-server; without it the HPA holds minReplicas. autoscaling: enabled: true minReplicas: 2 From 76d4062841f29cd236d99ab7ebac0d6bb1d013b1 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 3 Nov 2026 23:47:41 +0000 Subject: [PATCH 7/8] Validate AI Gateway HPA inputs Fail rendering when replica bounds or utilization targets are ones Kubernetes rejects, or the gateway has no CPU or memory request. Co-Authored-By: Claude Opus 5.5 --- braintrust/templates/ai-gateway-hpa.yaml | 16 +++++-- .../tests/ai-gateway-autoscaling_test.yaml | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml index fcd72fd..40a54df 100644 --- a/braintrust/templates/ai-gateway-hpa.yaml +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -1,5 +1,13 @@ {{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} {{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} +{{- $minReplicas := int (dig "minReplicas" 2 $autoscaling) }} +{{- $maxReplicas := int (dig "maxReplicas" 6 $autoscaling) }} +{{- $cpuTarget := int (dig "cpu" "targetAverageUtilization" 70 $autoscaling) }} +{{- $memoryTarget := int (dig "memory" "targetAverageUtilization" 75 $autoscaling) }} +{{- $requests := (.Values.aiGateway.resources | default dict).requests | default dict }} +{{- if not (and (ge $minReplicas 1) (ge $maxReplicas $minReplicas) (gt $cpuTarget 0) (gt $memoryTarget 0) $requests.cpu $requests.memory) }} +{{- fail "aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory" }} +{{- end }} --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -15,8 +23,8 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ .Values.aiGateway.name }} - minReplicas: {{ dig "minReplicas" 2 $autoscaling }} - maxReplicas: {{ dig "maxReplicas" 6 $autoscaling }} + minReplicas: {{ $minReplicas }} + maxReplicas: {{ $maxReplicas }} metrics: - type: ContainerResource containerResource: @@ -24,14 +32,14 @@ spec: container: ai-gateway target: type: Utilization - averageUtilization: {{ dig "cpu" "targetAverageUtilization" 70 $autoscaling }} + averageUtilization: {{ $cpuTarget }} - type: ContainerResource containerResource: name: memory container: ai-gateway target: type: Utilization - averageUtilization: {{ dig "memory" "targetAverageUtilization" 75 $autoscaling }} + averageUtilization: {{ $memoryTarget }} behavior: scaleUp: stabilizationWindowSeconds: 60 diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index bbf7176..7fe6f9d 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -77,6 +77,51 @@ tests: path: spec.metrics[1].containerResource.target.averageUtilization value: 80 + - it: rejects zero minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects maxReplicas below minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.maxReplicas: 1 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects an empty CPU target + set: + aiGateway.enabled: true + aiGateway.autoscaling.cpu.targetAverageUtilization: "" + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects a zero memory target + set: + aiGateway.enabled: true + aiGateway.autoscaling.memory.targetAverageUtilization: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects a missing memory request + set: + aiGateway.enabled: true + aiGateway.resources.requests.memory: null + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + - it: renders no HPA when autoscaling is disabled set: aiGateway.enabled: true From 34f88b3a73f5fc9665554fdd5eeb39d5e5529ddd Mon Sep 17 00:00:00 2001 From: Alex R Date: Sun, 8 Nov 2026 19:28:21 +0000 Subject: [PATCH 8/8] Default AI Gateway HPA bounds from aiGateway.replicas minReplicas now defaults to aiGateway.replicas and maxReplicas to max(6, minReplicas), so a custom replica count is no longer reset to 2-6. The bounds leave values.yaml so the template default applies. README notes the 30-60s dip on enabling autoscaling. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 4 +- braintrust/templates/ai-gateway-hpa.yaml | 5 +- .../tests/ai-gateway-autoscaling_test.yaml | 100 ++++++++++++++++++ braintrust/values.yaml | 7 +- 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index b7fe255..5ca193e 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -280,9 +280,9 @@ When enabled for a pool, that pool's `replicas` setting is ignored and the HPA c ## AI Gateway Autoscaling -The AI Gateway autoscales by default. An HPA runs 2-6 replicas at 70% CPU and 75% memory, the same as the AWS module. It needs metrics-server, which GKE and AKS include; without it the gateway stays at 2. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas`. +The AI Gateway autoscales by default at 70% CPU and 75% memory, the same as the AWS module. `minReplicas` defaults to `aiGateway.replicas` (2) and `maxReplicas` to the larger of 6 and `minReplicas`, so a custom `replicas` count is kept as the floor. It needs metrics-server, which GKE and AKS include; without it the gateway stays at `minReplicas`. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas`. -Turning on autoscaling for a running gateway briefly drops it to 1 replica until the HPA scales it back up. +Turning on autoscaling for a running gateway briefly drops it to 1 replica, for roughly 30-60 seconds: up to about 15 seconds for the HPA to restore the count, then image pull and the readiness delay. Setting `aiGateway.autoscaling.enabled: false` only postpones this until autoscaling is turned on. ## API workload isolation diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml index 40a54df..500f2b1 100644 --- a/braintrust/templates/ai-gateway-hpa.yaml +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -1,7 +1,8 @@ {{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} {{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} -{{- $minReplicas := int (dig "minReplicas" 2 $autoscaling) }} -{{- $maxReplicas := int (dig "maxReplicas" 6 $autoscaling) }} +{{- /* Bounds default from replicas so a custom count is not reset to 2-6. */}} +{{- $minReplicas := int (dig "minReplicas" (dig "replicas" 2 .Values.aiGateway) $autoscaling) }} +{{- $maxReplicas := int (dig "maxReplicas" (max 6 $minReplicas) $autoscaling) }} {{- $cpuTarget := int (dig "cpu" "targetAverageUtilization" 70 $autoscaling) }} {{- $memoryTarget := int (dig "memory" "targetAverageUtilization" 75 $autoscaling) }} {{- $requests := (.Values.aiGateway.resources | default dict).requests | default dict }} diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml index 7fe6f9d..7c14eb8 100644 --- a/braintrust/tests/ai-gateway-autoscaling_test.yaml +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -77,6 +77,91 @@ tests: path: spec.metrics[1].containerResource.target.averageUtilization value: 80 + - it: keeps a custom replicas count as the floor + set: + aiGateway.enabled: true + aiGateway.replicas: 4 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 6 + + - it: raises maxReplicas to a replicas count above 6 + set: + aiGateway.enabled: true + aiGateway.replicas: 10 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 10 + - equal: + path: spec.maxReplicas + value: 10 + + - it: raises maxReplicas to a minReplicas above 6 + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 8 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 8 + - equal: + path: spec.maxReplicas + value: 8 + + - it: prefers explicit bounds over replicas + set: + aiGateway.enabled: true + aiGateway.replicas: 10 + aiGateway.autoscaling.minReplicas: 3 + aiGateway.autoscaling.maxReplicas: 5 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 3 + - equal: + path: spec.maxReplicas + value: 5 + + - it: uses 2 replicas when replicas is null + set: + aiGateway.enabled: true + aiGateway.replicas: null + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + + - it: rejects zero replicas + set: + aiGateway.enabled: true + aiGateway.replicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects empty replicas + set: + aiGateway.enabled: true + aiGateway.replicas: "" + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + - it: rejects zero minReplicas set: aiGateway.enabled: true @@ -173,6 +258,21 @@ tests: path: spec.maxReplicas value: 20 + - it: keeps a reused custom replicas count as the floor + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.replicas: 4 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 6 + - it: keeps fixed replicas when reused values disable autoscaling values: - __fixtures__/base-values.yaml diff --git a/braintrust/values.yaml b/braintrust/values.yaml index cb5ba9a..a5aed8b 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -324,13 +324,14 @@ aiGateway: deployment: {} service: {} pod: {} - # Ignored when autoscaling.enabled is true. + # With autoscaling, the default minReplicas. replicas: 2 # Needs metrics-server; without it the HPA holds minReplicas. autoscaling: enabled: true - minReplicas: 2 - maxReplicas: 6 + # minReplicas defaults to replicas, maxReplicas to max(6, minReplicas). + # minReplicas: 2 + # maxReplicas: 6 cpu: targetAverageUtilization: 70 memory: