diff --git a/braintrust/README.md b/braintrust/README.md index 5adb943..5ca193e 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -278,6 +278,12 @@ api: When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. +## AI Gateway Autoscaling + +The AI Gateway autoscales by default at 70% CPU and 75% memory, the same as the AWS module. `minReplicas` defaults to `aiGateway.replicas` (2) and `maxReplicas` to the larger of 6 and `minReplicas`, so a custom `replicas` count is kept as the floor. It needs metrics-server, which GKE and AKS include; without it the gateway stays at `minReplicas`. Set `aiGateway.autoscaling.enabled: false` to use a fixed `aiGateway.replicas`. + +Turning on autoscaling for a running gateway briefly drops it to 1 replica, for roughly 30-60 seconds: up to about 15 seconds for the HPA to restore the count, then image pull and the readiness delay. Setting `aiGateway.autoscaling.enabled: false` only postpones this until autoscaling is turned on. + ## API workload isolation `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest` diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index e78b254..ef18994 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -14,7 +14,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if not (dig "enabled" true (.Values.aiGateway.autoscaling | default dict)) }} replicas: {{ .Values.aiGateway.replicas }} + {{- end }} strategy: type: RollingUpdate rollingUpdate: diff --git a/braintrust/templates/ai-gateway-hpa.yaml b/braintrust/templates/ai-gateway-hpa.yaml new file mode 100644 index 0000000..500f2b1 --- /dev/null +++ b/braintrust/templates/ai-gateway-hpa.yaml @@ -0,0 +1,49 @@ +{{- $autoscaling := .Values.aiGateway.autoscaling | default dict }} +{{- if and .Values.aiGateway.enabled (dig "enabled" true $autoscaling) }} +{{- /* Bounds default from replicas so a custom count is not reset to 2-6. */}} +{{- $minReplicas := int (dig "minReplicas" (dig "replicas" 2 .Values.aiGateway) $autoscaling) }} +{{- $maxReplicas := int (dig "maxReplicas" (max 6 $minReplicas) $autoscaling) }} +{{- $cpuTarget := int (dig "cpu" "targetAverageUtilization" 70 $autoscaling) }} +{{- $memoryTarget := int (dig "memory" "targetAverageUtilization" 75 $autoscaling) }} +{{- $requests := (.Values.aiGateway.resources | default dict).requests | default dict }} +{{- if not (and (ge $minReplicas 1) (ge $maxReplicas $minReplicas) (gt $cpuTarget 0) (gt $memoryTarget 0) $requests.cpu $requests.memory) }} +{{- fail "aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory" }} +{{- end }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Values.aiGateway.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (merge (deepCopy .Values.aiGateway.labels) .Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.aiGateway.name }} + minReplicas: {{ $minReplicas }} + maxReplicas: {{ $maxReplicas }} + metrics: + - type: ContainerResource + containerResource: + name: cpu + container: ai-gateway + target: + type: Utilization + averageUtilization: {{ $cpuTarget }} + - type: ContainerResource + containerResource: + name: memory + container: ai-gateway + target: + type: Utilization + averageUtilization: {{ $memoryTarget }} + behavior: + scaleUp: + stabilizationWindowSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 +{{- end }} diff --git a/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml new file mode 100644 index 0000000..7179721 --- /dev/null +++ b/braintrust/tests/__fixtures__/pre-ai-gateway-autoscaling-values.yaml @@ -0,0 +1,6 @@ +# Simulate values retained by `helm upgrade --reuse-values` from a chart release +# that predates AI Gateway autoscaling. An explicit null removes the new chart +# default during Helm value coalescing. +aiGateway: + enabled: true + autoscaling: null diff --git a/braintrust/tests/ai-gateway-autoscaling_test.yaml b/braintrust/tests/ai-gateway-autoscaling_test.yaml new file mode 100644 index 0000000..7c14eb8 --- /dev/null +++ b/braintrust/tests/ai-gateway-autoscaling_test.yaml @@ -0,0 +1,319 @@ +suite: AI Gateway autoscaling +templates: + - ai-gateway-hpa.yaml + - ai-gateway-deployment.yaml + - ai-gateway-configmap.yaml +values: + - __fixtures__/base-values.yaml +release: + namespace: braintrust +tests: + - it: renders no HPA when the gateway is disabled + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders an HPA with AWS-parity defaults + set: + aiGateway.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: spec.scaleTargetRef.name + value: braintrust-ai-gateway + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + - equal: + path: spec.metrics[0].containerResource + value: + name: cpu + container: ai-gateway + target: + type: Utilization + averageUtilization: 70 + - equal: + path: spec.metrics[1].containerResource + value: + name: memory + container: ai-gateway + target: + type: Utilization + averageUtilization: 75 + + - it: lets the HPA own replicas by default + set: + aiGateway.enabled: true + template: ai-gateway-deployment.yaml + asserts: + - notExists: + path: spec.replicas + + - it: applies custom bounds and targets + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 4 + aiGateway.autoscaling.maxReplicas: 40 + aiGateway.autoscaling.cpu.targetAverageUtilization: 60 + aiGateway.autoscaling.memory.targetAverageUtilization: 80 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 40 + - equal: + path: spec.metrics[0].containerResource.target.averageUtilization + value: 60 + - equal: + path: spec.metrics[1].containerResource.target.averageUtilization + value: 80 + + - it: keeps a custom replicas count as the floor + set: + aiGateway.enabled: true + aiGateway.replicas: 4 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 6 + + - it: raises maxReplicas to a replicas count above 6 + set: + aiGateway.enabled: true + aiGateway.replicas: 10 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 10 + - equal: + path: spec.maxReplicas + value: 10 + + - it: raises maxReplicas to a minReplicas above 6 + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 8 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 8 + - equal: + path: spec.maxReplicas + value: 8 + + - it: prefers explicit bounds over replicas + set: + aiGateway.enabled: true + aiGateway.replicas: 10 + aiGateway.autoscaling.minReplicas: 3 + aiGateway.autoscaling.maxReplicas: 5 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 3 + - equal: + path: spec.maxReplicas + value: 5 + + - it: uses 2 replicas when replicas is null + set: + aiGateway.enabled: true + aiGateway.replicas: null + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + + - it: rejects zero replicas + set: + aiGateway.enabled: true + aiGateway.replicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects empty replicas + set: + aiGateway.enabled: true + aiGateway.replicas: "" + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects zero minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.minReplicas: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects maxReplicas below minReplicas + set: + aiGateway.enabled: true + aiGateway.autoscaling.maxReplicas: 1 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects an empty CPU target + set: + aiGateway.enabled: true + aiGateway.autoscaling.cpu.targetAverageUtilization: "" + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects a zero memory target + set: + aiGateway.enabled: true + aiGateway.autoscaling.memory.targetAverageUtilization: 0 + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: rejects a missing memory request + set: + aiGateway.enabled: true + aiGateway.resources.requests.memory: null + template: ai-gateway-hpa.yaml + asserts: + - failedTemplate: + errorMessage: aiGateway.autoscaling needs 1 <= minReplicas <= maxReplicas, positive CPU and memory targets, and aiGateway.resources.requests for CPU and memory + + - it: renders no HPA when autoscaling is disabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: false + template: ai-gateway-hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: keeps fixed replicas when autoscaling is disabled + set: + aiGateway.enabled: true + aiGateway.autoscaling.enabled: false + aiGateway.replicas: 3 + template: ai-gateway-deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 3 + + - it: uses default bounds for reused values from an older chart + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + - equal: + path: spec.metrics[1].containerResource.target.averageUtilization + value: 75 + + - it: fills defaults around a partial --set on reused values + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.autoscaling.maxReplicas: 20 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 20 + + - it: keeps a reused custom replicas count as the floor + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.replicas: 4 + template: ai-gateway-hpa.yaml + asserts: + - equal: + path: spec.minReplicas + value: 4 + - equal: + path: spec.maxReplicas + value: 6 + + - it: keeps fixed replicas when reused values disable autoscaling + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-ai-gateway-autoscaling-values.yaml + set: + aiGateway.autoscaling.enabled: false + template: ai-gateway-deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: renders the HPA on aws + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + aiGateway.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + + - it: renders the HPA on azure + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml + set: + aiGateway.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler + + - it: renders the HPA on google without the AutoscalingMetric API + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + set: + aiGateway.enabled: true + template: ai-gateway-hpa.yaml + asserts: + - isKind: + of: HorizontalPodAutoscaler diff --git a/braintrust/tests/ai-gateway-deployment_test.yaml b/braintrust/tests/ai-gateway-deployment_test.yaml index af84e6f..b6ad304 100644 --- a/braintrust/tests/ai-gateway-deployment_test.yaml +++ b/braintrust/tests/ai-gateway-deployment_test.yaml @@ -19,6 +19,7 @@ tests: - __fixtures__/base-values.yaml set: aiGateway.enabled: true + aiGateway.autoscaling.enabled: false aiGateway.image.repository: test/gateway aiGateway.image.tag: v1.0.0 release: diff --git a/braintrust/tests/labels-merge-autoscaling_test.yaml b/braintrust/tests/labels-merge-autoscaling_test.yaml index a04fb5c..ffd365e 100644 --- a/braintrust/tests/labels-merge-autoscaling_test.yaml +++ b/braintrust/tests/labels-merge-autoscaling_test.yaml @@ -2,6 +2,7 @@ suite: test label merge isolation for API autoscaling resources templates: - api-autoscaling-metric.yaml - api-hpa.yaml + - ai-gateway-hpa.yaml - brainstore-reader-configmap.yaml capabilities: apiVersions: @@ -84,3 +85,55 @@ tests: value: reader-config - isNull: path: metadata.labels["api-specific"] + + - it: should give the AI Gateway HPA only global and gateway-specific labels + template: ai-gateway-hpa.yaml + values: + - __fixtures__/base-values.yaml + set: + aiGateway.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + aiGateway.labels.tier: gateway-tier + aiGateway.labels.gateway-specific: gateway-autoscaling + api.labels.api-specific: api-autoscaling + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: gateway-tier + - equal: + path: metadata.labels["gateway-specific"] + value: gateway-autoscaling + - isNull: + path: metadata.labels["api-specific"] + + - it: should not leak AI Gateway labels into another rendered component + template: brainstore-reader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + aiGateway.enabled: true + global.labels.shared: global-autoscaling + global.labels.tier: global-tier + aiGateway.labels.tier: gateway-tier + aiGateway.labels.gateway-specific: gateway-autoscaling + brainstore.reader.labels.reader-specific: reader-config + release: + namespace: braintrust + asserts: + - equal: + path: metadata.labels.shared + value: global-autoscaling + - equal: + path: metadata.labels.tier + value: global-tier + - equal: + path: metadata.labels["reader-specific"] + value: reader-config + - isNull: + path: metadata.labels["gateway-specific"] diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d3b85fa..a5aed8b 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -324,7 +324,18 @@ aiGateway: deployment: {} service: {} pod: {} + # With autoscaling, the default minReplicas. replicas: 2 + # Needs metrics-server; without it the HPA holds minReplicas. + autoscaling: + enabled: true + # minReplicas defaults to replicas, maxReplicas to max(6, minReplicas). + # minReplicas: 2 + # maxReplicas: 6 + cpu: + targetAverageUtilization: 70 + memory: + targetAverageUtilization: 75 image: repository: public.ecr.aws/braintrust/gateway tag: v2.21.0