From bf2a552c156e039355ca71eb60462f14bf848ab5 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Fri, 6 Nov 2026 15:47:21 -0500 Subject: [PATCH 01/13] Gate Helm deployments on a release migration Job --- braintrust/README.md | 11 + braintrust/files/migration-gate.py | 60 ++ braintrust/files/migration-rollback.py | 80 ++ braintrust/templates/_api-deployment.tpl | 23 +- braintrust/templates/_migrations.tpl | 91 +++ .../templates/ai-gateway-deployment.yaml | 21 +- ...rainstore-automationwriter-deployment.yaml | 10 +- .../brainstore-fastreader-deployment.yaml | 10 +- .../brainstore-reader-deployment.yaml | 10 +- .../brainstore-writer-deployment.yaml | 10 +- .../templates/loop-runtime-deployment.yaml | 11 + braintrust/templates/migration-gate.yaml | 71 ++ braintrust/templates/migration-job.yaml | 53 ++ braintrust/templates/migration-rollback.yaml | 100 +++ ...instore-automationwriter-routing_test.yaml | 3 + .../brainstore-automationwriter_test.yaml | 3 + .../tests/brainstore-fastreader_test.yaml | 3 + braintrust/tests/brainstore-reader_test.yaml | 3 + .../tests/brainstore-startup-gate_test.yaml | 3 + braintrust/tests/brainstore-writer_test.yaml | 3 + braintrust/tests/migrations_test.yaml | 734 ++++++++++++++++++ braintrust/values.yaml | 17 + docs/migration-job.md | 57 ++ tests/test_migration_gate.py | 129 +++ 24 files changed, 1504 insertions(+), 12 deletions(-) create mode 100644 braintrust/files/migration-gate.py create mode 100644 braintrust/files/migration-rollback.py create mode 100644 braintrust/templates/_migrations.tpl create mode 100644 braintrust/templates/migration-gate.yaml create mode 100644 braintrust/templates/migration-job.yaml create mode 100644 braintrust/templates/migration-rollback.yaml create mode 100644 braintrust/tests/migrations_test.yaml create mode 100644 docs/migration-job.md create mode 100644 tests/test_migration_gate.py diff --git a/braintrust/README.md b/braintrust/README.md index ac71725..83363b8 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -548,3 +548,14 @@ Example values files for different cloud providers and configurations are locate - `examples/google-autopilot-cel/values.yaml`: GKE Autopilot deployment with CEL-friendly security settings. - `examples/google-api-isolation-autoscaling/values.yaml`: Minimal example for API workload isolation and per-pool autoscaling on GKE (combine with an Autopilot or Standard values file). - `examples/google-standard/values.yaml`: GKE Standard deployment. + +## Migration ordering + +By default, a dedicated migration Job runs for each Helm release revision. All new +Deployment pods wait for its success before starting application containers. API +pods then run the existing Brainstore compatibility gate. The Job uses the shared +`api.image` and API identity, configuration, secret mounts and resources. + +Set `migrations.enabled: false` to retain the previous API-startup migration behavior. +See [migration lifecycle and configuration matrix](../docs/migration-job.md) for +retention, cleanup, rollback, timeout, and Helm-controller requirements. diff --git a/braintrust/files/migration-gate.py b/braintrust/files/migration-gate.py new file mode 100644 index 0000000..a40276d --- /dev/null +++ b/braintrust/files/migration-gate.py @@ -0,0 +1,60 @@ +"""Wait for the exact release's migration Job; never infer success from absence.""" + +import os +import sys +import time +import urllib.error +import urllib.parse + +from brainstore_gate import KubernetesClient + + +def job_complete(job): + if job.get("metadata", {}).get("deletionTimestamp"): + return False + conditions = job.get("status", {}).get("conditions", []) + if any(c.get("type") in {"Failed", "FailureTarget"} and c.get("status") == "True" + for c in conditions): + raise RuntimeError("migration Job failed; inspect its logs and retry with a new Helm upgrade") + return any(c.get("type") == "Complete" and c.get("status") == "True" for c in conditions) + + +def wait_for_job(client, name, timeout, interval, + clock=time.monotonic, sleep=time.sleep, log=print): + deadline = clock() + timeout + path = f"/apis/batch/v1/namespaces/{client.namespace}/jobs/{urllib.parse.quote(name, safe='')}" + reason = "Job has not completed" + while clock() < deadline: + try: + complete = job_complete(client.get(path, deadline)) + if complete and clock() < deadline: + log(f"Migration Job {name} completed successfully", flush=True) + return + reason = "Job has not completed" + except urllib.error.HTTPError as error: + # Job and RBAC may be applied after the Deployment, including 403/404. + reason = f"Kubernetes API returned HTTP {error.code}" + error.close() + except (OSError, ValueError, KeyError, TypeError) as error: + reason = f"Kubernetes observation failed: {type(error).__name__}" + log(f"Waiting for migration Job {name}: {reason}", flush=True) + sleep(min(interval, max(0, deadline - clock()))) + raise TimeoutError(f"Migration Job {name} wait timed out after {timeout}s: {reason}") + + +def main(): + timeout = int(os.environ["MIGRATION_TIMEOUT_SECONDS"]) + interval = int(os.environ["MIGRATION_POLL_SECONDS"]) + name = os.environ["MIGRATION_JOB_NAME"] + if not name or interval <= 0 or timeout <= interval: + raise ValueError("invalid migration gate configuration") + client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/migration-gate") + wait_for_job(client, name, timeout, interval) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + print(str(error), file=sys.stderr, flush=True) + sys.exit(1) diff --git a/braintrust/files/migration-rollback.py b/braintrust/files/migration-rollback.py new file mode 100644 index 0000000..3497379 --- /dev/null +++ b/braintrust/files/migration-rollback.py @@ -0,0 +1,80 @@ +"""Remove the target revision's retained resources before Helm restores them. + +Helm 3 cannot update a retained object absent from the current release manifest. +Delete only the named target revision; foreground Job deletion waits for its +migration pods to terminate before the rollback can recreate that Job. +""" +import json +import os +from pathlib import Path +import ssl +import time +import urllib.error +import urllib.parse +import urllib.request + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + raise urllib.error.HTTPError(request.full_url, code, message, headers, response) + + +class Client: + def __init__(self): + host = os.environ["KUBERNETES_SERVICE_HOST"] + if ":" in host and not host.startswith("["): + host = f"[{host}]" + self.base = f"https://{host}:{os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443')}" + self.credentials = Path("/var/run/secrets/kubernetes.io/serviceaccount") + self.opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), NoRedirect(), + urllib.request.HTTPSHandler(context=ssl.create_default_context( + cafile=str(self.credentials / "ca.crt"))), + ) + + def request(self, method, path, deadline): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("rollback migration cleanup timed out") + token = (self.credentials / "token").read_text().strip() + payload = json.dumps({"propagationPolicy": "Foreground"}).encode() if method == "DELETE" else None + request = urllib.request.Request(self.base + path, method=method, data=payload, + headers={"Authorization": f"Bearer {token}", + "Content-Type": "application/json"}) + try: + with self.opener.open(request, timeout=min(5, remaining)): + return True + except urllib.error.HTTPError as error: + missing = error.code == 404 + error.close() + if missing: + return False + # Do not log response bodies or credentials. + raise RuntimeError(f"Kubernetes cleanup {method} returned HTTP {error.code}") from None + + +def resource_paths(namespace, name): + namespace = urllib.parse.quote(namespace, safe="") + name = urllib.parse.quote(name, safe="") + return [ + f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}", + f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", + f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", + ] + + +def cleanup(client, namespace, name, timeout, clock=time.monotonic, sleep=time.sleep): + deadline = clock() + timeout + for path in resource_paths(namespace, name): + client.request("DELETE", path, deadline) + while client.request("GET", path, deadline): + if clock() >= deadline: + raise TimeoutError("rollback migration cleanup timed out") + sleep(min(2, max(0, deadline - clock()))) + + +if __name__ == "__main__": + cleanup(Client(), os.environ["POD_NAMESPACE"], os.environ["MIGRATION_JOB_NAME"], + int(os.environ["MIGRATION_TIMEOUT_SECONDS"])) diff --git a/braintrust/templates/_api-deployment.tpl b/braintrust/templates/_api-deployment.tpl index 62433ff..8ef4a4d 100644 --- a/braintrust/templates/_api-deployment.tpl +++ b/braintrust/templates/_api-deployment.tpl @@ -3,7 +3,12 @@ {{- $root := .root -}} {{- $api := .api -}} {{- $role := .role -}} +{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} {{- $startupGate := include "braintrust.brainstoreStartupGate.config" $root | fromYaml -}} +{{- if .migrationJob -}} +{{- $_ := set $startupGate "enabled" false -}} +{{- $_ := set $migration "enabled" false -}} +{{- end -}} {{- if $startupGate.enabled -}} {{- include "braintrust.brainstoreStartupGate.validateApi" (dict "api" $api) -}} {{- end -}} @@ -44,8 +49,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if $startupGate.enabled }} - progressDeadlineSeconds: {{ add (int $startupGate.timeoutSeconds) 300 }} + {{- if or $startupGate.enabled $migration.enabled }} + progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} {{- end }} {{- if not (dig "autoscaling" "enabled" false $api) }} replicas: {{ $api.replicas }} @@ -78,7 +83,7 @@ spec: {{- end }} spec: serviceAccountName: {{ $api.serviceAccount.name }} - {{- if $startupGate.enabled }} + {{- if or $startupGate.enabled $migration.enabled }} # Only the init container mounts the projected Kubernetes API credential. automountServiceAccountToken: false {{- end }} @@ -107,8 +112,12 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $startupGate.enabled }} + {{- if or $startupGate.enabled $migration.enabled }} initContainers: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.initContainer" $root | nindent 8 }} + {{- end }} + {{- if $startupGate.enabled }} - name: wait-for-brainstore image: {{ $startupGate.image | quote }} imagePullPolicy: IfNotPresent @@ -145,6 +154,7 @@ spec: - name: brainstore-gate-credentials mountPath: /var/run/brainstore-gate readOnly: true + {{- end }} {{- end }} containers: - name: api @@ -259,7 +269,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if or $startupGate.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} + {{- if or $migration.enabled $startupGate.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" $root | nindent 8 }} + {{- end }} {{- if $startupGate.enabled }} - name: brainstore-gate-code configMap: diff --git a/braintrust/templates/_migrations.tpl b/braintrust/templates/_migrations.tpl new file mode 100644 index 0000000..fad6d17 --- /dev/null +++ b/braintrust/templates/_migrations.tpl @@ -0,0 +1,91 @@ +{{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} +{{- define "braintrust.migrations.config" -}} +{{- $defaults := dict + "enabled" true + "timeoutSeconds" 1200 + "pollIntervalSeconds" 5 + "backoffLimit" 2 + "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" + "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) +-}} +{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} +{{- if $config.enabled -}} +{{- $_ := required "migrations.image is required" $config.image -}} +{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} +{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} +{{- fail "migrations timing values must be positive integers" -}} +{{- end -}} +{{- end -}} +{{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} +{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} +{{- end -}} +{{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} +{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} +{{- end -}} +{{- end -}} +{{- toYaml $config -}} +{{- end -}} + +{{/* Preserve the revision suffix even for long release names. */}} +{{- define "braintrust.migrations.name" -}} +{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} +{{- end -}} + +{{- define "braintrust.migrations.gatewayAccount" -}} +{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} +{{- end -}} + +{{- define "braintrust.migrations.initContainer" -}} +{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +- name: wait-for-migrations + image: {{ $config.image | quote }} + imagePullPolicy: IfNotPresent + command: ["python3", "-B", "-u", "/opt/migration-gate/gate.py"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $config.resources | nindent 4 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MIGRATION_JOB_NAME + value: {{ include "braintrust.migrations.name" . | quote }} + - name: MIGRATION_TIMEOUT_SECONDS + value: {{ $config.timeoutSeconds | quote }} + - name: MIGRATION_POLL_SECONDS + value: {{ $config.pollIntervalSeconds | quote }} + volumeMounts: + - name: migration-gate-code + mountPath: /opt/migration-gate + readOnly: true + - name: migration-gate-credentials + mountPath: /var/run/migration-gate + readOnly: true +{{- end -}} + +{{- define "braintrust.migrations.volumes" -}} +- name: migration-gate-code + configMap: + name: {{ include "braintrust.migrations.name" . }}-gate +- name: migration-gate-credentials + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt +{{- end -}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index e78b254..7383a21 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{- if .Values.aiGateway.enabled }} --- apiVersion: apps/v1 @@ -14,6 +15,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if $migration.enabled }} + progressDeadlineSeconds: {{ add 600 (int $migration.timeoutSeconds) }} + {{- end }} replicas: {{ .Values.aiGateway.replicas }} strategy: type: RollingUpdate @@ -36,6 +40,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + {{- if $migration.enabled }} + serviceAccountName: {{ include "braintrust.migrations.gatewayAccount" . }} + automountServiceAccountToken: false + {{- end }} {{- with .Values.aiGateway.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -48,6 +56,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: ai-gateway image: "{{ .Values.aiGateway.image.repository }}:{{ .Values.aiGateway.image.tag }}" @@ -90,9 +102,12 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- with .Values.aiGateway.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- else }} + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} + {{- if .Values.aiGateway.extraVolumes }} + {{- toYaml .Values.aiGateway.extraVolumes | nindent 8 }} + {{- else if not $migration.enabled }} [] {{- end }} {{- end }} diff --git a/braintrust/templates/brainstore-automationwriter-deployment.yaml b/braintrust/templates/brainstore-automationwriter-deployment.yaml index e8fd45a..440948b 100644 --- a/braintrust/templates/brainstore-automationwriter-deployment.yaml +++ b/braintrust/templates/brainstore-automationwriter-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{/* Omit this Deployment when brainstore.automationwriter is absent. helm upgrade --reuse-values from a chart that predates the pool does not merge the new @@ -23,7 +24,7 @@ metadata: spec: replicas: {{ $aw.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -79,6 +80,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: brainstore-automationwriter image: "{{ .Values.brainstore.image.repository }}:{{ .Values.brainstore.image.tag }}" @@ -176,6 +181,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} - name: cache-volume {{- /* Azure local disk is required only once this pool has replicas. A disabled pool must render without a size. */}} {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver (gt (int ($aw.replicas | default 0)) 0) }} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 6ca1d5e..92d2c34 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.fastreader) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.fastreader" "config" $rollout) }} apiVersion: apps/v1 @@ -16,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.fastreader.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -72,6 +73,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: brainstore-fastreader image: "{{ .Values.brainstore.image.repository }}:{{ .Values.brainstore.image.tag }}" @@ -161,6 +166,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} ephemeral: diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index f0342a7..d606e9c 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.reader) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.reader" "config" $rollout) }} apiVersion: apps/v1 @@ -16,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.reader.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -72,6 +73,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: brainstore-reader image: "{{ .Values.brainstore.image.repository }}:{{ .Values.brainstore.image.tag }}" @@ -161,6 +166,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} ephemeral: diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index f42b692..eb1b235 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.writer) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.writer" "config" $rollout) }} apiVersion: apps/v1 @@ -16,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.writer.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -72,6 +73,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: brainstore-writer image: "{{ .Values.brainstore.image.repository }}:{{ .Values.brainstore.image.tag }}" @@ -161,6 +166,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} ephemeral: diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index e82e4b4..f0bde32 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -1,3 +1,4 @@ +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} {{- $autoscaling := include "braintrust.loopRuntime.autoscaling" . | fromYaml -}} @@ -16,6 +17,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if $migration.enabled }} + progressDeadlineSeconds: {{ add 600 (int $migration.timeoutSeconds) }} + {{- end }} {{- if not $autoscaling }} replicas: {{ ternary 1 .Values.loopRuntime.replicas (has .Values.loopRuntime.replicas (list nil "")) }} {{- end }} @@ -61,6 +65,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $migration.enabled }} + initContainers: + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- end }} containers: - name: loop-runtime image: "{{ .Values.loopRuntime.image.repository }}:{{ .Values.loopRuntime.image.tag }}" @@ -148,6 +156,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if $migration.enabled }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- end }} - name: state emptyDir: sizeLimit: 2Gi diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml new file mode 100644 index 0000000..dfaddc2 --- /dev/null +++ b/braintrust/templates/migration-gate.yaml @@ -0,0 +1,71 @@ +{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- if $config.enabled -}} +{{- $name := include "braintrust.migrations.name" . -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-gate + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + braintrust.dev/migration-release: {{ .Release.Name | quote }} +data: + gate.py: | + {{- .Files.Get "files/migration-gate.py" | nindent 4 }} + brainstore_gate.py: | + {{- .Files.Get "files/brainstore-startup-gate.py" | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + braintrust.dev/migration-release: {{ .Release.Name | quote }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + resourceNames: [{{ $name | quote }}] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + braintrust.dev/migration-release: {{ .Release.Name | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + {{- $accounts := dict .Values.brainstore.serviceAccount.name true -}} + {{- range $pool := include "braintrust.apiPools" . | fromYamlArray -}} + {{- $_ := set $accounts $pool.config.serviceAccount.name true -}} + {{- end -}} + {{- if include "braintrust.loopRuntime.enabled" . -}} + {{- $_ := set $accounts .Values.loopRuntime.serviceAccount.name true -}} + {{- end -}} + {{- if .Values.aiGateway.enabled -}} + {{- $_ := set $accounts (include "braintrust.migrations.gatewayAccount" .) true -}} + {{- end -}} + {{- range $account := keys $accounts | sortAlpha }} + - kind: ServiceAccount + name: {{ $account }} + namespace: {{ include "braintrust.namespace" $ }} + {{- end }} +{{- if .Values.aiGateway.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "braintrust.migrations.gatewayAccount" . }} + namespace: {{ include "braintrust.namespace" . }} +{{- end }} +{{- end }} diff --git a/braintrust/templates/migration-job.yaml b/braintrust/templates/migration-job.yaml new file mode 100644 index 0000000..5aee29a --- /dev/null +++ b/braintrust/templates/migration-job.yaml @@ -0,0 +1,53 @@ +{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- if $config.enabled -}} +{{- $name := include "braintrust.migrations.name" . -}} +{{- $api := include "braintrust.apiPoolConfig" (dict "root" . "overrides" (dict)) | fromYaml -}} +{{/* Reuse the API pod contract so identity, TLS, secrets, security and scheduling +stay in sync. The migrationJob flag suppresses both startup gates. Only the main +container is retained: sidecars and HTTP probes must not keep this Job alive. */}} +{{- $deployment := include "braintrust.apiDeployment" (dict "root" . "api" $api "role" "migration" "migrationJob" true) | fromYaml -}} +{{- $pod := $deployment.spec.template -}} +{{- $container := first $pod.spec.containers -}} +{{- range $key := list "ports" "livenessProbe" "readinessProbe" "startupProbe" -}} +{{- $_ := unset $container $key -}} +{{- end -}} +{{- $_ := set $container "name" "migrate" -}} +{{- $_ := set $container "command" (list "/braintrust/venv/bin/python" "/braintrust/api-schema/lambda_function.py" "--execute") -}} +{{- $_ := set $container "envFrom" (list (dict "configMapRef" (dict "name" (printf "%s-config" $name)))) -}} +{{- $_ := set $pod.spec "containers" (list $container) -}} +{{- $_ := set $pod.spec "restartPolicy" "Never" -}} +{{- $_ := set $pod.spec "automountServiceAccountToken" false -}} +{{- $_ := unset $pod.spec "topologySpreadConstraints" -}} +{{/* A migration pod must never match API Service / Deployment selectors. */}} +{{- $_ := set $pod.metadata.labels "app" $name -}} +{{- $_ := set $pod.metadata.labels "braintrust.dev/api-pool" "migration" -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.api.labels) (dict "braintrust.dev/migration-release" .Release.Name) | toYaml | nindent 4 }} +spec: + backoffLimit: {{ $config.backoffLimit }} + activeDeadlineSeconds: {{ $config.timeoutSeconds }} + template: + {{- toYaml $pod | nindent 4 }} +--- +{{/* Snapshot non-secret configuration for this attempt rather than reading a +mutable API ConfigMap after a subsequent upgrade. Secrets remain Secret refs. */}} +{{- $apiConfig := include (print .Template.BasePath "/api-configmap.yaml") . | fromYaml }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + braintrust.dev/migration-release: {{ .Release.Name | quote }} +data: + {{- toYaml $apiConfig.data | nindent 2 }} +{{- end }} diff --git a/braintrust/templates/migration-rollback.yaml b/braintrust/templates/migration-rollback.yaml new file mode 100644 index 0000000..6a36ee8 --- /dev/null +++ b/braintrust/templates/migration-rollback.yaml @@ -0,0 +1,100 @@ +{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- if $config.enabled -}} +{{- $name := include "braintrust.migrations.name" . -}} +{{- $hook := printf "%s-reset" $name -}} +{{/* Hook manifests belong to the target release, so the name is the original +revision's name, not the new rollback revision. No hook runs during install or +upgrade. Helm executes these by weight, then deletes successful hooks in reverse. */}} +{{- range $kind := list "ServiceAccount" "Role" "RoleBinding" }} +--- +apiVersion: {{ ternary "v1" "rbac.authorization.k8s.io/v1" (eq $kind "ServiceAccount") }} +kind: {{ $kind }} +metadata: + name: {{ $hook }} + namespace: {{ include "braintrust.namespace" $ }} + annotations: + helm.sh/hook: pre-rollback + helm.sh/hook-weight: {{ index (dict "ServiceAccount" "-3" "Role" "-2" "RoleBinding" "-1") $kind | quote }} + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +{{- if eq $kind "Role" }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + resourceNames: [{{ $name | quote }}] + verbs: ["get", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{ printf "%s-config" $name | quote }}, {{ printf "%s-gate" $name | quote }}] + verbs: ["get", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + resourceNames: [{{ $name | quote }}] + verbs: ["get", "delete"] +{{- else if eq $kind "RoleBinding" }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $hook }} +subjects: + - kind: ServiceAccount + name: {{ $hook }} + namespace: {{ include "braintrust.namespace" $ }} +{{- end }} +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $hook }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/hook: pre-rollback + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ $config.backoffLimit }} + activeDeadlineSeconds: {{ $config.timeoutSeconds }} + template: + spec: + serviceAccountName: {{ $hook }} + restartPolicy: Never + {{- with .Values.api.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cleanup + image: {{ $config.image | quote }} + command: + - python3 + - -B + - -u + - -c + - | + {{- .Files.Get "files/migration-rollback.py" | nindent 14 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $config.resources | nindent 12 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MIGRATION_JOB_NAME + value: {{ $name | quote }} + - name: MIGRATION_TIMEOUT_SECONDS + value: {{ $config.timeoutSeconds | quote }} +{{- end }} diff --git a/braintrust/tests/brainstore-automationwriter-routing_test.yaml b/braintrust/tests/brainstore-automationwriter-routing_test.yaml index 44b3d5d..7f53de4 100644 --- a/braintrust/tests/brainstore-automationwriter-routing_test.yaml +++ b/braintrust/tests/brainstore-automationwriter-routing_test.yaml @@ -3,6 +3,9 @@ templates: - api-configmap.yaml - api-deployment.yaml - brainstore-startup-gate-rbac.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should preserve writer fallback when the pool is disabled template: api-configmap.yaml diff --git a/braintrust/tests/brainstore-automationwriter_test.yaml b/braintrust/tests/brainstore-automationwriter_test.yaml index 6f83e00..734b686 100644 --- a/braintrust/tests/brainstore-automationwriter_test.yaml +++ b/braintrust/tests/brainstore-automationwriter_test.yaml @@ -2,6 +2,9 @@ suite: test Brainstore Automation Writer deployment template templates: - brainstore-automationwriter-deployment.yaml - brainstore-automationwriter-configmap.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should render Brainstore Automation Writer deployment with default values template: brainstore-automationwriter-deployment.yaml diff --git a/braintrust/tests/brainstore-fastreader_test.yaml b/braintrust/tests/brainstore-fastreader_test.yaml index 295bb3f..d4fbf4b 100644 --- a/braintrust/tests/brainstore-fastreader_test.yaml +++ b/braintrust/tests/brainstore-fastreader_test.yaml @@ -2,6 +2,9 @@ suite: test Brainstore Fast Reader deployment template templates: - brainstore-fastreader-deployment.yaml - brainstore-fastreader-configmap.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should render Brainstore Fast Reader deployment with default values template: brainstore-fastreader-deployment.yaml diff --git a/braintrust/tests/brainstore-reader_test.yaml b/braintrust/tests/brainstore-reader_test.yaml index 5575af9..cea2e21 100644 --- a/braintrust/tests/brainstore-reader_test.yaml +++ b/braintrust/tests/brainstore-reader_test.yaml @@ -2,6 +2,9 @@ suite: test Brainstore Reader deployment template templates: - brainstore-reader-deployment.yaml - brainstore-reader-configmap.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should render Brainstore Reader deployment with default values template: brainstore-reader-deployment.yaml diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index 758da25..b28d086 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -7,6 +7,9 @@ templates: - api-serviceaccount.yaml - brainstore-reader-deployment.yaml - brainstore-reader-configmap.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should gate API startup by default template: api-deployment.yaml diff --git a/braintrust/tests/brainstore-writer_test.yaml b/braintrust/tests/brainstore-writer_test.yaml index d2af988..b0b66ca 100644 --- a/braintrust/tests/brainstore-writer_test.yaml +++ b/braintrust/tests/brainstore-writer_test.yaml @@ -2,6 +2,9 @@ suite: test Brainstore Writer deployment template templates: - brainstore-writer-deployment.yaml - brainstore-writer-configmap.yaml +# Existing workload behavior without the independent migration gate. +set: + migrations.enabled: false tests: - it: should render Brainstore Writer deployment with default values template: brainstore-writer-deployment.yaml diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml new file mode 100644 index 0000000..1a1db8f --- /dev/null +++ b/braintrust/tests/migrations_test.yaml @@ -0,0 +1,734 @@ +suite: release migration Job and Deployment dependencies +templates: + - migration-rollback.yaml + - migration-job.yaml + - migration-gate.yaml + - api-deployment.yaml + - api-configmap.yaml + - brainstore-reader-deployment.yaml + - brainstore-reader-configmap.yaml + - brainstore-writer-deployment.yaml + - brainstore-writer-configmap.yaml + - brainstore-fastreader-deployment.yaml + - brainstore-fastreader-configmap.yaml + - brainstore-automationwriter-deployment.yaml + - brainstore-automationwriter-configmap.yaml + - ai-gateway-deployment.yaml + - ai-gateway-configmap.yaml + - loop-runtime-deployment.yaml + - loop-runtime-configmap.yaml + - api-serviceaccount.yaml +release: + name: test + namespace: workload + revision: 7 +tests: + - it: runs a regular bounded migration Job using the target API image + template: migration-job.yaml + documentIndex: 0 + asserts: + - isKind: + of: Job + - matchRegex: + path: metadata.name + pattern: '^test-[a-f0-9]{8}-migrate-r7$' + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - isNull: + path: metadata.annotations["helm.sh/hook"] + - isNull: + path: spec.ttlSecondsAfterFinished + - equal: + path: spec.activeDeadlineSeconds + value: 1200 + - equal: + path: spec.backoffLimit + value: 2 + - equal: + path: spec.template.spec.restartPolicy + value: Never + - equal: + path: spec.template.spec.containers[0].command + value: [/braintrust/venv/bin/python, /braintrust/api-schema/lambda_function.py, --execute] + - equal: + path: spec.template.spec.containers[0].image + value: public.ecr.aws/braintrust/standalone-api:v2.22.0 + - isNull: + path: spec.template.spec.initContainers + - isNull: + path: spec.template.spec.containers[0].readinessProbe + - isNull: + path: spec.template.spec.containers[0].livenessProbe + - isNull: + path: spec.template.spec.containers[0].ports + - notEqual: + path: spec.template.metadata.labels.app + value: braintrust-api + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + + - it: snapshots configuration for the attempt without copying Secrets + template: migration-job.yaml + documentIndex: 1 + set: + global.orgName: migration-org + asserts: + - isKind: + of: ConfigMap + - matchRegex: + path: metadata.name + pattern: '-migrate-r7-config$' + - equal: + path: data.ORG_NAME + value: migration-org + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - isNull: + path: data.PG_URL + + - it: waits for migrations before the Brainstore compatibility gate + template: api-deployment.yaml + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-migrations + - equal: + path: spec.template.spec.initContainers[1].name + value: wait-for-brainstore + - matchRegex: + path: spec.template.spec.initContainers[0].env[1].value + pattern: '-migrate-r7$' + - equal: + path: spec.progressDeadlineSeconds + value: 2700 + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + + - it: grants only get on the exact revision Job + template: migration-gate.yaml + documentIndex: 1 + asserts: + - isKind: + of: Role + - equal: + path: rules[0].apiGroups + value: [batch] + - equal: + path: rules[0].resources + value: [jobs] + - equal: + path: rules[0].verbs + value: [get] + - lengthEqual: + path: rules + count: 1 + - matchRegex: + path: rules[0].resourceNames[0] + pattern: '-migrate-r7$' + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + + - it: keeps revision-bound helper code for old replicas and rollback + template: migration-gate.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: metadata.name + pattern: '-migrate-r7-gate$' + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - isNotNull: + path: data["gate.py"] + - isNotNull: + path: data["brainstore_gate.py"] + + - it: binds all pool identities and the Brainstore account in the actual namespace + template: migration-gate.yaml + documentIndex: 2 + set: + global.createNamespace: true + global.namespace: actual-namespace + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.serviceAccount.name: ingest + api.workloadIsolation.background.serviceAccount.name: background + asserts: + - equal: + path: metadata.namespace + value: actual-namespace + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - contains: + path: subjects + content: {kind: ServiceAccount, name: ingest, namespace: actual-namespace} + - contains: + path: subjects + content: {kind: ServiceAccount, name: background, namespace: actual-namespace} + - contains: + path: subjects + content: {kind: ServiceAccount, name: brainstore, namespace: actual-namespace} + + - it: inherits API environment CA security scheduling and volumes without sidecars + template: migration-job.yaml + documentIndex: 0 + set: + api.customCA.enabled: true + api.tmpVolume.enabled: true + api.nodeSelector.migration-node: "yes" + api.securityContext.readOnlyRootFilesystem: true + api.extraEnvVars: + - {name: MIGRATION_VERSION_TABLE, value: custom_version} + api.extraContainers: + - {name: forever-sidecar, image: busybox} + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + - equal: + path: spec.template.spec.nodeSelector.migration-node + value: "yes" + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - contains: + path: spec.template.spec.containers[0].env + content: {name: MIGRATION_VERSION_TABLE, value: custom_version} + - contains: + path: spec.template.spec.containers[0].env + content: {name: SSL_CERT_FILE, value: /etc/braintrust/runtime-ca/ca-bundle.pem} + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: {name: custom-ca-bundle, mountPath: /etc/braintrust/runtime-ca, readOnly: true} + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: {name: tmp-volume, mountPath: /tmp} + - contains: + path: spec.template.spec.containers[0].env + content: + name: PG_URL + valueFrom: + secretKeyRef: {name: braintrust-secrets, key: PG_URL} + + - it: mounts Azure CSI secrets independently of application startup + template: migration-job.yaml + documentIndex: 0 + set: + cloud: azure + azure.enableAzureKeyVaultDriver: true + azure.enableAzureContainerStorageDriver: false + azure.keyVaultName: test-vault + asserts: + - equal: + path: spec.template.metadata.labels["azure.workload.identity/use"] + value: "true" + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: {name: secrets-store-inline, mountPath: /mnt/secrets-store, readOnly: true} + - contains: + path: spec.template.spec.volumes + content: + name: secrets-store-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: {secretProviderClass: test-vault} + + - it: preserves Google native workload identity + template: migration-job.yaml + documentIndex: 0 + set: + cloud: google + api.enableGcsAuth: true + api.serviceAccount.googleServiceAccount: api@example.iam.gserviceaccount.com + brainstore.serviceAccount.googleServiceAccount: brainstore@example.iam.gserviceaccount.com + asserts: + - equal: + path: spec.template.metadata.labels["gke-workload-identity/use"] + value: "true" + - equal: + path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] + value: api@example.iam.gserviceaccount.com + + - it: does not mutate API service labels while rendering the Job + template: api-serviceaccount.yaml + set: + global.labels.shared: global + api.labels.api-only: api + brainstore.reader.labels.reader-only: reader + asserts: + - equal: + path: metadata.labels.shared + value: global + - equal: + path: metadata.labels["api-only"] + value: api + - isNull: + path: metadata.labels["braintrust.dev/migration-release"] + - isNull: + path: metadata.labels["reader-only"] + + - it: does not leak Brainstore labels into the Job + template: migration-job.yaml + documentIndex: 0 + set: + global.labels.shared: global + api.labels.api-only: api + brainstore.reader.labels.reader-only: reader + asserts: + - equal: + path: metadata.labels.shared + value: global + - equal: + path: metadata.labels["api-only"] + value: api + - isNull: + path: metadata.labels["reader-only"] + + - it: gates the reader pool + template: brainstore-reader-deployment.yaml + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} + + - it: gates the writer pool + template: brainstore-writer-deployment.yaml + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} + + - it: gates the fastreader pool + template: brainstore-fastreader-deployment.yaml + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} + + - it: gates the automationwriter pool + template: brainstore-automationwriter-deployment.yaml + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} + + - it: gates isolated API pool 0 + template: api-deployment.yaml + documentIndex: 0 + set: {"api.workloadIsolation.enabled": true} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} + + - it: gates isolated API pool 1 + template: api-deployment.yaml + documentIndex: 1 + set: {"api.workloadIsolation.enabled": true} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} + + - it: gates isolated API pool 2 + template: api-deployment.yaml + documentIndex: 2 + set: {"api.workloadIsolation.enabled": true} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} + + - it: gates the optional AI Gateway + template: ai-gateway-deployment.yaml + set: {"aiGateway.enabled": true} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.serviceAccountName", "value": "test-migration-gate"}} + - {"equal": {"path": "spec.template.spec.automountServiceAccountToken", "value": false}} + + - it: gates the optional Loop runtime + values: + - __fixtures__/loop-runtime-values.yaml + template: loop-runtime-deployment.yaml + set: {"cloud": "google", "loopRuntime.enabled": true, "loopRuntime.serviceAccount.googleServiceAccount": "loop@example.iam.gserviceaccount.com", "brainstore.serviceAccount.googleServiceAccount": "bs@example.iam.gserviceaccount.com"} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + + - it: binds the Loop runtime identity + values: + - __fixtures__/loop-runtime-values.yaml + template: migration-gate.yaml + documentIndex: 2 + set: {"cloud": "google", "loopRuntime.enabled": true, "loopRuntime.serviceAccount.googleServiceAccount": "loop@example.iam.gserviceaccount.com", "brainstore.serviceAccount.googleServiceAccount": "bs@example.iam.gserviceaccount.com"} + asserts: + - {"contains": {"path": "subjects", "content": {"kind": "ServiceAccount", "name": "braintrust-loop-runtime", "namespace": "workload"}}} + + - it: omits migration-job.yaml when disabled + template: migration-job.yaml + set: {"migrations.enabled": false} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: omits migration-gate.yaml when disabled + template: migration-gate.yaml + set: {"migrations.enabled": false} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: removes migration wait when disabled in api-deployment.yaml + template: api-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: removes migration wait when disabled in brainstore-reader-deployment.yaml + template: brainstore-reader-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: removes migration wait when disabled in brainstore-writer-deployment.yaml + template: brainstore-writer-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: removes migration wait when disabled in brainstore-fastreader-deployment.yaml + template: brainstore-fastreader-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: removes migration wait when disabled in brainstore-automationwriter-deployment.yaml + template: brainstore-automationwriter-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: removes migration wait when disabled in ai-gateway-deployment.yaml + template: ai-gateway-deployment.yaml + set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + asserts: + - {"isNull": {"path": "spec.template.spec.initContainers"}} + + - it: gates APIs independently of the compatibility gate + template: api-deployment.yaml + set: {"api.brainstoreStartupGate.enabled": false} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"lengthEqual": {"path": "spec.template.spec.initContainers", "count": 1}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1500}} + + - it: defaults a missing or null feature map on reused values + template: api-deployment.yaml + set: {"migrations": null} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + + - it: merges partial reused values with defaults + template: migration-job.yaml + documentIndex: 0 + set: {"migrations": null, "migrations.backoffLimit": 0} + asserts: + - {"equal": {"path": "spec.backoffLimit", "value": 0}} + - {"equal": {"path": "spec.activeDeadlineSeconds", "value": 1200}} + + - it: allows zero retries + template: migration-job.yaml + documentIndex: 0 + set: {"migrations.backoffLimit": 0} + asserts: + - {"equal": {"path": "spec.backoffLimit", "value": 0}} + + - it: uses custom timing and a mirrored helper image + template: api-deployment.yaml + set: {"migrations.timeoutSeconds": 1800, "migrations.pollIntervalSeconds": 10, "migrations.image": "mirror/python:3.13"} + asserts: + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 3300}} + - {"equal": {"path": "spec.template.spec.initContainers[0].image", "value": "mirror/python:3.13"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[2].value", "value": "1800"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[3].value", "value": "10"}} + + - it: preserves API image tag v2.22.0 + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "v2.22.0", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.22.0"}} + + - it: preserves API image tag 2.22.0 + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "2.22.0", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:2.22.0"}} + + - it: preserves API image tag abcdef0123456789abcdef0123456789abcdef0123 + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "abcdef0123456789abcdef0123456789abcdef0123", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:abcdef0123456789abcdef0123456789abcdef0123"}} + + - it: preserves API image tag deadbeef + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "deadbeef", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:deadbeef"}} + + - it: preserves API image tag 1234567 + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "1234567", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:1234567"}} + + - it: preserves API image tag v2.23.0-rc.1 + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "v2.23.0-rc.1", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.23.0-rc.1"}} + + - it: preserves API image tag custom + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "custom", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:custom"}} + + - it: preserves API image tag v2.22.0@sha256:abc + template: migration-job.yaml + documentIndex: 0 + set: {"api.image.tag": "v2.22.0@sha256:abc", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} + asserts: + - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.22.0@sha256:abc"}} + + - it: rejects invalid timeoutSeconds 0 + template: migration-rollback.yaml + set: {"migrations.timeoutSeconds": 0} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: rejects invalid timeoutSeconds + template: migration-rollback.yaml + set: {"migrations.timeoutSeconds": ""} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: defaults a null timeoutSeconds after Helm coalesces values + template: migration-job.yaml + documentIndex: 0 + set: {migrations.timeoutSeconds: null} + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1200 + + - it: rejects invalid timeoutSeconds -1 + template: migration-rollback.yaml + set: {"migrations.timeoutSeconds": -1} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: rejects invalid pollIntervalSeconds 0 + template: migration-rollback.yaml + set: {"migrations.pollIntervalSeconds": 0} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: rejects invalid pollIntervalSeconds + template: migration-rollback.yaml + set: {"migrations.pollIntervalSeconds": ""} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: defaults a null pollIntervalSeconds after Helm coalesces values + template: migration-job.yaml + documentIndex: 0 + set: {migrations.pollIntervalSeconds: null} + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1200 + + - it: rejects invalid pollIntervalSeconds -1 + template: migration-rollback.yaml + set: {"migrations.pollIntervalSeconds": -1} + asserts: + - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + + - it: rejects a poll interval outside the budget + template: migration-rollback.yaml + set: {"migrations.pollIntervalSeconds": 1200} + asserts: + - {"failedTemplate": {"errorMessage": "migrations.timeoutSeconds must exceed pollIntervalSeconds"}} + + - it: rejects an empty helper image + template: migration-rollback.yaml + set: {"migrations.image": ""} + asserts: + - {"failedTemplate": {"errorMessage": "migrations.image is required"}} + + - it: does not validate unused disabled settings + template: migration-job.yaml + set: {"migrations.enabled": false, "migrations.image": "", "migrations.timeoutSeconds": 0} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: rejects a negative retry count + template: migration-rollback.yaml + set: {"migrations.backoffLimit": -1} + asserts: + - {"failedTemplate": {"errorMessage": "migrations.backoffLimit must be a nonnegative integer"}} + + - it: binds a distinct Job to release revision 1 + template: migration-job.yaml + documentIndex: 0 + release: {"name": "test", "revision": 1} + asserts: + - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r1$"}} + + - it: binds pods to release revision 1 + template: api-deployment.yaml + release: {"name": "test", "revision": 1} + asserts: + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r1$"}} + + - it: binds a distinct Job to release revision 8 + template: migration-job.yaml + documentIndex: 0 + release: {"name": "test", "revision": 8} + asserts: + - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r8$"}} + + - it: binds pods to release revision 8 + template: api-deployment.yaml + release: {"name": "test", "revision": 8} + asserts: + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r8$"}} + + - it: binds a distinct Job to release revision 101 + template: migration-job.yaml + documentIndex: 0 + release: {"name": "test", "revision": 101} + asserts: + - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} + + - it: binds pods to release revision 101 + template: api-deployment.yaml + release: {"name": "test", "revision": 101} + asserts: + - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r101$"}} + + - it: preserves revision suffix with long release names + template: migration-job.yaml + documentIndex: 0 + release: {"name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision": 101} + asserts: + - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,56}$"}} + - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} + + - it: preserves reader replica behavior for 0 + template: brainstore-reader-deployment.yaml + set: {"brainstore.reader.replicas": 0} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + + - it: preserves reader replica behavior for None + template: brainstore-reader-deployment.yaml + set: {"brainstore.reader.replicas": null} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + + - it: preserves reader replica behavior for + template: brainstore-reader-deployment.yaml + set: {"brainstore.reader.replicas": ""} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + + - it: keeps missing automation writer omitted + template: brainstore-automationwriter-deployment.yaml + set: {"brainstore.automationwriter": null} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: gates a partially specified automation writer + template: brainstore-automationwriter-deployment.yaml + set: {"brainstore.automationwriter": {"replicas": 1}} + asserts: + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.replicas", "value": 1}} + + - it: keeps disabled optional workloads omitted + template: ai-gateway-deployment.yaml + set: {"aiGateway.enabled": false} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: keeps disabled Loop runtime omitted + template: loop-runtime-deployment.yaml + set: {"loopRuntime.enabled": false} + asserts: + - {"hasDocuments": {"count": 0}} + + - it: prepares target revision rollback cleanup without install or upgrade hooks + template: migration-rollback.yaml + documentIndex: 3 + asserts: + - isKind: + of: Job + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-rollback + - equal: + path: metadata.annotations["helm.sh/hook-delete-policy"] + value: before-hook-creation,hook-succeeded + - matchRegex: + path: spec.template.spec.containers[0].env[1].value + pattern: '-migrate-r7$' + - matchRegex: + path: spec.template.spec.serviceAccountName + pattern: '-migrate-r7-reset$' + - equal: + path: spec.activeDeadlineSeconds + value: 1200 + + - it: limits rollback cleanup to the exact target revision + template: migration-rollback.yaml + documentIndex: 1 + asserts: + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "-2" + - equal: + path: rules[0].verbs + value: [get, delete] + - equal: + path: rules[1].verbs + value: [get, delete] + - equal: + path: rules[2].verbs + value: [get, delete] + - equal: + path: rules[0].resourceNames + value: [test-9f86d081-migrate-r7] + - equal: + path: rules[1].resourceNames + value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate] + - equal: + path: rules[2].resourceNames + value: [test-9f86d081-migrate-r7] + + - it: does not render rollback hooks when migrations are disabled + template: migration-rollback.yaml + set: + migrations.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index f655142..933c6ee 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -820,3 +820,20 @@ virtualService: host: "braintrust-api" # Should match api.service.name or api.name port: number: 8000 # Should match api.service.port + +# Run schema migrations once per Helm revision before any new Deployment +# application containers start. See docs/migration-job.md for retention/rollback. +migrations: + enabled: true + timeoutSeconds: 1200 + pollIntervalSeconds: 5 + backoffLimit: 2 + # Small read-only init-container helper; migration itself uses api.image. + image: "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi diff --git a/docs/migration-job.md b/docs/migration-job.md new file mode 100644 index 0000000..da7a0d6 --- /dev/null +++ b/docs/migration-job.md @@ -0,0 +1,57 @@ +# Release migration Job + +## Input matrix and intended behavior + +| Input | Behavior | +| --- | --- | +| Feature enabled (default), missing/null map, partial reused values | One regular Job per Helm release revision; all chart Deployment pods wait for that revision's successful Job | +| Feature disabled; enabled to disabled | No new migration resources or waits; API startup continues to run migrations as before | +| Disabled to enabled | New revision Job and pod gates are created | +| API image release tags with/without v, full/short/numeric SHA, prerelease, custom tags, digest | Pass image through unchanged; existing Brainstore version gate still requires its explicit minimum for non-release tags | +| AWS, Google, Azure; provider examples | Job uses shared API identity, configuration, secrets, CA, scheduling and security settings | +| Azure Key Vault CSI enabled | Migration Job mounts CSI secrets itself before application pods start | +| API workload isolation on/off; custom pool identities | One shared API-image migrator; every pool waits, with read-only Job RBAC for each account | +| Brainstore roles enabled/zero/null replicas; automation writer missing/partial map | Preserve existing replica/default rules; every rendered pod template carries the wait | +| Optional gateway and Loop runtime enabled/disabled | Every rendered Deployment waits; disabled workloads stay omitted | +| Missing/null timing; empty/zero timing; empty image | Missing/null use defaults after Helm coalescing; empty/zero timing and empty image fail when enabled; disabled configuration does not validate unused settings | +| Upgrade/retry, old ReplicaSet replacement, rollback | Distinct revision names; keep completed Jobs and their config and read permissions; rollback cleans and recreates the target revision resources, rerunning migrations | +| Migration failure, missing Job, API errors | Fail closed; bounded waits and Job execution; application containers remain stopped | +| Global/API/role labels | No mutation or leakage into other workloads; Job must not match API Service selectors | + +## Lifecycle + +This feature requires Helm install/upgrade release revisions. Plain `helm template` +plus apply (including render-only GitOps controllers) does not increment revisions; +use a Helm release controller rather than repeatedly applying revision 1. + +A release snapshots API ConfigMap data and runs the shared `api.image` migration +entry point. Pool-specific API image/config overrides must remain schema-compatible +with that shared image and database. The API startup migration check remains for +compatibility with other deployment methods. SQL changes must be compatible with +old running pods; rollback does not undo database changes. Existing rollout +strategies still govern availability (Recreate and nonzero maxUnavailable can +remove old capacity while replacements wait). + +Jobs have no TTL. Jobs, migration configuration, gate code and revision-specific +RBAC use `helm.sh/resource-policy: keep`, so old ReplicaSets and rollback revisions +can still start pods. These resources also survive uninstall. Delete retained +resources only after no Pod/ReplicaSet references that revision and it is no longer +needed for rollback. Before reinstalling the same release name, clean up its +retained resources after ensuring no old workload references them. Do not delete +the active revision's Job: future replacements/autoscaling need its success record. +A failed Job is retried with a new `helm upgrade` revision after fixing its cause. + +Native Helm rollback executes a `pre-rollback` cleanup hook with a separate service +account allowed only to get/delete the target revision's Job, two ConfigMaps, Role +and RoleBinding. It waits for foreground Job deletion before Helm recreates the +target resources and reruns the migration check. This avoids the Helm 3 retained- +resource rollback error (https://github.com/helm/helm/issues/13142). Existing running +application containers keep running, but replacements can wait during cleanup. +Do not use `--no-hooks` on rollback. Automatic rollback follows the same hook path. +Schema changes are never reversed. Helm rollback must have time for cleanup plus +migration plus application readiness. Failed cleanup blocks rollback; its hook Job +and logs remain for diagnosis. + +Allow enough Helm timeout for migration plus Brainstore rollout and the existing +API compatibility gate. `--wait --wait-for-jobs` makes Helm report migration failure; +without waits, Helm can return before migration/rollout completion. diff --git a/tests/test_migration_gate.py b/tests/test_migration_gate.py new file mode 100644 index 0000000..2766de5 --- /dev/null +++ b/tests/test_migration_gate.py @@ -0,0 +1,129 @@ +import importlib.util +from pathlib import Path +import sys +import unittest +from unittest.mock import Mock +import urllib.error + +FILES = Path(__file__).resolve().parents[1] / "braintrust/files" +# The chart mounts both files together under these import names. +for name, file in [("brainstore_gate", "brainstore-startup-gate.py"), ("migration_gate", "migration-gate.py"), ("migration_rollback", "migration-rollback.py")]: + spec = importlib.util.spec_from_file_location(name, FILES / file) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) +gate = sys.modules["migration_gate"] + + +def job(*conditions, **metadata): + return {"metadata": metadata, "status": {"conditions": [ + {"type": c, "status": "True"} for c in conditions + ]}} + + +class JobGateTests(unittest.TestCase): + def wait(self, observations, timeout=20): + now = [0] + def sleep(seconds): + now[0] += seconds + client = Mock(namespace="test-namespace") + client.get.side_effect = observations + gate.wait_for_job(client, "release-migrate-r2", timeout, 5, + clock=lambda: now[0], sleep=sleep, log=Mock()) + return client, now[0] + + def test_only_complete_condition_counts_as_success(self): + for resource in ({}, job(), job("SuccessCriteriaMet"), + {"status": {"succeeded": 1}}, + {"status": {"conditions": [{"type": "Complete", "status": "False"}]}}, + job("Complete", deletionTimestamp="now")): + with self.subTest(resource=resource): + self.assertFalse(gate.job_complete(resource)) + self.assertTrue(gate.job_complete(job("Complete"))) + + def test_failed_job_is_terminal_even_with_complete_condition(self): + for condition in ("Failed", "FailureTarget"): + with self.assertRaisesRegex(RuntimeError, "migration Job failed"): + gate.job_complete(job(condition, "Complete")) + + def test_waits_for_exact_revision_and_retries_missing_job_and_rbac(self): + observations = [urllib.error.HTTPError("url", code, "error", {}, None) + for code in (404, 403, 500)] + client, elapsed = self.wait(observations + [job("Complete")]) + self.assertEqual(elapsed, 15) + self.assertEqual(client.get.call_count, 4) + for call in client.get.call_args_list: + self.assertEqual(call.args, ( + "/apis/batch/v1/namespaces/test-namespace/jobs/release-migrate-r2", 20)) + + def test_running_job_then_success(self): + _, elapsed = self.wait([job(), job("Complete")]) + self.assertEqual(elapsed, 5) + + def test_timeout_never_allows_app_to_start(self): + with self.assertRaisesRegex(TimeoutError, "timed out"): + self.wait([job()] * 4) + + def test_failed_job_never_allows_app_to_start(self): + with self.assertRaisesRegex(RuntimeError, "migration Job failed"): + self.wait([job("Failed")]) + + def test_transient_observation_errors_retry(self): + self.wait([OSError("network"), ValueError("bad JSON"), job("Complete")]) + + def test_late_success_does_not_escape_deadline(self): + now = [0] + client = Mock(namespace="ns") + def late(*args): + now[0] = 21 + return job("Complete") + client.get.side_effect = late + with self.assertRaises(TimeoutError): + gate.wait_for_job(client, "job", 20, 5, clock=lambda: now[0], sleep=Mock(), log=Mock()) + + +class RollbackCleanupTests(unittest.TestCase): + def test_cleanup_only_target_revision_and_waits_for_job_deletion(self): + rollback = sys.modules["migration_rollback"] + paths = rollback.resource_paths("ns", "target-r7") + client = Mock() + # DELETE succeeds; first GET still sees foreground deletion; next is gone. + client.request.side_effect = [True, True, False] + [True, False] * 4 + now = [0] + def sleep(seconds): + now[0] += seconds + rollback.cleanup(client, "ns", "target-r7", 10, clock=lambda: now[0], sleep=sleep) + calls = client.request.call_args_list + self.assertEqual([c.args[1] for c in calls if c.args[0] == "DELETE"], paths) + self.assertEqual(calls[0].args, ("DELETE", paths[0], 10)) + self.assertEqual(calls[1].args, ("GET", paths[0], 10)) + self.assertEqual(calls[2].args, ("GET", paths[0], 10)) + self.assertEqual(now[0], 2) + + def test_cleanup_of_absent_resources_is_idempotent(self): + rollback = sys.modules["migration_rollback"] + client = Mock() + client.request.return_value = False + rollback.cleanup(client, "ns", "target-r7", 10, clock=lambda: 0, sleep=Mock()) + self.assertEqual(client.request.call_count, 10) + + def test_cleanup_fails_closed_on_stuck_deletion(self): + rollback = sys.modules["migration_rollback"] + client = Mock() + client.request.return_value = True + now = [0] + def sleep(seconds): + now[0] += seconds + with self.assertRaises(TimeoutError): + rollback.cleanup(client, "ns", "target-r7", 3, clock=lambda: now[0], sleep=sleep) + self.assertEqual(now[0], 3) + + def test_cleanup_permission_errors_stop_rollback(self): + client = Mock() + client.request.side_effect = RuntimeError("forbidden") + with self.assertRaises(RuntimeError): + sys.modules["migration_rollback"].cleanup(client, "ns", "target-r7", 10) + + +if __name__ == "__main__": + unittest.main() From ffdf8f49adfe2cac8645df10b86dfad98b6cf8eb Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Wed, 11 Nov 2026 17:19:44 -0500 Subject: [PATCH 02/13] Gate API startup on completed Brainstore release rollouts --- braintrust/README.md | 33 +- braintrust/files/brainstore-rollout.py | 138 +++++ .../files/{migration-gate.py => job-gate.py} | 22 +- braintrust/files/migration-rollback.py | 72 ++- braintrust/templates/_api-deployment.tpl | 7 +- braintrust/templates/_migrations.tpl | 91 ---- braintrust/templates/_release-gates.tpl | 156 ++++++ .../templates/ai-gateway-deployment.yaml | 4 +- ...rainstore-automationwriter-deployment.yaml | 12 +- .../brainstore-fastreader-deployment.yaml | 12 +- .../brainstore-reader-deployment.yaml | 12 +- braintrust/templates/brainstore-rollout.yaml | 135 +++++ .../brainstore-writer-deployment.yaml | 12 +- .../templates/loop-runtime-deployment.yaml | 4 +- braintrust/templates/migration-gate.yaml | 15 +- braintrust/templates/migration-rollback.yaml | 21 +- .../tests/brainstore-rollout-labels_test.yaml | 38 ++ braintrust/tests/brainstore-rollout_test.yaml | 160 ++++++ braintrust/tests/migrations_test.yaml | 176 +------ braintrust/values.yaml | 8 +- docs/migration-job.md | 116 +++-- tests/test_brainstore_rollout.py | 474 ++++++++++++++++++ tests/test_job_gate.py | 286 +++++++++++ tests/test_migration_gate.py | 129 ----- 24 files changed, 1612 insertions(+), 521 deletions(-) create mode 100644 braintrust/files/brainstore-rollout.py rename braintrust/files/{migration-gate.py => job-gate.py} (73%) delete mode 100644 braintrust/templates/_migrations.tpl create mode 100644 braintrust/templates/_release-gates.tpl create mode 100644 braintrust/templates/brainstore-rollout.yaml create mode 100644 braintrust/tests/brainstore-rollout-labels_test.yaml create mode 100644 braintrust/tests/brainstore-rollout_test.yaml create mode 100644 tests/test_brainstore_rollout.py create mode 100644 tests/test_job_gate.py delete mode 100644 tests/test_migration_gate.py diff --git a/braintrust/README.md b/braintrust/README.md index 83363b8..819241f 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -469,7 +469,8 @@ The API receives `BRAINSTORE_AUTOMATION_WRITER_URL` whenever the pool has replicas. A compatible API release decides which queries use this endpoint. With no replicas, the URL is omitted and the API falls back to the regular writers. -The API startup gate covers readers, writers, and fast readers. +The live version-compatibility check covers readers, writers, and fast readers. +The release rollout barrier also waits for every rendered automation writer. The include/exclude split follows `brainstore.automationwriter.replicas` in the Helm values. `kubectl scale` or an autoscaler changes the live pod count only; @@ -549,13 +550,23 @@ Example values files for different cloud providers and configurations are locate - `examples/google-api-isolation-autoscaling/values.yaml`: Minimal example for API workload isolation and per-pool autoscaling on GKE (combine with an Autopilot or Standard values file). - `examples/google-standard/values.yaml`: GKE Standard deployment. -## Migration ordering - -By default, a dedicated migration Job runs for each Helm release revision. All new -Deployment pods wait for its success before starting application containers. API -pods then run the existing Brainstore compatibility gate. The Job uses the shared -`api.image` and API identity, configuration, secret mounts and resources. - -Set `migrations.enabled: false` to retain the previous API-startup migration behavior. -See [migration lifecycle and configuration matrix](../docs/migration-job.md) for -retention, cleanup, rollback, timeout, and Helm-controller requirements. +## Release ordering + +By default, each Helm release runs migrations first, rolls out the exact +Brainstore image and configuration, then starts new API application containers. +Old Ready writers cannot satisfy the rollout barrier for an environment-only +change. All API pools share the same revision-specific rollout Job and then run +the existing live version-compatibility check. + +Migration and rollout Jobs share a read-only wait helper but remain separate to +avoid a dependency cycle. The migrator uses the shared `api.image`, identity, +configuration, secrets, and resources; the rollout observer only reads Kubernetes +Deployments and Pods. Completed phase records are retained for replacement pods +and reset together during rollback. + +`migrations.enabled: false` retains the previous API-startup migration behavior +and compatibility-only gate. Disabling `api.brainstoreStartupGate.enabled` leaves +migration-only ordering. Gates control application startup, not when Helm updates +Deployment specs or mutable configuration objects. +See [release lifecycle and configuration matrix](../docs/migration-job.md) for +retention, rollback, timeouts, configuration limits, and capacity requirements. diff --git a/braintrust/files/brainstore-rollout.py b/braintrust/files/brainstore-rollout.py new file mode 100644 index 0000000..f23a3ff --- /dev/null +++ b/braintrust/files/brainstore-rollout.py @@ -0,0 +1,138 @@ +"""Observe the exact Brainstore configuration for one Helm release. + +ROLLOUT_TARGETS is a JSON array of deployment/container/revision/image/configChecksum +objects. POD_NAMESPACE selects the namespace; ROLLOUT_TIMEOUT_SECONDS and +ROLLOUT_POLL_SECONDS bound the entire fleet observation. Kubernetes credentials +are projected at /var/run/migration-gate; brainstore_gate.py supplies shared helpers. +""" + +import json +import os +import sys +import time +import urllib.error +import urllib.parse + +from brainstore_gate import KubernetesClient, TERMINAL_PHASES, container_image, ready + + +RELEASE_ANNOTATION = "braintrust.dev/release-revision" +CONFIG_ANNOTATION = "checksum/config" + + +def configuration_mismatch(resource, target): + annotations = resource.get("metadata", {}).get("annotations", {}) + if annotations.get(RELEASE_ANNOTATION) != target["revision"]: + return "release revision" + if annotations.get(CONFIG_ANNOTATION) != target["configChecksum"]: + return "config checksum" + if container_image(resource.get("spec", {}), target["container"]) != target["image"]: + return "image" + return None + + +def check_deployment(deployment, pods, target): + """Return a wait reason until the desired spec and every backend have rolled.""" + metadata = deployment.get("metadata", {}) + spec = deployment.get("spec", {}) + status = deployment.get("status", {}) + name = target["deployment"] + if metadata.get("deletionTimestamp"): + return f"{name}: deployment is being deleted" + mismatch = configuration_mismatch(spec.get("template", {}), target) + if mismatch: + return f"{name}: deployment template {mismatch} does not match the expected release" + if status.get("observedGeneration", 0) < metadata.get("generation", 1): + return f"{name}: controller has not observed the desired generation" + + # Scaling is independent of Helm: use the current spec, preserving explicit zero. + desired = spec.get("replicas", 1) + live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] + for pod in live: + pod_name = pod.get("metadata", {}).get("name", "unknown") + if pod.get("metadata", {}).get("deletionTimestamp"): + return f"{name}: terminating Pod {pod_name} still exists" + mismatch = configuration_mismatch(pod, target) + if mismatch: + return f"{name}: Pod {pod_name} {mismatch} does not match the expected release" + if len(live) != desired: + return f"{name}: waiting for exactly {desired} active service backends ({len(live)} remain)" + ready_count = sum(ready(pod, target["container"]) for pod in live) + if ready_count != desired: + return f"{name}: waiting for {desired} ready updated Pods ({ready_count} ready)" + + # Availability includes minReadySeconds; total replicas excludes old surges. + # Kubernetes omits zero-valued counters, so a fully scaled-down role can pass. + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + actual = status.get(counter, 0) + if actual != desired: + return f"{name}: rollout incomplete ({counter}={actual}, desired={desired})" + return None + + +def check_fleet(client, targets, deadline): + for target in targets: + deployment = client.deployment(target["deployment"], deadline) + # Services route by app, not owner reference. Include stray and terminating + # Pods, including those from a previous ReplicaSet or release revision. + pods = client.pods("app=" + target["deployment"], deadline) + reason = check_deployment(deployment, pods, target) + if reason: + return reason + return None + + +def wait_for_fleet(client, targets, timeout, interval, + clock=time.monotonic, sleep=time.sleep, log=print): + deadline = clock() + timeout + successes = 0 + last_reason = "no observation yet" + while clock() < deadline: + try: + reason = check_fleet(client, targets, deadline) + except urllib.error.HTTPError as error: + # The deployment or read-only RBAC may not have been applied yet. + # Never include API response bodies, credentials, or query strings. + path = urllib.parse.urlsplit(error.url).path[:256] + reason = f"Kubernetes API returned HTTP {error.code} for {path}" + error.close() + except (OSError, ValueError, KeyError, TypeError) as error: + reason = f"Kubernetes observation failed: {type(error).__name__}" + if clock() >= deadline: + last_reason = reason or "deadline expired before rollout confirmation" + break + if reason is None: + successes += 1 + if successes == 2: + log("Brainstore rollout complete: exact release configuration is ready", flush=True) + return + last_reason = "confirming exact release configuration (1/2)" + else: + successes = 0 + last_reason = reason + log(f"Brainstore rollout waiting: {last_reason}", flush=True) + sleep(min(interval, max(0, deadline - clock()))) + raise TimeoutError(f"Brainstore rollout timed out after {timeout}s: {last_reason}") + + +def main(): + targets = json.loads(os.environ["ROLLOUT_TARGETS"]) + timeout = int(os.environ["ROLLOUT_TIMEOUT_SECONDS"]) + interval = int(os.environ["ROLLOUT_POLL_SECONDS"]) + fields = ("deployment", "container", "revision", "image", "configChecksum") + if (not isinstance(targets, list) or not targets + or any(not isinstance(target, dict) + or any(not isinstance(target.get(field), str) or not target[field] + for field in fields) for target in targets) + or timeout <= 0 or interval <= 0 or interval >= timeout): + raise ValueError("invalid Brainstore rollout configuration") + client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/migration-gate") + wait_for_fleet(client, targets, timeout, interval) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError) as error: + print(str(error), file=sys.stderr, flush=True) + sys.exit(1) diff --git a/braintrust/files/migration-gate.py b/braintrust/files/job-gate.py similarity index 73% rename from braintrust/files/migration-gate.py rename to braintrust/files/job-gate.py index a40276d..99af236 100644 --- a/braintrust/files/migration-gate.py +++ b/braintrust/files/job-gate.py @@ -1,4 +1,4 @@ -"""Wait for the exact release's migration Job; never infer success from absence.""" +"""Wait for the exact release Job; never infer success from absence.""" import os import sys @@ -10,12 +10,12 @@ def job_complete(job): - if job.get("metadata", {}).get("deletionTimestamp"): - return False conditions = job.get("status", {}).get("conditions", []) if any(c.get("type") in {"Failed", "FailureTarget"} and c.get("status") == "True" for c in conditions): - raise RuntimeError("migration Job failed; inspect its logs and retry with a new Helm upgrade") + raise RuntimeError("Job failed; inspect its logs and retry with a new Helm upgrade") + if job.get("metadata", {}).get("deletionTimestamp"): + return False return any(c.get("type") == "Complete" and c.get("status") == "True" for c in conditions) @@ -28,7 +28,7 @@ def wait_for_job(client, name, timeout, interval, try: complete = job_complete(client.get(path, deadline)) if complete and clock() < deadline: - log(f"Migration Job {name} completed successfully", flush=True) + log(f"Job {name} completed successfully", flush=True) return reason = "Job has not completed" except urllib.error.HTTPError as error: @@ -37,17 +37,17 @@ def wait_for_job(client, name, timeout, interval, error.close() except (OSError, ValueError, KeyError, TypeError) as error: reason = f"Kubernetes observation failed: {type(error).__name__}" - log(f"Waiting for migration Job {name}: {reason}", flush=True) + log(f"Waiting for Job {name}: {reason}", flush=True) sleep(min(interval, max(0, deadline - clock()))) - raise TimeoutError(f"Migration Job {name} wait timed out after {timeout}s: {reason}") + raise TimeoutError(f"Job {name} wait timed out after {timeout}s: {reason}") def main(): - timeout = int(os.environ["MIGRATION_TIMEOUT_SECONDS"]) - interval = int(os.environ["MIGRATION_POLL_SECONDS"]) - name = os.environ["MIGRATION_JOB_NAME"] + timeout = int(os.environ["JOB_TIMEOUT_SECONDS"]) + interval = int(os.environ["JOB_POLL_SECONDS"]) + name = os.environ["JOB_NAME"] if not name or interval <= 0 or timeout <= interval: - raise ValueError("invalid migration gate configuration") + raise ValueError("invalid Job gate configuration") client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/migration-gate") wait_for_job(client, name, timeout, interval) diff --git a/braintrust/files/migration-rollback.py b/braintrust/files/migration-rollback.py index 3497379..d9c20cd 100644 --- a/braintrust/files/migration-rollback.py +++ b/braintrust/files/migration-rollback.py @@ -1,8 +1,8 @@ """Remove the target revision's retained resources before Helm restores them. Helm 3 cannot update a retained object absent from the current release manifest. -Delete only the named target revision; foreground Job deletion waits for its -migration pods to terminate before the rollback can recreate that Job. +Delete only the named target revision; foreground Job deletion waits for all of +its migration and rollout pods before rollback can recreate either Job. """ import json import os @@ -35,7 +35,7 @@ def __init__(self): def request(self, method, path, deadline): remaining = deadline - time.monotonic() if remaining <= 0: - raise TimeoutError("rollback migration cleanup timed out") + raise TimeoutError("rollback release cleanup timed out") token = (self.credentials / "token").read_text().strip() payload = json.dumps({"propagationPolicy": "Foreground"}).encode() if method == "DELETE" else None request = urllib.request.Request(self.base + path, method=method, data=payload, @@ -53,28 +53,60 @@ def request(self, method, path, deadline): raise RuntimeError(f"Kubernetes cleanup {method} returned HTTP {error.code}") from None -def resource_paths(namespace, name): +def resource_paths(namespace, names): namespace = urllib.parse.quote(namespace, safe="") - name = urllib.parse.quote(name, safe="") - return [ - f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}", - f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", - f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", - ] + names = [urllib.parse.quote(name, safe="") for name in names] + jobs = [f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}" for name in names] + retained = [] + for name in names: + retained.extend([ + f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", + f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", + ]) + return jobs + retained -def cleanup(client, namespace, name, timeout, clock=time.monotonic, sleep=time.sleep): +def cleanup(client, namespace, names, timeout, clock=time.monotonic, sleep=time.sleep): deadline = clock() + timeout - for path in resource_paths(namespace, name): - client.request("DELETE", path, deadline) - while client.request("GET", path, deadline): - if clock() >= deadline: - raise TimeoutError("rollback migration cleanup timed out") + paths = resource_paths(namespace, names) + jobs, retained = paths[:len(names)], paths[len(names):] + + def check_deadline(): + if clock() >= deadline: + raise TimeoutError("rollback release cleanup timed out") + + def request(method, path): + check_deadline() + exists = client.request(method, path, deadline) + check_deadline() + return exists + + def wait_until_deleted(path): + while request("GET", path): sleep(min(2, max(0, deadline - clock()))) + # Stop both phases before removing any config or permissions they rely on. + for path in jobs: + request("DELETE", path) + for path in jobs: + wait_until_deleted(path) + for path in retained: + request("DELETE", path) + wait_until_deleted(path) + + +def main(): + names = json.loads(os.environ["RELEASE_JOB_NAMES"]) + if not isinstance(names, list) or not names or any( + not isinstance(name, str) or not name for name in names): + raise ValueError("RELEASE_JOB_NAMES must be a nonempty list of Job names") + timeout = int(os.environ["RELEASE_TIMEOUT_SECONDS"]) + if timeout <= 0: + raise ValueError("RELEASE_TIMEOUT_SECONDS must be positive") + cleanup(Client(), os.environ["POD_NAMESPACE"], names, timeout) + if __name__ == "__main__": - cleanup(Client(), os.environ["POD_NAMESPACE"], os.environ["MIGRATION_JOB_NAME"], - int(os.environ["MIGRATION_TIMEOUT_SECONDS"])) + main() diff --git a/braintrust/templates/_api-deployment.tpl b/braintrust/templates/_api-deployment.tpl index 8ef4a4d..b5e3bdd 100644 --- a/braintrust/templates/_api-deployment.tpl +++ b/braintrust/templates/_api-deployment.tpl @@ -9,6 +9,7 @@ {{- $_ := set $startupGate "enabled" false -}} {{- $_ := set $migration "enabled" false -}} {{- end -}} +{{- $strictRollout := and $migration.enabled $startupGate.enabled -}} {{- if $startupGate.enabled -}} {{- include "braintrust.brainstoreStartupGate.validateApi" (dict "api" $api) -}} {{- end -}} @@ -50,7 +51,7 @@ metadata: {{- end }} spec: {{- if or $startupGate.enabled $migration.enabled }} - progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) (ternary (int $startupGate.timeoutSeconds) 0 $strictRollout) }} {{- end }} {{- if not (dig "autoscaling" "enabled" false $api) }} replicas: {{ $api.replicas }} @@ -115,7 +116,7 @@ spec: {{- if or $startupGate.enabled $migration.enabled }} initContainers: {{- if $migration.enabled }} - {{- include "braintrust.migrations.initContainer" $root | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" $root "phase" (ternary "rollout" "migration" $strictRollout)) | nindent 8 }} {{- end }} {{- if $startupGate.enabled }} - name: wait-for-brainstore @@ -271,7 +272,7 @@ spec: volumes: {{- if or $migration.enabled $startupGate.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" $root | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" $root | nindent 8 }} {{- end }} {{- if $startupGate.enabled }} - name: brainstore-gate-code diff --git a/braintrust/templates/_migrations.tpl b/braintrust/templates/_migrations.tpl deleted file mode 100644 index fad6d17..0000000 --- a/braintrust/templates/_migrations.tpl +++ /dev/null @@ -1,91 +0,0 @@ -{{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} -{{- define "braintrust.migrations.config" -}} -{{- $defaults := dict - "enabled" true - "timeoutSeconds" 1200 - "pollIntervalSeconds" 5 - "backoffLimit" 2 - "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" - "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) --}} -{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} -{{- if $config.enabled -}} -{{- $_ := required "migrations.image is required" $config.image -}} -{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} -{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} -{{- fail "migrations timing values must be positive integers" -}} -{{- end -}} -{{- end -}} -{{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} -{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} -{{- end -}} -{{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} -{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} -{{- end -}} -{{- end -}} -{{- toYaml $config -}} -{{- end -}} - -{{/* Preserve the revision suffix even for long release names. */}} -{{- define "braintrust.migrations.name" -}} -{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} -{{- end -}} - -{{- define "braintrust.migrations.gatewayAccount" -}} -{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} -{{- end -}} - -{{- define "braintrust.migrations.initContainer" -}} -{{- $config := include "braintrust.migrations.config" . | fromYaml -}} -- name: wait-for-migrations - image: {{ $config.image | quote }} - imagePullPolicy: IfNotPresent - command: ["python3", "-B", "-u", "/opt/migration-gate/gate.py"] - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - resources: - {{- toYaml $config.resources | nindent 4 }} - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MIGRATION_JOB_NAME - value: {{ include "braintrust.migrations.name" . | quote }} - - name: MIGRATION_TIMEOUT_SECONDS - value: {{ $config.timeoutSeconds | quote }} - - name: MIGRATION_POLL_SECONDS - value: {{ $config.pollIntervalSeconds | quote }} - volumeMounts: - - name: migration-gate-code - mountPath: /opt/migration-gate - readOnly: true - - name: migration-gate-credentials - mountPath: /var/run/migration-gate - readOnly: true -{{- end -}} - -{{- define "braintrust.migrations.volumes" -}} -- name: migration-gate-code - configMap: - name: {{ include "braintrust.migrations.name" . }}-gate -- name: migration-gate-credentials - projected: - sources: - - serviceAccountToken: - path: token - expirationSeconds: 3600 - - configMap: - name: kube-root-ca.crt - items: - - key: ca.crt - path: ca.crt -{{- end -}} diff --git a/braintrust/templates/_release-gates.tpl b/braintrust/templates/_release-gates.tpl new file mode 100644 index 0000000..9df2ee7 --- /dev/null +++ b/braintrust/templates/_release-gates.tpl @@ -0,0 +1,156 @@ +{{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} +{{- define "braintrust.migrations.config" -}} +{{- $defaults := dict + "enabled" true + "timeoutSeconds" 1200 + "pollIntervalSeconds" 5 + "backoffLimit" 2 + "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" + "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) +-}} +{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} +{{- if $config.enabled -}} +{{- $_ := required "migrations.image is required" $config.image -}} +{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} +{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} +{{- fail "migrations timing values must be positive integers" -}} +{{- end -}} +{{- end -}} +{{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} +{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} +{{- end -}} +{{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} +{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} +{{- end -}} +{{- end -}} +{{- toYaml $config -}} +{{- end -}} + +{{/* Preserve the revision suffix even for long release names. */}} +{{- define "braintrust.migrations.name" -}} +{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} +{{- end -}} + +{{- define "braintrust.migrations.gatewayAccount" -}} +{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} +{{- end -}} + +{{/* Two completed Jobs preserve release ordering without pinning old API +replacements to a Brainstore rollout that has since been superseded. */}} +{{- define "braintrust.rollout.enabled" -}} +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} +{{- if and $migration.enabled $gate.enabled -}}true{{- end -}} +{{- end -}} + +{{- define "braintrust.rollout.name" -}} +{{- printf "%s-%s-rollout-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} +{{- end -}} + +{{- define "braintrust.rollout.serviceAccount" -}} +{{- printf "%s-%s-rollout" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) -}} +{{- end -}} + +{{/* Reserve rollout identity/checksum only while the exact release gate is on. +Merge annotations before rendering so user input cannot create duplicate keys. */}} +{{- define "braintrust.rollout.podAnnotations" -}} +{{- $root := .root -}} +{{- $checksum := include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum -}} +{{- $annotations := dict "checksum/config" $checksum -}} +{{- if eq $root.Values.cloud "google" -}} +{{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} +{{- end -}} +{{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} +{{- if include "braintrust.rollout.enabled" $root -}} +{{- $_ := set $annotations "checksum/config" $checksum -}} +{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} +{{- end -}} +{{- toYaml $annotations -}} +{{- end -}} + + +{{/* The revision binds image, inline env and all other Pod template changes; +the existing checksum binds ConfigMap-backed environment without copying secrets. */}} +{{- define "braintrust.rollout.targets" -}} +{{- $targets := list -}} +{{- $image := printf "%s:%s" .Values.brainstore.image.repository .Values.brainstore.image.tag -}} +{{- range $role := list "reader" "writer" "fastreader" -}} +{{- $config := index $.Values.brainstore $role -}} +{{- $checksum := include (printf "%s/brainstore-%s-configmap.yaml" $.Template.BasePath $role) $ | sha256sum -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role) "revision" (toString $.Release.Revision) "image" $image "configChecksum" $checksum) -}} +{{- end -}} +{{- if .Values.brainstore.automationwriter -}} +{{- $config := include "braintrust.automationWriter.config" . | fromYaml -}} +{{- $checksum := include (print .Template.BasePath "/brainstore-automationwriter-configmap.yaml") . | sha256sum -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" "brainstore-automationwriter" "revision" (toString .Release.Revision) "image" $image "configChecksum" $checksum) -}} +{{- end -}} +{{- toJson $targets -}} +{{- end -}} + +{{/* One read-only Job gate serves both release phases. The rollout gate includes +the migration budget because its Job first waits for migrations itself. */}} +{{- define "braintrust.releaseJob.initContainer" -}} +{{- $root := .root -}} +{{- $config := include "braintrust.migrations.config" $root | fromYaml -}} +{{- $job := include "braintrust.migrations.name" $root -}} +{{- $name := "wait-for-migrations" -}} +{{- $timeout := int $config.timeoutSeconds -}} +{{- if eq (.phase | default "migration") "rollout" -}} +{{- $gate := include "braintrust.brainstoreStartupGate.config" $root | fromYaml -}} +{{- $timeout = int (add $timeout (int $gate.timeoutSeconds)) -}} +{{- $config = $gate -}} +{{- $job = include "braintrust.rollout.name" $root -}} +{{- $name = "wait-for-brainstore-rollout" -}} +{{- end -}} +- name: {{ $name }} + image: {{ $config.image | quote }} + imagePullPolicy: IfNotPresent + command: ["python3", "-B", "-u", "/opt/migration-gate/gate.py"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $config.resources | nindent 4 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JOB_NAME + value: {{ $job | quote }} + - name: JOB_TIMEOUT_SECONDS + value: {{ $timeout | quote }} + - name: JOB_POLL_SECONDS + value: {{ $config.pollIntervalSeconds | quote }} + volumeMounts: + - name: migration-gate-code + mountPath: /opt/migration-gate + readOnly: true + - name: migration-gate-credentials + mountPath: /var/run/migration-gate + readOnly: true +{{- end -}} + +{{- define "braintrust.releaseJob.volumes" -}} +- name: migration-gate-code + configMap: + name: {{ include "braintrust.migrations.name" . }}-gate +- name: migration-gate-credentials + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt +{{- end -}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index 7383a21..d07d279 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: ai-gateway @@ -103,7 +103,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} {{- if .Values.aiGateway.extraVolumes }} {{- toYaml .Values.aiGateway.extraVolumes | nindent 8 }} diff --git a/braintrust/templates/brainstore-automationwriter-deployment.yaml b/braintrust/templates/brainstore-automationwriter-deployment.yaml index 440948b..1cda872 100644 --- a/braintrust/templates/brainstore-automationwriter-deployment.yaml +++ b/braintrust/templates/brainstore-automationwriter-deployment.yaml @@ -48,13 +48,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/brainstore-automationwriter-configmap.yaml") . | sha256sum }} - {{- if eq .Values.cloud "google" }} - iam.gke.io/gcp-service-account: {{ required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" .Values.brainstore.serviceAccount.googleServiceAccount }} - {{- end }} - {{- with $aw.annotations.pod }} - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "automationwriter" "config" $aw) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with $aw.podSecurityContext }} @@ -82,7 +76,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: brainstore-automationwriter @@ -182,7 +176,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- /* Azure local disk is required only once this pool has replicas. A disabled pool must render without a size. */}} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 92d2c34..699b8ef 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -41,13 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/brainstore-fastreader-configmap.yaml") . | sha256sum }} - {{- if eq .Values.cloud "google" }} - iam.gke.io/gcp-service-account: {{ required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" .Values.brainstore.serviceAccount.googleServiceAccount }} - {{- end }} - {{- with .Values.brainstore.fastreader.annotations.pod }} - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "fastreader" "config" .Values.brainstore.fastreader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.fastreader.podSecurityContext }} @@ -75,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: brainstore-fastreader @@ -167,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index d606e9c..ca39064 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -41,13 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/brainstore-reader-configmap.yaml") . | sha256sum }} - {{- if eq .Values.cloud "google" }} - iam.gke.io/gcp-service-account: {{ required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" .Values.brainstore.serviceAccount.googleServiceAccount }} - {{- end }} - {{- with .Values.brainstore.reader.annotations.pod }} - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "reader" "config" .Values.brainstore.reader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.reader.podSecurityContext }} @@ -75,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: brainstore-reader @@ -167,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/brainstore-rollout.yaml b/braintrust/templates/brainstore-rollout.yaml new file mode 100644 index 0000000..5e9db08 --- /dev/null +++ b/braintrust/templates/brainstore-rollout.yaml @@ -0,0 +1,135 @@ +{{- if include "braintrust.rollout.enabled" . -}} +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} +{{- $name := include "braintrust.rollout.name" . -}} +{{- $account := include "braintrust.rollout.serviceAccount" . -}} +{{- $targets := include "braintrust.rollout.targets" . | fromJsonArray -}} +{{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (dict "braintrust.dev/migration-release" .Release.Name) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- toYaml $labels | nindent 4 }} +spec: + backoffLimit: {{ $migration.backoffLimit }} + activeDeadlineSeconds: {{ add (int $migration.timeoutSeconds) (int $gate.timeoutSeconds) }} + template: + metadata: + labels: + {{- mergeOverwrite (deepCopy $labels) (dict "app" $name) | toYaml | nindent 8 }} + spec: + serviceAccountName: {{ $account }} + automountServiceAccountToken: false + restartPolicy: Never + {{- with .Values.api.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + containers: + - name: wait-for-rollout + image: {{ $gate.image | quote }} + imagePullPolicy: IfNotPresent + command: ["python3", "-B", "-u", "/opt/migration-gate/brainstore_rollout.py"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $gate.resources | nindent 12 }} + envFrom: + - configMapRef: + name: {{ $name }}-config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLLOUT_TIMEOUT_SECONDS + value: {{ $gate.timeoutSeconds | quote }} + - name: ROLLOUT_POLL_SECONDS + value: {{ $gate.pollIntervalSeconds | quote }} + volumeMounts: + - name: migration-gate-code + mountPath: /opt/migration-gate + readOnly: true + - name: migration-gate-credentials + mountPath: /var/run/migration-gate + readOnly: true + volumes: + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- toYaml $labels | nindent 4 }} +data: + ROLLOUT_TARGETS: {{ toJson $targets | quote }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $account }} + namespace: {{ include "braintrust.namespace" . }} + labels: + {{- toYaml $labels | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- toYaml $labels | nindent 4 }} +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: + {{- range $target := $targets }} + - {{ $target.deployment | quote }} + {{- end }} + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ include "braintrust.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- toYaml $labels | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $account }} + namespace: {{ include "braintrust.namespace" . }} +{{- end }} diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index eb1b235..95ec164 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -41,13 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/brainstore-writer-configmap.yaml") . | sha256sum }} - {{- if eq .Values.cloud "google" }} - iam.gke.io/gcp-service-account: {{ required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" .Values.brainstore.serviceAccount.googleServiceAccount }} - {{- end }} - {{- with .Values.brainstore.writer.annotations.pod }} - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "writer" "config" .Values.brainstore.writer) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.writer.podSecurityContext }} @@ -75,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: brainstore-writer @@ -167,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index f0bde32..9cc628e 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -67,7 +67,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} {{- end }} containers: - name: loop-runtime @@ -157,7 +157,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.migrations.volumes" . | nindent 8 }} + {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} {{- end }} - name: state emptyDir: diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml index dfaddc2..fd51aaa 100644 --- a/braintrust/templates/migration-gate.yaml +++ b/braintrust/templates/migration-gate.yaml @@ -12,9 +12,13 @@ metadata: braintrust.dev/migration-release: {{ .Release.Name | quote }} data: gate.py: | - {{- .Files.Get "files/migration-gate.py" | nindent 4 }} + {{- .Files.Get "files/job-gate.py" | nindent 4 }} brainstore_gate.py: | {{- .Files.Get "files/brainstore-startup-gate.py" | nindent 4 }} + {{- if include "braintrust.rollout.enabled" . }} + brainstore_rollout.py: | + {{- .Files.Get "files/brainstore-rollout.py" | nindent 4 }} + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -28,7 +32,11 @@ metadata: rules: - apiGroups: ["batch"] resources: ["jobs"] - resourceNames: [{{ $name | quote }}] + resourceNames: + - {{ $name | quote }} + {{- if include "braintrust.rollout.enabled" . }} + - {{ include "braintrust.rollout.name" . | quote }} + {{- end }} verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -46,6 +54,9 @@ roleRef: name: {{ $name }} subjects: {{- $accounts := dict .Values.brainstore.serviceAccount.name true -}} + {{- if include "braintrust.rollout.enabled" . -}} + {{- $_ := set $accounts (include "braintrust.rollout.serviceAccount" .) true -}} + {{- end -}} {{- range $pool := include "braintrust.apiPools" . | fromYamlArray -}} {{- $_ := set $accounts $pool.config.serviceAccount.name true -}} {{- end -}} diff --git a/braintrust/templates/migration-rollback.yaml b/braintrust/templates/migration-rollback.yaml index 6a36ee8..a067fbe 100644 --- a/braintrust/templates/migration-rollback.yaml +++ b/braintrust/templates/migration-rollback.yaml @@ -1,6 +1,15 @@ {{- $config := include "braintrust.migrations.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} +{{- $jobs := list $name -}} +{{- if include "braintrust.rollout.enabled" . -}} +{{- $jobs = append $jobs (include "braintrust.rollout.name" .) -}} +{{- end -}} +{{- $configNames := list -}} +{{- range $job := $jobs -}} +{{- $configNames = append $configNames (printf "%s-config" $job) -}} +{{- $configNames = append $configNames (printf "%s-gate" $job) -}} +{{- end -}} {{- $hook := printf "%s-reset" $name -}} {{/* Hook manifests belong to the target release, so the name is the original revision's name, not the new rollback revision. No hook runs during install or @@ -20,15 +29,15 @@ metadata: rules: - apiGroups: ["batch"] resources: ["jobs"] - resourceNames: [{{ $name | quote }}] + resourceNames: {{ toJson $jobs }} verbs: ["get", "delete"] - apiGroups: [""] resources: ["configmaps"] - resourceNames: [{{ printf "%s-config" $name | quote }}, {{ printf "%s-gate" $name | quote }}] + resourceNames: {{ toJson $configNames }} verbs: ["get", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] - resourceNames: [{{ $name | quote }}] + resourceNames: {{ toJson $jobs }} verbs: ["get", "delete"] {{- else if eq $kind "RoleBinding" }} roleRef: @@ -93,8 +102,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: MIGRATION_JOB_NAME - value: {{ $name | quote }} - - name: MIGRATION_TIMEOUT_SECONDS + - name: RELEASE_JOB_NAMES + value: {{ toJson $jobs | quote }} + - name: RELEASE_TIMEOUT_SECONDS value: {{ $config.timeoutSeconds | quote }} {{- end }} diff --git a/braintrust/tests/brainstore-rollout-labels_test.yaml b/braintrust/tests/brainstore-rollout-labels_test.yaml new file mode 100644 index 0000000..d3ac3db --- /dev/null +++ b/braintrust/tests/brainstore-rollout-labels_test.yaml @@ -0,0 +1,38 @@ +suite: rollout observer label isolation +release: + name: test + namespace: workload + revision: 7 +templates: + - brainstore-rollout.yaml + - brainstore-reader-configmap.yaml + - brainstore-writer-configmap.yaml + - brainstore-fastreader-configmap.yaml + - brainstore-automationwriter-configmap.yaml +tests: + - it: prevents observer pods from becoming API backends through global labels + template: brainstore-rollout.yaml + documentIndex: 0 + set: + global.labels: + app: braintrust-api + shared: global + api.labels.api-only: api + brainstore.reader.labels.reader-only: reader + asserts: + - notEqual: + path: spec.template.metadata.labels.app + value: braintrust-api + - equal: + path: spec.template.metadata.labels.shared + value: global + - isNull: + path: spec.template.metadata.labels["api-only"] + - isNull: + path: spec.template.metadata.labels["reader-only"] + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + - isNull: + path: spec.template.spec.containers[0].envFrom[0].secretRef + diff --git a/braintrust/tests/brainstore-rollout_test.yaml b/braintrust/tests/brainstore-rollout_test.yaml new file mode 100644 index 0000000..bce2b20 --- /dev/null +++ b/braintrust/tests/brainstore-rollout_test.yaml @@ -0,0 +1,160 @@ +suite: exact release rollout boundaries +release: + name: test + namespace: workload + revision: 7 +templates: + - brainstore-rollout.yaml + - api-deployment.yaml + - api-configmap.yaml + - brainstore-reader-deployment.yaml + - brainstore-reader-configmap.yaml + - brainstore-writer-configmap.yaml + - brainstore-fastreader-configmap.yaml + - brainstore-automationwriter-configmap.yaml +tests: + - it: omits the observer when migrations are disabled + template: brainstore-rollout.yaml + set: + migrations.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: omits exact rollout ordering when the startup gate is disabled + template: brainstore-rollout.yaml + set: + api.brainstoreStartupGate.enabled: false + api.brainstoreStartupGate.timeoutSeconds: 0 + api.brainstoreStartupGate.image: "" + asserts: + - hasDocuments: + count: 0 + + - it: allows rollback to recreate the observer identity after a disabled revision + template: brainstore-rollout.yaml + documentIndex: 2 + asserts: + - isKind: + of: ServiceAccount + - isNull: + path: metadata.annotations["helm.sh/resource-policy"] + + - it: keeps compatibility-only startup when migrations are disabled + template: api-deployment.yaml + set: + migrations.enabled: false + asserts: + - lengthEqual: + path: spec.template.spec.initContainers + count: 1 + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-brainstore + + - it: restores full ordering for null feature maps on reused values + template: brainstore-rollout.yaml + set: + migrations: null + api.brainstoreStartupGate: null + asserts: + - hasDocuments: + count: 5 + + - it: budgets migration and rollout separately without premature API expiry + template: api-deployment.yaml + set: + migrations.timeoutSeconds: 900 + api.brainstoreStartupGate.timeoutSeconds: 300 + asserts: + - equal: + path: spec.template.spec.initContainers[0].env[2].value + value: "1200" + - equal: + path: spec.progressDeadlineSeconds + value: 1800 + + - it: gives the observer time for migration before its rollout observation + template: brainstore-rollout.yaml + documentIndex: 0 + set: + migrations.timeoutSeconds: 900 + api.brainstoreStartupGate.timeoutSeconds: 300 + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1200 + - isNull: + path: spec.ttlSecondsAfterFinished + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - isNull: + path: metadata.annotations["helm.sh/hook"] + + - it: prevents custom annotations from spoofing the expected release or config + template: brainstore-reader-deployment.yaml + set: + brainstore.reader.annotations.pod: + braintrust.dev/release-revision: forged + checksum/config: forged + custom: preserved + asserts: + - equal: + path: spec.template.metadata.annotations["braintrust.dev/release-revision"] + value: "7" + - notEqual: + path: spec.template.metadata.annotations["checksum/config"] + value: forged + - equal: + path: spec.template.metadata.annotations.custom + value: preserved + + - it: does not force strict revision identity after opting out of migrations + template: brainstore-reader-deployment.yaml + set: + migrations.enabled: false + asserts: + - isNull: + path: spec.template.metadata.annotations["braintrust.dev/release-revision"] + - isNull: + path: spec.template.spec.initContainers + + - it: restricts observation to namespace pod listing and named deployments + template: brainstore-rollout.yaml + documentIndex: 3 + set: + global.createNamespace: true + global.namespace: custom-namespace + brainstore.reader.name: custom-reader + brainstore.writer.name: custom-writer + brainstore.fastreader.name: custom-fastreader + brainstore.automationwriter: null + asserts: + - equal: + path: metadata.namespace + value: custom-namespace + - lengthEqual: + path: rules + count: 2 + - equal: + path: rules[0].apiGroups + value: [apps] + - equal: + path: rules[0].resources + value: [deployments] + - equal: + path: rules[0].resourceNames + value: [custom-reader, custom-writer, custom-fastreader] + - equal: + path: rules[0].verbs + value: [get] + - equal: + path: rules[1].apiGroups + value: [""] + - equal: + path: rules[1].resources + value: [pods] + - equal: + path: rules[1].verbs + value: [list] diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index 1a1db8f..0a2bbbb 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -89,26 +89,23 @@ tests: - isNull: path: data.PG_URL - - it: waits for migrations before the Brainstore compatibility gate + - it: holds APIs behind the completed rollout before the compatibility check template: api-deployment.yaml asserts: - equal: path: spec.template.spec.initContainers[0].name - value: wait-for-migrations + value: wait-for-brainstore-rollout - equal: path: spec.template.spec.initContainers[1].name value: wait-for-brainstore - matchRegex: path: spec.template.spec.initContainers[0].env[1].value - pattern: '-migrate-r7$' - - equal: - path: spec.progressDeadlineSeconds - value: 2700 + pattern: '-rollout-r7$' - equal: path: spec.template.spec.automountServiceAccountToken value: false - - it: grants only get on the exact revision Job + - it: restricts workload gates to reading only the two exact release Jobs template: migration-gate.yaml documentIndex: 1 asserts: @@ -129,24 +126,12 @@ tests: - matchRegex: path: rules[0].resourceNames[0] pattern: '-migrate-r7$' - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - - it: keeps revision-bound helper code for old replicas and rollback - template: migration-gate.yaml - documentIndex: 0 - asserts: - matchRegex: - path: metadata.name - pattern: '-migrate-r7-gate$' + path: rules[0].resourceNames[1] + pattern: '-rollout-r7$' - equal: path: metadata.annotations["helm.sh/resource-policy"] value: keep - - isNotNull: - path: data["gate.py"] - - isNotNull: - path: data["brainstore_gate.py"] - it: binds all pool identities and the Brainstore account in the actual namespace template: migration-gate.yaml @@ -323,7 +308,7 @@ tests: documentIndex: 0 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates isolated API pool 1 @@ -331,7 +316,7 @@ tests: documentIndex: 1 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates isolated API pool 2 @@ -339,7 +324,7 @@ tests: documentIndex: 2 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates the optional AI Gateway @@ -427,7 +412,7 @@ tests: template: api-deployment.yaml set: {"migrations": null} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} - it: merges partial reused values with defaults template: migration-job.yaml @@ -445,81 +430,25 @@ tests: - {"equal": {"path": "spec.backoffLimit", "value": 0}} - it: uses custom timing and a mirrored helper image - template: api-deployment.yaml + template: brainstore-reader-deployment.yaml set: {"migrations.timeoutSeconds": 1800, "migrations.pollIntervalSeconds": 10, "migrations.image": "mirror/python:3.13"} asserts: - - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 3300}} - {"equal": {"path": "spec.template.spec.initContainers[0].image", "value": "mirror/python:3.13"}} - {"equal": {"path": "spec.template.spec.initContainers[0].env[2].value", "value": "1800"}} - {"equal": {"path": "spec.template.spec.initContainers[0].env[3].value", "value": "10"}} - - it: preserves API image tag v2.22.0 - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "v2.22.0", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.22.0"}} - - - it: preserves API image tag 2.22.0 - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "2.22.0", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:2.22.0"}} - - - it: preserves API image tag abcdef0123456789abcdef0123456789abcdef0123 - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "abcdef0123456789abcdef0123456789abcdef0123", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:abcdef0123456789abcdef0123456789abcdef0123"}} - - - it: preserves API image tag deadbeef - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "deadbeef", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:deadbeef"}} - - - it: preserves API image tag 1234567 - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "1234567", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:1234567"}} - - - it: preserves API image tag v2.23.0-rc.1 - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "v2.23.0-rc.1", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.23.0-rc.1"}} - - - it: preserves API image tag custom - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "custom", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:custom"}} - - - it: preserves API image tag v2.22.0@sha256:abc - template: migration-job.yaml - documentIndex: 0 - set: {"api.image.tag": "v2.22.0@sha256:abc", "api.brainstoreStartupGate.minimumVersion": "2.22.0"} - asserts: - - {"equal": {"path": "spec.template.spec.containers[0].image", "value": "public.ecr.aws/braintrust/standalone-api:v2.22.0@sha256:abc"}} - it: rejects invalid timeoutSeconds 0 template: migration-rollback.yaml set: {"migrations.timeoutSeconds": 0} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: rejects invalid timeoutSeconds template: migration-rollback.yaml set: {"migrations.timeoutSeconds": ""} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: defaults a null timeoutSeconds after Helm coalesces values template: migration-job.yaml @@ -534,19 +463,19 @@ tests: template: migration-rollback.yaml set: {"migrations.timeoutSeconds": -1} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: rejects invalid pollIntervalSeconds 0 template: migration-rollback.yaml set: {"migrations.pollIntervalSeconds": 0} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: rejects invalid pollIntervalSeconds template: migration-rollback.yaml set: {"migrations.pollIntervalSeconds": ""} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: defaults a null pollIntervalSeconds after Helm coalesces values template: migration-job.yaml @@ -561,19 +490,19 @@ tests: template: migration-rollback.yaml set: {"migrations.pollIntervalSeconds": -1} asserts: - - {"failedTemplate": {"errorMessage": "migrations timing values must be positive integers"}} + - failedTemplate: {} - it: rejects a poll interval outside the budget template: migration-rollback.yaml set: {"migrations.pollIntervalSeconds": 1200} asserts: - - {"failedTemplate": {"errorMessage": "migrations.timeoutSeconds must exceed pollIntervalSeconds"}} + - failedTemplate: {} - it: rejects an empty helper image template: migration-rollback.yaml set: {"migrations.image": ""} asserts: - - {"failedTemplate": {"errorMessage": "migrations.image is required"}} + - failedTemplate: {} - it: does not validate unused disabled settings template: migration-job.yaml @@ -585,46 +514,7 @@ tests: template: migration-rollback.yaml set: {"migrations.backoffLimit": -1} asserts: - - {"failedTemplate": {"errorMessage": "migrations.backoffLimit must be a nonnegative integer"}} - - - it: binds a distinct Job to release revision 1 - template: migration-job.yaml - documentIndex: 0 - release: {"name": "test", "revision": 1} - asserts: - - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r1$"}} - - - it: binds pods to release revision 1 - template: api-deployment.yaml - release: {"name": "test", "revision": 1} - asserts: - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r1$"}} - - - it: binds a distinct Job to release revision 8 - template: migration-job.yaml - documentIndex: 0 - release: {"name": "test", "revision": 8} - asserts: - - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r8$"}} - - - it: binds pods to release revision 8 - template: api-deployment.yaml - release: {"name": "test", "revision": 8} - asserts: - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r8$"}} - - - it: binds a distinct Job to release revision 101 - template: migration-job.yaml - documentIndex: 0 - release: {"name": "test", "revision": 101} - asserts: - - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} - - - it: binds pods to release revision 101 - template: api-deployment.yaml - release: {"name": "test", "revision": 101} - asserts: - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r101$"}} + - failedTemplate: {} - it: preserves revision suffix with long release names template: migration-job.yaml @@ -634,24 +524,6 @@ tests: - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,56}$"}} - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} - - it: preserves reader replica behavior for 0 - template: brainstore-reader-deployment.yaml - set: {"brainstore.reader.replicas": 0} - asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - - it: preserves reader replica behavior for None - template: brainstore-reader-deployment.yaml - set: {"brainstore.reader.replicas": null} - asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - - it: preserves reader replica behavior for - template: brainstore-reader-deployment.yaml - set: {"brainstore.reader.replicas": ""} - asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - it: keeps missing automation writer omitted template: brainstore-automationwriter-deployment.yaml set: {"brainstore.automationwriter": null} @@ -691,7 +563,7 @@ tests: value: before-hook-creation,hook-succeeded - matchRegex: path: spec.template.spec.containers[0].env[1].value - pattern: '-migrate-r7$' + pattern: '^\["test-9f86d081-migrate-r7","test-9f86d081-rollout-r7"\]$' - matchRegex: path: spec.template.spec.serviceAccountName pattern: '-migrate-r7-reset$' @@ -717,13 +589,13 @@ tests: value: [get, delete] - equal: path: rules[0].resourceNames - value: [test-9f86d081-migrate-r7] + value: [test-9f86d081-migrate-r7, test-9f86d081-rollout-r7] - equal: path: rules[1].resourceNames - value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate] + value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate, test-9f86d081-rollout-r7-config, test-9f86d081-rollout-r7-gate] - equal: path: rules[2].resourceNames - value: [test-9f86d081-migrate-r7] + value: [test-9f86d081-migrate-r7, test-9f86d081-rollout-r7] - it: does not render rollback hooks when migrations are disabled template: migration-rollback.yaml diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 933c6ee..1c37c93 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -102,8 +102,8 @@ btqlAuditLogsBestEffortOrgIds: [] api: name: "braintrust-api" - # Experimental: hold new API processes until compatible Brainstore capacity - # is available. Shared by all API pools. Enable for a canary before promotion. + # Hold new API processes until compatible Brainstore capacity is available. + # With migrations enabled, first require this release's exact Brainstore rollout. # Requires stable release tags (vX.Y.Z); opaque custom tags cannot prove compatibility. # Only the init container mounts a Kubernetes API token when enabled; custom # sidecars needing Kubernetes API access must project their own token. @@ -821,8 +821,8 @@ virtualService: port: number: 8000 # Should match api.service.port -# Run schema migrations once per Helm revision before any new Deployment -# application containers start. See docs/migration-job.md for retention/rollback. +# Run schema migrations once per Helm revision before new application containers +# start. API startup then waits for the exact Brainstore rollout. See docs/migration-job.md. migrations: enabled: true timeoutSeconds: 1200 diff --git a/docs/migration-job.md b/docs/migration-job.md index da7a0d6..6e062bc 100644 --- a/docs/migration-job.md +++ b/docs/migration-job.md @@ -1,57 +1,69 @@ -# Release migration Job +# Release ordering: migrations, Brainstore, then API + +The default startup order is: + +1. A revision-specific migration Job runs the shared API image's migration entry point. +2. New Brainstore application containers wait for that Job, then roll out. +3. A separate revision-specific rollout Job verifies the exact reader/writer configuration is fully deployed. +4. New API application containers wait for the rollout Job, then run the existing live version-compatibility check. + +Two Jobs are necessary: if a single Job waited for Brainstore before reporting migration success, Brainstore would wait for that same Job forever. Both phases share one read-only Job-wait helper and revision-scoped code/RBAC. Gateway and Loop runtime retain their migration-only wait. ## Input matrix and intended behavior | Input | Behavior | | --- | --- | -| Feature enabled (default), missing/null map, partial reused values | One regular Job per Helm release revision; all chart Deployment pods wait for that revision's successful Job | -| Feature disabled; enabled to disabled | No new migration resources or waits; API startup continues to run migrations as before | -| Disabled to enabled | New revision Job and pod gates are created | -| API image release tags with/without v, full/short/numeric SHA, prerelease, custom tags, digest | Pass image through unchanged; existing Brainstore version gate still requires its explicit minimum for non-release tags | -| AWS, Google, Azure; provider examples | Job uses shared API identity, configuration, secrets, CA, scheduling and security settings | -| Azure Key Vault CSI enabled | Migration Job mounts CSI secrets itself before application pods start | -| API workload isolation on/off; custom pool identities | One shared API-image migrator; every pool waits, with read-only Job RBAC for each account | -| Brainstore roles enabled/zero/null replicas; automation writer missing/partial map | Preserve existing replica/default rules; every rendered pod template carries the wait | -| Optional gateway and Loop runtime enabled/disabled | Every rendered Deployment waits; disabled workloads stay omitted | -| Missing/null timing; empty/zero timing; empty image | Missing/null use defaults after Helm coalescing; empty/zero timing and empty image fail when enabled; disabled configuration does not validate unused settings | -| Upgrade/retry, old ReplicaSet replacement, rollback | Distinct revision names; keep completed Jobs and their config and read permissions; rollback cleans and recreates the target revision resources, rerunning migrations | -| Migration failure, missing Job, API errors | Fail closed; bounded waits and Job execution; application containers remain stopped | -| Global/API/role labels | No mutation or leakage into other workloads; Job must not match API Service selectors | - -## Lifecycle - -This feature requires Helm install/upgrade release revisions. Plain `helm template` -plus apply (including render-only GitOps controllers) does not increment revisions; -use a Helm release controller rather than repeatedly applying revision 1. - -A release snapshots API ConfigMap data and runs the shared `api.image` migration -entry point. Pool-specific API image/config overrides must remain schema-compatible -with that shared image and database. The API startup migration check remains for -compatibility with other deployment methods. SQL changes must be compatible with -old running pods; rollback does not undo database changes. Existing rollout -strategies still govern availability (Recreate and nonzero maxUnavailable can -remove old capacity while replacements wait). - -Jobs have no TTL. Jobs, migration configuration, gate code and revision-specific -RBAC use `helm.sh/resource-policy: keep`, so old ReplicaSets and rollback revisions -can still start pods. These resources also survive uninstall. Delete retained -resources only after no Pod/ReplicaSet references that revision and it is no longer -needed for rollback. Before reinstalling the same release name, clean up its -retained resources after ensuring no old workload references them. Do not delete -the active revision's Job: future replacements/autoscaling need its success record. -A failed Job is retried with a new `helm upgrade` revision after fixing its cause. - -Native Helm rollback executes a `pre-rollback` cleanup hook with a separate service -account allowed only to get/delete the target revision's Job, two ConfigMaps, Role -and RoleBinding. It waits for foreground Job deletion before Helm recreates the -target resources and reruns the migration check. This avoids the Helm 3 retained- -resource rollback error (https://github.com/helm/helm/issues/13142). Existing running -application containers keep running, but replacements can wait during cleanup. -Do not use `--no-hooks` on rollback. Automatic rollback follows the same hook path. -Schema changes are never reversed. Helm rollback must have time for cleanup plus -migration plus application readiness. Failed cleanup blocks rollback; its hook Job -and logs remain for diagnosis. - -Allow enough Helm timeout for migration plus Brainstore rollout and the existing -API compatibility gate. `--wait --wait-for-jobs` makes Helm report migration failure; -without waits, Helm can return before migration/rollout completion. +| Migrations and API Brainstore startup gate enabled (default), missing/null maps, partial reused values | Migration Job followed by exact Brainstore rollout Job; API waits for rollout success and live compatibility | +| `migrations.enabled: false`, startup gate enabled | No release Jobs; existing live version-compatibility check and API-startup migration behavior | +| Migrations enabled, `api.brainstoreStartupGate.enabled: false` | Migration Job only; all new application containers wait for migrations, without exact Brainstore rollout ordering | +| Both disabled | No chart-managed startup barriers | +| Disabled to enabled | New revision Jobs and pod gates are created | +| Same image with changed inline env or ConfigMap-backed env | Old Ready pods cannot satisfy the new release's rollout barrier | +| API or Brainstore release tags, custom tags, numeric/full/short SHAs, prerelease, tagged digest | Exact rollout identity does not parse versions; the separate API compatibility check still needs a stable minimum for non-release tags | +| AWS, Google, Azure; provider examples | Migrator retains shared API identity, secrets, CA mounts, resources, and scheduling; rollout observer has no cloud identity or secret access | +| Azure Key Vault CSI enabled | Migration Job mounts CSI secrets itself before application containers start | +| API workload isolation/custom pool identities | Every API pool waits for the same rollout Job using its own read-only Job permission | +| Reader, writer, fastreader, rendered automationwriter | Every rendered role participates, including disabled roles that must finish scaling down | +| Missing/null replica count, explicit zero | Missing/null uses Kubernetes' default of one; zero stays zero and requires all non-terminal role pods to disappear | +| Automationwriter absent/null or partial reused map | Absent/null stays omitted; a partial map uses existing automationwriter defaults and participates if rendered | +| Old Deployment still fully Ready before Helm updates it | Expected release revision, image, and ConfigMap checksum must match before rollout state can count | +| Controller not caught up, pending replacements, insufficient Ready replicas, `minReadySeconds`, old/terminating/stray service backends | Barrier remains closed; existing healthy replicas do not stand in for updated replicas | +| Missing/null timing; empty/zero timing or helper image | Existing default/coalescing rules apply; invalid enabled settings fail rendering, unused disabled settings are not validated | +| Migration or rollout failure, missing Job, transient Kubernetes read/RBAC errors | Gates fail closed with bounded waits; completed Jobs are never inferred from absence | +| Subsequent upgrade, superseded API replacement | Each revision has distinct Jobs; old API replacements use their retained rollout success, not an obsolete desired configuration | +| Rollback | Cleanup resets both target-revision Jobs before removing their config/RBAC; migrations and exact rollout verification run again | +| Global/API/role labels and custom pod annotations | Inputs are not mutated or leaked between workloads; observer/migrator pods cannot match API service selectors; managed rollout identity/checksum cannot be overridden | + +## Exact rollout contract + +The chart stamps each rendered Brainstore Pod template with `braintrust.dev/release-revision`. Each Helm revision already changes its migration Job dependency, forcing replacement even for same-image changes. The rollout Job snapshots the expected deployment/container names, release revision, exact image string, and rendered role ConfigMap checksum. + +For each role it checks the Deployment template against those expectations, requires the controller to have observed the desired generation, and waits for all desired replicas to be updated, Ready, and Available. It also lists every service-selected pod, including stray pods, and rejects previous revisions/configurations and terminating live pods. Two successful fleet observations are required within one deadline; this is an observation barrier, not a lock on later changes. + +The existing live compatibility check remains intentionally separate. An older API can restart while a later compatible Brainstore release rolls out: its retained rollout Job records the completed prerequisite, while the compatibility check protects its current live connections. Do not make older API replacements wait for their old deployment revision to become current again. + +## Configuration and availability limits + +This feature requires real Helm install/upgrade revisions. Repeated `helm template | kubectl apply` does not advance revision identity and is unsupported. + +The guarantee is **application-container startup order**, not Kubernetes object mutation order. Helm still applies Deployment specs and ConfigMaps together. New containers read their ConfigMap-backed environment only after their init gates finish, but mutable mounted files, externally changed Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migration deployment and versioned configuration/Secrets if no object or old workload may see changed configuration before migration completion. After changing an external Secret, perform a Helm upgrade to roll pods; the chart does not watch or snapshot secret values. Pin image digests when immutable image identity is required. + +Pool-specific API image/config overrides must remain schema-compatible with the shared migrator image and database. SQL changes must be backward-compatible with existing running workloads; rollback never reverses schema changes. + +Existing rollout strategies still govern availability and spare capacity. `Recreate` and nonzero `maxUnavailable` can remove old capacity while replacements wait. With RollingUpdate and `maxUnavailable: 0`, old API replicas remain while the new API waits; exact ordering does not solve insufficient surge nodes. Local SSD cache warm-up is governed by readiness and `minReadySeconds`, not inferred from pod creation alone. + +## Retention and rollback + +Neither release Job has a TTL. Completed Jobs, their configuration, helper code, and per-revision RBAC use `helm.sh/resource-policy: keep`, so replacement pods and rollback revisions retain their prerequisites. Those phase records survive uninstall. The rollout observer uses a stable, namespace-only ServiceAccount that is not retained: disabling the gate removes it, allowing Helm 3 to recreate it cleanly when rolling back to an enabled revision. Old API replacements read the retained Job using their own API identity, not the observer account. + +Delete retained resources only after no Pod/ReplicaSet references the revision and it is no longer needed for rollback. Do not delete the active revision's completed Jobs: replacements and autoscaling need those success records. Before reinstalling the same release name, deliberately clean up retained resources after confirming no old workloads reference them. Retry failed attempts with a new `helm upgrade` revision after fixing the cause. + +Native Helm rollback runs the target revision's `pre-rollback` cleanup hook. Its separate identity can get/delete only that target's two Jobs and corresponding ConfigMaps, Roles, and RoleBindings. Cleanup issues deletion for both Jobs, waits for foreground deletion of both, then removes retained config/RBAC. Helm then recreates the target resources and reruns both phases. Migration-only releases clean up just their migration phase. This avoids the Helm 3 retained-resource restoration error (https://github.com/helm/helm/issues/13142) without overlapping the target's old and new Jobs. + +Do not use `--no-hooks` on rollback. Automatic rollback follows the same hook path. Existing running application containers continue according to their rollout strategy; replacement pods can wait during cleanup. A failed cleanup blocks rollback and leaves its hook logs for diagnosis. + +## Timeouts and verification + +`migrations.timeoutSeconds` bounds the migration phase. `api.brainstoreStartupGate.timeoutSeconds` bounds exact rollout observation and the existing compatibility check. The rollout Job and its API wait include the migration plus rollout budgets; API progress deadlines also include the subsequent compatibility budget and startup margin. + +Allow enough Helm timeout for migration, node allocation/cache readiness, and API startup. Use `--wait --wait-for-jobs` to observe migration/rollout failure; without waits Helm can return before these phases finish. Inspect the revision-specific rollout Job's logs to identify the role and incomplete condition. Pending replacement writer capacity must keep the new API blocked even when an old compatible writer remains Ready. diff --git a/tests/test_brainstore_rollout.py b/tests/test_brainstore_rollout.py new file mode 100644 index 0000000..03c4751 --- /dev/null +++ b/tests/test_brainstore_rollout.py @@ -0,0 +1,474 @@ +import importlib.util +import io +import os +from pathlib import Path +import sys +import unittest +from unittest.mock import Mock, patch +import urllib.error +import urllib.parse + + +FILES = Path(__file__).resolve().parents[1] / "braintrust/files" +shared_spec = importlib.util.spec_from_file_location( + "brainstore_gate", FILES / "brainstore-startup-gate.py") +shared = importlib.util.module_from_spec(shared_spec) +shared_spec.loader.exec_module(shared) +spec = importlib.util.spec_from_file_location("rollout", FILES / "brainstore-rollout.py") +rollout = importlib.util.module_from_spec(spec) +with patch.dict(sys.modules, {"brainstore_gate": shared}): + spec.loader.exec_module(rollout) + + +def target(role="reader", **overrides): + return { + "deployment": "custom-" + role, + "container": "brainstore-" + role, + "revision": "17", + "image": "registry.example:5000/brainstore:custom-build", + "configChecksum": "b" * 64, + **overrides, + } + + +def pod(expected=None, name="new", is_ready=True, **metadata): + expected = target() if expected is None else expected + return { + "metadata": { + "name": name, + "labels": {"app": expected["deployment"]}, + "annotations": { + "braintrust.dev/release-revision": expected["revision"], + "checksum/config": expected["configChecksum"], + }, + **metadata, + }, + "spec": {"containers": [{"name": expected["container"], "image": expected["image"]}]}, + "status": { + "phase": "Running", + "conditions": [{"type": "Ready", "status": "True" if is_ready else "False"}], + "containerStatuses": [{ + "name": expected["container"], "image": expected["image"], + "ready": is_ready, "state": {"running": {}}, + }], + }, + } + + +def deployment(expected=None, replicas=1): + expected = target() if expected is None else expected + current = pod(expected) + return { + "metadata": {"name": expected["deployment"], "generation": 7}, + "spec": { + "replicas": replicas, + "template": {"metadata": {"annotations": current["metadata"]["annotations"]}, + "spec": current["spec"]}, + }, + "status": { + "observedGeneration": 7, + "updatedReplicas": replicas, + "readyReplicas": replicas, + "availableReplicas": replicas, + "replicas": replicas, + }, + } + + +class DeploymentTests(unittest.TestCase): + def check(self, resource=None, pods=None, expected=None): + expected = target() if expected is None else expected + return rollout.check_deployment( + deployment(expected) if resource is None else resource, + [pod(expected)] if pods is None else pods, expected) + + def test_exact_ready_release_passes(self): + self.assertIsNone(self.check()) + + def test_old_fully_ready_spec_does_not_pass_before_helm_updates_it(self): + old = target(revision="16") + self.assertIsNotNone(self.check(deployment(old), [pod(old)])) + + def test_config_only_change_requires_updated_template_and_pods(self): + old = target(configChecksum="a" * 64) + self.assertIsNotNone(self.check(deployment(old), [pod(old)])) + self.assertIsNotNone(self.check(pods=[pod(old, "old")])) + self.assertIsNone(self.check()) + + def test_old_ready_writer_cannot_release_a_pending_same_image_replacement(self): + expected = target("writer") + old = target("writer", configChecksum="a" * 64) + pending = pod(expected, "replacement", False) + pending["status"]["phase"] = "Pending" + resource = deployment(expected) + resource["status"]["updatedReplicas"] = 0 + self.assertIsNotNone(self.check(resource, [pod(old, "old-writer"), pending], expected)) + self.assertIsNotNone(self.check(pods=[pending], expected=expected)) + + def test_revision_only_env_rollout_cannot_use_previous_ready_pod(self): + old = target(revision="16") + self.assertIsNotNone(self.check(pods=[pod(old, "old")])) + + def test_missing_and_wrong_identity_fields_fail_closed_on_template_and_pods(self): + fields = ("braintrust.dev/release-revision", "checksum/config") + for location in ("template", "pod"): + for annotation in fields: + for value in (None, "wrong"): + with self.subTest(location=location, annotation=annotation, value=value): + resource = deployment() + current = pod() + subject = resource["spec"]["template"] if location == "template" else current + if value is None: + del subject["metadata"]["annotations"][annotation] + else: + subject["metadata"]["annotations"][annotation] = value + self.assertIsNotNone(self.check(resource, [current])) + + def test_image_must_match_exactly_in_template_and_pod_spec(self): + for location in ("template", "pod"): + with self.subTest(location=location): + resource = deployment() + current = pod() + subject = resource["spec"]["template"] if location == "template" else current + subject["spec"]["containers"][0]["image"] = "registry.example/brainstore:other" + self.assertIsNotNone(self.check(resource, [current])) + + def test_exact_images_do_not_need_semver_tags(self): + for image in ("registry/brainstore:v2.15.0", "registry/brainstore:latest", + "registry/brainstore:deadbeef", "registry/brainstore:v2.15.0-rc.1", + "registry:5000/brainstore@sha256:" + "a" * 64, + "registry/brainstore:custom@sha256:" + "b" * 64): + with self.subTest(image=image): + self.assertIsNone(self.check(expected=target(image=image))) + + def test_runtime_image_alias_neither_blocks_nor_overrides_the_pod_spec(self): + current = pod() + current["status"]["containerStatuses"][0]["image"] = "containerd://sha256:resolved" + self.assertIsNone(self.check(pods=[current])) + current["spec"]["containers"][0]["image"] = "registry/brainstore:old" + self.assertIsNotNone(self.check(pods=[current])) + + def test_missing_expected_spec_container_fails_closed(self): + for location in ("template", "pod"): + with self.subTest(location=location): + resource = deployment() + current = pod() + subject = resource["spec"]["template"] if location == "template" else current + subject["spec"]["containers"][0]["name"] = "sidecar" + self.assertIsNotNone(self.check(resource, [current])) + + def test_current_controller_observation_is_required(self): + for observed in (None, 6): + with self.subTest(observed=observed): + resource = deployment() + if observed is None: + del resource["status"]["observedGeneration"] + else: + resource["status"]["observedGeneration"] = observed + self.assertIsNotNone(self.check(resource)) + + def test_deleted_deployment_blocks(self): + resource = deployment() + resource["metadata"]["deletionTimestamp"] = "2026-11-01T00:00:00Z" + self.assertIsNotNone(self.check(resource)) + + def test_each_controller_replica_counter_must_equal_desired(self): + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + for count in (None, 0, 2): + with self.subTest(counter=counter, count=count): + resource = deployment() + if count is None: + del resource["status"][counter] + else: + resource["status"][counter] = count + self.assertIsNotNone(self.check(resource)) + + def test_min_ready_seconds_requires_controller_availability(self): + resource = deployment() + resource["spec"]["minReadySeconds"] = 30 + resource["status"]["availableReplicas"] = 0 + self.assertIsNotNone(self.check(resource)) + resource["status"]["availableReplicas"] = 1 + self.assertIsNone(self.check(resource)) + + def test_current_spec_scaling_is_honored_without_target_replica_snapshot(self): + resource = deployment(replicas=3) + self.assertIsNone(self.check(resource, [pod(name=str(index)) for index in range(3)])) + self.assertIsNotNone(self.check(resource, [pod()])) + self.assertIsNotNone(self.check(pods=[pod(), pod(name="surge")])) + + def test_missing_replicas_defaults_to_one(self): + resource = deployment() + del resource["spec"]["replicas"] + self.assertIsNone(self.check(resource)) + self.assertIsNotNone(self.check(resource, [])) + + def test_explicit_zero_requires_no_live_backends_and_zero_status_counts(self): + resource = deployment(replicas=0) + self.assertIsNone(self.check(resource, [])) + self.assertIsNotNone(self.check(resource, [pod()])) + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + with self.subTest(counter=counter): + stale = deployment(replicas=0) + stale["status"][counter] = 1 + self.assertIsNotNone(self.check(stale, [])) + resource["status"] = {"observedGeneration": 7} + self.assertIsNone(self.check(resource, [])) + + def test_zero_still_requires_current_release_template_and_observation(self): + resource = deployment(target(revision="16"), replicas=0) + self.assertIsNotNone(self.check(resource, [])) + resource = deployment(replicas=0) + resource["status"]["observedGeneration"] = 6 + self.assertIsNotNone(self.check(resource, [])) + + def test_terminal_pods_are_ignored_even_when_old_or_terminating(self): + for phase in ("Failed", "Succeeded"): + with self.subTest(phase=phase): + old = pod(target(revision="16"), "old", False, + deletionTimestamp="2026-11-01T00:00:00Z") + old["status"]["phase"] = phase + self.assertIsNone(self.check(pods=[pod(), old])) + self.assertIsNone(self.check(deployment(replicas=0), [old])) + + def test_terminating_matching_backend_blocks_even_if_unready(self): + for is_ready in (True, False): + with self.subTest(is_ready=is_ready): + terminating = pod(name="terminating", is_ready=is_ready, + deletionTimestamp="2026-11-01T00:00:00Z") + self.assertIsNotNone(self.check(pods=[pod(), terminating])) + self.assertIsNotNone(self.check(deployment(replicas=0), [terminating])) + + def test_ready_requires_running_expected_container_and_pod_condition(self): + for missing in ("phase", "condition", "container", "running", "ready"): + with self.subTest(missing=missing): + current = pod() + if missing == "phase": + current["status"]["phase"] = "Pending" + elif missing == "condition": + current["status"]["conditions"] = [] + elif missing == "container": + current["status"]["containerStatuses"][0]["name"] = "sidecar" + elif missing == "running": + current["status"]["containerStatuses"][0]["state"] = {"waiting": {}} + else: + current["status"]["containerStatuses"][0]["ready"] = False + self.assertIsNotNone(self.check(pods=[current])) + + +class FleetTests(unittest.TestCase): + def test_every_enabled_role_uses_its_custom_service_selector_and_current_spec(self): + targets = [target(role, deployment="tenant-pool-" + role) + for role in ("reader", "writer", "fastreader", "automationwriter")] + resources = {expected["deployment"]: deployment(expected, replicas=0 if index == 3 else 1) + for index, expected in enumerate(targets)} + backends = {"app=" + expected["deployment"]: [pod(expected)] if index != 3 else [] + for index, expected in enumerate(targets)} + client = Mock() + client.deployment.side_effect = lambda name, deadline: resources[name] + client.pods.side_effect = lambda selector, deadline: backends[selector] + self.assertIsNone(rollout.check_fleet(client, targets, 100)) + for expected in targets: + with self.subTest(role=expected["container"]): + selector = "app=" + expected["deployment"] + current = backends[selector] + backends[selector] = [pod({**expected, "revision": "16"}, "old")] + self.assertIsNotNone(rollout.check_fleet(client, targets, 100)) + backends[selector] = current + + def test_unready_optional_role_blocks_the_fleet(self): + targets = [target(role) for role in ("reader", "writer", "fastreader", "automationwriter")] + client = Mock() + client.deployment.side_effect = [deployment(expected) for expected in targets] + client.pods.side_effect = [[pod(expected, is_ready=index != 3)] + for index, expected in enumerate(targets)] + self.assertIsNotNone(rollout.check_fleet(client, targets, 100)) + + def test_old_stray_backend_on_later_api_page_blocks_without_owner_filtering(self): + expected = target(deployment="tenant-pool-reader") + with patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}), \ + patch.object(shared.ssl, "create_default_context"): + client = rollout.KubernetesClient("custom namespace", "/synthetic") + stray = pod(target(deployment=expected["deployment"], revision="16"), "stray") + resources = { + "/apis/apps/v1/namespaces/custom%20namespace/deployments/tenant-pool-reader": + deployment(expected), + } + pages = { + "": {"items": [pod(expected)], "metadata": {"continue": "next-page"}}, + "next-page": {"items": [stray]}, + } + + def get_resource(path, deadline): + if path in resources: + return resources[path] + parsed = urllib.parse.urlsplit(path) + query = urllib.parse.parse_qs(parsed.query) + if (parsed.path != "/api/v1/namespaces/custom%20namespace/pods" + or query.get("labelSelector") != ["app=" + expected["deployment"]]): + raise AssertionError("unexpected Kubernetes resource request") + return pages.pop(query.get("continue", [""])[0]) + + client.get = get_resource + self.assertIsNotNone(rollout.check_fleet(client, [expected], 100)) + + def test_extra_matching_stray_is_not_hidden_by_ready_controller_status(self): + client = Mock() + client.deployment.return_value = deployment() + client.pods.return_value = [pod(), pod(name="stray")] + self.assertIsNotNone(rollout.check_fleet(client, [target()], 100)) + + +class FakeClock: + def __init__(self): + self.now = 0 + self.sleeps = [] + + def clock(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += seconds + + +def healthy_client(): + client = Mock() + client.deployment.return_value = deployment() + client.pods.return_value = [pod()] + return client + + +class WaitTests(unittest.TestCase): + def wait(self, client, clock=None, timeout=60, interval=10, targets=None, log=None): + clock = FakeClock() if clock is None else clock + log = Mock() if log is None else log + rollout.wait_for_fleet(client, [target()] if targets is None else targets, + timeout, interval, clock.clock, clock.sleep, log) + return clock, log + + def test_requires_two_real_successful_fleet_observations(self): + client = healthy_client() + clock, _ = self.wait(client) + self.assertEqual(client.deployment.call_count, 2) + self.assertEqual(client.pods.call_count, 2) + self.assertEqual(clock.now, 10) + + def test_unhealthy_observation_resets_confirmation(self): + client = healthy_client() + client.pods.side_effect = [[pod()], [pod(is_ready=False)], [pod()], [pod()]] + clock, _ = self.wait(client) + self.assertEqual(client.pods.call_count, 4) + self.assertEqual(clock.now, 30) + + def test_transient_read_failure_resets_confirmation(self): + client = healthy_client() + client.deployment.side_effect = [deployment(), OSError("offline"), deployment(), deployment()] + clock, _ = self.wait(client) + self.assertEqual(client.deployment.call_count, 4) + self.assertEqual(client.pods.call_count, 3) + self.assertEqual(clock.now, 30) + + def test_missing_resources_rbac_throttling_and_server_errors_are_retried(self): + bodies = [io.StringIO("sensitive response body") for _ in range(4)] + errors = [urllib.error.HTTPError("https://api/deployments/reader?secret=hidden", code, + "sensitive reason", None, body) + for code, body in zip((404, 403, 429, 503), bodies)] + client = healthy_client() + client.deployment.side_effect = errors + [deployment(), deployment()] + clock, log = self.wait(client) + self.assertEqual(clock.now, 50) + self.assertTrue(all(body.closed for body in bodies)) + messages = "\n".join(call.args[0] for call in log.call_args_list) + self.assertNotIn("sensitive", messages) + self.assertNotIn("secret", messages) + + def test_read_and_decode_errors_are_retried_without_leaking_details(self): + client = healthy_client() + client.deployment.side_effect = [ + urllib.error.URLError("secret transport detail"), ValueError("secret JSON"), + KeyError("secret field"), TypeError("secret object"), deployment(), deployment(), + ] + clock, log = self.wait(client) + self.assertEqual(clock.now, 50) + messages = "\n".join(call.args[0] for call in log.call_args_list) + self.assertNotIn("secret", messages) + + def test_pod_list_failure_is_retried_instead_of_using_previous_ready_result(self): + client = healthy_client() + client.pods.side_effect = [[pod()], urllib.error.URLError("offline"), [pod()], [pod()]] + clock, _ = self.wait(client) + self.assertEqual(client.pods.call_count, 4) + self.assertEqual(clock.now, 30) + + def test_old_ready_config_never_completes_and_sleep_is_bounded(self): + client = healthy_client() + client.pods.return_value = [pod(target(configChecksum="a" * 64), "old")] + clock = FakeClock() + with self.assertRaises(TimeoutError): + self.wait(client, clock, timeout=25) + self.assertEqual(clock.now, 25) + self.assertEqual(clock.sleeps, [10, 10, 5]) + self.assertEqual(client.deployment.call_count, 3) + + def test_persistent_api_failure_times_out_with_bounded_diagnostics(self): + client = healthy_client() + client.deployment.side_effect = [urllib.error.HTTPError( + "https://api/" + "x" * 1000 + "?secret=hidden", 429, + "sensitive reason", None, io.StringIO("sensitive body")) for _ in range(3)] + clock = FakeClock() + with self.assertRaises(TimeoutError) as error: + self.wait(client, clock, timeout=25) + self.assertLess(len(str(error.exception)), 400) + self.assertNotIn("sensitive", str(error.exception)) + self.assertNotIn("secret", str(error.exception)) + self.assertEqual(clock.now, 25) + + def test_successful_response_after_deadline_cannot_complete(self): + client = healthy_client() + clock = FakeClock() + observations = iter((0, 9)) + + def observe(selector, deadline): + clock.now += next(observations) + return [pod()] + + client.pods.side_effect = observe + with self.assertRaises(TimeoutError): + self.wait(client, clock, timeout=10, interval=2) + self.assertEqual(client.pods.call_count, 2) + self.assertEqual(clock.now, 11) + + def test_all_roles_requests_and_confirmations_share_one_deadline(self): + targets = [target("reader"), target("writer")] + expected_by_name = {expected["deployment"]: expected for expected in targets} + client = Mock() + clock = FakeClock() + deadlines = [] + + def get_deployment(name, deadline): + deadlines.append(deadline) + clock.now += 1 + return deployment(expected_by_name[name]) + + def get_pods(selector, deadline): + deadlines.append(deadline) + clock.now += 1 + return [pod(expected_by_name[selector.removeprefix("app=")])] + + client.deployment.side_effect = get_deployment + client.pods.side_effect = get_pods + self.wait(client, clock, timeout=20, interval=3, targets=targets) + self.assertEqual(deadlines, [20] * 8) + self.assertEqual(clock.now, 11) + + def test_scaling_between_confirmations_uses_each_current_spec(self): + client = healthy_client() + client.deployment.side_effect = [deployment(replicas=2), deployment()] + client.pods.side_effect = [[pod(name="one"), pod(name="two")], [pod()]] + self.wait(client) + self.assertEqual(client.deployment.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_job_gate.py b/tests/test_job_gate.py new file mode 100644 index 0000000..f707e78 --- /dev/null +++ b/tests/test_job_gate.py @@ -0,0 +1,286 @@ +import importlib.util +import itertools +import json +import os +from pathlib import Path +import sys +import unittest +from unittest.mock import MagicMock, Mock, patch +import urllib.error + +FILES = Path(__file__).resolve().parents[1] / "braintrust/files" +# The chart mounts the shared runtime under these import names. +for name, file in [ + ("brainstore_gate", "brainstore-startup-gate.py"), + ("job_gate", "job-gate.py"), + ("migration_rollback", "migration-rollback.py"), +]: + spec = importlib.util.spec_from_file_location(name, FILES / file) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) +gate = sys.modules["job_gate"] +rollback = sys.modules["migration_rollback"] + + +def job(*conditions, **metadata): + return {"metadata": metadata, "status": {"conditions": [ + {"type": condition, "status": "True"} for condition in conditions + ]}} + + +class Clock: + def __init__(self): + self.now = 0 + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class JobGateTests(unittest.TestCase): + def wait(self, observations, name="release-migrate-r2", timeout=20): + clock = Clock() + client = Mock(namespace="test-namespace") + client.get.side_effect = observations + gate.wait_for_job(client, name, timeout, 5, + clock=clock, sleep=clock.sleep, log=Mock()) + return client, clock.now + + def test_only_complete_condition_counts_as_success(self): + for resource in ({}, job(), job("SuccessCriteriaMet"), + {"status": {"succeeded": 1}}, + {"status": {"conditions": [{"type": "Complete", "status": "False"}]}}, + job("Complete", deletionTimestamp="now")): + with self.subTest(resource=resource): + self.assertFalse(gate.job_complete(resource)) + self.assertTrue(gate.job_complete(job("Complete"))) + + def test_failure_takes_precedence_over_completion_and_deletion(self): + for condition in ("Failed", "FailureTarget"): + for metadata in ({}, {"deletionTimestamp": "now"}): + with self.subTest(condition=condition, metadata=metadata): + with self.assertRaises(RuntimeError): + gate.job_complete(job(condition, "Complete", **metadata)) + + def test_exact_job_wait_retries_missing_job_rbac_and_server_errors(self): + for name in ("release-migrate-r2", "release-rollout-r2"): + with self.subTest(name=name): + errors = [urllib.error.HTTPError("url", code, "error", {}, None) + for code in (404, 403, 500)] + client, _ = self.wait(errors + [job("Complete")], name=name) + self.assertEqual( + {call.args for call in client.get.call_args_list}, + {(f"/apis/batch/v1/namespaces/test-namespace/jobs/{name}", 20)}, + ) + + def test_retained_success_does_not_wait_for_another_observation(self): + for name in ("release-migrate-r1", "release-rollout-r1"): + client = Mock(namespace="ns") + client.get.return_value = job("Complete") + sleep = Mock() + gate.wait_for_job(client, name, 20, 5, clock=lambda: 0, + sleep=sleep, log=Mock()) + sleep.assert_not_called() + + def test_running_job_then_success(self): + _, elapsed = self.wait([job(), job("Complete")]) + self.assertEqual(elapsed, 5) + + def test_running_missing_or_forbidden_job_never_allows_startup(self): + for observation in (job(), urllib.error.HTTPError("url", 404, "", {}, None), + urllib.error.HTTPError("url", 403, "", {}, None)): + with self.subTest(observation=observation): + with self.assertRaises(TimeoutError): + self.wait(itertools.repeat(observation)) + + def test_failed_job_never_allows_startup(self): + with self.assertRaises(RuntimeError): + self.wait([job("Failed")]) + + def test_transient_observation_errors_retry(self): + self.wait([OSError("network"), ValueError("bad JSON"), job("Complete")]) + + def test_observation_errors_do_not_disclose_response_details(self): + clock = Clock() + client = Mock(namespace="ns") + client.get.side_effect = ValueError("secret response body") + log = Mock() + with self.assertRaises(TimeoutError) as raised: + gate.wait_for_job(client, "job", 6, 5, clock=clock, + sleep=clock.sleep, log=log) + self.assertNotIn("secret response body", str(raised.exception)) + self.assertNotIn("secret response body", str(log.call_args_list)) + + def test_late_success_does_not_escape_deadline(self): + clock = Clock() + client = Mock(namespace="ns") + def late(*args): + clock.now = 21 + return job("Complete") + client.get.side_effect = late + with self.assertRaises(TimeoutError): + gate.wait_for_job(client, "job", 20, 5, clock=clock, + sleep=clock.sleep, log=Mock()) + + +class RollbackCleanupTests(unittest.TestCase): + names = ["release-migrate-r7", "release-rollout-r7"] + + def test_resource_paths_escape_namespace_and_names(self): + paths = rollback.resource_paths("ns/other", ["name/other"]) + self.assertTrue(all("/namespaces/ns%2Fother/" in path for path in paths)) + self.assertTrue(all(path.rsplit("/", 1)[1].startswith("name%2Fother") + for path in paths)) + + def test_both_jobs_are_deleted_before_any_config_or_rbac_cleanup(self): + paths = rollback.resource_paths("ns", self.names) + jobs = paths[:2] + observations = {jobs[0]: iter([True, False]), + jobs[1]: iter([True, True, False])} + deleted, absent = set(), set() + clock = Clock() + def request(method, path, deadline): + self.assertIn(path, paths) + self.assertEqual(deadline, 10) + if path in jobs: + if method == "DELETE": + deleted.add(path) + return True + self.assertEqual(deleted, set(jobs)) + exists = next(observations[path]) + if not exists: + absent.add(path) + return exists + self.assertEqual(absent, set(jobs)) + return method == "DELETE" + client = Mock() + client.request.side_effect = request + rollback.cleanup(client, "ns", self.names, 10, clock=clock, sleep=clock.sleep) + self.assertEqual( + {call.args[1] for call in client.request.call_args_list if call.args[0] == "DELETE"}, + set(paths), + ) + + def test_cleanup_of_absent_resources_is_idempotent_for_one_or_both_jobs(self): + for names in (self.names[:1], self.names): + with self.subTest(names=names): + client = Mock() + client.request.return_value = False + sleep = Mock() + rollback.cleanup(client, "ns", names, 10, clock=lambda: 0, sleep=sleep) + self.assertEqual( + {call.args[1] for call in client.request.call_args_list + if call.args[0] == "DELETE"}, + set(rollback.resource_paths("ns", names)), + ) + sleep.assert_not_called() + + def test_either_stuck_job_blocks_all_config_and_rbac_cleanup(self): + jobs = rollback.resource_paths("ns", self.names)[:2] + for stuck in jobs: + with self.subTest(stuck=stuck): + client = Mock() + client.request.side_effect = lambda method, path, deadline: path == stuck + clock = Clock() + with self.assertRaises(TimeoutError): + rollback.cleanup(client, "ns", self.names, 3, + clock=clock, sleep=clock.sleep) + self.assertEqual(clock.now, 3) + self.assertTrue(all(call.args[1] in jobs + for call in client.request.call_args_list)) + self.assertEqual( + {call.args[1] for call in client.request.call_args_list + if call.args[0] == "DELETE"}, set(jobs), + ) + + def test_each_phase_shares_the_same_deadline(self): + paths = rollback.resource_paths("ns", self.names) + for late_path in (paths[1], paths[2]): + with self.subTest(late_path=late_path): + clock = Clock() + client = Mock() + def request(method, path, deadline): + self.assertEqual(deadline, 3) + if method == "GET" and path == late_path: + clock.now = 3 + return False + client.request.side_effect = request + with self.assertRaises(TimeoutError): + rollback.cleanup(client, "ns", self.names, 3, + clock=clock, sleep=clock.sleep) + self.assertEqual(client.request.call_args.args[1], late_path) + + def test_permission_or_transport_failure_stops_cleanup(self): + paths = rollback.resource_paths("ns", self.names) + for failure in (RuntimeError("forbidden"), OSError("network")): + for failing_path in paths[:3]: + with self.subTest(failure=failure, path=failing_path): + client = Mock() + def request(method, path, deadline): + if path == failing_path: + raise failure + return False + client.request.side_effect = request + with self.assertRaises(type(failure)): + rollback.cleanup(client, "ns", self.names, 10, + clock=lambda: 0, sleep=Mock()) + self.assertEqual(client.request.call_args.args[1], failing_path) + if failing_path in paths[:2]: + self.assertTrue(all(call.args[1] in paths[:2] + for call in client.request.call_args_list)) + + def test_main_rejects_non_list_or_empty_job_names(self): + for names in ("job", [], ["job", ""], ["job", None]): + env = {"RELEASE_JOB_NAMES": json.dumps(names), "RELEASE_TIMEOUT_SECONDS": "60"} + with self.subTest(names=names), patch.dict(os.environ, env, clear=True), \ + self.assertRaises(ValueError): + rollback.main() + + +class RollbackClientTests(unittest.TestCase): + def client(self): + client = rollback.Client.__new__(rollback.Client) + client.base = "https://kubernetes:443" + client.credentials = MagicMock() + (client.credentials / "token").read_text.return_value = "private-token\n" + client.opener = MagicMock() + return client + + def test_delete_uses_foreground_propagation_and_shared_request_budget(self): + client = self.client() + with patch.object(rollback.time, "monotonic", return_value=8): + self.assertTrue(client.request("DELETE", "/exact/path", 10)) + request = client.opener.open.call_args.args[0] + self.assertEqual(request.full_url, "https://kubernetes:443/exact/path") + self.assertEqual(json.loads(request.data), {"propagationPolicy": "Foreground"}) + self.assertEqual(request.get_header("Authorization"), "Bearer private-token") + self.assertEqual(client.opener.open.call_args.kwargs["timeout"], 2) + + def test_only_not_found_is_absence_and_error_bodies_are_not_disclosed(self): + for status in (404, 403, 500): + with self.subTest(status=status): + client = self.client() + body = Mock() + client.opener.open.side_effect = urllib.error.HTTPError( + "https://kubernetes", status, "private response", {}, body) + with patch.object(rollback.time, "monotonic", return_value=0): + if status == 404: + self.assertFalse(client.request("GET", "/exact/path", 10)) + else: + with self.assertRaises(RuntimeError) as raised: + client.request("GET", "/exact/path", 10) + self.assertNotIn("private", str(raised.exception)) + body.close.assert_called_once_with() + + def test_redirects_never_forward_credentials(self): + handler = rollback.NoRedirect() + with self.assertRaises(urllib.error.HTTPError): + handler.redirect_request(Mock(full_url="https://kubernetes"), Mock(), + 302, "redirect", {}, "https://elsewhere") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migration_gate.py b/tests/test_migration_gate.py deleted file mode 100644 index 2766de5..0000000 --- a/tests/test_migration_gate.py +++ /dev/null @@ -1,129 +0,0 @@ -import importlib.util -from pathlib import Path -import sys -import unittest -from unittest.mock import Mock -import urllib.error - -FILES = Path(__file__).resolve().parents[1] / "braintrust/files" -# The chart mounts both files together under these import names. -for name, file in [("brainstore_gate", "brainstore-startup-gate.py"), ("migration_gate", "migration-gate.py"), ("migration_rollback", "migration-rollback.py")]: - spec = importlib.util.spec_from_file_location(name, FILES / file) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) -gate = sys.modules["migration_gate"] - - -def job(*conditions, **metadata): - return {"metadata": metadata, "status": {"conditions": [ - {"type": c, "status": "True"} for c in conditions - ]}} - - -class JobGateTests(unittest.TestCase): - def wait(self, observations, timeout=20): - now = [0] - def sleep(seconds): - now[0] += seconds - client = Mock(namespace="test-namespace") - client.get.side_effect = observations - gate.wait_for_job(client, "release-migrate-r2", timeout, 5, - clock=lambda: now[0], sleep=sleep, log=Mock()) - return client, now[0] - - def test_only_complete_condition_counts_as_success(self): - for resource in ({}, job(), job("SuccessCriteriaMet"), - {"status": {"succeeded": 1}}, - {"status": {"conditions": [{"type": "Complete", "status": "False"}]}}, - job("Complete", deletionTimestamp="now")): - with self.subTest(resource=resource): - self.assertFalse(gate.job_complete(resource)) - self.assertTrue(gate.job_complete(job("Complete"))) - - def test_failed_job_is_terminal_even_with_complete_condition(self): - for condition in ("Failed", "FailureTarget"): - with self.assertRaisesRegex(RuntimeError, "migration Job failed"): - gate.job_complete(job(condition, "Complete")) - - def test_waits_for_exact_revision_and_retries_missing_job_and_rbac(self): - observations = [urllib.error.HTTPError("url", code, "error", {}, None) - for code in (404, 403, 500)] - client, elapsed = self.wait(observations + [job("Complete")]) - self.assertEqual(elapsed, 15) - self.assertEqual(client.get.call_count, 4) - for call in client.get.call_args_list: - self.assertEqual(call.args, ( - "/apis/batch/v1/namespaces/test-namespace/jobs/release-migrate-r2", 20)) - - def test_running_job_then_success(self): - _, elapsed = self.wait([job(), job("Complete")]) - self.assertEqual(elapsed, 5) - - def test_timeout_never_allows_app_to_start(self): - with self.assertRaisesRegex(TimeoutError, "timed out"): - self.wait([job()] * 4) - - def test_failed_job_never_allows_app_to_start(self): - with self.assertRaisesRegex(RuntimeError, "migration Job failed"): - self.wait([job("Failed")]) - - def test_transient_observation_errors_retry(self): - self.wait([OSError("network"), ValueError("bad JSON"), job("Complete")]) - - def test_late_success_does_not_escape_deadline(self): - now = [0] - client = Mock(namespace="ns") - def late(*args): - now[0] = 21 - return job("Complete") - client.get.side_effect = late - with self.assertRaises(TimeoutError): - gate.wait_for_job(client, "job", 20, 5, clock=lambda: now[0], sleep=Mock(), log=Mock()) - - -class RollbackCleanupTests(unittest.TestCase): - def test_cleanup_only_target_revision_and_waits_for_job_deletion(self): - rollback = sys.modules["migration_rollback"] - paths = rollback.resource_paths("ns", "target-r7") - client = Mock() - # DELETE succeeds; first GET still sees foreground deletion; next is gone. - client.request.side_effect = [True, True, False] + [True, False] * 4 - now = [0] - def sleep(seconds): - now[0] += seconds - rollback.cleanup(client, "ns", "target-r7", 10, clock=lambda: now[0], sleep=sleep) - calls = client.request.call_args_list - self.assertEqual([c.args[1] for c in calls if c.args[0] == "DELETE"], paths) - self.assertEqual(calls[0].args, ("DELETE", paths[0], 10)) - self.assertEqual(calls[1].args, ("GET", paths[0], 10)) - self.assertEqual(calls[2].args, ("GET", paths[0], 10)) - self.assertEqual(now[0], 2) - - def test_cleanup_of_absent_resources_is_idempotent(self): - rollback = sys.modules["migration_rollback"] - client = Mock() - client.request.return_value = False - rollback.cleanup(client, "ns", "target-r7", 10, clock=lambda: 0, sleep=Mock()) - self.assertEqual(client.request.call_count, 10) - - def test_cleanup_fails_closed_on_stuck_deletion(self): - rollback = sys.modules["migration_rollback"] - client = Mock() - client.request.return_value = True - now = [0] - def sleep(seconds): - now[0] += seconds - with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", "target-r7", 3, clock=lambda: now[0], sleep=sleep) - self.assertEqual(now[0], 3) - - def test_cleanup_permission_errors_stop_rollback(self): - client = Mock() - client.request.side_effect = RuntimeError("forbidden") - with self.assertRaises(RuntimeError): - sys.modules["migration_rollback"].cleanup(client, "ns", "target-r7", 10) - - -if __name__ == "__main__": - unittest.main() From 7dd2269cc49472d663d4888afce787f84444dae5 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 11:43:44 -0500 Subject: [PATCH 03/13] Simplify API startup to direct Brainstore rollout checks --- braintrust/README.md | 37 +- braintrust/files/brainstore-rollout.py | 138 ----- braintrust/files/brainstore-startup-gate.py | 45 +- braintrust/files/migration-rollback.py | 52 +- braintrust/templates/_api-deployment.tpl | 7 +- .../templates/_brainstore-startup-gate.tpl | 24 +- braintrust/templates/_migrations.tpl | 91 ++++ braintrust/templates/_release-gates.tpl | 156 ------ .../templates/ai-gateway-deployment.yaml | 4 +- ...rainstore-automationwriter-deployment.yaml | 6 +- .../brainstore-fastreader-deployment.yaml | 6 +- .../brainstore-reader-deployment.yaml | 6 +- braintrust/templates/brainstore-rollout.yaml | 135 ----- .../brainstore-startup-gate-rbac.yaml | 6 +- .../brainstore-writer-deployment.yaml | 6 +- .../templates/loop-runtime-deployment.yaml | 4 +- braintrust/templates/migration-gate.yaml | 10 - braintrust/templates/migration-rollback.yaml | 21 +- ...instore-automationwriter-routing_test.yaml | 52 +- .../tests/brainstore-rollout-labels_test.yaml | 38 -- braintrust/tests/brainstore-rollout_test.yaml | 160 ------ .../tests/brainstore-startup-gate_test.yaml | 2 +- braintrust/tests/migrations_test.yaml | 54 +- braintrust/values.yaml | 4 +- docs/migration-job.md | 85 ++-- tests/test_brainstore_rollout.py | 474 ------------------ tests/test_brainstore_startup_gate.py | 195 ++++++- tests/test_job_gate.py | 222 ++++---- 28 files changed, 602 insertions(+), 1438 deletions(-) delete mode 100644 braintrust/files/brainstore-rollout.py create mode 100644 braintrust/templates/_migrations.tpl delete mode 100644 braintrust/templates/_release-gates.tpl delete mode 100644 braintrust/templates/brainstore-rollout.yaml delete mode 100644 braintrust/tests/brainstore-rollout-labels_test.yaml delete mode 100644 braintrust/tests/brainstore-rollout_test.yaml delete mode 100644 tests/test_brainstore_rollout.py diff --git a/braintrust/README.md b/braintrust/README.md index 819241f..b3cddb5 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -469,8 +469,8 @@ The API receives `BRAINSTORE_AUTOMATION_WRITER_URL` whenever the pool has replicas. A compatible API release decides which queries use this endpoint. With no replicas, the URL is omitted and the API falls back to the regular writers. -The live version-compatibility check covers readers, writers, and fast readers. -The release rollout barrier also waits for every rendered automation writer. +The API startup gate covers every rendered reader and writer role, including +automation writers and pools that are scaling down to zero. The include/exclude split follows `brainstore.automationwriter.replicas` in the Helm values. `kubectl scale` or an autoscaler changes the live pod count only; @@ -552,21 +552,18 @@ Example values files for different cloud providers and configurations are locate ## Release ordering -By default, each Helm release runs migrations first, rolls out the exact -Brainstore image and configuration, then starts new API application containers. -Old Ready writers cannot satisfy the rollout barrier for an environment-only -change. All API pools share the same revision-specific rollout Job and then run -the existing live version-compatibility check. - -Migration and rollout Jobs share a read-only wait helper but remain separate to -avoid a dependency cycle. The migrator uses the shared `api.image`, identity, -configuration, secrets, and resources; the rollout observer only reads Kubernetes -Deployments and Pods. Completed phase records are retained for replacement pods -and reset together during rollback. - -`migrations.enabled: false` retains the previous API-startup migration behavior -and compatibility-only gate. Disabling `api.brainstoreStartupGate.enabled` leaves -migration-only ordering. Gates control application startup, not when Helm updates -Deployment specs or mutable configuration objects. -See [release lifecycle and configuration matrix](../docs/migration-job.md) for -retention, rollback, timeouts, configuration limits, and capacity requirements. +Each Helm release runs one migration Job. All new application containers wait +for it. The existing API startup gate then directly checks that the reader/writer +Deployments have observed the intended release and fully rolled out: updated, +Ready, Available, with no old or terminating pods. Same-image environment changes +cannot pass using old Ready writers. There is no separate rollout Job or snapshot. + +An older API replacement also waits during a later unfinished Brainstore rollout, +but accepts a later completed compatible release. This deliberately favors simpler +ordering over restarting that API while a future rollout is blocked on capacity. + +`migrations.enabled: false` keeps compatibility-only/API-startup migration behavior; +disabling `api.brainstoreStartupGate.enabled` leaves migration-only ordering. +These gates order container startup, not Helm's updates to configuration objects. +See [migration lifecycle and input matrix](../docs/migration-job.md) for rollback, +retention, timeouts, and configuration limits. diff --git a/braintrust/files/brainstore-rollout.py b/braintrust/files/brainstore-rollout.py deleted file mode 100644 index f23a3ff..0000000 --- a/braintrust/files/brainstore-rollout.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Observe the exact Brainstore configuration for one Helm release. - -ROLLOUT_TARGETS is a JSON array of deployment/container/revision/image/configChecksum -objects. POD_NAMESPACE selects the namespace; ROLLOUT_TIMEOUT_SECONDS and -ROLLOUT_POLL_SECONDS bound the entire fleet observation. Kubernetes credentials -are projected at /var/run/migration-gate; brainstore_gate.py supplies shared helpers. -""" - -import json -import os -import sys -import time -import urllib.error -import urllib.parse - -from brainstore_gate import KubernetesClient, TERMINAL_PHASES, container_image, ready - - -RELEASE_ANNOTATION = "braintrust.dev/release-revision" -CONFIG_ANNOTATION = "checksum/config" - - -def configuration_mismatch(resource, target): - annotations = resource.get("metadata", {}).get("annotations", {}) - if annotations.get(RELEASE_ANNOTATION) != target["revision"]: - return "release revision" - if annotations.get(CONFIG_ANNOTATION) != target["configChecksum"]: - return "config checksum" - if container_image(resource.get("spec", {}), target["container"]) != target["image"]: - return "image" - return None - - -def check_deployment(deployment, pods, target): - """Return a wait reason until the desired spec and every backend have rolled.""" - metadata = deployment.get("metadata", {}) - spec = deployment.get("spec", {}) - status = deployment.get("status", {}) - name = target["deployment"] - if metadata.get("deletionTimestamp"): - return f"{name}: deployment is being deleted" - mismatch = configuration_mismatch(spec.get("template", {}), target) - if mismatch: - return f"{name}: deployment template {mismatch} does not match the expected release" - if status.get("observedGeneration", 0) < metadata.get("generation", 1): - return f"{name}: controller has not observed the desired generation" - - # Scaling is independent of Helm: use the current spec, preserving explicit zero. - desired = spec.get("replicas", 1) - live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] - for pod in live: - pod_name = pod.get("metadata", {}).get("name", "unknown") - if pod.get("metadata", {}).get("deletionTimestamp"): - return f"{name}: terminating Pod {pod_name} still exists" - mismatch = configuration_mismatch(pod, target) - if mismatch: - return f"{name}: Pod {pod_name} {mismatch} does not match the expected release" - if len(live) != desired: - return f"{name}: waiting for exactly {desired} active service backends ({len(live)} remain)" - ready_count = sum(ready(pod, target["container"]) for pod in live) - if ready_count != desired: - return f"{name}: waiting for {desired} ready updated Pods ({ready_count} ready)" - - # Availability includes minReadySeconds; total replicas excludes old surges. - # Kubernetes omits zero-valued counters, so a fully scaled-down role can pass. - for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): - actual = status.get(counter, 0) - if actual != desired: - return f"{name}: rollout incomplete ({counter}={actual}, desired={desired})" - return None - - -def check_fleet(client, targets, deadline): - for target in targets: - deployment = client.deployment(target["deployment"], deadline) - # Services route by app, not owner reference. Include stray and terminating - # Pods, including those from a previous ReplicaSet or release revision. - pods = client.pods("app=" + target["deployment"], deadline) - reason = check_deployment(deployment, pods, target) - if reason: - return reason - return None - - -def wait_for_fleet(client, targets, timeout, interval, - clock=time.monotonic, sleep=time.sleep, log=print): - deadline = clock() + timeout - successes = 0 - last_reason = "no observation yet" - while clock() < deadline: - try: - reason = check_fleet(client, targets, deadline) - except urllib.error.HTTPError as error: - # The deployment or read-only RBAC may not have been applied yet. - # Never include API response bodies, credentials, or query strings. - path = urllib.parse.urlsplit(error.url).path[:256] - reason = f"Kubernetes API returned HTTP {error.code} for {path}" - error.close() - except (OSError, ValueError, KeyError, TypeError) as error: - reason = f"Kubernetes observation failed: {type(error).__name__}" - if clock() >= deadline: - last_reason = reason or "deadline expired before rollout confirmation" - break - if reason is None: - successes += 1 - if successes == 2: - log("Brainstore rollout complete: exact release configuration is ready", flush=True) - return - last_reason = "confirming exact release configuration (1/2)" - else: - successes = 0 - last_reason = reason - log(f"Brainstore rollout waiting: {last_reason}", flush=True) - sleep(min(interval, max(0, deadline - clock()))) - raise TimeoutError(f"Brainstore rollout timed out after {timeout}s: {last_reason}") - - -def main(): - targets = json.loads(os.environ["ROLLOUT_TARGETS"]) - timeout = int(os.environ["ROLLOUT_TIMEOUT_SECONDS"]) - interval = int(os.environ["ROLLOUT_POLL_SECONDS"]) - fields = ("deployment", "container", "revision", "image", "configChecksum") - if (not isinstance(targets, list) or not targets - or any(not isinstance(target, dict) - or any(not isinstance(target.get(field), str) or not target[field] - for field in fields) for target in targets) - or timeout <= 0 or interval <= 0 or interval >= timeout): - raise ValueError("invalid Brainstore rollout configuration") - client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/migration-gate") - wait_for_fleet(client, targets, timeout, interval) - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, KeyError, TypeError) as error: - print(str(error), file=sys.stderr, flush=True) - sys.exit(1) diff --git a/braintrust/files/brainstore-startup-gate.py b/braintrust/files/brainstore-startup-gate.py index cf4ff72..483d95b 100644 --- a/braintrust/files/brainstore-startup-gate.py +++ b/braintrust/files/brainstore-startup-gate.py @@ -1,4 +1,4 @@ -"""Wait for compatible Brainstore capacity before starting an API container. +"""Wait for compatible Brainstore capacity and required rollouts before API startup. Uses only Python's standard library and read-only Kubernetes API requests. The required version is fixed in the API Pod's environment, not a ConfigMap that @@ -76,13 +76,14 @@ def ready(pod, container): ) -def check_deployment(deployment, pods, container, minimum): - """Return a wait reason, or None when every potential backend is compatible. +def check_deployment(deployment, pods, container, minimum, minimum_revision=0): + """Return a wait reason, or None when compatible backends satisfy the gate. - We deliberately do not require an exact image or a finished *compatible* - rollout: older APIs must be able to restart while a later Brainstore release - rolls. Incompatible terminating Pods still block, because their connections - may remain usable until the process exits. + A release revision requires the current Deployment rollout to finish, even + for an older API restarting during a later compatible release. Without a + revision floor, compatible capacity suffices during an ongoing rollout. + Incompatible terminating Pods always block because connections may remain + usable until their process exits. Deployment rollouts replace Pods, so use each Pod's declared release tag. Container status can report a different alias or only a digest. Neither is @@ -99,12 +100,22 @@ def check_deployment(deployment, pods, container, minimum): if status.get("observedGeneration", 0) < metadata.get("generation", 1): return f"{name}: controller has not observed the desired generation" desired = spec.get("replicas", 1) + template = spec.get("template", {}) + revision_key = "braintrust.dev/release-revision" + revision = template.get("metadata", {}).get("annotations", {}).get(revision_key) + if minimum_revision > 0: + try: + current_revision = int(revision) + except (TypeError, ValueError): + return f"{name}: desired release revision is missing or invalid" + if current_revision < minimum_revision: + return f"{name}: waiting for release revision {minimum_revision} or later" live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] - if desired == 0: + if desired == 0 and minimum_revision == 0: return f"{name}: waiting for scaled-down Pods to disappear" if live else None # Check desired state too: compatible Pods are insufficient if the # Deployment is about to replace them with an incompatible image. - image = container_image(spec.get("template", {}).get("spec", {}), container) + image = container_image(template.get("spec", {}), container) if not compatible(image, minimum): return f"{name}: desired image {image!r} is below or outside the required release" for pod in live: @@ -112,9 +123,21 @@ def check_deployment(deployment, pods, container, minimum): if not compatible(image, minimum): pod_name = pod.get("metadata", {}).get("name", "unknown") return f"{name}: incompatible Pod {pod_name} still exists ({image!r})" + if minimum_revision > 0: + pod_metadata = pod.get("metadata", {}) + if pod_metadata.get("annotations", {}).get(revision_key) != revision: + return f"{name}: waiting for Pods from the current release revision" + if pod_metadata.get("deletionTimestamp"): + return f"{name}: waiting for terminating Pods to disappear" # Require both current Pod readiness and Deployment availability; the # latter also accounts for the Deployment's minReadySeconds setting. ready_count = sum(ready(pod, container) for pod in live) + if minimum_revision > 0: + counters = ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas") + if any(status.get(counter, 0) != desired for counter in counters): + return f"{name}: waiting for all {desired} replicas to complete the current rollout" + if len(live) != desired or ready_count != desired: + return f"{name}: waiting for exactly {desired} current Ready Pods" if ready_count < desired or status.get("availableReplicas", 0) < desired: return f"{name}: waiting for {desired} available compatible Pods ({ready_count} ready)" return None @@ -181,7 +204,9 @@ def check_fleet(client, targets, minimum, deadline): # The chart's Services select app=. Include stray Pods # with that label too; they can receive traffic regardless of ownership. pods = client.pods("app=" + target["deployment"], deadline) - reason = check_deployment(deployment, pods, target["container"], minimum) + reason = check_deployment( + deployment, pods, target["container"], minimum, target.get("minimumRevision", 0), + ) if reason: return reason return None diff --git a/braintrust/files/migration-rollback.py b/braintrust/files/migration-rollback.py index d9c20cd..6c55a78 100644 --- a/braintrust/files/migration-rollback.py +++ b/braintrust/files/migration-rollback.py @@ -2,7 +2,7 @@ Helm 3 cannot update a retained object absent from the current release manifest. Delete only the named target revision; foreground Job deletion waits for all of -its migration and rollout pods before rollback can recreate either Job. +its migration pods before rollback can recreate the Job. """ import json import os @@ -53,25 +53,21 @@ def request(self, method, path, deadline): raise RuntimeError(f"Kubernetes cleanup {method} returned HTTP {error.code}") from None -def resource_paths(namespace, names): +def resource_paths(namespace, name): namespace = urllib.parse.quote(namespace, safe="") - names = [urllib.parse.quote(name, safe="") for name in names] - jobs = [f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}" for name in names] - retained = [] - for name in names: - retained.extend([ - f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", - f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", - ]) - return jobs + retained - - -def cleanup(client, namespace, names, timeout, clock=time.monotonic, sleep=time.sleep): + name = urllib.parse.quote(name, safe="") + return [ + f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}", + f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", + f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", + f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", + ] + + +def cleanup(client, namespace, name, timeout, clock=time.monotonic, sleep=time.sleep): deadline = clock() + timeout - paths = resource_paths(namespace, names) - jobs, retained = paths[:len(names)], paths[len(names):] + paths = resource_paths(namespace, name) def check_deadline(): if clock() >= deadline: @@ -87,25 +83,19 @@ def wait_until_deleted(path): while request("GET", path): sleep(min(2, max(0, deadline - clock()))) - # Stop both phases before removing any config or permissions they rely on. - for path in jobs: - request("DELETE", path) - for path in jobs: - wait_until_deleted(path) - for path in retained: + # Stop the migration before removing any config or permissions it relies on. + request("DELETE", paths[0]) + wait_until_deleted(paths[0]) + for path in paths[1:]: request("DELETE", path) wait_until_deleted(path) def main(): - names = json.loads(os.environ["RELEASE_JOB_NAMES"]) - if not isinstance(names, list) or not names or any( - not isinstance(name, str) or not name for name in names): - raise ValueError("RELEASE_JOB_NAMES must be a nonempty list of Job names") - timeout = int(os.environ["RELEASE_TIMEOUT_SECONDS"]) + timeout = int(os.environ["JOB_TIMEOUT_SECONDS"]) if timeout <= 0: - raise ValueError("RELEASE_TIMEOUT_SECONDS must be positive") - cleanup(Client(), os.environ["POD_NAMESPACE"], names, timeout) + raise ValueError("JOB_TIMEOUT_SECONDS must be positive") + cleanup(Client(), os.environ["POD_NAMESPACE"], os.environ["JOB_NAME"], timeout) if __name__ == "__main__": diff --git a/braintrust/templates/_api-deployment.tpl b/braintrust/templates/_api-deployment.tpl index b5e3bdd..8ef4a4d 100644 --- a/braintrust/templates/_api-deployment.tpl +++ b/braintrust/templates/_api-deployment.tpl @@ -9,7 +9,6 @@ {{- $_ := set $startupGate "enabled" false -}} {{- $_ := set $migration "enabled" false -}} {{- end -}} -{{- $strictRollout := and $migration.enabled $startupGate.enabled -}} {{- if $startupGate.enabled -}} {{- include "braintrust.brainstoreStartupGate.validateApi" (dict "api" $api) -}} {{- end -}} @@ -51,7 +50,7 @@ metadata: {{- end }} spec: {{- if or $startupGate.enabled $migration.enabled }} - progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) (ternary (int $startupGate.timeoutSeconds) 0 $strictRollout) }} + progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} {{- end }} {{- if not (dig "autoscaling" "enabled" false $api) }} replicas: {{ $api.replicas }} @@ -116,7 +115,7 @@ spec: {{- if or $startupGate.enabled $migration.enabled }} initContainers: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.initContainer" (dict "root" $root "phase" (ternary "rollout" "migration" $strictRollout)) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" $root | nindent 8 }} {{- end }} {{- if $startupGate.enabled }} - name: wait-for-brainstore @@ -272,7 +271,7 @@ spec: volumes: {{- if or $migration.enabled $startupGate.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" $root | nindent 8 }} + {{- include "braintrust.migrations.volumes" $root | nindent 8 }} {{- end }} {{- if $startupGate.enabled }} - name: brainstore-gate-code diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index 5f2543c..4fc37ce 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -40,9 +40,15 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- define "braintrust.brainstoreStartupGate.targets" -}} {{- $targets := list -}} +{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $revision := ternary (int .Release.Revision) 0 $migration.enabled -}} {{- range $role := list "reader" "writer" "fastreader" -}} {{- $config := index $.Values.brainstore $role -}} -{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role)) -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role) "minimumRevision" $revision) -}} +{{- end -}} +{{- if .Values.brainstore.automationwriter -}} +{{- $config := include "braintrust.automationWriter.config" . | fromYaml -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" "brainstore-automationwriter" "minimumRevision" $revision) -}} {{- end -}} {{- toJson $targets -}} {{- end -}} @@ -56,3 +62,19 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- fail "api.brainstoreStartupGate requires nonzero maxSurge for every API pool" -}} {{- end -}} {{- end -}} + +{{/* Release identity prevents a previous fully Ready generation from passing +before Helm updates the desired image or configuration. */}} +{{- define "braintrust.brainstoreStartupGate.podAnnotations" -}} +{{- $root := .root -}} +{{- $annotations := dict "checksum/config" (include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum) -}} +{{- if eq $root.Values.cloud "google" -}} +{{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} +{{- end -}} +{{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} +{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} +{{- if $migration.enabled -}} +{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} +{{- end -}} +{{- toYaml $annotations -}} +{{- end -}} diff --git a/braintrust/templates/_migrations.tpl b/braintrust/templates/_migrations.tpl new file mode 100644 index 0000000..0e1e7a8 --- /dev/null +++ b/braintrust/templates/_migrations.tpl @@ -0,0 +1,91 @@ +{{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} +{{- define "braintrust.migrations.config" -}} +{{- $defaults := dict + "enabled" true + "timeoutSeconds" 1200 + "pollIntervalSeconds" 5 + "backoffLimit" 2 + "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" + "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) +-}} +{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} +{{- if $config.enabled -}} +{{- $_ := required "migrations.image is required" $config.image -}} +{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} +{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} +{{- fail "migrations timing values must be positive integers" -}} +{{- end -}} +{{- end -}} +{{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} +{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} +{{- end -}} +{{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} +{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} +{{- end -}} +{{- end -}} +{{- toYaml $config -}} +{{- end -}} + +{{/* Preserve the revision suffix even for long release names. */}} +{{- define "braintrust.migrations.name" -}} +{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} +{{- end -}} + +{{- define "braintrust.migrations.gatewayAccount" -}} +{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} +{{- end -}} + +{{- define "braintrust.migrations.initContainer" -}} +{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +- name: wait-for-migrations + image: {{ $config.image | quote }} + imagePullPolicy: IfNotPresent + command: ["python3", "-B", "-u", "/opt/migration-gate/gate.py"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $config.resources | nindent 4 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JOB_NAME + value: {{ include "braintrust.migrations.name" . | quote }} + - name: JOB_TIMEOUT_SECONDS + value: {{ $config.timeoutSeconds | quote }} + - name: JOB_POLL_SECONDS + value: {{ $config.pollIntervalSeconds | quote }} + volumeMounts: + - name: migration-gate-code + mountPath: /opt/migration-gate + readOnly: true + - name: migration-gate-credentials + mountPath: /var/run/migration-gate + readOnly: true +{{- end -}} + +{{- define "braintrust.migrations.volumes" -}} +- name: migration-gate-code + configMap: + name: {{ include "braintrust.migrations.name" . }}-gate +- name: migration-gate-credentials + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt +{{- end -}} diff --git a/braintrust/templates/_release-gates.tpl b/braintrust/templates/_release-gates.tpl deleted file mode 100644 index 9df2ee7..0000000 --- a/braintrust/templates/_release-gates.tpl +++ /dev/null @@ -1,156 +0,0 @@ -{{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} -{{- define "braintrust.migrations.config" -}} -{{- $defaults := dict - "enabled" true - "timeoutSeconds" 1200 - "pollIntervalSeconds" 5 - "backoffLimit" 2 - "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" - "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) --}} -{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} -{{- if $config.enabled -}} -{{- $_ := required "migrations.image is required" $config.image -}} -{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} -{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} -{{- fail "migrations timing values must be positive integers" -}} -{{- end -}} -{{- end -}} -{{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} -{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} -{{- end -}} -{{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} -{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} -{{- end -}} -{{- end -}} -{{- toYaml $config -}} -{{- end -}} - -{{/* Preserve the revision suffix even for long release names. */}} -{{- define "braintrust.migrations.name" -}} -{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} -{{- end -}} - -{{- define "braintrust.migrations.gatewayAccount" -}} -{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} -{{- end -}} - -{{/* Two completed Jobs preserve release ordering without pinning old API -replacements to a Brainstore rollout that has since been superseded. */}} -{{- define "braintrust.rollout.enabled" -}} -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} -{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} -{{- if and $migration.enabled $gate.enabled -}}true{{- end -}} -{{- end -}} - -{{- define "braintrust.rollout.name" -}} -{{- printf "%s-%s-rollout-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} -{{- end -}} - -{{- define "braintrust.rollout.serviceAccount" -}} -{{- printf "%s-%s-rollout" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) -}} -{{- end -}} - -{{/* Reserve rollout identity/checksum only while the exact release gate is on. -Merge annotations before rendering so user input cannot create duplicate keys. */}} -{{- define "braintrust.rollout.podAnnotations" -}} -{{- $root := .root -}} -{{- $checksum := include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum -}} -{{- $annotations := dict "checksum/config" $checksum -}} -{{- if eq $root.Values.cloud "google" -}} -{{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} -{{- end -}} -{{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} -{{- if include "braintrust.rollout.enabled" $root -}} -{{- $_ := set $annotations "checksum/config" $checksum -}} -{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} -{{- end -}} -{{- toYaml $annotations -}} -{{- end -}} - - -{{/* The revision binds image, inline env and all other Pod template changes; -the existing checksum binds ConfigMap-backed environment without copying secrets. */}} -{{- define "braintrust.rollout.targets" -}} -{{- $targets := list -}} -{{- $image := printf "%s:%s" .Values.brainstore.image.repository .Values.brainstore.image.tag -}} -{{- range $role := list "reader" "writer" "fastreader" -}} -{{- $config := index $.Values.brainstore $role -}} -{{- $checksum := include (printf "%s/brainstore-%s-configmap.yaml" $.Template.BasePath $role) $ | sha256sum -}} -{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role) "revision" (toString $.Release.Revision) "image" $image "configChecksum" $checksum) -}} -{{- end -}} -{{- if .Values.brainstore.automationwriter -}} -{{- $config := include "braintrust.automationWriter.config" . | fromYaml -}} -{{- $checksum := include (print .Template.BasePath "/brainstore-automationwriter-configmap.yaml") . | sha256sum -}} -{{- $targets = append $targets (dict "deployment" $config.name "container" "brainstore-automationwriter" "revision" (toString .Release.Revision) "image" $image "configChecksum" $checksum) -}} -{{- end -}} -{{- toJson $targets -}} -{{- end -}} - -{{/* One read-only Job gate serves both release phases. The rollout gate includes -the migration budget because its Job first waits for migrations itself. */}} -{{- define "braintrust.releaseJob.initContainer" -}} -{{- $root := .root -}} -{{- $config := include "braintrust.migrations.config" $root | fromYaml -}} -{{- $job := include "braintrust.migrations.name" $root -}} -{{- $name := "wait-for-migrations" -}} -{{- $timeout := int $config.timeoutSeconds -}} -{{- if eq (.phase | default "migration") "rollout" -}} -{{- $gate := include "braintrust.brainstoreStartupGate.config" $root | fromYaml -}} -{{- $timeout = int (add $timeout (int $gate.timeoutSeconds)) -}} -{{- $config = $gate -}} -{{- $job = include "braintrust.rollout.name" $root -}} -{{- $name = "wait-for-brainstore-rollout" -}} -{{- end -}} -- name: {{ $name }} - image: {{ $config.image | quote }} - imagePullPolicy: IfNotPresent - command: ["python3", "-B", "-u", "/opt/migration-gate/gate.py"] - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - resources: - {{- toYaml $config.resources | nindent 4 }} - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: JOB_NAME - value: {{ $job | quote }} - - name: JOB_TIMEOUT_SECONDS - value: {{ $timeout | quote }} - - name: JOB_POLL_SECONDS - value: {{ $config.pollIntervalSeconds | quote }} - volumeMounts: - - name: migration-gate-code - mountPath: /opt/migration-gate - readOnly: true - - name: migration-gate-credentials - mountPath: /var/run/migration-gate - readOnly: true -{{- end -}} - -{{- define "braintrust.releaseJob.volumes" -}} -- name: migration-gate-code - configMap: - name: {{ include "braintrust.migrations.name" . }}-gate -- name: migration-gate-credentials - projected: - sources: - - serviceAccountToken: - path: token - expirationSeconds: 3600 - - configMap: - name: kube-root-ca.crt - items: - - key: ca.crt - path: ca.crt -{{- end -}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index d07d279..7383a21 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -58,7 +58,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: ai-gateway @@ -103,7 +103,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} {{- if .Values.aiGateway.extraVolumes }} {{- toYaml .Values.aiGateway.extraVolumes | nindent 8 }} diff --git a/braintrust/templates/brainstore-automationwriter-deployment.yaml b/braintrust/templates/brainstore-automationwriter-deployment.yaml index 1cda872..15e69c9 100644 --- a/braintrust/templates/brainstore-automationwriter-deployment.yaml +++ b/braintrust/templates/brainstore-automationwriter-deployment.yaml @@ -48,7 +48,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "automationwriter" "config" $aw) | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "automationwriter" "config" $aw) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with $aw.podSecurityContext }} @@ -76,7 +76,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: brainstore-automationwriter @@ -176,7 +176,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- /* Azure local disk is required only once this pool has replicas. A disabled pool must render without a size. */}} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 699b8ef..bdf0949 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "fastreader" "config" .Values.brainstore.fastreader) | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "fastreader" "config" .Values.brainstore.fastreader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.fastreader.podSecurityContext }} @@ -69,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: brainstore-fastreader @@ -161,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index ca39064..7e54ef0 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "reader" "config" .Values.brainstore.reader) | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "reader" "config" .Values.brainstore.reader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.reader.podSecurityContext }} @@ -69,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: brainstore-reader @@ -161,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/brainstore-rollout.yaml b/braintrust/templates/brainstore-rollout.yaml deleted file mode 100644 index 5e9db08..0000000 --- a/braintrust/templates/brainstore-rollout.yaml +++ /dev/null @@ -1,135 +0,0 @@ -{{- if include "braintrust.rollout.enabled" . -}} -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} -{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} -{{- $name := include "braintrust.rollout.name" . -}} -{{- $account := include "braintrust.rollout.serviceAccount" . -}} -{{- $targets := include "braintrust.rollout.targets" . | fromJsonArray -}} -{{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (dict "braintrust.dev/migration-release" .Release.Name) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $name }} - namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep - labels: - {{- toYaml $labels | nindent 4 }} -spec: - backoffLimit: {{ $migration.backoffLimit }} - activeDeadlineSeconds: {{ add (int $migration.timeoutSeconds) (int $gate.timeoutSeconds) }} - template: - metadata: - labels: - {{- mergeOverwrite (deepCopy $labels) (dict "app" $name) | toYaml | nindent 8 }} - spec: - serviceAccountName: {{ $account }} - automountServiceAccountToken: false - restartPolicy: Never - {{- with .Values.api.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.api.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} - containers: - - name: wait-for-rollout - image: {{ $gate.image | quote }} - imagePullPolicy: IfNotPresent - command: ["python3", "-B", "-u", "/opt/migration-gate/brainstore_rollout.py"] - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - resources: - {{- toYaml $gate.resources | nindent 12 }} - envFrom: - - configMapRef: - name: {{ $name }}-config - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROLLOUT_TIMEOUT_SECONDS - value: {{ $gate.timeoutSeconds | quote }} - - name: ROLLOUT_POLL_SECONDS - value: {{ $gate.pollIntervalSeconds | quote }} - volumeMounts: - - name: migration-gate-code - mountPath: /opt/migration-gate - readOnly: true - - name: migration-gate-credentials - mountPath: /var/run/migration-gate - readOnly: true - volumes: - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ $name }}-config - namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep - labels: - {{- toYaml $labels | nindent 4 }} -data: - ROLLOUT_TARGETS: {{ toJson $targets | quote }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $account }} - namespace: {{ include "braintrust.namespace" . }} - labels: - {{- toYaml $labels | nindent 4 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $name }} - namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep - labels: - {{- toYaml $labels | nindent 4 }} -rules: - - apiGroups: ["apps"] - resources: ["deployments"] - resourceNames: - {{- range $target := $targets }} - - {{ $target.deployment | quote }} - {{- end }} - verbs: ["get"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $name }} - namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep - labels: - {{- toYaml $labels | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $name }} -subjects: - - kind: ServiceAccount - name: {{ $account }} - namespace: {{ include "braintrust.namespace" . }} -{{- end }} diff --git a/braintrust/templates/brainstore-startup-gate-rbac.yaml b/braintrust/templates/brainstore-startup-gate-rbac.yaml index b81e8a2..c44d3ff 100644 --- a/braintrust/templates/brainstore-startup-gate-rbac.yaml +++ b/braintrust/templates/brainstore-startup-gate-rbac.yaml @@ -13,9 +13,9 @@ rules: - apiGroups: ["apps"] resources: ["deployments"] resourceNames: - - {{ .Values.brainstore.reader.name | quote }} - - {{ .Values.brainstore.writer.name | quote }} - - {{ .Values.brainstore.fastreader.name | quote }} + {{- range $target := include "braintrust.brainstoreStartupGate.targets" . | fromJsonArray }} + - {{ $target.deployment | quote }} + {{- end }} verbs: ["get"] - apiGroups: [""] resources: ["pods"] diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index 95ec164..ea7c7f6 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.rollout.podAnnotations" (dict "root" . "role" "writer" "config" .Values.brainstore.writer) | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "writer" "config" .Values.brainstore.writer) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.writer.podSecurityContext }} @@ -69,7 +69,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: brainstore-writer @@ -161,7 +161,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume {{- if and (eq .Values.cloud "azure") .Values.azure.enableAzureContainerStorageDriver }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 9cc628e..f0bde32 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -67,7 +67,7 @@ spec: {{- end }} {{- if $migration.enabled }} initContainers: - {{- include "braintrust.releaseJob.initContainer" (dict "root" .) | nindent 8 }} + {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} containers: - name: loop-runtime @@ -157,7 +157,7 @@ spec: {{- end }} volumes: {{- if $migration.enabled }} - {{- include "braintrust.releaseJob.volumes" . | nindent 8 }} + {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: state emptyDir: diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml index fd51aaa..5ecb048 100644 --- a/braintrust/templates/migration-gate.yaml +++ b/braintrust/templates/migration-gate.yaml @@ -15,10 +15,6 @@ data: {{- .Files.Get "files/job-gate.py" | nindent 4 }} brainstore_gate.py: | {{- .Files.Get "files/brainstore-startup-gate.py" | nindent 4 }} - {{- if include "braintrust.rollout.enabled" . }} - brainstore_rollout.py: | - {{- .Files.Get "files/brainstore-rollout.py" | nindent 4 }} - {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -34,9 +30,6 @@ rules: resources: ["jobs"] resourceNames: - {{ $name | quote }} - {{- if include "braintrust.rollout.enabled" . }} - - {{ include "braintrust.rollout.name" . | quote }} - {{- end }} verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -54,9 +47,6 @@ roleRef: name: {{ $name }} subjects: {{- $accounts := dict .Values.brainstore.serviceAccount.name true -}} - {{- if include "braintrust.rollout.enabled" . -}} - {{- $_ := set $accounts (include "braintrust.rollout.serviceAccount" .) true -}} - {{- end -}} {{- range $pool := include "braintrust.apiPools" . | fromYamlArray -}} {{- $_ := set $accounts $pool.config.serviceAccount.name true -}} {{- end -}} diff --git a/braintrust/templates/migration-rollback.yaml b/braintrust/templates/migration-rollback.yaml index a067fbe..30b246c 100644 --- a/braintrust/templates/migration-rollback.yaml +++ b/braintrust/templates/migration-rollback.yaml @@ -1,15 +1,6 @@ {{- $config := include "braintrust.migrations.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} -{{- $jobs := list $name -}} -{{- if include "braintrust.rollout.enabled" . -}} -{{- $jobs = append $jobs (include "braintrust.rollout.name" .) -}} -{{- end -}} -{{- $configNames := list -}} -{{- range $job := $jobs -}} -{{- $configNames = append $configNames (printf "%s-config" $job) -}} -{{- $configNames = append $configNames (printf "%s-gate" $job) -}} -{{- end -}} {{- $hook := printf "%s-reset" $name -}} {{/* Hook manifests belong to the target release, so the name is the original revision's name, not the new rollback revision. No hook runs during install or @@ -29,15 +20,15 @@ metadata: rules: - apiGroups: ["batch"] resources: ["jobs"] - resourceNames: {{ toJson $jobs }} + resourceNames: [{{ $name | quote }}] verbs: ["get", "delete"] - apiGroups: [""] resources: ["configmaps"] - resourceNames: {{ toJson $configNames }} + resourceNames: [{{ printf "%s-config" $name | quote }}, {{ printf "%s-gate" $name | quote }}] verbs: ["get", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] - resourceNames: {{ toJson $jobs }} + resourceNames: [{{ $name | quote }}] verbs: ["get", "delete"] {{- else if eq $kind "RoleBinding" }} roleRef: @@ -102,8 +93,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: RELEASE_JOB_NAMES - value: {{ toJson $jobs | quote }} - - name: RELEASE_TIMEOUT_SECONDS + - name: JOB_NAME + value: {{ $name | quote }} + - name: JOB_TIMEOUT_SECONDS value: {{ $config.timeoutSeconds | quote }} {{- end }} diff --git a/braintrust/tests/brainstore-automationwriter-routing_test.yaml b/braintrust/tests/brainstore-automationwriter-routing_test.yaml index 7f53de4..76428e3 100644 --- a/braintrust/tests/brainstore-automationwriter-routing_test.yaml +++ b/braintrust/tests/brainstore-automationwriter-routing_test.yaml @@ -89,35 +89,6 @@ tests: path: data.BRAINSTORE_AUTOMATION_WRITER_URL value: http://custom-pool.created-namespace:4000 - - it: should preserve existing API startup targets with automation writers enabled - template: api-deployment.yaml - set: - <<: *enabled - brainstore.automationwriter.name: custom-pool - asserts: &startup-targets - - contains: - path: spec.template.spec.initContainers[0].env - content: - name: BRAINSTORE_DEPLOYMENTS - value: '[{"container":"brainstore-reader","deployment":"brainstore-reader"},{"container":"brainstore-writer","deployment":"brainstore-writer"},{"container":"brainstore-fastreader","deployment":"brainstore-fastreader"}]' - - - it: should preserve background API startup targets with automation writers enabled - template: api-deployment.yaml - documentIndex: 2 - set: - <<: *enabled - api.workloadIsolation.enabled: true - brainstore.automationwriter.name: custom-pool - asserts: *startup-targets - - - it: should preserve ingest API startup targets with automation writers enabled - template: api-deployment.yaml - documentIndex: 1 - set: - <<: *enabled - api.workloadIsolation.enabled: true - brainstore.automationwriter.name: custom-pool - asserts: *startup-targets - it: should preserve gate permissions with a custom automation deployment enabled template: brainstore-startup-gate-rbac.yaml @@ -128,7 +99,7 @@ tests: asserts: - equal: path: rules[0].resourceNames - value: [brainstore-reader, brainstore-writer, brainstore-fastreader] + value: [brainstore-reader, brainstore-writer, brainstore-fastreader, custom-pool] - it: should preserve gate permissions with a partial reused pool enabled template: brainstore-startup-gate-rbac.yaml @@ -138,25 +109,10 @@ tests: asserts: - equal: path: rules[0].resourceNames - value: [brainstore-reader, brainstore-writer, brainstore-fastreader] - - - it: should omit disabled automation writers from gate targets - template: api-deployment.yaml - asserts: *startup-targets + value: [brainstore-reader, brainstore-writer, brainstore-fastreader, brainstore-automationwriter] - - it: should omit a missing reused pool from gate targets - template: api-deployment.yaml - values: - - __fixtures__/pre-automationwriter-values.yaml - asserts: *startup-targets - - - it: should omit null replicas from gate targets - template: api-deployment.yaml - set: - brainstore.automationwriter.replicas: null - asserts: *startup-targets - - it: should omit empty replicas from gate permissions + - it: should track a rendered zero-replica pool until it finishes draining template: brainstore-startup-gate-rbac.yaml documentIndex: 0 set: @@ -164,7 +120,7 @@ tests: asserts: - equal: path: rules[0].resourceNames - value: [brainstore-reader, brainstore-writer, brainstore-fastreader] + value: [brainstore-reader, brainstore-writer, brainstore-fastreader, brainstore-automationwriter] - it: should still allow an explicitly disabled startup gate template: api-deployment.yaml diff --git a/braintrust/tests/brainstore-rollout-labels_test.yaml b/braintrust/tests/brainstore-rollout-labels_test.yaml deleted file mode 100644 index d3ac3db..0000000 --- a/braintrust/tests/brainstore-rollout-labels_test.yaml +++ /dev/null @@ -1,38 +0,0 @@ -suite: rollout observer label isolation -release: - name: test - namespace: workload - revision: 7 -templates: - - brainstore-rollout.yaml - - brainstore-reader-configmap.yaml - - brainstore-writer-configmap.yaml - - brainstore-fastreader-configmap.yaml - - brainstore-automationwriter-configmap.yaml -tests: - - it: prevents observer pods from becoming API backends through global labels - template: brainstore-rollout.yaml - documentIndex: 0 - set: - global.labels: - app: braintrust-api - shared: global - api.labels.api-only: api - brainstore.reader.labels.reader-only: reader - asserts: - - notEqual: - path: spec.template.metadata.labels.app - value: braintrust-api - - equal: - path: spec.template.metadata.labels.shared - value: global - - isNull: - path: spec.template.metadata.labels["api-only"] - - isNull: - path: spec.template.metadata.labels["reader-only"] - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - isNull: - path: spec.template.spec.containers[0].envFrom[0].secretRef - diff --git a/braintrust/tests/brainstore-rollout_test.yaml b/braintrust/tests/brainstore-rollout_test.yaml deleted file mode 100644 index bce2b20..0000000 --- a/braintrust/tests/brainstore-rollout_test.yaml +++ /dev/null @@ -1,160 +0,0 @@ -suite: exact release rollout boundaries -release: - name: test - namespace: workload - revision: 7 -templates: - - brainstore-rollout.yaml - - api-deployment.yaml - - api-configmap.yaml - - brainstore-reader-deployment.yaml - - brainstore-reader-configmap.yaml - - brainstore-writer-configmap.yaml - - brainstore-fastreader-configmap.yaml - - brainstore-automationwriter-configmap.yaml -tests: - - it: omits the observer when migrations are disabled - template: brainstore-rollout.yaml - set: - migrations.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: omits exact rollout ordering when the startup gate is disabled - template: brainstore-rollout.yaml - set: - api.brainstoreStartupGate.enabled: false - api.brainstoreStartupGate.timeoutSeconds: 0 - api.brainstoreStartupGate.image: "" - asserts: - - hasDocuments: - count: 0 - - - it: allows rollback to recreate the observer identity after a disabled revision - template: brainstore-rollout.yaml - documentIndex: 2 - asserts: - - isKind: - of: ServiceAccount - - isNull: - path: metadata.annotations["helm.sh/resource-policy"] - - - it: keeps compatibility-only startup when migrations are disabled - template: api-deployment.yaml - set: - migrations.enabled: false - asserts: - - lengthEqual: - path: spec.template.spec.initContainers - count: 1 - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore - - - it: restores full ordering for null feature maps on reused values - template: brainstore-rollout.yaml - set: - migrations: null - api.brainstoreStartupGate: null - asserts: - - hasDocuments: - count: 5 - - - it: budgets migration and rollout separately without premature API expiry - template: api-deployment.yaml - set: - migrations.timeoutSeconds: 900 - api.brainstoreStartupGate.timeoutSeconds: 300 - asserts: - - equal: - path: spec.template.spec.initContainers[0].env[2].value - value: "1200" - - equal: - path: spec.progressDeadlineSeconds - value: 1800 - - - it: gives the observer time for migration before its rollout observation - template: brainstore-rollout.yaml - documentIndex: 0 - set: - migrations.timeoutSeconds: 900 - api.brainstoreStartupGate.timeoutSeconds: 300 - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 1200 - - isNull: - path: spec.ttlSecondsAfterFinished - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - isNull: - path: metadata.annotations["helm.sh/hook"] - - - it: prevents custom annotations from spoofing the expected release or config - template: brainstore-reader-deployment.yaml - set: - brainstore.reader.annotations.pod: - braintrust.dev/release-revision: forged - checksum/config: forged - custom: preserved - asserts: - - equal: - path: spec.template.metadata.annotations["braintrust.dev/release-revision"] - value: "7" - - notEqual: - path: spec.template.metadata.annotations["checksum/config"] - value: forged - - equal: - path: spec.template.metadata.annotations.custom - value: preserved - - - it: does not force strict revision identity after opting out of migrations - template: brainstore-reader-deployment.yaml - set: - migrations.enabled: false - asserts: - - isNull: - path: spec.template.metadata.annotations["braintrust.dev/release-revision"] - - isNull: - path: spec.template.spec.initContainers - - - it: restricts observation to namespace pod listing and named deployments - template: brainstore-rollout.yaml - documentIndex: 3 - set: - global.createNamespace: true - global.namespace: custom-namespace - brainstore.reader.name: custom-reader - brainstore.writer.name: custom-writer - brainstore.fastreader.name: custom-fastreader - brainstore.automationwriter: null - asserts: - - equal: - path: metadata.namespace - value: custom-namespace - - lengthEqual: - path: rules - count: 2 - - equal: - path: rules[0].apiGroups - value: [apps] - - equal: - path: rules[0].resources - value: [deployments] - - equal: - path: rules[0].resourceNames - value: [custom-reader, custom-writer, custom-fastreader] - - equal: - path: rules[0].verbs - value: [get] - - equal: - path: rules[1].apiGroups - value: [""] - - equal: - path: rules[1].resources - value: [pods] - - equal: - path: rules[1].verbs - value: [list] diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index b28d086..8d406da 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -214,7 +214,7 @@ tests: value: - apiGroups: [apps] resources: [deployments] - resourceNames: [custom-reader, brainstore-writer, brainstore-fastreader] + resourceNames: [custom-reader, brainstore-writer, brainstore-fastreader, brainstore-automationwriter] verbs: [get] - apiGroups: [""] resources: [pods] diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index 0a2bbbb..5e5f4f1 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -89,23 +89,23 @@ tests: - isNull: path: data.PG_URL - - it: holds APIs behind the completed rollout before the compatibility check + - it: waits for migrations before directly checking the Brainstore rollout template: api-deployment.yaml asserts: - equal: path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore-rollout + value: wait-for-migrations - equal: path: spec.template.spec.initContainers[1].name value: wait-for-brainstore - matchRegex: path: spec.template.spec.initContainers[0].env[1].value - pattern: '-rollout-r7$' + pattern: '-migrate-r7$' - equal: path: spec.template.spec.automountServiceAccountToken value: false - - it: restricts workload gates to reading only the two exact release Jobs + - it: restricts workload gates to reading only the exact migration Job template: migration-gate.yaml documentIndex: 1 asserts: @@ -126,9 +126,9 @@ tests: - matchRegex: path: rules[0].resourceNames[0] pattern: '-migrate-r7$' - - matchRegex: - path: rules[0].resourceNames[1] - pattern: '-rollout-r7$' + - lengthEqual: + path: rules[0].resourceNames + count: 1 - equal: path: metadata.annotations["helm.sh/resource-policy"] value: keep @@ -308,7 +308,7 @@ tests: documentIndex: 0 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates isolated API pool 1 @@ -316,7 +316,7 @@ tests: documentIndex: 1 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates isolated API pool 2 @@ -324,7 +324,7 @@ tests: documentIndex: 2 set: {"api.workloadIsolation.enabled": true} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - it: gates the optional AI Gateway @@ -412,7 +412,7 @@ tests: template: api-deployment.yaml set: {"migrations": null} asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-brainstore-rollout"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - it: merges partial reused values with defaults template: migration-job.yaml @@ -563,7 +563,7 @@ tests: value: before-hook-creation,hook-succeeded - matchRegex: path: spec.template.spec.containers[0].env[1].value - pattern: '^\["test-9f86d081-migrate-r7","test-9f86d081-rollout-r7"\]$' + pattern: '^test-9f86d081-migrate-r7$' - matchRegex: path: spec.template.spec.serviceAccountName pattern: '-migrate-r7-reset$' @@ -589,13 +589,13 @@ tests: value: [get, delete] - equal: path: rules[0].resourceNames - value: [test-9f86d081-migrate-r7, test-9f86d081-rollout-r7] + value: [test-9f86d081-migrate-r7] - equal: path: rules[1].resourceNames - value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate, test-9f86d081-rollout-r7-config, test-9f86d081-rollout-r7-gate] + value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate] - equal: path: rules[2].resourceNames - value: [test-9f86d081-migrate-r7, test-9f86d081-rollout-r7] + value: [test-9f86d081-migrate-r7] - it: does not render rollback hooks when migrations are disabled template: migration-rollback.yaml @@ -604,3 +604,27 @@ tests: asserts: - hasDocuments: count: 0 + + - it: reserves the release marker without discarding custom pod annotations + template: brainstore-reader-deployment.yaml + set: + brainstore.reader.annotations.pod: + braintrust.dev/release-revision: forged + custom: preserved + asserts: + - equal: + path: spec.template.metadata.annotations["braintrust.dev/release-revision"] + value: "7" + - equal: + path: spec.template.metadata.annotations.custom + value: preserved + + - it: gives the API time for migration and one direct rollout check + template: api-deployment.yaml + set: + migrations.timeoutSeconds: 900 + api.brainstoreStartupGate.timeoutSeconds: 300 + asserts: + - equal: + path: spec.progressDeadlineSeconds + value: 1500 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 1c37c93..ed645e7 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -103,14 +103,14 @@ btqlAuditLogsBestEffortOrgIds: [] api: name: "braintrust-api" # Hold new API processes until compatible Brainstore capacity is available. - # With migrations enabled, first require this release's exact Brainstore rollout. + # With migrations enabled, first require fully updated reader/writer Deployments. # Requires stable release tags (vX.Y.Z); opaque custom tags cannot prove compatibility. # Only the init container mounts a Kubernetes API token when enabled; custom # sidecars needing Kubernetes API access must project their own token. brainstoreStartupGate: enabled: true # Empty derives the floor from each pool's api.image.tag. Kept in each API Pod so - # older APIs can restart against newer Brainstore releases in the same major. + # older APIs can start after a later compatible Brainstore rollout completes. minimumVersion: "" image: "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" timeoutSeconds: 1200 diff --git a/docs/migration-job.md b/docs/migration-job.md index 6e062bc..cda9c32 100644 --- a/docs/migration-job.md +++ b/docs/migration-job.md @@ -1,69 +1,50 @@ -# Release ordering: migrations, Brainstore, then API +# Release ordering -The default startup order is: +**Migrations complete → readers/writers finish rolling out → API starts.** -1. A revision-specific migration Job runs the shared API image's migration entry point. -2. New Brainstore application containers wait for that Job, then roll out. -3. A separate revision-specific rollout Job verifies the exact reader/writer configuration is fully deployed. -4. New API application containers wait for the rollout Job, then run the existing live version-compatibility check. +One regular migration Job runs the shared API image's migration entry point for each Helm revision. Every new application container waits for it. The existing API startup init container then checks Brainstore Deployments and Pods directly. There is no rollout Job, observer ServiceAccount, or rollout-configuration snapshot. -Two Jobs are necessary: if a single Job waited for Brainstore before reporting migration success, Brainstore would wait for that same Job forever. Both phases share one read-only Job-wait helper and revision-scoped code/RBAC. Gateway and Loop runtime retain their migration-only wait. +## What the API checks -## Input matrix and intended behavior +For every rendered reader, writer, fastreader, and automationwriter: -| Input | Behavior | -| --- | --- | -| Migrations and API Brainstore startup gate enabled (default), missing/null maps, partial reused values | Migration Job followed by exact Brainstore rollout Job; API waits for rollout success and live compatibility | -| `migrations.enabled: false`, startup gate enabled | No release Jobs; existing live version-compatibility check and API-startup migration behavior | -| Migrations enabled, `api.brainstoreStartupGate.enabled: false` | Migration Job only; all new application containers wait for migrations, without exact Brainstore rollout ordering | -| Both disabled | No chart-managed startup barriers | -| Disabled to enabled | New revision Jobs and pod gates are created | -| Same image with changed inline env or ConfigMap-backed env | Old Ready pods cannot satisfy the new release's rollout barrier | -| API or Brainstore release tags, custom tags, numeric/full/short SHAs, prerelease, tagged digest | Exact rollout identity does not parse versions; the separate API compatibility check still needs a stable minimum for non-release tags | -| AWS, Google, Azure; provider examples | Migrator retains shared API identity, secrets, CA mounts, resources, and scheduling; rollout observer has no cloud identity or secret access | -| Azure Key Vault CSI enabled | Migration Job mounts CSI secrets itself before application containers start | -| API workload isolation/custom pool identities | Every API pool waits for the same rollout Job using its own read-only Job permission | -| Reader, writer, fastreader, rendered automationwriter | Every rendered role participates, including disabled roles that must finish scaling down | -| Missing/null replica count, explicit zero | Missing/null uses Kubernetes' default of one; zero stays zero and requires all non-terminal role pods to disappear | -| Automationwriter absent/null or partial reused map | Absent/null stays omitted; a partial map uses existing automationwriter defaults and participates if rendered | -| Old Deployment still fully Ready before Helm updates it | Expected release revision, image, and ConfigMap checksum must match before rollout state can count | -| Controller not caught up, pending replacements, insufficient Ready replicas, `minReadySeconds`, old/terminating/stray service backends | Barrier remains closed; existing healthy replicas do not stand in for updated replicas | -| Missing/null timing; empty/zero timing or helper image | Existing default/coalescing rules apply; invalid enabled settings fail rendering, unused disabled settings are not validated | -| Migration or rollout failure, missing Job, transient Kubernetes read/RBAC errors | Gates fail closed with bounded waits; completed Jobs are never inferred from absence | -| Subsequent upgrade, superseded API replacement | Each revision has distinct Jobs; old API replacements use their retained rollout success, not an obsolete desired configuration | -| Rollback | Cleanup resets both target-revision Jobs before removing their config/RBAC; migrations and exact rollout verification run again | -| Global/API/role labels and custom pod annotations | Inputs are not mutated or leaked between workloads; observer/migrator pods cannot match API service selectors; managed rollout identity/checksum cannot be overridden | - -## Exact rollout contract - -The chart stamps each rendered Brainstore Pod template with `braintrust.dev/release-revision`. Each Helm revision already changes its migration Job dependency, forcing replacement even for same-image changes. The rollout Job snapshots the expected deployment/container names, release revision, exact image string, and rendered role ConfigMap checksum. - -For each role it checks the Deployment template against those expectations, requires the controller to have observed the desired generation, and waits for all desired replicas to be updated, Ready, and Available. It also lists every service-selected pod, including stray pods, and rejects previous revisions/configurations and terminating live pods. Two successful fleet observations are required within one deadline; this is an observation barrier, not a lock on later changes. +- The Deployment pod-template release marker is this API's Helm revision or newer. +- `status.observedGeneration >= metadata.generation`. +- Updated, Ready, Available, and total replica counts equal the current desired count. +- All service-selected pods belong to that current rollout and are Ready; no old or terminating live pods remain. +- Desired and running Brainstore images satisfy this API pool's version floor. -The existing live compatibility check remains intentionally separate. An older API can restart while a later compatible Brainstore release rolls out: its retained rollout Job records the completed prerequisite, while the compatibility check protects its current live connections. Do not make older API replacements wait for their old deployment revision to become current again. +The release marker prevents the previous fully Ready Deployment from passing before Helm updates it. Deployment generation counters are checked within each Deployment, not compared between API and Brainstore. Pod-template changes cover images and inline environment; the existing ConfigMap checksum covers chart-managed environment. Zero-replica roles must finish draining. Two successful fleet observations share one bounded deadline. -## Configuration and availability limits +**Tradeoff:** an older API replacement also waits during a newer unfinished Brainstore rollout. It accepts a newer completed compatible release rather than waiting for its old configuration to become current again. Insufficient surge capacity can therefore block replacement APIs as well as the new release. -This feature requires real Helm install/upgrade revisions. Repeated `helm template | kubectl apply` does not advance revision identity and is unsupported. +## Input matrix -The guarantee is **application-container startup order**, not Kubernetes object mutation order. Helm still applies Deployment specs and ConfigMaps together. New containers read their ConfigMap-backed environment only after their init gates finish, but mutable mounted files, externally changed Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migration deployment and versioned configuration/Secrets if no object or old workload may see changed configuration before migration completion. After changing an external Secret, perform a Helm upgrade to roll pods; the chart does not watch or snapshot secret values. Pin image digests when immutable image identity is required. - -Pool-specific API image/config overrides must remain schema-compatible with the shared migrator image and database. SQL changes must be backward-compatible with existing running workloads; rollback never reverses schema changes. - -Existing rollout strategies still govern availability and spare capacity. `Recreate` and nonzero `maxUnavailable` can remove old capacity while replacements wait. With RollingUpdate and `maxUnavailable: 0`, old API replicas remain while the new API waits; exact ordering does not solve insufficient surge nodes. Local SSD cache warm-up is governed by readiness and `minReadySeconds`, not inferred from pod creation alone. +| Input | Behavior | +| --- | --- | +| Both gates enabled; missing/null maps or partial reused values | Migration Job, then direct completed-rollout and compatibility checks | +| `migrations.enabled: false` | No migration Job; previous compatible-capacity gate and API-startup migration behavior | +| `api.brainstoreStartupGate.enabled: false` | Migration-only startup ordering; both disabled means no chart-managed gates | +| Same image with changed env/config; stale controller observation; old Ready writer/new Pending writer | New API remains blocked | +| Older API, newer Brainstore release | Wait during rollout; pass after the newer compatible rollout completes | +| Replica count missing/null/zero; optional automationwriter absent/partial | Preserve Kubernetes/default pool behavior; zero roles drain, absent pools stay omitted | +| Custom role names, namespaces, API pools | Checks and read-only permissions follow the rendered workload names and actual namespace | +| Release tags with/without v, SHA/numeric/custom/prerelease/digest API tags | Existing image handling is unchanged; opaque API tags require an explicit stable minimumVersion | +| AWS/Google/Azure, Azure CSI, CA mounts, custom scheduling/security | Migrator inherits the shared API contract and independently mounts its required secrets; startup checks only read Kubernetes state | +| Missing/null timing or invalid enabled timing/image | Existing defaults apply; invalid enabled values fail rendering, unused disabled settings are ignored | +| Failed/missing migration Job, API errors, old/stray/terminating pods, readiness dwell | Fail closed within the configured budgets; no missing resource implies success | +| Upgrade/retry/rollback and replacement pods | Distinct migration Jobs; keep completed migration records; rollback resets only the target migration phase | ## Retention and rollback -Neither release Job has a TTL. Completed Jobs, their configuration, helper code, and per-revision RBAC use `helm.sh/resource-policy: keep`, so replacement pods and rollback revisions retain their prerequisites. Those phase records survive uninstall. The rollout observer uses a stable, namespace-only ServiceAccount that is not retained: disabling the gate removes it, allowing Helm 3 to recreate it cleanly when rolling back to an enabled revision. Old API replacements read the retained Job using their own API identity, not the observer account. - -Delete retained resources only after no Pod/ReplicaSet references the revision and it is no longer needed for rollback. Do not delete the active revision's completed Jobs: replacements and autoscaling need those success records. Before reinstalling the same release name, deliberately clean up retained resources after confirming no old workloads reference them. Retry failed attempts with a new `helm upgrade` revision after fixing the cause. +The migration Job, its snapshot configuration, helper code, and exact-Job read RBAC are retained without a TTL for replacement pods and rollback. They survive uninstall. Do not delete records still referenced by active pods, ReplicaSets, or a needed rollback revision. Retry a failed migration with a new Helm upgrade after fixing its cause. -Native Helm rollback runs the target revision's `pre-rollback` cleanup hook. Its separate identity can get/delete only that target's two Jobs and corresponding ConfigMaps, Roles, and RoleBindings. Cleanup issues deletion for both Jobs, waits for foreground deletion of both, then removes retained config/RBAC. Helm then recreates the target resources and reruns both phases. Migration-only releases clean up just their migration phase. This avoids the Helm 3 retained-resource restoration error (https://github.com/helm/helm/issues/13142) without overlapping the target's old and new Jobs. +The target revision's `pre-rollback` hook deletes that migration Job and waits for foreground deletion before removing its config/RBAC. Helm then recreates the target phase. This avoids Helm 3 retained-resource restoration errors. Do not use `--no-hooks`; failed cleanup blocks rollback. SQL changes are never reversed and must remain compatible with existing workloads. -Do not use `--no-hooks` on rollback. Automatic rollback follows the same hook path. Existing running application containers continue according to their rollout strategy; replacement pods can wait during cleanup. A failed cleanup blocks rollback and leaves its hook logs for diagnosis. +## Limits and operation -## Timeouts and verification +These gates order **application-container startup**, not Kubernetes object updates. Helm still submits Deployment specs and ConfigMaps together. Mutable mounted configuration, externally updated Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migrations and versioned configuration if no object or old workload may observe changes before migrations finish. Perform a Helm upgrade after an external Secret change to roll pods; secret values are not snapshotted. -`migrations.timeoutSeconds` bounds the migration phase. `api.brainstoreStartupGate.timeoutSeconds` bounds exact rollout observation and the existing compatibility check. The rollout Job and its API wait include the migration plus rollout budgets; API progress deadlines also include the subsequent compatibility budget and startup margin. +Use real Helm revisions, not repeated `helm template | kubectl apply`. Pool-specific API overrides must remain schema-compatible with the shared migrator image/database. Existing rollout strategies determine availability; strict startup ordering does not provide spare nodes or guarantee cache warm-up beyond configured readiness and `minReadySeconds`. -Allow enough Helm timeout for migration, node allocation/cache readiness, and API startup. Use `--wait --wait-for-jobs` to observe migration/rollout failure; without waits Helm can return before these phases finish. Inspect the revision-specific rollout Job's logs to identify the role and incomplete condition. Pending replacement writer capacity must keep the new API blocked even when an old compatible writer remains Ready. +`migrations.timeoutSeconds` bounds migration waiting. `api.brainstoreStartupGate.timeoutSeconds` bounds direct rollout observation. API progress deadlines include both budgets plus startup margin. Use sufficient Helm timeout and `--wait --wait-for-jobs`; otherwise Helm can return before migrations or rollout complete. Inspect API `wait-for-brainstore` init logs for the role and condition holding startup. diff --git a/tests/test_brainstore_rollout.py b/tests/test_brainstore_rollout.py deleted file mode 100644 index 03c4751..0000000 --- a/tests/test_brainstore_rollout.py +++ /dev/null @@ -1,474 +0,0 @@ -import importlib.util -import io -import os -from pathlib import Path -import sys -import unittest -from unittest.mock import Mock, patch -import urllib.error -import urllib.parse - - -FILES = Path(__file__).resolve().parents[1] / "braintrust/files" -shared_spec = importlib.util.spec_from_file_location( - "brainstore_gate", FILES / "brainstore-startup-gate.py") -shared = importlib.util.module_from_spec(shared_spec) -shared_spec.loader.exec_module(shared) -spec = importlib.util.spec_from_file_location("rollout", FILES / "brainstore-rollout.py") -rollout = importlib.util.module_from_spec(spec) -with patch.dict(sys.modules, {"brainstore_gate": shared}): - spec.loader.exec_module(rollout) - - -def target(role="reader", **overrides): - return { - "deployment": "custom-" + role, - "container": "brainstore-" + role, - "revision": "17", - "image": "registry.example:5000/brainstore:custom-build", - "configChecksum": "b" * 64, - **overrides, - } - - -def pod(expected=None, name="new", is_ready=True, **metadata): - expected = target() if expected is None else expected - return { - "metadata": { - "name": name, - "labels": {"app": expected["deployment"]}, - "annotations": { - "braintrust.dev/release-revision": expected["revision"], - "checksum/config": expected["configChecksum"], - }, - **metadata, - }, - "spec": {"containers": [{"name": expected["container"], "image": expected["image"]}]}, - "status": { - "phase": "Running", - "conditions": [{"type": "Ready", "status": "True" if is_ready else "False"}], - "containerStatuses": [{ - "name": expected["container"], "image": expected["image"], - "ready": is_ready, "state": {"running": {}}, - }], - }, - } - - -def deployment(expected=None, replicas=1): - expected = target() if expected is None else expected - current = pod(expected) - return { - "metadata": {"name": expected["deployment"], "generation": 7}, - "spec": { - "replicas": replicas, - "template": {"metadata": {"annotations": current["metadata"]["annotations"]}, - "spec": current["spec"]}, - }, - "status": { - "observedGeneration": 7, - "updatedReplicas": replicas, - "readyReplicas": replicas, - "availableReplicas": replicas, - "replicas": replicas, - }, - } - - -class DeploymentTests(unittest.TestCase): - def check(self, resource=None, pods=None, expected=None): - expected = target() if expected is None else expected - return rollout.check_deployment( - deployment(expected) if resource is None else resource, - [pod(expected)] if pods is None else pods, expected) - - def test_exact_ready_release_passes(self): - self.assertIsNone(self.check()) - - def test_old_fully_ready_spec_does_not_pass_before_helm_updates_it(self): - old = target(revision="16") - self.assertIsNotNone(self.check(deployment(old), [pod(old)])) - - def test_config_only_change_requires_updated_template_and_pods(self): - old = target(configChecksum="a" * 64) - self.assertIsNotNone(self.check(deployment(old), [pod(old)])) - self.assertIsNotNone(self.check(pods=[pod(old, "old")])) - self.assertIsNone(self.check()) - - def test_old_ready_writer_cannot_release_a_pending_same_image_replacement(self): - expected = target("writer") - old = target("writer", configChecksum="a" * 64) - pending = pod(expected, "replacement", False) - pending["status"]["phase"] = "Pending" - resource = deployment(expected) - resource["status"]["updatedReplicas"] = 0 - self.assertIsNotNone(self.check(resource, [pod(old, "old-writer"), pending], expected)) - self.assertIsNotNone(self.check(pods=[pending], expected=expected)) - - def test_revision_only_env_rollout_cannot_use_previous_ready_pod(self): - old = target(revision="16") - self.assertIsNotNone(self.check(pods=[pod(old, "old")])) - - def test_missing_and_wrong_identity_fields_fail_closed_on_template_and_pods(self): - fields = ("braintrust.dev/release-revision", "checksum/config") - for location in ("template", "pod"): - for annotation in fields: - for value in (None, "wrong"): - with self.subTest(location=location, annotation=annotation, value=value): - resource = deployment() - current = pod() - subject = resource["spec"]["template"] if location == "template" else current - if value is None: - del subject["metadata"]["annotations"][annotation] - else: - subject["metadata"]["annotations"][annotation] = value - self.assertIsNotNone(self.check(resource, [current])) - - def test_image_must_match_exactly_in_template_and_pod_spec(self): - for location in ("template", "pod"): - with self.subTest(location=location): - resource = deployment() - current = pod() - subject = resource["spec"]["template"] if location == "template" else current - subject["spec"]["containers"][0]["image"] = "registry.example/brainstore:other" - self.assertIsNotNone(self.check(resource, [current])) - - def test_exact_images_do_not_need_semver_tags(self): - for image in ("registry/brainstore:v2.15.0", "registry/brainstore:latest", - "registry/brainstore:deadbeef", "registry/brainstore:v2.15.0-rc.1", - "registry:5000/brainstore@sha256:" + "a" * 64, - "registry/brainstore:custom@sha256:" + "b" * 64): - with self.subTest(image=image): - self.assertIsNone(self.check(expected=target(image=image))) - - def test_runtime_image_alias_neither_blocks_nor_overrides_the_pod_spec(self): - current = pod() - current["status"]["containerStatuses"][0]["image"] = "containerd://sha256:resolved" - self.assertIsNone(self.check(pods=[current])) - current["spec"]["containers"][0]["image"] = "registry/brainstore:old" - self.assertIsNotNone(self.check(pods=[current])) - - def test_missing_expected_spec_container_fails_closed(self): - for location in ("template", "pod"): - with self.subTest(location=location): - resource = deployment() - current = pod() - subject = resource["spec"]["template"] if location == "template" else current - subject["spec"]["containers"][0]["name"] = "sidecar" - self.assertIsNotNone(self.check(resource, [current])) - - def test_current_controller_observation_is_required(self): - for observed in (None, 6): - with self.subTest(observed=observed): - resource = deployment() - if observed is None: - del resource["status"]["observedGeneration"] - else: - resource["status"]["observedGeneration"] = observed - self.assertIsNotNone(self.check(resource)) - - def test_deleted_deployment_blocks(self): - resource = deployment() - resource["metadata"]["deletionTimestamp"] = "2026-11-01T00:00:00Z" - self.assertIsNotNone(self.check(resource)) - - def test_each_controller_replica_counter_must_equal_desired(self): - for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): - for count in (None, 0, 2): - with self.subTest(counter=counter, count=count): - resource = deployment() - if count is None: - del resource["status"][counter] - else: - resource["status"][counter] = count - self.assertIsNotNone(self.check(resource)) - - def test_min_ready_seconds_requires_controller_availability(self): - resource = deployment() - resource["spec"]["minReadySeconds"] = 30 - resource["status"]["availableReplicas"] = 0 - self.assertIsNotNone(self.check(resource)) - resource["status"]["availableReplicas"] = 1 - self.assertIsNone(self.check(resource)) - - def test_current_spec_scaling_is_honored_without_target_replica_snapshot(self): - resource = deployment(replicas=3) - self.assertIsNone(self.check(resource, [pod(name=str(index)) for index in range(3)])) - self.assertIsNotNone(self.check(resource, [pod()])) - self.assertIsNotNone(self.check(pods=[pod(), pod(name="surge")])) - - def test_missing_replicas_defaults_to_one(self): - resource = deployment() - del resource["spec"]["replicas"] - self.assertIsNone(self.check(resource)) - self.assertIsNotNone(self.check(resource, [])) - - def test_explicit_zero_requires_no_live_backends_and_zero_status_counts(self): - resource = deployment(replicas=0) - self.assertIsNone(self.check(resource, [])) - self.assertIsNotNone(self.check(resource, [pod()])) - for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): - with self.subTest(counter=counter): - stale = deployment(replicas=0) - stale["status"][counter] = 1 - self.assertIsNotNone(self.check(stale, [])) - resource["status"] = {"observedGeneration": 7} - self.assertIsNone(self.check(resource, [])) - - def test_zero_still_requires_current_release_template_and_observation(self): - resource = deployment(target(revision="16"), replicas=0) - self.assertIsNotNone(self.check(resource, [])) - resource = deployment(replicas=0) - resource["status"]["observedGeneration"] = 6 - self.assertIsNotNone(self.check(resource, [])) - - def test_terminal_pods_are_ignored_even_when_old_or_terminating(self): - for phase in ("Failed", "Succeeded"): - with self.subTest(phase=phase): - old = pod(target(revision="16"), "old", False, - deletionTimestamp="2026-11-01T00:00:00Z") - old["status"]["phase"] = phase - self.assertIsNone(self.check(pods=[pod(), old])) - self.assertIsNone(self.check(deployment(replicas=0), [old])) - - def test_terminating_matching_backend_blocks_even_if_unready(self): - for is_ready in (True, False): - with self.subTest(is_ready=is_ready): - terminating = pod(name="terminating", is_ready=is_ready, - deletionTimestamp="2026-11-01T00:00:00Z") - self.assertIsNotNone(self.check(pods=[pod(), terminating])) - self.assertIsNotNone(self.check(deployment(replicas=0), [terminating])) - - def test_ready_requires_running_expected_container_and_pod_condition(self): - for missing in ("phase", "condition", "container", "running", "ready"): - with self.subTest(missing=missing): - current = pod() - if missing == "phase": - current["status"]["phase"] = "Pending" - elif missing == "condition": - current["status"]["conditions"] = [] - elif missing == "container": - current["status"]["containerStatuses"][0]["name"] = "sidecar" - elif missing == "running": - current["status"]["containerStatuses"][0]["state"] = {"waiting": {}} - else: - current["status"]["containerStatuses"][0]["ready"] = False - self.assertIsNotNone(self.check(pods=[current])) - - -class FleetTests(unittest.TestCase): - def test_every_enabled_role_uses_its_custom_service_selector_and_current_spec(self): - targets = [target(role, deployment="tenant-pool-" + role) - for role in ("reader", "writer", "fastreader", "automationwriter")] - resources = {expected["deployment"]: deployment(expected, replicas=0 if index == 3 else 1) - for index, expected in enumerate(targets)} - backends = {"app=" + expected["deployment"]: [pod(expected)] if index != 3 else [] - for index, expected in enumerate(targets)} - client = Mock() - client.deployment.side_effect = lambda name, deadline: resources[name] - client.pods.side_effect = lambda selector, deadline: backends[selector] - self.assertIsNone(rollout.check_fleet(client, targets, 100)) - for expected in targets: - with self.subTest(role=expected["container"]): - selector = "app=" + expected["deployment"] - current = backends[selector] - backends[selector] = [pod({**expected, "revision": "16"}, "old")] - self.assertIsNotNone(rollout.check_fleet(client, targets, 100)) - backends[selector] = current - - def test_unready_optional_role_blocks_the_fleet(self): - targets = [target(role) for role in ("reader", "writer", "fastreader", "automationwriter")] - client = Mock() - client.deployment.side_effect = [deployment(expected) for expected in targets] - client.pods.side_effect = [[pod(expected, is_ready=index != 3)] - for index, expected in enumerate(targets)] - self.assertIsNotNone(rollout.check_fleet(client, targets, 100)) - - def test_old_stray_backend_on_later_api_page_blocks_without_owner_filtering(self): - expected = target(deployment="tenant-pool-reader") - with patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}), \ - patch.object(shared.ssl, "create_default_context"): - client = rollout.KubernetesClient("custom namespace", "/synthetic") - stray = pod(target(deployment=expected["deployment"], revision="16"), "stray") - resources = { - "/apis/apps/v1/namespaces/custom%20namespace/deployments/tenant-pool-reader": - deployment(expected), - } - pages = { - "": {"items": [pod(expected)], "metadata": {"continue": "next-page"}}, - "next-page": {"items": [stray]}, - } - - def get_resource(path, deadline): - if path in resources: - return resources[path] - parsed = urllib.parse.urlsplit(path) - query = urllib.parse.parse_qs(parsed.query) - if (parsed.path != "/api/v1/namespaces/custom%20namespace/pods" - or query.get("labelSelector") != ["app=" + expected["deployment"]]): - raise AssertionError("unexpected Kubernetes resource request") - return pages.pop(query.get("continue", [""])[0]) - - client.get = get_resource - self.assertIsNotNone(rollout.check_fleet(client, [expected], 100)) - - def test_extra_matching_stray_is_not_hidden_by_ready_controller_status(self): - client = Mock() - client.deployment.return_value = deployment() - client.pods.return_value = [pod(), pod(name="stray")] - self.assertIsNotNone(rollout.check_fleet(client, [target()], 100)) - - -class FakeClock: - def __init__(self): - self.now = 0 - self.sleeps = [] - - def clock(self): - return self.now - - def sleep(self, seconds): - self.sleeps.append(seconds) - self.now += seconds - - -def healthy_client(): - client = Mock() - client.deployment.return_value = deployment() - client.pods.return_value = [pod()] - return client - - -class WaitTests(unittest.TestCase): - def wait(self, client, clock=None, timeout=60, interval=10, targets=None, log=None): - clock = FakeClock() if clock is None else clock - log = Mock() if log is None else log - rollout.wait_for_fleet(client, [target()] if targets is None else targets, - timeout, interval, clock.clock, clock.sleep, log) - return clock, log - - def test_requires_two_real_successful_fleet_observations(self): - client = healthy_client() - clock, _ = self.wait(client) - self.assertEqual(client.deployment.call_count, 2) - self.assertEqual(client.pods.call_count, 2) - self.assertEqual(clock.now, 10) - - def test_unhealthy_observation_resets_confirmation(self): - client = healthy_client() - client.pods.side_effect = [[pod()], [pod(is_ready=False)], [pod()], [pod()]] - clock, _ = self.wait(client) - self.assertEqual(client.pods.call_count, 4) - self.assertEqual(clock.now, 30) - - def test_transient_read_failure_resets_confirmation(self): - client = healthy_client() - client.deployment.side_effect = [deployment(), OSError("offline"), deployment(), deployment()] - clock, _ = self.wait(client) - self.assertEqual(client.deployment.call_count, 4) - self.assertEqual(client.pods.call_count, 3) - self.assertEqual(clock.now, 30) - - def test_missing_resources_rbac_throttling_and_server_errors_are_retried(self): - bodies = [io.StringIO("sensitive response body") for _ in range(4)] - errors = [urllib.error.HTTPError("https://api/deployments/reader?secret=hidden", code, - "sensitive reason", None, body) - for code, body in zip((404, 403, 429, 503), bodies)] - client = healthy_client() - client.deployment.side_effect = errors + [deployment(), deployment()] - clock, log = self.wait(client) - self.assertEqual(clock.now, 50) - self.assertTrue(all(body.closed for body in bodies)) - messages = "\n".join(call.args[0] for call in log.call_args_list) - self.assertNotIn("sensitive", messages) - self.assertNotIn("secret", messages) - - def test_read_and_decode_errors_are_retried_without_leaking_details(self): - client = healthy_client() - client.deployment.side_effect = [ - urllib.error.URLError("secret transport detail"), ValueError("secret JSON"), - KeyError("secret field"), TypeError("secret object"), deployment(), deployment(), - ] - clock, log = self.wait(client) - self.assertEqual(clock.now, 50) - messages = "\n".join(call.args[0] for call in log.call_args_list) - self.assertNotIn("secret", messages) - - def test_pod_list_failure_is_retried_instead_of_using_previous_ready_result(self): - client = healthy_client() - client.pods.side_effect = [[pod()], urllib.error.URLError("offline"), [pod()], [pod()]] - clock, _ = self.wait(client) - self.assertEqual(client.pods.call_count, 4) - self.assertEqual(clock.now, 30) - - def test_old_ready_config_never_completes_and_sleep_is_bounded(self): - client = healthy_client() - client.pods.return_value = [pod(target(configChecksum="a" * 64), "old")] - clock = FakeClock() - with self.assertRaises(TimeoutError): - self.wait(client, clock, timeout=25) - self.assertEqual(clock.now, 25) - self.assertEqual(clock.sleeps, [10, 10, 5]) - self.assertEqual(client.deployment.call_count, 3) - - def test_persistent_api_failure_times_out_with_bounded_diagnostics(self): - client = healthy_client() - client.deployment.side_effect = [urllib.error.HTTPError( - "https://api/" + "x" * 1000 + "?secret=hidden", 429, - "sensitive reason", None, io.StringIO("sensitive body")) for _ in range(3)] - clock = FakeClock() - with self.assertRaises(TimeoutError) as error: - self.wait(client, clock, timeout=25) - self.assertLess(len(str(error.exception)), 400) - self.assertNotIn("sensitive", str(error.exception)) - self.assertNotIn("secret", str(error.exception)) - self.assertEqual(clock.now, 25) - - def test_successful_response_after_deadline_cannot_complete(self): - client = healthy_client() - clock = FakeClock() - observations = iter((0, 9)) - - def observe(selector, deadline): - clock.now += next(observations) - return [pod()] - - client.pods.side_effect = observe - with self.assertRaises(TimeoutError): - self.wait(client, clock, timeout=10, interval=2) - self.assertEqual(client.pods.call_count, 2) - self.assertEqual(clock.now, 11) - - def test_all_roles_requests_and_confirmations_share_one_deadline(self): - targets = [target("reader"), target("writer")] - expected_by_name = {expected["deployment"]: expected for expected in targets} - client = Mock() - clock = FakeClock() - deadlines = [] - - def get_deployment(name, deadline): - deadlines.append(deadline) - clock.now += 1 - return deployment(expected_by_name[name]) - - def get_pods(selector, deadline): - deadlines.append(deadline) - clock.now += 1 - return [pod(expected_by_name[selector.removeprefix("app=")])] - - client.deployment.side_effect = get_deployment - client.pods.side_effect = get_pods - self.wait(client, clock, timeout=20, interval=3, targets=targets) - self.assertEqual(deadlines, [20] * 8) - self.assertEqual(clock.now, 11) - - def test_scaling_between_confirmations_uses_each_current_spec(self): - client = healthy_client() - client.deployment.side_effect = [deployment(replicas=2), deployment()] - client.pods.side_effect = [[pod(name="one"), pod(name="two")], [pod()]] - self.wait(client) - self.assertEqual(client.deployment.call_count, 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_brainstore_startup_gate.py b/tests/test_brainstore_startup_gate.py index 3575496..6ce0b2f 100644 --- a/tests/test_brainstore_startup_gate.py +++ b/tests/test_brainstore_startup_gate.py @@ -14,24 +14,31 @@ spec.loader.exec_module(gate) -def pod(version="v2.15.0", uid="new", is_ready=True, **metadata): +def pod(version="v2.15.0", uid="new", is_ready=True, revision=None, + container="brainstore-reader", **metadata): + if revision is not None: + metadata["annotations"] = {"braintrust.dev/release-revision": str(revision)} return { "metadata": {"name": uid, "uid": uid, **metadata}, - "spec": {"containers": [{"name": "brainstore-reader", "image": "registry/brainstore:" + version}]}, + "spec": {"containers": [{"name": container, "image": "registry/brainstore:" + version}]}, "status": { "phase": "Running", "conditions": [{"type": "Ready", "status": "True" if is_ready else "False"}], - "containerStatuses": [{"name": "brainstore-reader", "image": "registry/brainstore:" + version, + "containerStatuses": [{"name": container, "image": "registry/brainstore:" + version, "ready": is_ready, "state": {"running": {}}}], }, } -def deployment(version="v2.15.0", replicas=1): +def deployment(version="v2.15.0", replicas=1, revision=None, container="brainstore-reader"): return { "metadata": {"name": "reader", "uid": "reader-deployment", "generation": 2}, - "spec": {"replicas": replicas, "template": {"spec": pod(version)["spec"]}}, - "status": {"observedGeneration": 2, "availableReplicas": replicas}, + "spec": {"replicas": replicas, "template": { + "metadata": {"annotations": pod(revision=revision)["metadata"].get("annotations", {})}, + "spec": pod(version, container=container)["spec"], + }}, + "status": {"observedGeneration": 2, "updatedReplicas": replicas, + "readyReplicas": replicas, "availableReplicas": replicas, "replicas": replicas}, } @@ -142,6 +149,128 @@ def test_pod_ready_without_expected_running_container_is_insufficient(self): self.assertIn("available compatible", self.check(pods=[current])) +class StrictRolloutTests(unittest.TestCase): + def check(self, resource=None, pods=None, minimum_revision=8): + return gate.check_deployment( + resource if resource is not None else deployment(revision=8), + pods if pods is not None else [pod(revision=8)], + "brainstore-reader", (2, 15, 0), minimum_revision, + ) + + def test_same_image_release_waits_for_current_pods(self): + old = pod(revision=7) + current = pod(revision=8, is_ready=False) + self.assertIsNotNone(self.check(pods=[old, current])) + self.assertIsNotNone(self.check(pods=[old])) + current = pod(revision=8) + self.assertIsNone(self.check(pods=[current])) + + def test_same_release_env_change_waits_for_updated_replicas(self): + resource = deployment(revision=8) + resource["metadata"]["generation"] = 3 + resource["status"]["observedGeneration"] = 3 + resource["spec"]["template"]["spec"]["containers"][0]["env"] = [ + {"name": "BRAINSTORE_CONFIG", "value": "new-config"}, + ] + resource["status"]["updatedReplicas"] = 0 + self.assertIsNotNone(self.check(resource)) + resource["status"]["updatedReplicas"] = 1 + current = pod(revision=8) + current["spec"] = resource["spec"]["template"]["spec"] + self.assertIsNone(self.check(resource, [current])) + + def test_old_complete_generation_cannot_pass_before_helm_update(self): + self.assertIsNotNone(self.check(deployment(revision=7), [pod(revision=7)])) + + def test_revision_floor_rejects_missing_malformed_and_old_markers(self): + for revision in (None, "", "invalid", "8.0", 7): + with self.subTest(revision=revision): + self.assertIsNotNone(self.check(deployment(revision=revision))) + + def test_requires_current_controller_observation(self): + resource = deployment(revision=8) + resource["status"]["observedGeneration"] = 1 + self.assertIsNotNone(self.check(resource)) + + def test_every_replica_counter_must_equal_current_desired_count(self): + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + for count in (None, 0, 2): + with self.subTest(counter=counter, count=count): + resource = deployment(revision=8) + if count is None: + del resource["status"][counter] + else: + resource["status"][counter] = count + self.assertIsNotNone(self.check(resource)) + + def test_requires_exactly_desired_live_ready_current_pods(self): + for pods in ([], [pod(revision=8, is_ready=False)], + [pod(revision=8), pod(uid="surge", revision=8)], + [pod(revision=8, deletionTimestamp="2026-09-30T12:00:00Z")], + [pod(revision=8), pod(uid="terminating", revision=8, is_ready=False, + deletionTimestamp="2026-09-30T12:00:00Z")]): + with self.subTest(pods=pods): + self.assertIsNotNone(self.check(pods=pods)) + current = pod(revision=8) + current["status"]["containerStatuses"] = [] + self.assertIsNotNone(self.check(pods=[current])) + + def test_stray_service_backend_is_not_exempt_from_revision_check(self): + stray = pod(uid="stray", revision=7) + self.assertIsNotNone(self.check(pods=[stray])) + del stray["metadata"]["annotations"] + self.assertIsNotNone(self.check(pods=[stray])) + + def test_terminal_old_pods_do_not_block_completed_rollout(self): + for phase in ("Succeeded", "Failed"): + with self.subTest(phase=phase): + old = pod("v2.14.0", revision=7) + old["status"]["phase"] = phase + self.assertIsNone(self.check(pods=[pod(revision=8), old])) + + def test_min_ready_seconds_requires_available_replicas(self): + resource = deployment(revision=8) + resource["spec"]["minReadySeconds"] = 60 + resource["status"]["availableReplicas"] = 0 + self.assertIsNotNone(self.check(resource)) + resource["status"]["availableReplicas"] = 1 + self.assertIsNone(self.check(resource)) + + def test_zero_role_requires_release_identity_and_completed_scale_down(self): + resource = deployment(replicas=0, revision=8) + self.assertIsNone(self.check(resource, [])) + self.assertIsNotNone(self.check(resource, [pod(revision=8)])) + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + with self.subTest(counter=counter): + resource["status"][counter] = 1 + self.assertIsNotNone(self.check(resource, [])) + del resource["status"][counter] + self.assertIsNone(self.check(resource, [])) + for revision in (None, 7): + with self.subTest(revision=revision): + self.assertIsNotNone(self.check(deployment(replicas=0, revision=revision), [])) + resource["status"]["observedGeneration"] = 1 + self.assertIsNotNone(self.check(resource, [])) + + def test_uses_current_replica_count_after_scaling(self): + resource = deployment(replicas=2, revision=8) + self.assertIsNotNone(self.check(resource, [pod(revision=8)])) + self.assertIsNone(self.check(resource, [pod(revision=8), pod(uid="second", revision=8)])) + + def test_older_api_accepts_later_completed_but_not_unfinished_rollout(self): + resource = deployment("v2.16.0", revision=9) + self.assertIsNone(self.check(resource, [pod("v2.16.0", revision=9)])) + self.assertIsNotNone(self.check(resource, [pod(revision=8)])) + resource["status"]["updatedReplicas"] = 0 + self.assertIsNotNone(self.check(resource, [pod("v2.16.0", revision=9)])) + + def test_strict_mode_preserves_desired_and_pod_image_compatibility(self): + for version in ("v2.14.0", "v3.0.0", "latest", "v2.15.0-rc.1"): + with self.subTest(version=version): + self.assertIsNotNone(self.check(deployment(version, revision=8))) + self.assertIsNotNone(self.check(pods=[pod(version, revision=8)])) + + class FakeClock: def __init__(self): self.now = 0 @@ -236,6 +365,60 @@ def test_checks_all_three_roles_and_includes_stray_service_backends(self): self.assertIn("stray", reason) + def test_strict_writer_waits_across_same_image_and_later_rollouts(self): + container = "brainstore-writer" + resources = [deployment(revision=revision, container=container) + for revision in (7, 8, 8, 9, 9, 9)] + for resource in resources: + resource["metadata"]["name"] = "writer" + resources[1]["status"]["updatedReplicas"] = 0 + resources[3]["status"]["updatedReplicas"] = 0 + old = pod(revision=7, container=container) + current = pod(revision=8, container=container) + later = pod(revision=9, container=container) + client = Mock() + client.deployment.side_effect = resources + client.pods.side_effect = [ + [old], [old, pod(revision=8, is_ready=False, container=container)], + [current], [current, pod(revision=9, is_ready=False, container=container)], + [later], [later], + ] + targets = [{"deployment": "writer", "container": container, "minimumRevision": 8}] + clock = FakeClock() + gate.wait_for_fleet(client, targets, (2, 15, 0), 120, 15, + clock.clock, clock.sleep, Mock()) + self.assertEqual(clock.now, 75) + + def test_strict_observation_errors_reset_confirmation(self): + client = Mock() + resource = deployment(revision=8) + client.deployment.side_effect = [ + urllib.error.HTTPError("https://api", 403, "synthetic", None, None), + resource, OSError("synthetic"), resource, resource, + ] + client.pods.return_value = [pod(revision=8)] + targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] + clock = FakeClock() + gate.wait_for_fleet(client, targets, (2, 15, 0), 90, 15, + clock.clock, clock.sleep, Mock()) + self.assertEqual(clock.now, 60) + + def test_strict_completed_observation_cannot_release_api_after_deadline(self): + clock = FakeClock() + + def observe(name, deadline): + clock.sleep(30) + return deployment(revision=8) + + client = Mock() + client.deployment.side_effect = observe + client.pods.return_value = [pod(revision=8)] + targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] + with self.assertRaises(TimeoutError): + gate.wait_for_fleet(client, targets, (2, 15, 0), 60, 15, + clock.clock, clock.sleep, Mock()) + + class ClientTests(unittest.TestCase): def client(self, directory, host="10.0.0.1"): with patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": host}), \ diff --git a/tests/test_job_gate.py b/tests/test_job_gate.py index f707e78..25bc52a 100644 --- a/tests/test_job_gate.py +++ b/tests/test_job_gate.py @@ -1,7 +1,6 @@ import importlib.util import itertools import json -import os from pathlib import Path import sys import unittest @@ -66,24 +65,23 @@ def test_failure_takes_precedence_over_completion_and_deletion(self): gate.job_complete(job(condition, "Complete", **metadata)) def test_exact_job_wait_retries_missing_job_rbac_and_server_errors(self): - for name in ("release-migrate-r2", "release-rollout-r2"): - with self.subTest(name=name): - errors = [urllib.error.HTTPError("url", code, "error", {}, None) - for code in (404, 403, 500)] - client, _ = self.wait(errors + [job("Complete")], name=name) - self.assertEqual( - {call.args for call in client.get.call_args_list}, - {(f"/apis/batch/v1/namespaces/test-namespace/jobs/{name}", 20)}, - ) + errors = [urllib.error.HTTPError("url", code, "error", {}, None) + for code in (404, 403, 500)] + client, elapsed = self.wait(errors + [job("Complete")]) + self.assertEqual(elapsed, 15) + self.assertEqual( + {call.args for call in client.get.call_args_list}, + {("/apis/batch/v1/namespaces/test-namespace/jobs/release-migrate-r2", 20)}, + ) def test_retained_success_does_not_wait_for_another_observation(self): - for name in ("release-migrate-r1", "release-rollout-r1"): - client = Mock(namespace="ns") - client.get.return_value = job("Complete") - sleep = Mock() - gate.wait_for_job(client, name, 20, 5, clock=lambda: 0, - sleep=sleep, log=Mock()) - sleep.assert_not_called() + client = Mock(namespace="ns") + client.get.return_value = job("Complete") + sleep = Mock() + gate.wait_for_job(client, "release-migrate-r1", 20, 5, clock=lambda: 0, + sleep=sleep, log=Mock()) + self.assertEqual(client.get.call_count, 1) + sleep.assert_not_called() def test_running_job_then_success(self): _, elapsed = self.wait([job(), job("Complete")]) @@ -101,7 +99,10 @@ def test_failed_job_never_allows_startup(self): self.wait([job("Failed")]) def test_transient_observation_errors_retry(self): - self.wait([OSError("network"), ValueError("bad JSON"), job("Complete")]) + client, elapsed = self.wait([OSError("network"), ValueError("bad JSON"), + job("Complete")]) + self.assertEqual(client.get.call_count, 3) + self.assertEqual(elapsed, 10) def test_observation_errors_do_not_disclose_response_details(self): clock = Clock() @@ -127,118 +128,133 @@ def late(*args): class RollbackCleanupTests(unittest.TestCase): - names = ["release-migrate-r7", "release-rollout-r7"] - - def test_resource_paths_escape_namespace_and_names(self): - paths = rollback.resource_paths("ns/other", ["name/other"]) - self.assertTrue(all("/namespaces/ns%2Fother/" in path for path in paths)) - self.assertTrue(all(path.rsplit("/", 1)[1].startswith("name%2Fother") - for path in paths)) - - def test_both_jobs_are_deleted_before_any_config_or_rbac_cleanup(self): - paths = rollback.resource_paths("ns", self.names) - jobs = paths[:2] - observations = {jobs[0]: iter([True, False]), - jobs[1]: iter([True, True, False])} - deleted, absent = set(), set() + name = "release-migrate-r7" + + def test_resource_paths_escape_and_limit_cleanup_to_the_target_migration(self): + self.assertEqual(rollback.resource_paths("ns/other", "name/other"), [ + "/apis/batch/v1/namespaces/ns%2Fother/jobs/name%2Fother", + "/api/v1/namespaces/ns%2Fother/configmaps/name%2Fother-config", + "/api/v1/namespaces/ns%2Fother/configmaps/name%2Fother-gate", + "/apis/rbac.authorization.k8s.io/v1/namespaces/ns%2Fother/rolebindings/name%2Fother", + "/apis/rbac.authorization.k8s.io/v1/namespaces/ns%2Fother/roles/name%2Fother", + ]) + + def test_job_is_absent_before_any_config_or_rbac_cleanup(self): + paths = rollback.resource_paths("ns", self.name) + job_path = paths[0] + observations = iter([True, False]) + deleted = absent = False clock = Clock() def request(method, path, deadline): + nonlocal deleted, absent self.assertIn(path, paths) self.assertEqual(deadline, 10) - if path in jobs: + if path == job_path: if method == "DELETE": - deleted.add(path) + deleted = True return True - self.assertEqual(deleted, set(jobs)) - exists = next(observations[path]) - if not exists: - absent.add(path) + self.assertTrue(deleted) + exists = next(observations) + absent = not exists return exists - self.assertEqual(absent, set(jobs)) + self.assertTrue(absent) return method == "DELETE" client = Mock() client.request.side_effect = request - rollback.cleanup(client, "ns", self.names, 10, clock=clock, sleep=clock.sleep) + rollback.cleanup(client, "ns", self.name, 10, clock=clock, sleep=clock.sleep) + expected = [("DELETE", job_path, 10), ("GET", job_path, 10), + ("GET", job_path, 10)] + expected.extend((method, path, 10) for path in paths[1:] + for method in ("DELETE", "GET")) + self.assertEqual([call.args for call in client.request.call_args_list], expected) + self.assertEqual(clock.now, 2) + + def test_cleanup_of_absent_resources_is_idempotent(self): + client = Mock() + client.request.return_value = False + sleep = Mock() + rollback.cleanup(client, "ns", self.name, 10, clock=lambda: 0, sleep=sleep) self.assertEqual( - {call.args[1] for call in client.request.call_args_list if call.args[0] == "DELETE"}, - set(paths), + [call.args for call in client.request.call_args_list], + [(method, path, 10) for path in rollback.resource_paths("ns", self.name) + for method in ("DELETE", "GET")], ) + sleep.assert_not_called() - def test_cleanup_of_absent_resources_is_idempotent_for_one_or_both_jobs(self): - for names in (self.names[:1], self.names): - with self.subTest(names=names): - client = Mock() - client.request.return_value = False - sleep = Mock() - rollback.cleanup(client, "ns", names, 10, clock=lambda: 0, sleep=sleep) - self.assertEqual( - {call.args[1] for call in client.request.call_args_list - if call.args[0] == "DELETE"}, - set(rollback.resource_paths("ns", names)), - ) - sleep.assert_not_called() - - def test_either_stuck_job_blocks_all_config_and_rbac_cleanup(self): - jobs = rollback.resource_paths("ns", self.names)[:2] - for stuck in jobs: - with self.subTest(stuck=stuck): - client = Mock() - client.request.side_effect = lambda method, path, deadline: path == stuck - clock = Clock() - with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", self.names, 3, - clock=clock, sleep=clock.sleep) - self.assertEqual(clock.now, 3) - self.assertTrue(all(call.args[1] in jobs - for call in client.request.call_args_list)) - self.assertEqual( - {call.args[1] for call in client.request.call_args_list - if call.args[0] == "DELETE"}, set(jobs), - ) - - def test_each_phase_shares_the_same_deadline(self): - paths = rollback.resource_paths("ns", self.names) - for late_path in (paths[1], paths[2]): + def test_stuck_job_blocks_all_config_and_rbac_cleanup(self): + job_path = rollback.resource_paths("ns", self.name)[0] + client = Mock() + client.request.return_value = True + clock = Clock() + with self.assertRaises(TimeoutError): + rollback.cleanup(client, "ns", self.name, 3, + clock=clock, sleep=clock.sleep) + self.assertEqual(clock.now, 3) + self.assertEqual( + [call.args for call in client.request.call_args_list], + [("DELETE", job_path, 3), ("GET", job_path, 3), ("GET", job_path, 3)], + ) + + def test_config_cleanup_uses_remaining_job_deadline(self): + paths = rollback.resource_paths("ns", self.name) + observations = iter([True, False]) + client = Mock() + clock = Clock() + def request(method, path, deadline): + self.assertEqual(deadline, 5) + if method == "GET" and path == paths[0]: + return next(observations) + return True + client.request.side_effect = request + with self.assertRaises(TimeoutError): + rollback.cleanup(client, "ns", self.name, 5, + clock=clock, sleep=clock.sleep) + self.assertEqual(clock.now, 5) + self.assertEqual( + [call.args[1] for call in client.request.call_args_list + if call.args[0] == "DELETE"], + paths[:2], + ) + self.assertEqual(client.request.call_args.args[:2], ("GET", paths[1])) + + def test_late_absence_does_not_escape_shared_deadline(self): + paths = rollback.resource_paths("ns", self.name) + for late_path in paths: with self.subTest(late_path=late_path): clock = Clock() client = Mock() def request(method, path, deadline): self.assertEqual(deadline, 3) - if method == "GET" and path == late_path: - clock.now = 3 + if method == "GET": + clock.now = 3 if path == late_path else clock.now + 0.25 return False client.request.side_effect = request with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", self.names, 3, + rollback.cleanup(client, "ns", self.name, 3, clock=clock, sleep=clock.sleep) - self.assertEqual(client.request.call_args.args[1], late_path) + self.assertEqual(client.request.call_args.args[:2], ("GET", late_path)) def test_permission_or_transport_failure_stops_cleanup(self): - paths = rollback.resource_paths("ns", self.names) + paths = rollback.resource_paths("ns", self.name) for failure in (RuntimeError("forbidden"), OSError("network")): - for failing_path in paths[:3]: - with self.subTest(failure=failure, path=failing_path): - client = Mock() - def request(method, path, deadline): - if path == failing_path: - raise failure - return False - client.request.side_effect = request - with self.assertRaises(type(failure)): - rollback.cleanup(client, "ns", self.names, 10, - clock=lambda: 0, sleep=Mock()) - self.assertEqual(client.request.call_args.args[1], failing_path) - if failing_path in paths[:2]: - self.assertTrue(all(call.args[1] in paths[:2] + for failing_path in paths: + for failing_method in ("DELETE", "GET"): + with self.subTest(failure=failure, path=failing_path, + method=failing_method): + client = Mock() + def request(method, path, deadline): + if (method, path) == (failing_method, failing_path): + raise failure + return False + client.request.side_effect = request + with self.assertRaises(type(failure)): + rollback.cleanup(client, "ns", self.name, 10, + clock=lambda: 0, sleep=Mock()) + self.assertEqual(client.request.call_args.args[:2], + (failing_method, failing_path)) + self.assertTrue(all(call.args[1] in paths[:paths.index(failing_path) + 1] for call in client.request.call_args_list)) - def test_main_rejects_non_list_or_empty_job_names(self): - for names in ("job", [], ["job", ""], ["job", None]): - env = {"RELEASE_JOB_NAMES": json.dumps(names), "RELEASE_TIMEOUT_SECONDS": "60"} - with self.subTest(names=names), patch.dict(os.environ, env, clear=True), \ - self.assertRaises(ValueError): - rollback.main() - class RollbackClientTests(unittest.TestCase): def client(self): From 37b786ee7dd8c197e4667020d927cfb26876f287 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 11:56:00 -0500 Subject: [PATCH 04/13] Document retained migration cleanup before reinstall --- docs/migration-job.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/migration-job.md b/docs/migration-job.md index cda9c32..7f2e62d 100644 --- a/docs/migration-job.md +++ b/docs/migration-job.md @@ -41,6 +41,20 @@ The migration Job, its snapshot configuration, helper code, and exact-Job read R The target revision's `pre-rollback` hook deletes that migration Job and waits for foreground deletion before removing its config/RBAC. Helm then recreates the target phase. This avoids Helm 3 retained-resource restoration errors. Do not use `--no-hooks`; failed cleanup blocks rollback. SQL changes are never reversed and must remain compatible with existing workloads. +### Reinstalling after uninstall + +Before reinstalling the same release name in the same namespace, stop all old workloads and delete its retained migration records. Helm restarts revision numbering at 1: reusing old records can skip migrations against a new database, reuse a failed Job, or fail on an immutable Job template. + +```sh +release= +namespace= +selector="braintrust.dev/migration-release=$release" +kubectl -n "$namespace" delete job -l "$selector" --cascade=foreground --wait=true +kubectl -n "$namespace" delete configmap,role,rolebinding -l "$selector" --wait=true +``` + +Run this only after uninstall, never against records needed by an active release. Wait for both commands to succeed before reinstalling; deleting the Job first prevents its migrator from outliving its configuration. + ## Limits and operation These gates order **application-container startup**, not Kubernetes object updates. Helm still submits Deployment specs and ConfigMaps together. Mutable mounted configuration, externally updated Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migrations and versioned configuration if no object or old workload may observe changes before migrations finish. Perform a Helm upgrade after an external Secret change to roll pods; secret values are not snapshotted. From 39d61b2e0786dceaeab1ae4ede4cc218f1ea4baf Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 12:17:22 -0500 Subject: [PATCH 05/13] Move general Brainstore pod annotations into shared helpers --- .../templates/_brainstore-startup-gate.tpl | 15 --------------- braintrust/templates/_helpers.tpl | 16 ++++++++++++++++ .../brainstore-automationwriter-deployment.yaml | 2 +- .../brainstore-fastreader-deployment.yaml | 2 +- .../templates/brainstore-reader-deployment.yaml | 2 +- .../templates/brainstore-writer-deployment.yaml | 2 +- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index 4fc37ce..5f9e32d 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -63,18 +63,3 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- end -}} {{- end -}} -{{/* Release identity prevents a previous fully Ready generation from passing -before Helm updates the desired image or configuration. */}} -{{- define "braintrust.brainstoreStartupGate.podAnnotations" -}} -{{- $root := .root -}} -{{- $annotations := dict "checksum/config" (include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum) -}} -{{- if eq $root.Values.cloud "google" -}} -{{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} -{{- end -}} -{{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} -{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} -{{- if $migration.enabled -}} -{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} -{{- end -}} -{{- toYaml $annotations -}} -{{- end -}} diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 69d87b8..cf21076 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -289,6 +289,22 @@ and normal writable-layer/log overhead. {{- toYaml $resources -}} {{- end -}} +{{/* Merge Brainstore pod annotations, reserving release identity for rollout checks. */}} +{{- define "braintrust.brainstore.podAnnotations" -}} +{{- $root := .root -}} +{{- $annotations := dict "checksum/config" (include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum) -}} +{{- if eq $root.Values.cloud "google" -}} +{{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} +{{- end -}} +{{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} +{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} +{{- if $migration.enabled -}} +{{/* Release identity rejects a previous Ready generation before Helm updates it. */}} +{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} +{{- end -}} +{{- toYaml $annotations -}} +{{- end -}} + {{/* Defaults for the optional automation writer pool. diff --git a/braintrust/templates/brainstore-automationwriter-deployment.yaml b/braintrust/templates/brainstore-automationwriter-deployment.yaml index 15e69c9..85b195e 100644 --- a/braintrust/templates/brainstore-automationwriter-deployment.yaml +++ b/braintrust/templates/brainstore-automationwriter-deployment.yaml @@ -48,7 +48,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "automationwriter" "config" $aw) | nindent 8 }} + {{- include "braintrust.brainstore.podAnnotations" (dict "root" . "role" "automationwriter" "config" $aw) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with $aw.podSecurityContext }} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index bdf0949..82d040e 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "fastreader" "config" .Values.brainstore.fastreader) | nindent 8 }} + {{- include "braintrust.brainstore.podAnnotations" (dict "root" . "role" "fastreader" "config" .Values.brainstore.fastreader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.fastreader.podSecurityContext }} diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index 7e54ef0..99343fe 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "reader" "config" .Values.brainstore.reader) | nindent 8 }} + {{- include "braintrust.brainstore.podAnnotations" (dict "root" . "role" "reader" "config" .Values.brainstore.reader) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.reader.podSecurityContext }} diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index ea7c7f6..0def40d 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "braintrust.brainstoreStartupGate.podAnnotations" (dict "root" . "role" "writer" "config" .Values.brainstore.writer) | nindent 8 }} + {{- include "braintrust.brainstore.podAnnotations" (dict "root" . "role" "writer" "config" .Values.brainstore.writer) | nindent 8 }} spec: serviceAccountName: {{ .Values.brainstore.serviceAccount.name }} {{- with .Values.brainstore.writer.podSecurityContext }} From 022a270de4a1639857d683aa4895b48f06bf2c2c Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 12:23:58 -0500 Subject: [PATCH 06/13] Unify migrations and Brainstore rollout under one ordering switch --- braintrust/README.md | 32 +-- braintrust/files/brainstore-startup-gate.py | 83 +++---- braintrust/templates/_api-deployment.tpl | 81 +----- .../templates/_brainstore-startup-gate.tpl | 73 +++--- ...igrations.tpl => _deployment-ordering.tpl} | 15 +- braintrust/templates/_helpers.tpl | 4 +- .../templates/ai-gateway-deployment.yaml | 15 +- ...rainstore-automationwriter-deployment.yaml | 8 +- .../brainstore-fastreader-deployment.yaml | 8 +- .../brainstore-reader-deployment.yaml | 8 +- .../brainstore-startup-gate-configmap.yaml | 15 -- .../brainstore-startup-gate-rbac.yaml | 8 +- .../brainstore-writer-deployment.yaml | 8 +- .../templates/loop-runtime-deployment.yaml | 14 +- braintrust/templates/migration-gate.yaml | 2 +- braintrust/templates/migration-job.yaml | 4 +- braintrust/templates/migration-rollback.yaml | 2 +- ...instore-automationwriter-routing_test.yaml | 5 +- .../brainstore-automationwriter_test.yaml | 4 +- .../tests/brainstore-fastreader_test.yaml | 4 +- braintrust/tests/brainstore-reader_test.yaml | 4 +- .../tests/brainstore-startup-gate_test.yaml | 232 ++++-------------- braintrust/tests/brainstore-writer_test.yaml | 4 +- braintrust/tests/migrations_test.yaml | 123 ++++++---- braintrust/values.yaml | 31 +-- docs/migration-job.md | 29 +-- tests/test_brainstore_startup_gate.py | 214 +++++++++------- 27 files changed, 438 insertions(+), 592 deletions(-) rename braintrust/templates/{_migrations.tpl => _deployment-ordering.tpl} (84%) delete mode 100644 braintrust/templates/brainstore-startup-gate-configmap.yaml diff --git a/braintrust/README.md b/braintrust/README.md index b3cddb5..1fd7994 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -552,18 +552,20 @@ Example values files for different cloud providers and configurations are locate ## Release ordering -Each Helm release runs one migration Job. All new application containers wait -for it. The existing API startup gate then directly checks that the reader/writer -Deployments have observed the intended release and fully rolled out: updated, -Ready, Available, with no old or terminating pods. Same-image environment changes -cannot pass using old Ready writers. There is no separate rollout Job or snapshot. - -An older API replacement also waits during a later unfinished Brainstore rollout, -but accepts a later completed compatible release. This deliberately favors simpler -ordering over restarting that API while a future rollout is blocked on capacity. - -`migrations.enabled: false` keeps compatibility-only/API-startup migration behavior; -disabling `api.brainstoreStartupGate.enabled` leaves migration-only ordering. -These gates order container startup, not Helm's updates to configuration objects. -See [migration lifecycle and input matrix](../docs/migration-job.md) for rollback, -retention, timeouts, and configuration limits. +`deploymentOrdering.enabled: true` (default) runs one migration Job per Helm +revision. Brainstore waits for migrations; all API pools, optional AI Gateway, +and optional Loop Runtime then wait for the full compatible Brainstore rollout. +Old or terminating live pods cannot satisfy the check, including same-image +configuration changes. `deploymentOrdering.enabled: false` removes all +chart-managed ordering resources and waits; there is no intermediate mode. + +The shared `deploymentOrdering.timeoutSeconds` bounds each phase separately; +`pollIntervalSeconds` controls observations. `minimumVersion` optionally sets a +shared stable floor; otherwise each API pool uses its image release and the +Gateway/Loop use the shared API release. + +Older replacement workloads also wait during a later unfinished Brainstore +rollout, accepting a newer completed compatible release. Capacity shortages can +therefore delay restarts. Ordering covers container startup, not Helm's object +updates. See [migration lifecycle and input matrix](../docs/migration-job.md) +for rollback, retained-record reinstall cleanup, and configuration limits. diff --git a/braintrust/files/brainstore-startup-gate.py b/braintrust/files/brainstore-startup-gate.py index 483d95b..f6842f1 100644 --- a/braintrust/files/brainstore-startup-gate.py +++ b/braintrust/files/brainstore-startup-gate.py @@ -1,8 +1,8 @@ -"""Wait for compatible Brainstore capacity and required rollouts before API startup. +"""Wait for compatible Brainstore rollouts before application startup. Uses only Python's standard library and read-only Kubernetes API requests. -The required version is fixed in the API Pod's environment, not a ConfigMap that -changes underneath an older API Pod when the next release is installed. +The required version is fixed in the application's Pod environment, not a +ConfigMap that changes underneath an older Pod when the next release is installed. """ import json @@ -76,14 +76,13 @@ def ready(pod, container): ) -def check_deployment(deployment, pods, container, minimum, minimum_revision=0): - """Return a wait reason, or None when compatible backends satisfy the gate. +def check_deployment(deployment, pods, container, minimum, minimum_revision): + """Return a wait reason, or None when the compatible rollout is complete. - A release revision requires the current Deployment rollout to finish, even - for an older API restarting during a later compatible release. Without a - revision floor, compatible capacity suffices during an ongoing rollout. - Incompatible terminating Pods always block because connections may remain - usable until their process exits. + The current Deployment rollout must finish, even for an older application + restarting during a later compatible release. All live Service backends + must belong to that rollout, and terminating Pods must disappear because + connections may remain usable until their process exits. Deployment rollouts replace Pods, so use each Pod's declared release tag. Container status can report a different alias or only a digest. Neither is @@ -94,6 +93,8 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision=0): spec = deployment.get("spec", {}) status = deployment.get("status", {}) name = metadata.get("name", "unknown") + if minimum_revision <= 0: + return f"{name}: minimum release revision must be positive" if metadata.get("deletionTimestamp"): return f"{name}: deployment is being deleted" # Do not trust availability status from before the latest spec update. @@ -103,43 +104,39 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision=0): template = spec.get("template", {}) revision_key = "braintrust.dev/release-revision" revision = template.get("metadata", {}).get("annotations", {}).get(revision_key) - if minimum_revision > 0: - try: - current_revision = int(revision) - except (TypeError, ValueError): - return f"{name}: desired release revision is missing or invalid" - if current_revision < minimum_revision: - return f"{name}: waiting for release revision {minimum_revision} or later" + try: + current_revision = int(revision) + except (TypeError, ValueError): + return f"{name}: desired release revision is missing or invalid" + if current_revision < minimum_revision: + return f"{name}: waiting for release revision {minimum_revision} or later" live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] - if desired == 0 and minimum_revision == 0: - return f"{name}: waiting for scaled-down Pods to disappear" if live else None - # Check desired state too: compatible Pods are insufficient if the - # Deployment is about to replace them with an incompatible image. - image = container_image(template.get("spec", {}), container) - if not compatible(image, minimum): - return f"{name}: desired image {image!r} is below or outside the required release" + if desired == 0: + if live: + return f"{name}: waiting for scaled-down Pods to disappear" + else: + # Compatible Pods cannot compensate for an incompatible desired image. + image = container_image(template.get("spec", {}), container) + if not compatible(image, minimum): + return f"{name}: desired image {image!r} is below or outside the required release" for pod in live: image = container_image(pod.get("spec", {}), container) if not compatible(image, minimum): pod_name = pod.get("metadata", {}).get("name", "unknown") return f"{name}: incompatible Pod {pod_name} still exists ({image!r})" - if minimum_revision > 0: - pod_metadata = pod.get("metadata", {}) - if pod_metadata.get("annotations", {}).get(revision_key) != revision: - return f"{name}: waiting for Pods from the current release revision" - if pod_metadata.get("deletionTimestamp"): - return f"{name}: waiting for terminating Pods to disappear" + pod_metadata = pod.get("metadata", {}) + if pod_metadata.get("annotations", {}).get(revision_key) != revision: + return f"{name}: waiting for Pods from the current release revision" + if pod_metadata.get("deletionTimestamp"): + return f"{name}: waiting for terminating Pods to disappear" # Require both current Pod readiness and Deployment availability; the # latter also accounts for the Deployment's minReadySeconds setting. ready_count = sum(ready(pod, container) for pod in live) - if minimum_revision > 0: - counters = ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas") - if any(status.get(counter, 0) != desired for counter in counters): - return f"{name}: waiting for all {desired} replicas to complete the current rollout" - if len(live) != desired or ready_count != desired: - return f"{name}: waiting for exactly {desired} current Ready Pods" - if ready_count < desired or status.get("availableReplicas", 0) < desired: - return f"{name}: waiting for {desired} available compatible Pods ({ready_count} ready)" + counters = ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas") + if any(status.get(counter, 0) != desired for counter in counters): + return f"{name}: waiting for all {desired} replicas to complete the current rollout" + if len(live) != desired or ready_count != desired: + return f"{name}: waiting for exactly {desired} current Ready Pods" return None @@ -205,7 +202,7 @@ def check_fleet(client, targets, minimum, deadline): # with that label too; they can receive traffic regardless of ownership. pods = client.pods("app=" + target["deployment"], deadline) reason = check_deployment( - deployment, pods, target["container"], minimum, target.get("minimumRevision", 0), + deployment, pods, target["container"], minimum, target["minimumRevision"], ) if reason: return reason @@ -222,7 +219,7 @@ def wait_for_fleet(client, targets, minimum, timeout, interval, try: reason = check_fleet(client, targets, minimum, deadline) except urllib.error.HTTPError as error: - # RBAC and Deployments may be applied after the API Pod is created. + # RBAC and Deployments may be applied after the application Pod is created. path = urllib.parse.urlsplit(error.url).path reason = f"Kubernetes API returned HTTP {error.code} for {path}" error.close() @@ -235,9 +232,9 @@ def wait_for_fleet(client, targets, minimum, timeout, interval, if reason is None: successes += 1 if successes == 2: - log("Brainstore startup gate complete: compatible capacity is available", flush=True) + log("Brainstore startup gate complete: compatible rollouts are complete", flush=True) return - last_reason = "confirming compatible capacity (1/2)" + last_reason = "confirming completed compatible rollouts (1/2)" else: # An unhealthy role or failed API read resets the confirmation count. successes = 0 @@ -263,5 +260,5 @@ def main(): main() except (OSError, ValueError, KeyError, TypeError) as error: print(str(error), file=sys.stderr, flush=True) - # Fail closed: Kubernetes retries this init container; the API stays stopped. + # Fail closed: Kubernetes retries this init container; the application stays stopped. sys.exit(1) diff --git a/braintrust/templates/_api-deployment.tpl b/braintrust/templates/_api-deployment.tpl index 8ef4a4d..ef952d3 100644 --- a/braintrust/templates/_api-deployment.tpl +++ b/braintrust/templates/_api-deployment.tpl @@ -3,13 +3,11 @@ {{- $root := .root -}} {{- $api := .api -}} {{- $role := .role -}} -{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} -{{- $startupGate := include "braintrust.brainstoreStartupGate.config" $root | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" $root | fromYaml -}} {{- if .migrationJob -}} -{{- $_ := set $startupGate "enabled" false -}} -{{- $_ := set $migration "enabled" false -}} +{{- $_ := set $ordering "enabled" false -}} {{- end -}} -{{- if $startupGate.enabled -}} +{{- if $ordering.enabled -}} {{- include "braintrust.brainstoreStartupGate.validateApi" (dict "api" $api) -}} {{- end -}} {{- $customCA := $api.customCA -}} @@ -49,8 +47,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if or $startupGate.enabled $migration.enabled }} - progressDeadlineSeconds: {{ add 300 (ternary (int $startupGate.timeoutSeconds) 0 $startupGate.enabled) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + {{- if $ordering.enabled }} + progressDeadlineSeconds: {{ add 300 (mul 2 (int $ordering.timeoutSeconds)) }} {{- end }} {{- if not (dig "autoscaling" "enabled" false $api) }} replicas: {{ $api.replicas }} @@ -72,7 +70,7 @@ spec: {{- end }} annotations: checksum/config: {{ include (print $root.Template.BasePath "/api-configmap.yaml") $root | sha256sum }} - {{- if $startupGate.enabled }} + {{- if $ordering.enabled }} checksum/brainstore-startup-gate: {{ $root.Files.Get "files/brainstore-startup-gate.py" | sha256sum }} {{- end }} {{- if and (eq $root.Values.cloud "google") $api.enableGcsAuth }} @@ -83,8 +81,8 @@ spec: {{- end }} spec: serviceAccountName: {{ $api.serviceAccount.name }} - {{- if or $startupGate.enabled $migration.enabled }} - # Only the init container mounts the projected Kubernetes API credential. + {{- if $ordering.enabled }} + # Only the init containers mount the projected Kubernetes API credential. automountServiceAccountToken: false {{- end }} {{- with $api.podSecurityContext }} @@ -112,49 +110,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if or $startupGate.enabled $migration.enabled }} + {{- if $ordering.enabled }} initContainers: - {{- if $migration.enabled }} {{- include "braintrust.migrations.initContainer" $root | nindent 8 }} - {{- end }} - {{- if $startupGate.enabled }} - - name: wait-for-brainstore - image: {{ $startupGate.image | quote }} - imagePullPolicy: IfNotPresent - command: ["python3", "-B", "-u", "/opt/brainstore-gate/gate.py"] - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - resources: - {{- toYaml $startupGate.resources | nindent 12 }} - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: BRAINSTORE_MINIMUM_VERSION - value: {{ include "braintrust.brainstoreStartupGate.minimumVersion" (dict "gate" $startupGate "api" $api) | quote }} - - name: BRAINSTORE_DEPLOYMENTS - value: {{ include "braintrust.brainstoreStartupGate.targets" $root | quote }} - - name: BRAINSTORE_GATE_TIMEOUT_SECONDS - value: {{ $startupGate.timeoutSeconds | quote }} - - name: BRAINSTORE_GATE_POLL_SECONDS - value: {{ $startupGate.pollIntervalSeconds | quote }} - volumeMounts: - - name: brainstore-gate-code - mountPath: /opt/brainstore-gate - readOnly: true - - name: brainstore-gate-credentials - mountPath: /var/run/brainstore-gate - readOnly: true - {{- end }} + {{- include "braintrust.brainstoreStartupGate.initContainer" (dict "root" $root "api" $api) | nindent 8 }} {{- end }} containers: - name: api @@ -269,26 +228,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if or $migration.enabled $startupGate.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} - {{- if $migration.enabled }} + {{- if or $ordering.enabled $api.tmpVolume.enabled (and (eq $root.Values.cloud "azure") $root.Values.azure.enableAzureKeyVaultDriver) $customCA.enabled $api.extraVolumes }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" $root | nindent 8 }} {{- end }} - {{- if $startupGate.enabled }} - - name: brainstore-gate-code - configMap: - name: {{ include "braintrust.brainstoreStartupGate.name" $root }} - - name: brainstore-gate-credentials - projected: - sources: - - serviceAccountToken: - path: token - expirationSeconds: 3600 - - configMap: - name: kube-root-ca.crt - items: - - key: ca.crt - path: ca.crt - {{- end }} {{- if $api.tmpVolume.enabled }} - name: tmp-volume emptyDir: diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index 5f9e32d..c5a2a04 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -1,35 +1,10 @@ -{{/* Shared configuration. Keep defaults for helm upgrade --reuse-values. */}} -{{- define "braintrust.brainstoreStartupGate.config" -}} -{{- $defaults := dict - "enabled" true - "minimumVersion" "" - "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" - "timeoutSeconds" 1200 - "pollIntervalSeconds" 15 - "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) --}} -{{- $gate := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.api.brainstoreStartupGate | default dict)) -}} -{{- if $gate.enabled -}} -{{- $_ := required "api.brainstoreStartupGate.image is required" $gate.image -}} -{{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} -{{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $gate $field))) -}} -{{- fail "api.brainstoreStartupGate timing values must be positive integers" -}} -{{- end -}} -{{- end -}} -{{- if ge (int $gate.pollIntervalSeconds) (int $gate.timeoutSeconds) -}} -{{- fail "api.brainstoreStartupGate timeoutSeconds must exceed pollIntervalSeconds for two observations" -}} -{{- end -}} -{{- end -}} -{{- toYaml $gate -}} -{{- end -}} - {{/* The floor belongs to the API, not the Brainstore image being requested. An API-only image bump must not silently accept the still-old Brainstore fleet. */}} {{- define "braintrust.brainstoreStartupGate.minimumVersion" -}} {{- $minimum := .gate.minimumVersion | default (first (splitList "@" .api.image.tag)) -}} {{- if not (regexMatch "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" $minimum) -}} -{{- fail "api.brainstoreStartupGate requires a stable API release tag or an explicit stable minimumVersion" -}} +{{- fail "deploymentOrdering requires a stable API release tag or an explicit stable minimumVersion" -}} {{- end -}} {{- trimPrefix "v" $minimum -}} {{- end -}} @@ -40,8 +15,7 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- define "braintrust.brainstoreStartupGate.targets" -}} {{- $targets := list -}} -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} -{{- $revision := ternary (int .Release.Revision) 0 $migration.enabled -}} +{{- $revision := int .Release.Revision -}} {{- range $role := list "reader" "writer" "fastreader" -}} {{- $config := index $.Values.brainstore $role -}} {{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role) "minimumRevision" $revision) -}} @@ -56,10 +30,49 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- define "braintrust.brainstoreStartupGate.validateApi" -}} {{- $rolling := .api.strategy.rollingUpdate | default dict -}} {{- if or (ne .api.strategy.type "RollingUpdate") (not (has (toString $rolling.maxUnavailable) (list "0" "0%"))) -}} -{{- fail "api.brainstoreStartupGate requires RollingUpdate with maxUnavailable: 0 for every API pool" -}} +{{- fail "deploymentOrdering requires RollingUpdate with maxUnavailable: 0 for every API pool" -}} {{- end -}} {{- if has (toString $rolling.maxSurge) (list "0" "0%" "") -}} -{{- fail "api.brainstoreStartupGate requires nonzero maxSurge for every API pool" -}} +{{- fail "deploymentOrdering requires nonzero maxSurge for every API pool" -}} {{- end -}} {{- end -}} +{{- define "braintrust.brainstoreStartupGate.initContainer" -}} +{{- $config := include "braintrust.deploymentOrdering.config" .root | fromYaml -}} +- name: wait-for-brainstore + image: {{ $config.image | quote }} + imagePullPolicy: IfNotPresent + command: ["python3", "-B", "-u", "/opt/migration-gate/brainstore_gate.py"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + {{- toYaml $config.resources | nindent 4 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: BRAINSTORE_MINIMUM_VERSION + value: {{ include "braintrust.brainstoreStartupGate.minimumVersion" (dict "gate" $config "api" .api) | quote }} + - name: BRAINSTORE_DEPLOYMENTS + value: {{ include "braintrust.brainstoreStartupGate.targets" .root | quote }} + - name: BRAINSTORE_GATE_TIMEOUT_SECONDS + value: {{ $config.timeoutSeconds | quote }} + - name: BRAINSTORE_GATE_POLL_SECONDS + value: {{ $config.pollIntervalSeconds | quote }} + volumeMounts: + - name: migration-gate-code + mountPath: /opt/migration-gate + readOnly: true + - name: migration-gate-credentials + mountPath: /var/run/brainstore-gate + readOnly: true +{{- end -}} diff --git a/braintrust/templates/_migrations.tpl b/braintrust/templates/_deployment-ordering.tpl similarity index 84% rename from braintrust/templates/_migrations.tpl rename to braintrust/templates/_deployment-ordering.tpl index 0e1e7a8..2e7ef68 100644 --- a/braintrust/templates/_migrations.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -1,26 +1,27 @@ {{/* Defaults also cover pre-feature helm upgrade --reuse-values. */}} -{{- define "braintrust.migrations.config" -}} +{{- define "braintrust.deploymentOrdering.config" -}} {{- $defaults := dict "enabled" true + "minimumVersion" "" "timeoutSeconds" 1200 "pollIntervalSeconds" 5 "backoffLimit" 2 "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) -}} -{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.migrations | default dict)) -}} +{{- $config := mergeOverwrite (deepCopy $defaults) (deepCopy (.Values.deploymentOrdering | default dict)) -}} {{- if $config.enabled -}} -{{- $_ := required "migrations.image is required" $config.image -}} +{{- $_ := required "deploymentOrdering.image is required" $config.image -}} {{- range $field := list "timeoutSeconds" "pollIntervalSeconds" -}} {{- if not (regexMatch "^[1-9][0-9]*$" (toString (index $config $field))) -}} -{{- fail "migrations timing values must be positive integers" -}} +{{- fail "deploymentOrdering timing values must be positive integers" -}} {{- end -}} {{- end -}} {{- if ge (int $config.pollIntervalSeconds) (int $config.timeoutSeconds) -}} -{{- fail "migrations.timeoutSeconds must exceed pollIntervalSeconds" -}} +{{- fail "deploymentOrdering.timeoutSeconds must exceed pollIntervalSeconds" -}} {{- end -}} {{- if not (regexMatch "^(0|[1-9][0-9]*)$" (toString $config.backoffLimit)) -}} -{{- fail "migrations.backoffLimit must be a nonnegative integer" -}} +{{- fail "deploymentOrdering.backoffLimit must be a nonnegative integer" -}} {{- end -}} {{- end -}} {{- toYaml $config -}} @@ -36,7 +37,7 @@ {{- end -}} {{- define "braintrust.migrations.initContainer" -}} -{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} - name: wait-for-migrations image: {{ $config.image | quote }} imagePullPolicy: IfNotPresent diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index cf21076..71c4f48 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -297,8 +297,8 @@ and normal writable-layer/log overhead. {{- $_ := set $annotations "iam.gke.io/gcp-service-account" (required "brainstore.serviceAccount.googleServiceAccount is required when cloud is google" $root.Values.brainstore.serviceAccount.googleServiceAccount) -}} {{- end -}} {{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} -{{- $migration := include "braintrust.migrations.config" $root | fromYaml -}} -{{- if $migration.enabled -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" $root | fromYaml -}} +{{- if $ordering.enabled -}} {{/* Release identity rejects a previous Ready generation before Helm updates it. */}} {{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} {{- end -}} diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index 7383a21..22b8148 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if .Values.aiGateway.enabled }} --- apiVersion: apps/v1 @@ -15,8 +15,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if $migration.enabled }} - progressDeadlineSeconds: {{ add 600 (int $migration.timeoutSeconds) }} + {{- if $ordering.enabled }} + progressDeadlineSeconds: {{ add 600 (mul 2 (int $ordering.timeoutSeconds)) }} {{- end }} replicas: {{ .Values.aiGateway.replicas }} strategy: @@ -40,7 +40,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} serviceAccountName: {{ include "braintrust.migrations.gatewayAccount" . }} automountServiceAccountToken: false {{- end }} @@ -56,9 +56,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.initContainer" (dict "root" . "api" .Values.api) | nindent 8 }} {{- end }} containers: - name: ai-gateway @@ -102,12 +103,12 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} {{- if .Values.aiGateway.extraVolumes }} {{- toYaml .Values.aiGateway.extraVolumes | nindent 8 }} - {{- else if not $migration.enabled }} + {{- else if not $ordering.enabled }} [] {{- end }} {{- end }} diff --git a/braintrust/templates/brainstore-automationwriter-deployment.yaml b/braintrust/templates/brainstore-automationwriter-deployment.yaml index 85b195e..4317255 100644 --- a/braintrust/templates/brainstore-automationwriter-deployment.yaml +++ b/braintrust/templates/brainstore-automationwriter-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{/* Omit this Deployment when brainstore.automationwriter is absent. helm upgrade --reuse-values from a chart that predates the pool does not merge the new @@ -24,7 +24,7 @@ metadata: spec: replicas: {{ $aw.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $ordering.timeoutSeconds) 0 $ordering.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -74,7 +74,7 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} @@ -175,7 +175,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 82d040e..0049fb5 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.fastreader) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.fastreader" "config" $rollout) }} apiVersion: apps/v1 @@ -17,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.fastreader.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $ordering.timeoutSeconds) 0 $ordering.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -67,7 +67,7 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} @@ -160,7 +160,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index 99343fe..2b984a3 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.reader) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.reader" "config" $rollout) }} apiVersion: apps/v1 @@ -17,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.reader.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $ordering.timeoutSeconds) 0 $ordering.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -67,7 +67,7 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} @@ -160,7 +160,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume diff --git a/braintrust/templates/brainstore-startup-gate-configmap.yaml b/braintrust/templates/brainstore-startup-gate-configmap.yaml deleted file mode 100644 index bd61aa8..0000000 --- a/braintrust/templates/brainstore-startup-gate-configmap.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} -{{- if $gate.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "braintrust.brainstoreStartupGate.name" . }} - namespace: {{ include "braintrust.namespace" . }} - {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.api.labels)) }} - labels: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - gate.py: | - {{- .Files.Get "files/brainstore-startup-gate.py" | nindent 4 }} -{{- end }} diff --git a/braintrust/templates/brainstore-startup-gate-rbac.yaml b/braintrust/templates/brainstore-startup-gate-rbac.yaml index c44d3ff..99b560c 100644 --- a/braintrust/templates/brainstore-startup-gate-rbac.yaml +++ b/braintrust/templates/brainstore-startup-gate-rbac.yaml @@ -1,4 +1,4 @@ -{{- $gate := include "braintrust.brainstoreStartupGate.config" . | fromYaml -}} +{{- $gate := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $gate.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -39,6 +39,12 @@ subjects: {{- range $pool := include "braintrust.apiPools" . | fromYamlArray -}} {{- $_ := set $accounts $pool.config.serviceAccount.name true -}} {{- end -}} + {{- if .Values.aiGateway.enabled -}} + {{- $_ := set $accounts (include "braintrust.migrations.gatewayAccount" .) true -}} + {{- end -}} + {{- if include "braintrust.loopRuntime.enabled" . -}} + {{- $_ := set $accounts .Values.loopRuntime.serviceAccount.name true -}} + {{- end -}} {{- range $name := keys $accounts | sortAlpha }} - kind: ServiceAccount name: {{ $name }} diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index 0def40d..d3c6abf 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.writer) | fromYaml -}} {{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.writer" "config" $rollout) }} apiVersion: apps/v1 @@ -17,7 +17,7 @@ metadata: spec: replicas: {{ .Values.brainstore.writer.replicas }} minReadySeconds: {{ $rollout.minReadySeconds }} - progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $migration.timeoutSeconds) 0 $migration.enabled) }} + progressDeadlineSeconds: {{ add (int $rollout.progressDeadlineSeconds) (ternary (int $ordering.timeoutSeconds) 0 $ordering.enabled) }} strategy: type: {{ $rollout.strategy.type }} {{- if eq $rollout.strategy.type "RollingUpdate" }} @@ -67,7 +67,7 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} {{- end }} @@ -160,7 +160,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: cache-volume diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index f0bde32..659ad4c 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -1,4 +1,4 @@ -{{- $migration := include "braintrust.migrations.config" . | fromYaml -}} +{{- $ordering := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} {{- $autoscaling := include "braintrust.loopRuntime.autoscaling" . | fromYaml -}} @@ -17,8 +17,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if $migration.enabled }} - progressDeadlineSeconds: {{ add 600 (int $migration.timeoutSeconds) }} + {{- if $ordering.enabled }} + progressDeadlineSeconds: {{ add 600 (mul 2 (int $ordering.timeoutSeconds)) }} {{- end }} {{- if not $autoscaling }} replicas: {{ ternary 1 .Values.loopRuntime.replicas (has .Values.loopRuntime.replicas (list nil "")) }} @@ -44,6 +44,9 @@ spec: {{- end }} spec: serviceAccountName: {{ .Values.loopRuntime.serviceAccount.name }} + {{- if $ordering.enabled }} + automountServiceAccountToken: false + {{- end }} # The runtime drains active turns for up to 90 seconds after SIGTERM. terminationGracePeriodSeconds: 120 securityContext: @@ -65,9 +68,10 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if $migration.enabled }} + {{- if $ordering.enabled }} initContainers: {{- include "braintrust.migrations.initContainer" . | nindent 8 }} + {{- include "braintrust.brainstoreStartupGate.initContainer" (dict "root" . "api" .Values.api) | nindent 8 }} {{- end }} containers: - name: loop-runtime @@ -156,7 +160,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if $migration.enabled }} + {{- if $ordering.enabled }} {{- include "braintrust.migrations.volumes" . | nindent 8 }} {{- end }} - name: state diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml index 5ecb048..f012158 100644 --- a/braintrust/templates/migration-gate.yaml +++ b/braintrust/templates/migration-gate.yaml @@ -1,4 +1,4 @@ -{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} apiVersion: v1 diff --git a/braintrust/templates/migration-job.yaml b/braintrust/templates/migration-job.yaml index 5aee29a..4291ac5 100644 --- a/braintrust/templates/migration-job.yaml +++ b/braintrust/templates/migration-job.yaml @@ -1,9 +1,9 @@ -{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} {{- $api := include "braintrust.apiPoolConfig" (dict "root" . "overrides" (dict)) | fromYaml -}} {{/* Reuse the API pod contract so identity, TLS, secrets, security and scheduling -stay in sync. The migrationJob flag suppresses both startup gates. Only the main +stay in sync. The migrationJob flag suppresses release ordering. Only the main container is retained: sidecars and HTTP probes must not keep this Job alive. */}} {{- $deployment := include "braintrust.apiDeployment" (dict "root" . "api" $api "role" "migration" "migrationJob" true) | fromYaml -}} {{- $pod := $deployment.spec.template -}} diff --git a/braintrust/templates/migration-rollback.yaml b/braintrust/templates/migration-rollback.yaml index 30b246c..20e536d 100644 --- a/braintrust/templates/migration-rollback.yaml +++ b/braintrust/templates/migration-rollback.yaml @@ -1,4 +1,4 @@ -{{- $config := include "braintrust.migrations.config" . | fromYaml -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} {{- $hook := printf "%s-reset" $name -}} diff --git a/braintrust/tests/brainstore-automationwriter-routing_test.yaml b/braintrust/tests/brainstore-automationwriter-routing_test.yaml index 76428e3..cbdfe87 100644 --- a/braintrust/tests/brainstore-automationwriter-routing_test.yaml +++ b/braintrust/tests/brainstore-automationwriter-routing_test.yaml @@ -3,9 +3,6 @@ templates: - api-configmap.yaml - api-deployment.yaml - brainstore-startup-gate-rbac.yaml -# Existing workload behavior without the independent migration gate. -set: - migrations.enabled: false tests: - it: should preserve writer fallback when the pool is disabled template: api-configmap.yaml @@ -126,7 +123,7 @@ tests: template: api-deployment.yaml set: <<: *enabled - api.brainstoreStartupGate.enabled: false + deploymentOrdering.enabled: false asserts: - isNull: path: spec.template.spec.initContainers diff --git a/braintrust/tests/brainstore-automationwriter_test.yaml b/braintrust/tests/brainstore-automationwriter_test.yaml index 734b686..d09f389 100644 --- a/braintrust/tests/brainstore-automationwriter_test.yaml +++ b/braintrust/tests/brainstore-automationwriter_test.yaml @@ -2,9 +2,9 @@ suite: test Brainstore Automation Writer deployment template templates: - brainstore-automationwriter-deployment.yaml - brainstore-automationwriter-configmap.yaml -# Existing workload behavior without the independent migration gate. +# Test workload details independently of release ordering. set: - migrations.enabled: false + deploymentOrdering.enabled: false tests: - it: should render Brainstore Automation Writer deployment with default values template: brainstore-automationwriter-deployment.yaml diff --git a/braintrust/tests/brainstore-fastreader_test.yaml b/braintrust/tests/brainstore-fastreader_test.yaml index d4fbf4b..ff915a5 100644 --- a/braintrust/tests/brainstore-fastreader_test.yaml +++ b/braintrust/tests/brainstore-fastreader_test.yaml @@ -2,9 +2,9 @@ suite: test Brainstore Fast Reader deployment template templates: - brainstore-fastreader-deployment.yaml - brainstore-fastreader-configmap.yaml -# Existing workload behavior without the independent migration gate. +# Test workload details independently of release ordering. set: - migrations.enabled: false + deploymentOrdering.enabled: false tests: - it: should render Brainstore Fast Reader deployment with default values template: brainstore-fastreader-deployment.yaml diff --git a/braintrust/tests/brainstore-reader_test.yaml b/braintrust/tests/brainstore-reader_test.yaml index cea2e21..c1d1747 100644 --- a/braintrust/tests/brainstore-reader_test.yaml +++ b/braintrust/tests/brainstore-reader_test.yaml @@ -2,9 +2,9 @@ suite: test Brainstore Reader deployment template templates: - brainstore-reader-deployment.yaml - brainstore-reader-configmap.yaml -# Existing workload behavior without the independent migration gate. +# Test workload details independently of release ordering. set: - migrations.enabled: false + deploymentOrdering.enabled: false tests: - it: should render Brainstore Reader deployment with default values template: brainstore-reader-deployment.yaml diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index 8d406da..5050890 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -2,127 +2,42 @@ suite: test Brainstore API startup gate templates: - api-deployment.yaml - api-configmap.yaml - - brainstore-startup-gate-configmap.yaml - brainstore-startup-gate-rbac.yaml - api-serviceaccount.yaml - brainstore-reader-deployment.yaml - brainstore-reader-configmap.yaml -# Existing workload behavior without the independent migration gate. -set: - migrations.enabled: false tests: - - it: should gate API startup by default - template: api-deployment.yaml - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore - - isNotNull: - path: spec.template.metadata.annotations["checksum/brainstore-startup-gate"] - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - it: should leave API startup unchanged when explicitly disabled template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: false + deploymentOrdering.enabled: false asserts: - isNull: path: spec.template.spec.initContainers - - isNull: - path: spec.template.metadata.annotations["checksum/brainstore-startup-gate"] - isNull: path: spec.template.spec.automountServiceAccountToken - it: should not create gate resources when disabled template: brainstore-startup-gate-rbac.yaml set: - api.brainstoreStartupGate.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should not create the helper ConfigMap when disabled - template: brainstore-startup-gate-configmap.yaml - set: - api.brainstoreStartupGate.enabled: false + deploymentOrdering.enabled: false asserts: - hasDocuments: count: 0 - - it: should gate reused values that do not include the new setting - template: api-deployment.yaml - set: - api.brainstoreStartupGate: null - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore - - - it: should keep the release floor in the API Pod environment - template: api-deployment.yaml - set: - api.brainstoreStartupGate.enabled: true - api.image.tag: v2.15.0 - brainstore.image.tag: v2.15.0 - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore - - equal: - path: spec.progressDeadlineSeconds - value: 1500 - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - contains: - path: spec.template.spec.initContainers[0].env - content: - name: BRAINSTORE_MINIMUM_VERSION - value: "2.15.0" - - contains: - path: spec.template.spec.initContainers[0].env - content: - name: BRAINSTORE_GATE_TIMEOUT_SECONDS - value: "1200" - - contains: - path: spec.template.spec.initContainers[0].env - content: - name: BRAINSTORE_GATE_POLL_SECONDS - value: "15" - - equal: - path: spec.template.spec.initContainers[0].env[0] - value: - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - equal: - path: spec.template.spec.initContainers[0].securityContext.runAsNonRoot - value: true - - equal: - path: spec.template.spec.initContainers[0].securityContext.readOnlyRootFilesystem - value: true - - equal: - path: spec.template.spec.volumes[1].projected.sources[0].serviceAccountToken.expirationSeconds - value: 3600 - - isNotNull: - path: spec.template.metadata.annotations["checksum/brainstore-startup-gate"] - - it: should allow an explicit minimum floor and mirrored helper image template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true - api.brainstoreStartupGate.minimumVersion: v2.15.0 - api.brainstoreStartupGate.image: registry.example/python:3.13 + deploymentOrdering.enabled: true + deploymentOrdering.minimumVersion: v2.15.0 + deploymentOrdering.image: registry.example/python:3.13 brainstore.image.tag: v2.15.1 asserts: - equal: - path: spec.template.spec.initContainers[0].image + path: spec.template.spec.initContainers[1].image value: registry.example/python:3.13 - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" @@ -130,7 +45,7 @@ tests: - it: should mount gate credentials only into the init container template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.tmpVolume.enabled: true api.extraContainers: - name: metrics-sidecar @@ -143,9 +58,9 @@ tests: path: spec.template.spec.automountServiceAccountToken value: false - contains: - path: spec.template.spec.initContainers[0].volumeMounts + path: spec.template.spec.initContainers[1].volumeMounts content: - name: brainstore-gate-credentials + name: migration-gate-credentials mountPath: /var/run/brainstore-gate readOnly: true - equal: @@ -162,48 +77,30 @@ tests: - it: should infer the API floor from a stable tag with a digest template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.image.tag: v2.15.0@sha256:abc asserts: - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" - - it: should gate each API pool even when a pool override tries to disable it - template: api-deployment.yaml - set: - api.brainstoreStartupGate.enabled: true - api.workloadIsolation.enabled: true - api.workloadIsolation.background.brainstoreStartupGate.enabled: false - documentIndex: 2 - asserts: - - equal: - path: metadata.name - value: braintrust-api-background - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-brainstore - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - it: should gate the ingest pool template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.workloadIsolation.enabled: true documentIndex: 1 asserts: - equal: - path: spec.template.spec.initContainers[0].name + path: spec.template.spec.initContainers[1].name value: wait-for-brainstore - it: should grant only named deployment reads and namespace pod lists template: brainstore-startup-gate-rbac.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true brainstore.reader.name: custom-reader documentIndex: 0 asserts: @@ -226,7 +123,7 @@ tests: name: gate-test namespace: install-namespace set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true global.createNamespace: true global.namespace: workload-namespace api.workloadIsolation.enabled: true @@ -246,32 +143,24 @@ tests: name: braintrust-api namespace: workload-namespace - - it: should mount helper code without changing Brainstore startup - template: brainstore-reader-deployment.yaml - set: - api.brainstoreStartupGate.enabled: true - asserts: - - isNull: - path: spec.template.spec.initContainers - - it: should reject opaque API tags when no minimum release can be inferred template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.image.tag: deadbeef asserts: - failedTemplate: - errorMessage: api.brainstoreStartupGate requires a stable API release tag or an explicit stable minimumVersion + errorMessage: deploymentOrdering requires a stable API release tag or an explicit stable minimumVersion - it: should protect an API-only image bump while Brainstore is still old template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.image.tag: v2.15.0 brainstore.image.tag: v2.14.0 asserts: - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" @@ -279,89 +168,71 @@ tests: - it: should derive the release floor independently for overridden API pools template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.workloadIsolation.enabled: true api.image.tag: v2.14.0 api.workloadIsolation.background.image.tag: v2.15.0 documentIndex: 2 asserts: - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" - - it: should reject invalid timing - template: brainstore-startup-gate-rbac.yaml - set: - api.brainstoreStartupGate.enabled: true - api.brainstoreStartupGate.timeoutSeconds: 0 - asserts: - - failedTemplate: - errorMessage: api.brainstoreStartupGate timing values must be positive integers - - it: should pass timing overrides to the checker and extend the progress deadline template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true - api.brainstoreStartupGate.timeoutSeconds: 1800 - api.brainstoreStartupGate.pollIntervalSeconds: 30 + deploymentOrdering.enabled: true + deploymentOrdering.timeoutSeconds: 1800 + deploymentOrdering.pollIntervalSeconds: 30 asserts: - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_GATE_TIMEOUT_SECONDS value: "1800" - contains: - path: spec.template.spec.initContainers[0].env + path: spec.template.spec.initContainers[1].env content: name: BRAINSTORE_GATE_POLL_SECONDS value: "30" - equal: path: spec.progressDeadlineSeconds - value: 2100 - - - it: should reject an insufficient confirmation budget - template: brainstore-startup-gate-rbac.yaml - set: - api.brainstoreStartupGate.enabled: true - api.brainstoreStartupGate.timeoutSeconds: 15 - asserts: - - failedTemplate: - errorMessage: api.brainstoreStartupGate timeoutSeconds must exceed pollIntervalSeconds for two observations + value: 3900 - it: should reject an API rollout that removes old serving capacity template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.strategy.rollingUpdate.maxUnavailable: 1 asserts: - failedTemplate: - errorMessage: "api.brainstoreStartupGate requires RollingUpdate with maxUnavailable: 0 for every API pool" + errorMessage: "deploymentOrdering requires RollingUpdate with maxUnavailable: 0 for every API pool" - it: should reject unsafe strategy overrides in isolated pools template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.workloadIsolation.enabled: true api.workloadIsolation.background.strategy.type: Recreate asserts: - failedTemplate: - errorMessage: "api.brainstoreStartupGate requires RollingUpdate with maxUnavailable: 0 for every API pool" + errorMessage: "deploymentOrdering requires RollingUpdate with maxUnavailable: 0 for every API pool" - it: should reject a rollout with no room for gated replacement pods template: api-deployment.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true api.strategy.rollingUpdate.maxSurge: 0 asserts: - failedTemplate: - errorMessage: api.brainstoreStartupGate requires nonzero maxSurge for every API pool + errorMessage: deploymentOrdering requires nonzero maxSurge for every API pool - - it: should not leak gate labels into Brainstore or mutate global labels - template: brainstore-startup-gate-configmap.yaml + - it: should preserve service account labels after gate rendering + template: api-serviceaccount.yaml set: - api.brainstoreStartupGate.enabled: true + deploymentOrdering.enabled: true global.labels.shared: global api.labels.api-only: api brainstore.reader.labels.reader-only: reader @@ -369,22 +240,23 @@ tests: - equal: path: metadata.labels.shared value: global - - equal: - path: metadata.labels["api-only"] - value: api - isNull: path: metadata.labels["reader-only"] - - it: should preserve service account labels after gate rendering - template: api-serviceaccount.yaml + - it: binds optional application identities for direct rollout reads + template: brainstore-startup-gate-rbac.yaml + documentIndex: 1 + values: + - __fixtures__/loop-runtime-values.yaml + release: + name: test + namespace: workload set: - api.brainstoreStartupGate.enabled: true - global.labels.shared: global - api.labels.api-only: api - brainstore.reader.labels.reader-only: reader + aiGateway.enabled: true asserts: - - equal: - path: metadata.labels.shared - value: global - - isNull: - path: metadata.labels["reader-only"] + - contains: + path: subjects + content: {kind: ServiceAccount, name: test-migration-gate, namespace: workload} + - contains: + path: subjects + content: {kind: ServiceAccount, name: braintrust-loop-runtime, namespace: workload} diff --git a/braintrust/tests/brainstore-writer_test.yaml b/braintrust/tests/brainstore-writer_test.yaml index b0b66ca..8a8c82a 100644 --- a/braintrust/tests/brainstore-writer_test.yaml +++ b/braintrust/tests/brainstore-writer_test.yaml @@ -2,9 +2,9 @@ suite: test Brainstore Writer deployment template templates: - brainstore-writer-deployment.yaml - brainstore-writer-configmap.yaml -# Existing workload behavior without the independent migration gate. +# Test workload details independently of release ordering. set: - migrations.enabled: false + deploymentOrdering.enabled: false tests: - it: should render Brainstore Writer deployment with default values template: brainstore-writer-deployment.yaml diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index 5e5f4f1..249d030 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -334,6 +334,8 @@ tests: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - {"equal": {"path": "spec.template.spec.serviceAccountName", "value": "test-migration-gate"}} - {"equal": {"path": "spec.template.spec.automountServiceAccountToken", "value": false}} + - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 3000}} - it: gates the optional Loop runtime values: @@ -342,6 +344,8 @@ tests: set: {"cloud": "google", "loopRuntime.enabled": true, "loopRuntime.serviceAccount.googleServiceAccount": "loop@example.iam.gserviceaccount.com", "brainstore.serviceAccount.googleServiceAccount": "bs@example.iam.gserviceaccount.com"} asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} + - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 3000}} - it: binds the Loop runtime identity values: @@ -354,84 +358,69 @@ tests: - it: omits migration-job.yaml when disabled template: migration-job.yaml - set: {"migrations.enabled": false} + set: {"deploymentOrdering.enabled": false} asserts: - {"hasDocuments": {"count": 0}} - it: omits migration-gate.yaml when disabled template: migration-gate.yaml - set: {"migrations.enabled": false} + set: {"deploymentOrdering.enabled": false} asserts: - {"hasDocuments": {"count": 0}} - it: removes migration wait when disabled in api-deployment.yaml template: api-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - it: removes migration wait when disabled in brainstore-reader-deployment.yaml template: brainstore-reader-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - it: removes migration wait when disabled in brainstore-writer-deployment.yaml template: brainstore-writer-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - it: removes migration wait when disabled in brainstore-fastreader-deployment.yaml template: brainstore-fastreader-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - it: removes migration wait when disabled in brainstore-automationwriter-deployment.yaml template: brainstore-automationwriter-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - it: removes migration wait when disabled in ai-gateway-deployment.yaml template: ai-gateway-deployment.yaml - set: {"migrations.enabled": false, "aiGateway.enabled": true, "api.brainstoreStartupGate.enabled": false} + set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} asserts: - {"isNull": {"path": "spec.template.spec.initContainers"}} - - it: gates APIs independently of the compatibility gate - template: api-deployment.yaml - set: {"api.brainstoreStartupGate.enabled": false} - asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"lengthEqual": {"path": "spec.template.spec.initContainers", "count": 1}} - - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1500}} - - it: defaults a missing or null feature map on reused values template: api-deployment.yaml - set: {"migrations": null} + set: {"deploymentOrdering": null} asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - it: merges partial reused values with defaults template: migration-job.yaml documentIndex: 0 - set: {"migrations": null, "migrations.backoffLimit": 0} + set: {"deploymentOrdering": null, "deploymentOrdering.backoffLimit": 0} asserts: - {"equal": {"path": "spec.backoffLimit", "value": 0}} - {"equal": {"path": "spec.activeDeadlineSeconds", "value": 1200}} - - it: allows zero retries - template: migration-job.yaml - documentIndex: 0 - set: {"migrations.backoffLimit": 0} - asserts: - - {"equal": {"path": "spec.backoffLimit", "value": 0}} - - it: uses custom timing and a mirrored helper image template: brainstore-reader-deployment.yaml - set: {"migrations.timeoutSeconds": 1800, "migrations.pollIntervalSeconds": 10, "migrations.image": "mirror/python:3.13"} + set: {"deploymentOrdering.timeoutSeconds": 1800, "deploymentOrdering.pollIntervalSeconds": 10, "deploymentOrdering.image": "mirror/python:3.13"} asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].image", "value": "mirror/python:3.13"}} - {"equal": {"path": "spec.template.spec.initContainers[0].env[2].value", "value": "1800"}} @@ -440,20 +429,21 @@ tests: - it: rejects invalid timeoutSeconds 0 template: migration-rollback.yaml - set: {"migrations.timeoutSeconds": 0} + set: {"deploymentOrdering.timeoutSeconds": 0} asserts: - - failedTemplate: {} + - failedTemplate: + errorMessage: deploymentOrdering timing values must be positive integers - it: rejects invalid timeoutSeconds template: migration-rollback.yaml - set: {"migrations.timeoutSeconds": ""} + set: {"deploymentOrdering.timeoutSeconds": ""} asserts: - failedTemplate: {} - it: defaults a null timeoutSeconds after Helm coalesces values template: migration-job.yaml documentIndex: 0 - set: {migrations.timeoutSeconds: null} + set: {deploymentOrdering.timeoutSeconds: null} asserts: - equal: path: spec.activeDeadlineSeconds @@ -461,60 +451,54 @@ tests: - it: rejects invalid timeoutSeconds -1 template: migration-rollback.yaml - set: {"migrations.timeoutSeconds": -1} + set: {"deploymentOrdering.timeoutSeconds": -1} asserts: - failedTemplate: {} - it: rejects invalid pollIntervalSeconds 0 template: migration-rollback.yaml - set: {"migrations.pollIntervalSeconds": 0} + set: {"deploymentOrdering.pollIntervalSeconds": 0} asserts: - failedTemplate: {} - it: rejects invalid pollIntervalSeconds template: migration-rollback.yaml - set: {"migrations.pollIntervalSeconds": ""} + set: {"deploymentOrdering.pollIntervalSeconds": ""} asserts: - failedTemplate: {} - - it: defaults a null pollIntervalSeconds after Helm coalesces values - template: migration-job.yaml - documentIndex: 0 - set: {migrations.pollIntervalSeconds: null} - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 1200 - - it: rejects invalid pollIntervalSeconds -1 template: migration-rollback.yaml - set: {"migrations.pollIntervalSeconds": -1} + set: {"deploymentOrdering.pollIntervalSeconds": -1} asserts: - failedTemplate: {} - it: rejects a poll interval outside the budget template: migration-rollback.yaml - set: {"migrations.pollIntervalSeconds": 1200} + set: {"deploymentOrdering.pollIntervalSeconds": 1200} asserts: - - failedTemplate: {} + - failedTemplate: + errorMessage: deploymentOrdering.timeoutSeconds must exceed pollIntervalSeconds - it: rejects an empty helper image template: migration-rollback.yaml - set: {"migrations.image": ""} + set: {"deploymentOrdering.image": ""} asserts: - - failedTemplate: {} + - failedTemplate: + errorMessage: deploymentOrdering.image is required - it: does not validate unused disabled settings template: migration-job.yaml - set: {"migrations.enabled": false, "migrations.image": "", "migrations.timeoutSeconds": 0} + set: {"deploymentOrdering.enabled": false, "deploymentOrdering.image": "", "deploymentOrdering.timeoutSeconds": 0} asserts: - {"hasDocuments": {"count": 0}} - it: rejects a negative retry count template: migration-rollback.yaml - set: {"migrations.backoffLimit": -1} + set: {"deploymentOrdering.backoffLimit": -1} asserts: - - failedTemplate: {} + - failedTemplate: + errorMessage: deploymentOrdering.backoffLimit must be a nonnegative integer - it: preserves revision suffix with long release names template: migration-job.yaml @@ -600,7 +584,7 @@ tests: - it: does not render rollback hooks when migrations are disabled template: migration-rollback.yaml set: - migrations.enabled: false + deploymentOrdering.enabled: false asserts: - hasDocuments: count: 0 @@ -622,9 +606,40 @@ tests: - it: gives the API time for migration and one direct rollout check template: api-deployment.yaml set: - migrations.timeoutSeconds: 900 - api.brainstoreStartupGate.timeoutSeconds: 300 + deploymentOrdering.timeoutSeconds: 900 asserts: - equal: path: spec.progressDeadlineSeconds - value: 1500 + value: 2100 + + - it: removes both Loop waits when ordering is disabled + template: loop-runtime-deployment.yaml + values: + - __fixtures__/loop-runtime-values.yaml + set: + deploymentOrdering.enabled: false + asserts: + - isNull: + path: spec.template.spec.initContainers + + - it: uses the shared API floor for Gateway startup + template: ai-gateway-deployment.yaml + set: + aiGateway.enabled: true + api.image.tag: v2.15.0 + asserts: + - contains: + path: spec.template.spec.initContainers[1].env + content: {name: BRAINSTORE_MINIMUM_VERSION, value: "2.15.0"} + + - it: applies the explicit shared floor to Loop startup + template: loop-runtime-deployment.yaml + values: + - __fixtures__/loop-runtime-values.yaml + set: + api.image.tag: deadbeef + deploymentOrdering.minimumVersion: v2.15.0 + asserts: + - contains: + path: spec.template.spec.initContainers[1].env + content: {name: BRAINSTORE_MINIMUM_VERSION, value: "2.15.0"} diff --git a/braintrust/values.yaml b/braintrust/values.yaml index ed645e7..e20b8ca 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -102,26 +102,6 @@ btqlAuditLogsBestEffortOrgIds: [] api: name: "braintrust-api" - # Hold new API processes until compatible Brainstore capacity is available. - # With migrations enabled, first require fully updated reader/writer Deployments. - # Requires stable release tags (vX.Y.Z); opaque custom tags cannot prove compatibility. - # Only the init container mounts a Kubernetes API token when enabled; custom - # sidecars needing Kubernetes API access must project their own token. - brainstoreStartupGate: - enabled: true - # Empty derives the floor from each pool's api.image.tag. Kept in each API Pod so - # older APIs can start after a later compatible Brainstore rollout completes. - minimumVersion: "" - image: "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" - timeoutSeconds: 1200 - pollIntervalSeconds: 15 - resources: - requests: - cpu: "25m" - memory: "32Mi" - limits: - cpu: "100m" - memory: "128Mi" labels: {} podLabels: {} annotations: @@ -821,10 +801,15 @@ virtualService: port: number: 8000 # Should match api.service.port -# Run schema migrations once per Helm revision before new application containers -# start. API startup then waits for the exact Brainstore rollout. See docs/migration-job.md. -migrations: +# Order startup as migrations -> completed Brainstore rollout -> all other apps. +# Disable to deploy everything without chart-managed startup ordering. +# See docs/migration-job.md for retention, rollback, and startup-only limits. +deploymentOrdering: enabled: true + # Empty derives the compatible Brainstore floor from each API pool's image tag. + # Opaque API tags require an explicit stable minimumVersion. + minimumVersion: "" + # Separate budget for each phase (migration Job/wait, then Brainstore rollout). timeoutSeconds: 1200 pollIntervalSeconds: 5 backoffLimit: 2 diff --git a/docs/migration-job.md b/docs/migration-job.md index 7f2e62d..ab89804 100644 --- a/docs/migration-job.md +++ b/docs/migration-job.md @@ -1,35 +1,34 @@ # Release ordering -**Migrations complete → readers/writers finish rolling out → API starts.** +**Migrations complete → Brainstore finishes rolling out → all other applications start.** -One regular migration Job runs the shared API image's migration entry point for each Helm revision. Every new application container waits for it. The existing API startup init container then checks Brainstore Deployments and Pods directly. There is no rollout Job, observer ServiceAccount, or rollout-configuration snapshot. +With `deploymentOrdering.enabled: true` (default), one regular migration Job runs the shared API image's migration entry point for each Helm revision. Brainstore waits for it; all API pools, optional AI Gateway, and optional Loop Runtime wait for migrations and then check Brainstore Deployments and Pods directly. There is no separate rollout Job or rollout-configuration snapshot. -## What the API checks +## What applications check For every rendered reader, writer, fastreader, and automationwriter: -- The Deployment pod-template release marker is this API's Helm revision or newer. +- The Deployment pod-template release marker is the waiting application's Helm revision or newer. - `status.observedGeneration >= metadata.generation`. - Updated, Ready, Available, and total replica counts equal the current desired count. - All service-selected pods belong to that current rollout and are Ready; no old or terminating live pods remain. -- Desired and running Brainstore images satisfy this API pool's version floor. +- Desired and running images in nonzero Brainstore pools satisfy the waiting application's version floor. The release marker prevents the previous fully Ready Deployment from passing before Helm updates it. Deployment generation counters are checked within each Deployment, not compared between API and Brainstore. Pod-template changes cover images and inline environment; the existing ConfigMap checksum covers chart-managed environment. Zero-replica roles must finish draining. Two successful fleet observations share one bounded deadline. -**Tradeoff:** an older API replacement also waits during a newer unfinished Brainstore rollout. It accepts a newer completed compatible release rather than waiting for its old configuration to become current again. Insufficient surge capacity can therefore block replacement APIs as well as the new release. +**Tradeoff:** older replacement API, Gateway, and Loop pods also wait during a newer unfinished Brainstore rollout. They accept a newer completed compatible release rather than waiting for their old configuration to become current again. Insufficient surge capacity can therefore block restarts as well as the new release. ## Input matrix | Input | Behavior | | --- | --- | -| Both gates enabled; missing/null maps or partial reused values | Migration Job, then direct completed-rollout and compatibility checks | -| `migrations.enabled: false` | No migration Job; previous compatible-capacity gate and API-startup migration behavior | -| `api.brainstoreStartupGate.enabled: false` | Migration-only startup ordering; both disabled means no chart-managed gates | -| Same image with changed env/config; stale controller observation; old Ready writer/new Pending writer | New API remains blocked | -| Older API, newer Brainstore release | Wait during rollout; pass after the newer compatible rollout completes | +| `deploymentOrdering.enabled: true`; missing/null map or partial reused values | Migration Job, then direct completed-rollout and compatibility checks | +| `deploymentOrdering.enabled: false` | No chart-managed Job, waits, RBAC, helper configuration, hooks, or release markers; no intermediate mode | +| Same image with changed env/config; stale controller observation; old Ready writer/new Pending writer | New API, Gateway, and Loop containers remain blocked | +| Older application, newer Brainstore release | Wait during rollout; pass after the newer compatible rollout completes | | Replica count missing/null/zero; optional automationwriter absent/partial | Preserve Kubernetes/default pool behavior; zero roles drain, absent pools stay omitted | | Custom role names, namespaces, API pools | Checks and read-only permissions follow the rendered workload names and actual namespace | -| Release tags with/without v, SHA/numeric/custom/prerelease/digest API tags | Existing image handling is unchanged; opaque API tags require an explicit stable minimumVersion | +| Stable API release tags with/without v or digest; SHA/numeric/custom/prerelease tags | Stable tags derive the floor; opaque API tags require explicit stable `deploymentOrdering.minimumVersion` | | AWS/Google/Azure, Azure CSI, CA mounts, custom scheduling/security | Migrator inherits the shared API contract and independently mounts its required secrets; startup checks only read Kubernetes state | | Missing/null timing or invalid enabled timing/image | Existing defaults apply; invalid enabled values fail rendering, unused disabled settings are ignored | | Failed/missing migration Job, API errors, old/stray/terminating pods, readiness dwell | Fail closed within the configured budgets; no missing resource implies success | @@ -59,6 +58,8 @@ Run this only after uninstall, never against records needed by an active release These gates order **application-container startup**, not Kubernetes object updates. Helm still submits Deployment specs and ConfigMaps together. Mutable mounted configuration, externally updated Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migrations and versioned configuration if no object or old workload may observe changes before migrations finish. Perform a Helm upgrade after an external Secret change to roll pods; secret values are not snapshotted. -Use real Helm revisions, not repeated `helm template | kubectl apply`. Pool-specific API overrides must remain schema-compatible with the shared migrator image/database. Existing rollout strategies determine availability; strict startup ordering does not provide spare nodes or guarantee cache warm-up beyond configured readiness and `minReadySeconds`. +Use real Helm revisions. Pool-specific API overrides must remain schema-compatible with the shared migrator image/database. Existing rollout strategies determine availability; strict startup ordering does not provide spare nodes or guarantee cache warm-up beyond configured readiness and `minReadySeconds`. -`migrations.timeoutSeconds` bounds migration waiting. `api.brainstoreStartupGate.timeoutSeconds` bounds direct rollout observation. API progress deadlines include both budgets plus startup margin. Use sufficient Helm timeout and `--wait --wait-for-jobs`; otherwise Helm can return before migrations or rollout complete. Inspect API `wait-for-brainstore` init logs for the role and condition holding startup. +`deploymentOrdering.timeoutSeconds` (default 1200) bounds each migration and rollout-wait phase separately; `pollIntervalSeconds` (default 5) must be smaller. `minimumVersion` is an optional shared stable floor; when empty each API pool derives its own image floor, and Gateway/Loop use the shared API image. `image`, `resources`, and `backoffLimit` configure the helpers and migration retry budget. API progress deadlines include two phase budgets plus 300 seconds; Gateway/Loop include two plus 600; Brainstore adds one phase budget to its configured rollout deadline. + +Use a sufficient Helm timeout and `--wait --wait-for-jobs`; otherwise Helm can return before migrations or rollout complete. Inspect `wait-for-migrations` and `wait-for-brainstore` init logs for the condition holding startup. diff --git a/tests/test_brainstore_startup_gate.py b/tests/test_brainstore_startup_gate.py index 6ce0b2f..e907319 100644 --- a/tests/test_brainstore_startup_gate.py +++ b/tests/test_brainstore_startup_gate.py @@ -14,7 +14,7 @@ spec.loader.exec_module(gate) -def pod(version="v2.15.0", uid="new", is_ready=True, revision=None, +def pod(version="v2.15.0", uid="new", is_ready=True, revision=8, container="brainstore-reader", **metadata): if revision is not None: metadata["annotations"] = {"braintrust.dev/release-revision": str(revision)} @@ -30,7 +30,7 @@ def pod(version="v2.15.0", uid="new", is_ready=True, revision=None, } -def deployment(version="v2.15.0", replicas=1, revision=None, container="brainstore-reader"): +def deployment(version="v2.15.0", replicas=1, revision=8, container="brainstore-reader"): return { "metadata": {"name": "reader", "uid": "reader-deployment", "generation": 2}, "spec": {"replicas": replicas, "template": { @@ -42,74 +42,33 @@ def deployment(version="v2.15.0", replicas=1, revision=None, container="brainsto } -class CompatibilityTests(unittest.TestCase): +class ImageCompatibilityTests(unittest.TestCase): def check(self, resource=None, pods=None, minimum="2.15.0"): - return gate.check_deployment(resource or deployment(), pods if pods is not None else [pod()], - "brainstore-reader", gate.release_version(minimum)) + return gate.check_deployment( + resource if resource is not None else deployment(), + pods if pods is not None else [pod()], + "brainstore-reader", gate.release_version(minimum), 8, + ) - def test_new_api_waits_when_helm_has_not_updated_brainstore_spec(self): - self.assertIn("desired image", self.check(deployment("v2.14.0"), [pod("v2.14.0")])) + def test_new_application_waits_for_compatible_desired_image(self): + self.assertIsNotNone(self.check(deployment("v2.14.0"))) - def test_new_api_waits_for_incompatible_old_pods(self): - self.assertIn("incompatible Pod old", self.check(pods=[pod(), pod("v2.14.0", "old")])) + def test_new_application_waits_for_incompatible_live_pods(self): + self.assertIsNotNone(self.check(pods=[pod(), pod("v2.14.0", "old")])) def test_incompatible_terminating_pod_blocks_even_when_unready(self): old = pod("v2.14.0", "old", False, deletionTimestamp="2026-09-30T12:00:00Z") - self.assertIn("incompatible Pod old", self.check(pods=[pod(), old])) - - def test_terminal_pods_do_not_block(self): - for phase in ("Succeeded", "Failed"): - with self.subTest(phase=phase): - old = pod("v2.14.0", "old", False) - old["status"]["phase"] = phase - self.assertIsNone(self.check(pods=[pod(), old])) + self.assertIsNotNone(self.check(pods=[pod(), old])) - def test_matching_ready_fleet_passes(self): - self.assertIsNone(self.check()) - - def test_requires_live_ready_capacity(self): - self.assertIn("available compatible", self.check(pods=[pod(is_ready=False)])) - self.assertIn("available compatible", self.check(deployment(replicas=2), [pod()])) - - def test_honors_deployment_min_ready_seconds_availability(self): - resource = deployment() - resource["status"]["availableReplicas"] = 0 - self.assertIn("available compatible", self.check(resource)) - - def test_old_api_can_restart_against_newer_brainstore(self): + def test_older_application_accepts_completed_newer_brainstore(self): self.assertIsNone(self.check(deployment("v2.16.0"), [pod("v2.16.0")], "2.15.0")) - def test_old_api_can_restart_during_compatible_future_rollout(self): - resource = deployment("v2.16.0") - pods = [pod("v2.15.0", "old"), pod("v2.16.0", "new", False)] - self.assertIsNone(self.check(resource, pods)) - - def test_new_api_still_blocks_during_same_future_rollout(self): - pods = [pod("v2.15.0", "old"), pod("v2.16.0", "new", False)] - self.assertIn("incompatible", self.check(deployment("v2.16.0"), pods, "2.16.0")) - - def test_compatible_terminating_pod_does_not_block_restart(self): - terminating = pod("v2.15.0", "old", False, deletionTimestamp="2026-09-30T12:00:00Z") - self.assertIsNone(self.check(pods=[pod(), terminating])) - - def test_zero_replica_role_waits_for_remaining_pods(self): - self.assertIsNone(self.check(deployment(replicas=0), [])) - self.assertIn("scaled-down", self.check(deployment(replicas=0), [pod()])) - - def test_unobserved_generation_does_not_pass(self): - resource = deployment() - resource["status"]["observedGeneration"] = 1 - self.assertIn("not observed", self.check(resource)) - - def test_deleted_deployment_does_not_pass(self): - resource = deployment() - resource["metadata"]["deletionTimestamp"] = "2026-09-30T12:00:00Z" - self.assertIn("being deleted", self.check(resource)) - def test_unknown_and_other_major_images_fail_closed(self): - for version in ("latest", "deadbeef", "v3.0.0", "v2.15.0-rc.1"): + for version in ("latest", "deadbeef", "0123456", "0" * 40, + "v3.0.0", "v2.15.0-rc.1"): with self.subTest(version=version): - self.assertIn("incompatible", self.check(pods=[pod(version)])) + self.assertIsNotNone(self.check(deployment(version))) + self.assertIsNotNone(self.check(pods=[pod(version)])) def test_registry_port_and_tagged_digest_are_supported(self): image = "localhost:5000/brainstore:v2.15.0@sha256:abc" @@ -129,7 +88,10 @@ def test_stable_release_parser(self): def test_missing_expected_container_fails_closed(self): wrong = pod() wrong["spec"]["containers"][0]["name"] = "sidecar" - self.assertIn("incompatible", self.check(pods=[wrong])) + self.assertIsNotNone(self.check(pods=[wrong])) + resource = deployment() + resource["spec"]["template"]["spec"]["containers"][0]["name"] = "sidecar" + self.assertIsNotNone(self.check(resource)) def test_runtime_image_aliases_do_not_block_a_compatible_deployment_pod(self): for image in ("registry/brainstore:deadbeef", "registry/brainstore@sha256:abc"): @@ -141,15 +103,10 @@ def test_runtime_image_aliases_do_not_block_a_compatible_deployment_pod(self): def test_runtime_alias_does_not_override_an_incompatible_pod_spec(self): current = pod("v2.14.0") current["status"]["containerStatuses"][0]["image"] = "registry/brainstore:v2.15.0" - self.assertIn("incompatible", self.check(pods=[current])) - - def test_pod_ready_without_expected_running_container_is_insufficient(self): - current = pod() - current["status"]["containerStatuses"] = [] - self.assertIn("available compatible", self.check(pods=[current])) + self.assertIsNotNone(self.check(pods=[current])) -class StrictRolloutTests(unittest.TestCase): +class RolloutTests(unittest.TestCase): def check(self, resource=None, pods=None, minimum_revision=8): return gate.check_deployment( resource if resource is not None else deployment(revision=8), @@ -187,11 +144,23 @@ def test_revision_floor_rejects_missing_malformed_and_old_markers(self): with self.subTest(revision=revision): self.assertIsNotNone(self.check(deployment(revision=revision))) + def test_revision_floor_is_required_and_cannot_be_disabled(self): + with self.assertRaises(TypeError): + gate.check_deployment(deployment(), [pod()], "brainstore-reader", (2, 15, 0)) + for minimum_revision in (0, -1): + with self.subTest(minimum_revision=minimum_revision): + self.assertIsNotNone(self.check(minimum_revision=minimum_revision)) + def test_requires_current_controller_observation(self): resource = deployment(revision=8) resource["status"]["observedGeneration"] = 1 self.assertIsNotNone(self.check(resource)) + def test_deleted_deployment_does_not_pass(self): + resource = deployment() + resource["metadata"]["deletionTimestamp"] = "2026-09-30T12:00:00Z" + self.assertIsNotNone(self.check(resource)) + def test_every_replica_counter_must_equal_current_desired_count(self): for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): for count in (None, 0, 2): @@ -211,9 +180,21 @@ def test_requires_exactly_desired_live_ready_current_pods(self): deletionTimestamp="2026-09-30T12:00:00Z")]): with self.subTest(pods=pods): self.assertIsNotNone(self.check(pods=pods)) - current = pod(revision=8) - current["status"]["containerStatuses"] = [] - self.assertIsNotNone(self.check(pods=[current])) + for status_change in ("missing", "not_running", "not_ready", "pod_not_ready", "pending"): + with self.subTest(status_change=status_change): + current = pod() + status = current["status"] + if status_change == "missing": + status["containerStatuses"] = [] + elif status_change == "not_running": + status["containerStatuses"][0]["state"] = {"waiting": {}} + elif status_change == "not_ready": + status["containerStatuses"][0]["ready"] = False + elif status_change == "pod_not_ready": + status["conditions"] = [] + else: + status["phase"] = "Pending" + self.assertIsNotNone(self.check(pods=[current])) def test_stray_service_backend_is_not_exempt_from_revision_check(self): stray = pod(uid="stray", revision=7) @@ -252,24 +233,33 @@ def test_zero_role_requires_release_identity_and_completed_scale_down(self): resource["status"]["observedGeneration"] = 1 self.assertIsNotNone(self.check(resource, [])) + def test_zero_role_does_not_require_an_inactive_image_or_container(self): + for version in ("v2.14.0", "v3.0.0", "deadbeef", "v2.15.0-rc.1"): + with self.subTest(version=version): + self.assertIsNone(self.check(deployment(version, replicas=0), [])) + resource = deployment(replicas=0) + resource["spec"]["template"]["spec"]["containers"] = [] + self.assertIsNone(self.check(resource, [])) + terminating = pod(is_ready=False, deletionTimestamp="2026-09-30T12:00:00Z") + self.assertIsNotNone(self.check(resource, [terminating])) + for phase in ("Succeeded", "Failed"): + with self.subTest(phase=phase): + terminal = pod("v2.14.0", revision=7) + terminal["status"]["phase"] = phase + self.assertIsNone(self.check(resource, [terminal])) + def test_uses_current_replica_count_after_scaling(self): resource = deployment(replicas=2, revision=8) self.assertIsNotNone(self.check(resource, [pod(revision=8)])) self.assertIsNone(self.check(resource, [pod(revision=8), pod(uid="second", revision=8)])) - def test_older_api_accepts_later_completed_but_not_unfinished_rollout(self): + def test_older_application_accepts_later_completed_but_not_unfinished_rollout(self): resource = deployment("v2.16.0", revision=9) self.assertIsNone(self.check(resource, [pod("v2.16.0", revision=9)])) self.assertIsNotNone(self.check(resource, [pod(revision=8)])) resource["status"]["updatedReplicas"] = 0 self.assertIsNotNone(self.check(resource, [pod("v2.16.0", revision=9)])) - def test_strict_mode_preserves_desired_and_pod_image_compatibility(self): - for version in ("v2.14.0", "v3.0.0", "latest", "v2.15.0-rc.1"): - with self.subTest(version=version): - self.assertIsNotNone(self.check(deployment(version, revision=8))) - self.assertIsNotNone(self.check(pods=[pod(version, revision=8)])) - class FakeClock: def __init__(self): @@ -287,7 +277,8 @@ def wait(self, observations, timeout=60): clock = FakeClock() log = Mock() with patch.object(gate, "check_fleet", side_effect=observations) as check: - gate.wait_for_fleet(Mock(), [{"deployment": "reader", "container": "brainstore-reader"}], + gate.wait_for_fleet(Mock(), [{"deployment": "reader", "container": "brainstore-reader", + "minimumRevision": 8}], (2, 15, 0), timeout, 15, clock.clock, clock.sleep, log) return clock, log, check @@ -306,40 +297,49 @@ def test_transient_missing_deployment_and_rbac_are_retried(self): _, _, check = self.wait(errors + [None, None]) self.assertEqual(check.call_count, 4) - def test_timeout_never_releases_api(self): + def test_failed_api_reads_reset_successful_observations(self): + for error in (OSError("synthetic"), ValueError("synthetic"), + KeyError("synthetic"), TypeError("synthetic")): + with self.subTest(error=type(error).__name__): + clock, _, check = self.wait([None, error, None, None]) + self.assertEqual(check.call_count, 4) + self.assertEqual(clock.now, 45) + + def test_timeout_never_releases_application(self): with self.assertRaisesRegex(TimeoutError, "old pod remains"): self.wait(["old pod remains"] * 3, timeout=30) - def test_kubernetes_api_failure_never_releases_api(self): + def test_kubernetes_api_failure_never_releases_application(self): with self.assertRaisesRegex(TimeoutError, "HTTP 429"): self.wait([urllib.error.HTTPError("https://api", 429, "synthetic", None, None)] * 3, 30) class FleetTests(unittest.TestCase): - def test_compatible_deployment_and_pod_churn_do_not_delay_an_old_api(self): + def test_completed_compatible_rollout_churn_can_confirm_an_older_application(self): client = Mock() - first = deployment("v2.15.0") - later = deployment("v2.16.0") + first = deployment("v2.15.0", revision=8) + later = deployment("v2.16.0", revision=9) later["metadata"]["generation"] = 3 later["status"]["observedGeneration"] = 3 client.deployment.side_effect = [first, later] - client.pods.side_effect = [[pod("v2.15.0", "old")], [pod("v2.16.0", "new")]] - targets = [{"deployment": "reader", "container": "brainstore-reader"}] + client.pods.side_effect = [[pod("v2.15.0", "old", revision=8)], + [pod("v2.16.0", "new", revision=9)]] + targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] clock = FakeClock() gate.wait_for_fleet(client, targets, (2, 15, 0), 60, 15, clock.clock, clock.sleep, Mock()) self.assertEqual(client.deployment.call_count, 2) self.assertEqual(clock.now, 15) - def test_checks_all_three_roles_and_includes_stray_service_backends(self): + def test_checks_all_roles_and_includes_stray_service_backends(self): client = Mock() targets = [] deployments = [] pod_sets = [] - for role in ("reader", "writer", "fastreader"): + for role in ("reader", "writer", "fastreader", "automationwriter"): name = "custom-" + role container = "brainstore-" + role - targets.append({"deployment": name, "container": container}) + targets.append({"deployment": name, "container": container, "minimumRevision": 8}) resource = deployment() resource["metadata"]["name"] = name resource["metadata"]["uid"] = name @@ -353,19 +353,35 @@ def test_checks_all_three_roles_and_includes_stray_service_backends(self): client.pods.side_effect = pod_sets reason = gate.check_fleet(client, targets, (2, 15, 0), 100) self.assertIsNone(reason) - self.assertEqual(client.deployment.call_count, 3) + self.assertEqual(client.deployment.call_count, 4) self.assertEqual([call.args[0] for call in client.pods.call_args_list], - ["app=custom-reader", "app=custom-writer", "app=custom-fastreader"]) + ["app=custom-reader", "app=custom-writer", + "app=custom-fastreader", "app=custom-automationwriter"]) + self.assertTrue(all(call.args[1] == 100 for call in client.deployment.call_args_list)) + self.assertTrue(all(call.args[1] == 100 for call in client.pods.call_args_list)) # A manually labelled Pod can receive Service traffic without belonging # to a ReplicaSet, so it must not be filtered out by owner references. stray = pod("v2.14.0", "stray") client.deployment.side_effect = deployments client.pods.side_effect = [[pod(), stray]] reason = gate.check_fleet(client, targets, (2, 15, 0), 100) - self.assertIn("stray", reason) - + self.assertIsNotNone(reason) - def test_strict_writer_waits_across_same_image_and_later_rollouts(self): + def test_missing_or_zero_revision_target_cannot_release_application(self): + for revision in (None, 0): + with self.subTest(revision=revision): + target = {"deployment": "reader", "container": "brainstore-reader"} + if revision is not None: + target["minimumRevision"] = revision + client = Mock() + client.deployment.return_value = deployment() + client.pods.return_value = [pod()] + clock = FakeClock() + with self.assertRaises(TimeoutError): + gate.wait_for_fleet(client, [target], (2, 15, 0), 30, 15, + clock.clock, clock.sleep, Mock()) + + def test_writer_waits_across_same_image_and_later_rollouts(self): container = "brainstore-writer" resources = [deployment(revision=revision, container=container) for revision in (7, 8, 8, 9, 9, 9)] @@ -389,7 +405,7 @@ def test_strict_writer_waits_across_same_image_and_later_rollouts(self): clock.clock, clock.sleep, Mock()) self.assertEqual(clock.now, 75) - def test_strict_observation_errors_reset_confirmation(self): + def test_observation_errors_reset_confirmation(self): client = Mock() resource = deployment(revision=8) client.deployment.side_effect = [ @@ -403,7 +419,7 @@ def test_strict_observation_errors_reset_confirmation(self): clock.clock, clock.sleep, Mock()) self.assertEqual(clock.now, 60) - def test_strict_completed_observation_cannot_release_api_after_deadline(self): + def test_completed_observation_cannot_release_application_after_deadline(self): clock = FakeClock() def observe(name, deadline): @@ -458,6 +474,14 @@ def test_reloads_rotating_token_and_bounds_request_timeout(self): self.assertEqual(calls[1].args[0].get_header("Authorization"), "Bearer synthetic-token-2") self.assertEqual(calls[0].kwargs["timeout"], 2) + def test_expired_budget_does_not_start_an_api_request(self): + client = self.client("/synthetic") + client.opener = Mock() + with patch.object(gate.time, "monotonic", return_value=10), \ + self.assertRaises(TimeoutError): + client.get("/test", 10) + client.opener.open.assert_not_called() + def test_pod_lists_are_paginated(self): client = self.client("/synthetic") client.get = Mock(side_effect=[ From 43a26b9c8c1420dbef1b9ac01c6b9df864f09b2b Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 13:37:24 -0500 Subject: [PATCH 07/13] Trim release ordering churn while preserving safety checks --- braintrust/files/brainstore-startup-gate.py | 13 +-- braintrust/files/migration-rollback.py | 4 +- .../templates/_brainstore-startup-gate.tpl | 7 +- braintrust/templates/_deployment-ordering.tpl | 2 +- braintrust/tests/api-autoscaling_test.yaml | 11 ++- .../tests/brainstore-startup-gate_test.yaml | 41 +++++++++ braintrust/tests/migrations_test.yaml | 29 +----- braintrust/values.yaml | 2 +- docs/migration-job.md | 4 +- tests/test_brainstore_startup_gate.py | 91 ++++++++++--------- 10 files changed, 122 insertions(+), 82 deletions(-) diff --git a/braintrust/files/brainstore-startup-gate.py b/braintrust/files/brainstore-startup-gate.py index f6842f1..6ca344a 100644 --- a/braintrust/files/brainstore-startup-gate.py +++ b/braintrust/files/brainstore-startup-gate.py @@ -195,21 +195,21 @@ def pods(self, selector, deadline): return items -def check_fleet(client, targets, minimum, deadline): +def check_fleet(client, targets, minimum, minimum_revision, deadline): for target in targets: deployment = client.deployment(target["deployment"], deadline) # The chart's Services select app=. Include stray Pods # with that label too; they can receive traffic regardless of ownership. pods = client.pods("app=" + target["deployment"], deadline) reason = check_deployment( - deployment, pods, target["container"], minimum, target["minimumRevision"], + deployment, pods, target["container"], minimum, minimum_revision, ) if reason: return reason return None -def wait_for_fleet(client, targets, minimum, timeout, interval, +def wait_for_fleet(client, targets, minimum, minimum_revision, timeout, interval, clock=time.monotonic, sleep=time.sleep, log=print): # All roles and API requests share one budget, unaffected by wall-clock changes. deadline = clock() + timeout @@ -217,7 +217,7 @@ def wait_for_fleet(client, targets, minimum, timeout, interval, last_reason = "no observation yet" while clock() < deadline: try: - reason = check_fleet(client, targets, minimum, deadline) + reason = check_fleet(client, targets, minimum, minimum_revision, deadline) except urllib.error.HTTPError as error: # RBAC and Deployments may be applied after the application Pod is created. path = urllib.parse.urlsplit(error.url).path @@ -246,13 +246,14 @@ def wait_for_fleet(client, targets, minimum, timeout, interval, def main(): minimum = release_version(os.environ["BRAINSTORE_MINIMUM_VERSION"]) + minimum_revision = int(os.environ["BRAINSTORE_MINIMUM_REVISION"]) targets = json.loads(os.environ["BRAINSTORE_DEPLOYMENTS"]) timeout = int(os.environ["BRAINSTORE_GATE_TIMEOUT_SECONDS"]) interval = int(os.environ["BRAINSTORE_GATE_POLL_SECONDS"]) - if not targets or timeout <= 0 or interval <= 0 or interval >= timeout: + if not targets or minimum_revision <= 0 or timeout <= 0 or interval <= 0 or interval >= timeout: raise ValueError("invalid startup gate configuration") client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/brainstore-gate") - wait_for_fleet(client, targets, minimum, timeout, interval) + wait_for_fleet(client, targets, minimum, minimum_revision, timeout, interval) if __name__ == "__main__": diff --git a/braintrust/files/migration-rollback.py b/braintrust/files/migration-rollback.py index 6c55a78..27363f8 100644 --- a/braintrust/files/migration-rollback.py +++ b/braintrust/files/migration-rollback.py @@ -84,9 +84,7 @@ def wait_until_deleted(path): sleep(min(2, max(0, deadline - clock()))) # Stop the migration before removing any config or permissions it relies on. - request("DELETE", paths[0]) - wait_until_deleted(paths[0]) - for path in paths[1:]: + for path in paths: request("DELETE", path) wait_until_deleted(path) diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index c5a2a04..0da9ee2 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -15,14 +15,13 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- define "braintrust.brainstoreStartupGate.targets" -}} {{- $targets := list -}} -{{- $revision := int .Release.Revision -}} {{- range $role := list "reader" "writer" "fastreader" -}} {{- $config := index $.Values.brainstore $role -}} -{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role) "minimumRevision" $revision) -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" (printf "brainstore-%s" $role)) -}} {{- end -}} {{- if .Values.brainstore.automationwriter -}} {{- $config := include "braintrust.automationWriter.config" . | fromYaml -}} -{{- $targets = append $targets (dict "deployment" $config.name "container" "brainstore-automationwriter" "minimumRevision" $revision) -}} +{{- $targets = append $targets (dict "deployment" $config.name "container" "brainstore-automationwriter") -}} {{- end -}} {{- toJson $targets -}} {{- end -}} @@ -68,6 +67,8 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. value: {{ $config.timeoutSeconds | quote }} - name: BRAINSTORE_GATE_POLL_SECONDS value: {{ $config.pollIntervalSeconds | quote }} + - name: BRAINSTORE_MINIMUM_REVISION + value: {{ .root.Release.Revision | quote }} volumeMounts: - name: migration-gate-code mountPath: /opt/migration-gate diff --git a/braintrust/templates/_deployment-ordering.tpl b/braintrust/templates/_deployment-ordering.tpl index 2e7ef68..6c9a26d 100644 --- a/braintrust/templates/_deployment-ordering.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -4,7 +4,7 @@ "enabled" true "minimumVersion" "" "timeoutSeconds" 1200 - "pollIntervalSeconds" 5 + "pollIntervalSeconds" 15 "backoffLimit" 2 "image" "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" "resources" (dict "requests" (dict "cpu" "25m" "memory" "32Mi") "limits" (dict "cpu" "100m" "memory" "128Mi")) diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index 8b7ce62..299d7f7 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -175,6 +175,9 @@ tests: containerPort: 8001 - equal: path: spec.template.spec.initContainers[0].name + value: wait-for-migrations + - equal: + path: spec.template.spec.initContainers[1].name value: wait-for-brainstore - contains: path: spec.template.spec.containers[0].env @@ -189,7 +192,7 @@ tests: set: cloud: google api.autoscaling.enabled: true - api.brainstoreStartupGate.enabled: false + deploymentOrdering.enabled: false api.extraEnvVars: - name: ENABLE_PROMETHEUS_METRICS value: "false" @@ -198,6 +201,8 @@ tests: release: namespace: "braintrust" asserts: + - isNull: + path: spec.template.spec.initContainers - equal: path: spec.template.spec.containers[0].env[-1].name value: ENABLE_PROMETHEUS_METRICS @@ -397,6 +402,10 @@ tests: template: api-deployment.yaml - equal: path: spec.template.spec.initContainers[0].name + value: wait-for-migrations + template: api-deployment.yaml + - equal: + path: spec.template.spec.initContainers[1].name value: wait-for-brainstore template: api-deployment.yaml - equal: diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index 5050890..18a750e 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -42,6 +42,29 @@ tests: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" + - it: should retain zero-replica roles in the rollout drain check + template: api-deployment.yaml + release: + revision: 7 + set: + brainstore.reader.name: drain-reader + brainstore.reader.replicas: 0 + brainstore.writer.name: drain-writer + brainstore.writer.replicas: 0 + brainstore.fastreader.name: drain-fastreader + brainstore.fastreader.replicas: 0 + brainstore.automationwriter.name: drain-automationwriter + brainstore.automationwriter.replicas: 0 + asserts: + - equal: + path: spec.template.spec.initContainers[1].env[2].value + value: '[{"container":"brainstore-reader","deployment":"drain-reader"},{"container":"brainstore-writer","deployment":"drain-writer"},{"container":"brainstore-fastreader","deployment":"drain-fastreader"},{"container":"brainstore-automationwriter","deployment":"drain-automationwriter"}]' + - contains: + path: spec.template.spec.initContainers[1].env + content: + name: BRAINSTORE_MINIMUM_REVISION + value: "7" + - it: should mount gate credentials only into the init container template: api-deployment.yaml set: @@ -57,6 +80,24 @@ tests: - equal: path: spec.template.spec.automountServiceAccountToken value: false + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.initContainers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.initContainers[1].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.initContainers[1].securityContext.readOnlyRootFilesystem + value: true + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: migration-gate-credentials + mountPath: /var/run/migration-gate + readOnly: true - contains: path: spec.template.spec.initContainers[1].volumeMounts content: diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index 249d030..a806639 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -26,6 +26,9 @@ tests: - it: runs a regular bounded migration Job using the target API image template: migration-job.yaml documentIndex: 0 + set: + api.image.repository: registry.example/custom-api + api.image.tag: v2.15.0 asserts: - isKind: of: Job @@ -53,7 +56,7 @@ tests: value: [/braintrust/venv/bin/python, /braintrust/api-schema/lambda_function.py, --execute] - equal: path: spec.template.spec.containers[0].image - value: public.ecr.aws/braintrust/standalone-api:v2.22.0 + value: registry.example/custom-api:v2.15.0 - isNull: path: spec.template.spec.initContainers - isNull: @@ -368,12 +371,6 @@ tests: asserts: - {"hasDocuments": {"count": 0}} - - it: removes migration wait when disabled in api-deployment.yaml - template: api-deployment.yaml - set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} - asserts: - - {"isNull": {"path": "spec.template.spec.initContainers"}} - - it: removes migration wait when disabled in brainstore-reader-deployment.yaml template: brainstore-reader-deployment.yaml set: {"deploymentOrdering.enabled": false, "aiGateway.enabled": true} @@ -508,12 +505,6 @@ tests: - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,56}$"}} - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} - - it: keeps missing automation writer omitted - template: brainstore-automationwriter-deployment.yaml - set: {"brainstore.automationwriter": null} - asserts: - - {"hasDocuments": {"count": 0}} - - it: gates a partially specified automation writer template: brainstore-automationwriter-deployment.yaml set: {"brainstore.automationwriter": {"replicas": 1}} @@ -521,18 +512,6 @@ tests: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - {"equal": {"path": "spec.replicas", "value": 1}} - - it: keeps disabled optional workloads omitted - template: ai-gateway-deployment.yaml - set: {"aiGateway.enabled": false} - asserts: - - {"hasDocuments": {"count": 0}} - - - it: keeps disabled Loop runtime omitted - template: loop-runtime-deployment.yaml - set: {"loopRuntime.enabled": false} - asserts: - - {"hasDocuments": {"count": 0}} - - it: prepares target revision rollback cleanup without install or upgrade hooks template: migration-rollback.yaml documentIndex: 3 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index e5dd61a..51905f8 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -822,7 +822,7 @@ deploymentOrdering: minimumVersion: "" # Separate budget for each phase (migration Job/wait, then Brainstore rollout). timeoutSeconds: 1200 - pollIntervalSeconds: 5 + pollIntervalSeconds: 15 backoffLimit: 2 # Small read-only init-container helper; migration itself uses api.image. image: "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" diff --git a/docs/migration-job.md b/docs/migration-job.md index ab89804..4368985 100644 --- a/docs/migration-job.md +++ b/docs/migration-job.md @@ -4,6 +4,8 @@ With `deploymentOrdering.enabled: true` (default), one regular migration Job runs the shared API image's migration entry point for each Helm revision. Brainstore waits for it; all API pools, optional AI Gateway, and optional Loop Runtime wait for migrations and then check Brainstore Deployments and Pods directly. There is no separate rollout Job or rollout-configuration snapshot. +This replaces `api.brainstoreStartupGate`; migrate previous opt-outs to `deploymentOrdering.enabled: false`. + ## What applications check For every rendered reader, writer, fastreader, and automationwriter: @@ -60,6 +62,6 @@ These gates order **application-container startup**, not Kubernetes object updat Use real Helm revisions. Pool-specific API overrides must remain schema-compatible with the shared migrator image/database. Existing rollout strategies determine availability; strict startup ordering does not provide spare nodes or guarantee cache warm-up beyond configured readiness and `minReadySeconds`. -`deploymentOrdering.timeoutSeconds` (default 1200) bounds each migration and rollout-wait phase separately; `pollIntervalSeconds` (default 5) must be smaller. `minimumVersion` is an optional shared stable floor; when empty each API pool derives its own image floor, and Gateway/Loop use the shared API image. `image`, `resources`, and `backoffLimit` configure the helpers and migration retry budget. API progress deadlines include two phase budgets plus 300 seconds; Gateway/Loop include two plus 600; Brainstore adds one phase budget to its configured rollout deadline. +`deploymentOrdering.timeoutSeconds` (default 1200) bounds each migration and rollout-wait phase separately; `pollIntervalSeconds` (default 15) must be smaller. `minimumVersion` is an optional shared stable floor; when empty each API pool derives its own image floor, and Gateway/Loop use the shared API image. `image`, `resources`, and `backoffLimit` configure the helpers and migration retry budget. API progress deadlines include two phase budgets plus 300 seconds; Gateway/Loop include two plus 600; Brainstore adds one phase budget to its configured rollout deadline. Use a sufficient Helm timeout and `--wait --wait-for-jobs`; otherwise Helm can return before migrations or rollout complete. Inspect `wait-for-migrations` and `wait-for-brainstore` init logs for the condition holding startup. diff --git a/tests/test_brainstore_startup_gate.py b/tests/test_brainstore_startup_gate.py index e907319..40a2ae9 100644 --- a/tests/test_brainstore_startup_gate.py +++ b/tests/test_brainstore_startup_gate.py @@ -42,7 +42,7 @@ def deployment(version="v2.15.0", replicas=1, revision=8, container="brainstore- } -class ImageCompatibilityTests(unittest.TestCase): +class CompatibilityTests(unittest.TestCase): def check(self, resource=None, pods=None, minimum="2.15.0"): return gate.check_deployment( resource if resource is not None else deployment(), @@ -50,17 +50,17 @@ def check(self, resource=None, pods=None, minimum="2.15.0"): "brainstore-reader", gate.release_version(minimum), 8, ) - def test_new_application_waits_for_compatible_desired_image(self): + def test_new_api_waits_when_helm_has_not_updated_brainstore_spec(self): self.assertIsNotNone(self.check(deployment("v2.14.0"))) - def test_new_application_waits_for_incompatible_live_pods(self): + def test_new_api_waits_for_incompatible_old_pods(self): self.assertIsNotNone(self.check(pods=[pod(), pod("v2.14.0", "old")])) def test_incompatible_terminating_pod_blocks_even_when_unready(self): old = pod("v2.14.0", "old", False, deletionTimestamp="2026-09-30T12:00:00Z") self.assertIsNotNone(self.check(pods=[pod(), old])) - def test_older_application_accepts_completed_newer_brainstore(self): + def test_old_api_can_restart_against_newer_brainstore(self): self.assertIsNone(self.check(deployment("v2.16.0"), [pod("v2.16.0")], "2.15.0")) def test_unknown_and_other_major_images_fail_closed(self): @@ -151,7 +151,7 @@ def test_revision_floor_is_required_and_cannot_be_disabled(self): with self.subTest(minimum_revision=minimum_revision): self.assertIsNotNone(self.check(minimum_revision=minimum_revision)) - def test_requires_current_controller_observation(self): + def test_unobserved_generation_does_not_pass(self): resource = deployment(revision=8) resource["status"]["observedGeneration"] = 1 self.assertIsNotNone(self.check(resource)) @@ -202,14 +202,14 @@ def test_stray_service_backend_is_not_exempt_from_revision_check(self): del stray["metadata"]["annotations"] self.assertIsNotNone(self.check(pods=[stray])) - def test_terminal_old_pods_do_not_block_completed_rollout(self): + def test_terminal_pods_do_not_block(self): for phase in ("Succeeded", "Failed"): with self.subTest(phase=phase): old = pod("v2.14.0", revision=7) old["status"]["phase"] = phase self.assertIsNone(self.check(pods=[pod(revision=8), old])) - def test_min_ready_seconds_requires_available_replicas(self): + def test_honors_deployment_min_ready_seconds_availability(self): resource = deployment(revision=8) resource["spec"]["minReadySeconds"] = 60 resource["status"]["availableReplicas"] = 0 @@ -277,9 +277,8 @@ def wait(self, observations, timeout=60): clock = FakeClock() log = Mock() with patch.object(gate, "check_fleet", side_effect=observations) as check: - gate.wait_for_fleet(Mock(), [{"deployment": "reader", "container": "brainstore-reader", - "minimumRevision": 8}], - (2, 15, 0), timeout, 15, clock.clock, clock.sleep, log) + gate.wait_for_fleet(Mock(), [{"deployment": "reader", "container": "brainstore-reader"}], + (2, 15, 0), 8, timeout, 15, clock.clock, clock.sleep, log) return clock, log, check def test_requires_two_successful_observations(self): @@ -305,17 +304,17 @@ def test_failed_api_reads_reset_successful_observations(self): self.assertEqual(check.call_count, 4) self.assertEqual(clock.now, 45) - def test_timeout_never_releases_application(self): + def test_timeout_never_releases_api(self): with self.assertRaisesRegex(TimeoutError, "old pod remains"): self.wait(["old pod remains"] * 3, timeout=30) - def test_kubernetes_api_failure_never_releases_application(self): + def test_kubernetes_api_failure_never_releases_api(self): with self.assertRaisesRegex(TimeoutError, "HTTP 429"): self.wait([urllib.error.HTTPError("https://api", 429, "synthetic", None, None)] * 3, 30) class FleetTests(unittest.TestCase): - def test_completed_compatible_rollout_churn_can_confirm_an_older_application(self): + def test_compatible_deployment_and_pod_churn_do_not_delay_an_old_api(self): client = Mock() first = deployment("v2.15.0", revision=8) later = deployment("v2.16.0", revision=9) @@ -324,9 +323,9 @@ def test_completed_compatible_rollout_churn_can_confirm_an_older_application(sel client.deployment.side_effect = [first, later] client.pods.side_effect = [[pod("v2.15.0", "old", revision=8)], [pod("v2.16.0", "new", revision=9)]] - targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] + targets = [{"deployment": "reader", "container": "brainstore-reader"}] clock = FakeClock() - gate.wait_for_fleet(client, targets, (2, 15, 0), 60, 15, + gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 60, 15, clock.clock, clock.sleep, Mock()) self.assertEqual(client.deployment.call_count, 2) self.assertEqual(clock.now, 15) @@ -339,7 +338,7 @@ def test_checks_all_roles_and_includes_stray_service_backends(self): for role in ("reader", "writer", "fastreader", "automationwriter"): name = "custom-" + role container = "brainstore-" + role - targets.append({"deployment": name, "container": container, "minimumRevision": 8}) + targets.append({"deployment": name, "container": container}) resource = deployment() resource["metadata"]["name"] = name resource["metadata"]["uid"] = name @@ -351,7 +350,7 @@ def test_checks_all_roles_and_includes_stray_service_backends(self): pod_sets.append([current]) client.deployment.side_effect = deployments client.pods.side_effect = pod_sets - reason = gate.check_fleet(client, targets, (2, 15, 0), 100) + reason = gate.check_fleet(client, targets, (2, 15, 0), 8, 100) self.assertIsNone(reason) self.assertEqual(client.deployment.call_count, 4) self.assertEqual([call.args[0] for call in client.pods.call_args_list], @@ -364,21 +363,19 @@ def test_checks_all_roles_and_includes_stray_service_backends(self): stray = pod("v2.14.0", "stray") client.deployment.side_effect = deployments client.pods.side_effect = [[pod(), stray]] - reason = gate.check_fleet(client, targets, (2, 15, 0), 100) + reason = gate.check_fleet(client, targets, (2, 15, 0), 8, 100) self.assertIsNotNone(reason) - def test_missing_or_zero_revision_target_cannot_release_application(self): - for revision in (None, 0): + def test_nonpositive_revision_floor_cannot_release_application(self): + for revision in (0, -1): with self.subTest(revision=revision): target = {"deployment": "reader", "container": "brainstore-reader"} - if revision is not None: - target["minimumRevision"] = revision client = Mock() client.deployment.return_value = deployment() client.pods.return_value = [pod()] clock = FakeClock() with self.assertRaises(TimeoutError): - gate.wait_for_fleet(client, [target], (2, 15, 0), 30, 15, + gate.wait_for_fleet(client, [target], (2, 15, 0), revision, 30, 15, clock.clock, clock.sleep, Mock()) def test_writer_waits_across_same_image_and_later_rollouts(self): @@ -399,26 +396,12 @@ def test_writer_waits_across_same_image_and_later_rollouts(self): [current], [current, pod(revision=9, is_ready=False, container=container)], [later], [later], ] - targets = [{"deployment": "writer", "container": container, "minimumRevision": 8}] + targets = [{"deployment": "writer", "container": container}] clock = FakeClock() - gate.wait_for_fleet(client, targets, (2, 15, 0), 120, 15, + gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 120, 15, clock.clock, clock.sleep, Mock()) self.assertEqual(clock.now, 75) - def test_observation_errors_reset_confirmation(self): - client = Mock() - resource = deployment(revision=8) - client.deployment.side_effect = [ - urllib.error.HTTPError("https://api", 403, "synthetic", None, None), - resource, OSError("synthetic"), resource, resource, - ] - client.pods.return_value = [pod(revision=8)] - targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] - clock = FakeClock() - gate.wait_for_fleet(client, targets, (2, 15, 0), 90, 15, - clock.clock, clock.sleep, Mock()) - self.assertEqual(clock.now, 60) - def test_completed_observation_cannot_release_application_after_deadline(self): clock = FakeClock() @@ -429,12 +412,38 @@ def observe(name, deadline): client = Mock() client.deployment.side_effect = observe client.pods.return_value = [pod(revision=8)] - targets = [{"deployment": "reader", "container": "brainstore-reader", "minimumRevision": 8}] + targets = [{"deployment": "reader", "container": "brainstore-reader"}] with self.assertRaises(TimeoutError): - gate.wait_for_fleet(client, targets, (2, 15, 0), 60, 15, + gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 60, 15, clock.clock, clock.sleep, Mock()) +class MainTests(unittest.TestCase): + def environment(self): + return { + "BRAINSTORE_MINIMUM_VERSION": "2.15.0", + "BRAINSTORE_MINIMUM_REVISION": "8", + "BRAINSTORE_DEPLOYMENTS": '[{"deployment":"reader","container":"brainstore-reader"}]', + "BRAINSTORE_GATE_TIMEOUT_SECONDS": "60", + "BRAINSTORE_GATE_POLL_SECONDS": "15", + "POD_NAMESPACE": "custom", + } + + def test_missing_invalid_or_nonpositive_revision_floor_fails_closed(self): + for revision in (None, "", "invalid", "8.0", "0", "-1"): + with self.subTest(revision=revision): + environment = self.environment() + if revision is None: + del environment["BRAINSTORE_MINIMUM_REVISION"] + else: + environment["BRAINSTORE_MINIMUM_REVISION"] = revision + with patch.dict(os.environ, environment, clear=True), \ + patch.object(gate, "wait_for_fleet") as wait, \ + self.assertRaises((KeyError, ValueError)): + gate.main() + wait.assert_not_called() + + class ClientTests(unittest.TestCase): def client(self, directory, host="10.0.0.1"): with patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": host}), \ From d460dc908b432bea6db1c47e0fe054b7cf2d7385 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 17:11:28 -0500 Subject: [PATCH 08/13] Consolidate release ordering guidance in chart README --- braintrust/README.md | 38 +++++++++++++----------- braintrust/values.yaml | 2 +- docs/migration-job.md | 67 ------------------------------------------ 3 files changed, 22 insertions(+), 85 deletions(-) delete mode 100644 docs/migration-job.md diff --git a/braintrust/README.md b/braintrust/README.md index 7315f55..34387ca 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -558,20 +558,24 @@ Example values files for different cloud providers and configurations are locate ## Release ordering -`deploymentOrdering.enabled: true` (default) runs one migration Job per Helm -revision. Brainstore waits for migrations; all API pools, optional AI Gateway, -and optional Loop Runtime then wait for the full compatible Brainstore rollout. -Old or terminating live pods cannot satisfy the check, including same-image -configuration changes. `deploymentOrdering.enabled: false` removes all -chart-managed ordering resources and waits; there is no intermediate mode. - -The shared `deploymentOrdering.timeoutSeconds` bounds each phase separately; -`pollIntervalSeconds` controls observations. `minimumVersion` optionally sets a -shared stable floor; otherwise each API pool uses its image release and the -Gateway/Loop use the shared API release. - -Older replacement workloads also wait during a later unfinished Brainstore -rollout, accepting a newer completed compatible release. Capacity shortages can -therefore delay restarts. Ordering covers container startup, not Helm's object -updates. See [migration lifecycle and input matrix](../docs/migration-job.md) -for rollback, retained-record reinstall cleanup, and configuration limits. +`deploymentOrdering.enabled` (default `true`) runs one migration Job per Helm +revision before Brainstore starts. All API pools, enabled AI Gateway, and enabled +Loop Runtime then wait for the full compatible Brainstore rollout and start in +parallel. Set it to `false` for unordered startup. This replaces +`api.brainstoreStartupGate`; migrate existing opt-outs to the new setting. + +Ordering gates container startup, not Kubernetes object updates. Older replacement +pods also wait during an unfinished Brainstore rollout. + +`deploymentOrdering.timeoutSeconds` bounds each phase; `minimumVersion` supplies +a stable compatibility floor for opaque API image tags. See [values.yaml](values.yaml) +for settings. Use `--wait --wait-for-jobs` with a sufficient Helm timeout. + +Migration records are retained for replacement pods. Retry failed migrations with +a new Helm upgrade. Rollback requires hooks and does not reverse SQL changes; +migrations must remain backward-compatible. + +After uninstall, before reusing a release name, delete Jobs labeled +`braintrust.dev/migration-release=` with foreground deletion and wait for +completion, then delete labeled ConfigMaps, Roles, and RoleBindings. Never delete +records still needed by active workloads or rollback revisions. diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 51905f8..eb70d44 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -814,7 +814,7 @@ virtualService: # Order startup as migrations -> completed Brainstore rollout -> all other apps. # Disable to deploy everything without chart-managed startup ordering. -# See docs/migration-job.md for retention, rollback, and startup-only limits. +# See README.md#release-ordering for configuration and operational limits. deploymentOrdering: enabled: true # Empty derives the compatible Brainstore floor from each API pool's image tag. diff --git a/docs/migration-job.md b/docs/migration-job.md deleted file mode 100644 index 4368985..0000000 --- a/docs/migration-job.md +++ /dev/null @@ -1,67 +0,0 @@ -# Release ordering - -**Migrations complete → Brainstore finishes rolling out → all other applications start.** - -With `deploymentOrdering.enabled: true` (default), one regular migration Job runs the shared API image's migration entry point for each Helm revision. Brainstore waits for it; all API pools, optional AI Gateway, and optional Loop Runtime wait for migrations and then check Brainstore Deployments and Pods directly. There is no separate rollout Job or rollout-configuration snapshot. - -This replaces `api.brainstoreStartupGate`; migrate previous opt-outs to `deploymentOrdering.enabled: false`. - -## What applications check - -For every rendered reader, writer, fastreader, and automationwriter: - -- The Deployment pod-template release marker is the waiting application's Helm revision or newer. -- `status.observedGeneration >= metadata.generation`. -- Updated, Ready, Available, and total replica counts equal the current desired count. -- All service-selected pods belong to that current rollout and are Ready; no old or terminating live pods remain. -- Desired and running images in nonzero Brainstore pools satisfy the waiting application's version floor. - -The release marker prevents the previous fully Ready Deployment from passing before Helm updates it. Deployment generation counters are checked within each Deployment, not compared between API and Brainstore. Pod-template changes cover images and inline environment; the existing ConfigMap checksum covers chart-managed environment. Zero-replica roles must finish draining. Two successful fleet observations share one bounded deadline. - -**Tradeoff:** older replacement API, Gateway, and Loop pods also wait during a newer unfinished Brainstore rollout. They accept a newer completed compatible release rather than waiting for their old configuration to become current again. Insufficient surge capacity can therefore block restarts as well as the new release. - -## Input matrix - -| Input | Behavior | -| --- | --- | -| `deploymentOrdering.enabled: true`; missing/null map or partial reused values | Migration Job, then direct completed-rollout and compatibility checks | -| `deploymentOrdering.enabled: false` | No chart-managed Job, waits, RBAC, helper configuration, hooks, or release markers; no intermediate mode | -| Same image with changed env/config; stale controller observation; old Ready writer/new Pending writer | New API, Gateway, and Loop containers remain blocked | -| Older application, newer Brainstore release | Wait during rollout; pass after the newer compatible rollout completes | -| Replica count missing/null/zero; optional automationwriter absent/partial | Preserve Kubernetes/default pool behavior; zero roles drain, absent pools stay omitted | -| Custom role names, namespaces, API pools | Checks and read-only permissions follow the rendered workload names and actual namespace | -| Stable API release tags with/without v or digest; SHA/numeric/custom/prerelease tags | Stable tags derive the floor; opaque API tags require explicit stable `deploymentOrdering.minimumVersion` | -| AWS/Google/Azure, Azure CSI, CA mounts, custom scheduling/security | Migrator inherits the shared API contract and independently mounts its required secrets; startup checks only read Kubernetes state | -| Missing/null timing or invalid enabled timing/image | Existing defaults apply; invalid enabled values fail rendering, unused disabled settings are ignored | -| Failed/missing migration Job, API errors, old/stray/terminating pods, readiness dwell | Fail closed within the configured budgets; no missing resource implies success | -| Upgrade/retry/rollback and replacement pods | Distinct migration Jobs; keep completed migration records; rollback resets only the target migration phase | - -## Retention and rollback - -The migration Job, its snapshot configuration, helper code, and exact-Job read RBAC are retained without a TTL for replacement pods and rollback. They survive uninstall. Do not delete records still referenced by active pods, ReplicaSets, or a needed rollback revision. Retry a failed migration with a new Helm upgrade after fixing its cause. - -The target revision's `pre-rollback` hook deletes that migration Job and waits for foreground deletion before removing its config/RBAC. Helm then recreates the target phase. This avoids Helm 3 retained-resource restoration errors. Do not use `--no-hooks`; failed cleanup blocks rollback. SQL changes are never reversed and must remain compatible with existing workloads. - -### Reinstalling after uninstall - -Before reinstalling the same release name in the same namespace, stop all old workloads and delete its retained migration records. Helm restarts revision numbering at 1: reusing old records can skip migrations against a new database, reuse a failed Job, or fail on an immutable Job template. - -```sh -release= -namespace= -selector="braintrust.dev/migration-release=$release" -kubectl -n "$namespace" delete job -l "$selector" --cascade=foreground --wait=true -kubectl -n "$namespace" delete configmap,role,rolebinding -l "$selector" --wait=true -``` - -Run this only after uninstall, never against records needed by an active release. Wait for both commands to succeed before reinstalling; deleting the Job first prevents its migrator from outliving its configuration. - -## Limits and operation - -These gates order **application-container startup**, not Kubernetes object updates. Helm still submits Deployment specs and ConfigMaps together. Mutable mounted configuration, externally updated Secrets, and old ReplicaSet replacements are not an atomic configuration transaction. Use separately staged migrations and versioned configuration if no object or old workload may observe changes before migrations finish. Perform a Helm upgrade after an external Secret change to roll pods; secret values are not snapshotted. - -Use real Helm revisions. Pool-specific API overrides must remain schema-compatible with the shared migrator image/database. Existing rollout strategies determine availability; strict startup ordering does not provide spare nodes or guarantee cache warm-up beyond configured readiness and `minReadySeconds`. - -`deploymentOrdering.timeoutSeconds` (default 1200) bounds each migration and rollout-wait phase separately; `pollIntervalSeconds` (default 15) must be smaller. `minimumVersion` is an optional shared stable floor; when empty each API pool derives its own image floor, and Gateway/Loop use the shared API image. `image`, `resources`, and `backoffLimit` configure the helpers and migration retry budget. API progress deadlines include two phase budgets plus 300 seconds; Gateway/Loop include two plus 600; Brainstore adds one phase budget to its configured rollout deadline. - -Use a sufficient Helm timeout and `--wait --wait-for-jobs`; otherwise Helm can return before migrations or rollout complete. Inspect `wait-for-migrations` and `wait-for-brainstore` init logs for the condition holding startup. From 7bed91c26d0c20fc4bc552f96c6b723b299221c9 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 17:39:07 -0500 Subject: [PATCH 09/13] Give AI Gateway a workload-owned service account --- braintrust/README.md | 3 + braintrust/templates/_deployment-ordering.tpl | 4 - braintrust/templates/_helpers.tpl | 5 ++ .../templates/ai-gateway-deployment.yaml | 2 +- .../templates/ai-gateway-serviceaccount.yaml | 15 ++++ .../brainstore-startup-gate-rbac.yaml | 2 +- braintrust/templates/migration-gate.yaml | 10 +-- .../tests/ai-gateway-serviceaccount_test.yaml | 75 +++++++++++++++++++ .../tests/brainstore-startup-gate_test.yaml | 3 +- .../labels-merge-serviceaccounts_test.yaml | 39 ++++++++++ braintrust/tests/migrations_test.yaml | 1 - braintrust/values.yaml | 3 + 12 files changed, 145 insertions(+), 17 deletions(-) create mode 100644 braintrust/templates/ai-gateway-serviceaccount.yaml create mode 100644 braintrust/tests/ai-gateway-serviceaccount_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 34387ca..00f2bd2 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -564,6 +564,9 @@ Loop Runtime then wait for the full compatible Brainstore rollout and start in parallel. Set it to `false` for unordered startup. This replaces `api.brainstoreStartupGate`; migrate existing opt-outs to the new setting. +Gateway has its own Kubernetes ServiceAccount regardless of ordering; configure +its name with `aiGateway.serviceAccount.name`. + Ordering gates container startup, not Kubernetes object updates. Older replacement pods also wait during an unfinished Brainstore rollout. diff --git a/braintrust/templates/_deployment-ordering.tpl b/braintrust/templates/_deployment-ordering.tpl index 6c9a26d..d7a3ccf 100644 --- a/braintrust/templates/_deployment-ordering.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -32,10 +32,6 @@ {{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} {{- end -}} -{{- define "braintrust.migrations.gatewayAccount" -}} -{{- printf "%s-migration-gate" (.Release.Name | trunc 48 | trimSuffix "-") -}} -{{- end -}} - {{- define "braintrust.migrations.initContainer" -}} {{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} - name: wait-for-migrations diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 71c4f48..f16d6f4 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -146,6 +146,11 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} +{{/* Also supports reused values that predate the Gateway ServiceAccount. */}} +{{- define "braintrust.aiGateway.serviceAccountName" -}} +{{- (.Values.aiGateway.serviceAccount | default dict).name | default "braintrust-ai-gateway" -}} +{{- end -}} + {{/* Whether Loop Runtime is enabled. Values reused from a chart release that predates Loop Runtime have no loopRuntime map, which means disabled. diff --git a/braintrust/templates/ai-gateway-deployment.yaml b/braintrust/templates/ai-gateway-deployment.yaml index 1192069..ff5f19d 100644 --- a/braintrust/templates/ai-gateway-deployment.yaml +++ b/braintrust/templates/ai-gateway-deployment.yaml @@ -42,8 +42,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + serviceAccountName: {{ include "braintrust.aiGateway.serviceAccountName" . }} {{- if $ordering.enabled }} - serviceAccountName: {{ include "braintrust.migrations.gatewayAccount" . }} automountServiceAccountToken: false {{- end }} {{- with .Values.aiGateway.nodeSelector }} diff --git a/braintrust/templates/ai-gateway-serviceaccount.yaml b/braintrust/templates/ai-gateway-serviceaccount.yaml new file mode 100644 index 0000000..068fe22 --- /dev/null +++ b/braintrust/templates/ai-gateway-serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.aiGateway.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "braintrust.aiGateway.serviceAccountName" . }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.aiGateway.labels)) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.aiGateway.annotations.serviceaccount }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/braintrust/templates/brainstore-startup-gate-rbac.yaml b/braintrust/templates/brainstore-startup-gate-rbac.yaml index 99b560c..611ac56 100644 --- a/braintrust/templates/brainstore-startup-gate-rbac.yaml +++ b/braintrust/templates/brainstore-startup-gate-rbac.yaml @@ -40,7 +40,7 @@ subjects: {{- $_ := set $accounts $pool.config.serviceAccount.name true -}} {{- end -}} {{- if .Values.aiGateway.enabled -}} - {{- $_ := set $accounts (include "braintrust.migrations.gatewayAccount" .) true -}} + {{- $_ := set $accounts (include "braintrust.aiGateway.serviceAccountName" .) true -}} {{- end -}} {{- if include "braintrust.loopRuntime.enabled" . -}} {{- $_ := set $accounts .Values.loopRuntime.serviceAccount.name true -}} diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml index f012158..5ffe7a5 100644 --- a/braintrust/templates/migration-gate.yaml +++ b/braintrust/templates/migration-gate.yaml @@ -54,19 +54,11 @@ subjects: {{- $_ := set $accounts .Values.loopRuntime.serviceAccount.name true -}} {{- end -}} {{- if .Values.aiGateway.enabled -}} - {{- $_ := set $accounts (include "braintrust.migrations.gatewayAccount" .) true -}} + {{- $_ := set $accounts (include "braintrust.aiGateway.serviceAccountName" .) true -}} {{- end -}} {{- range $account := keys $accounts | sortAlpha }} - kind: ServiceAccount name: {{ $account }} namespace: {{ include "braintrust.namespace" $ }} {{- end }} -{{- if .Values.aiGateway.enabled }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "braintrust.migrations.gatewayAccount" . }} - namespace: {{ include "braintrust.namespace" . }} -{{- end }} {{- end }} diff --git a/braintrust/tests/ai-gateway-serviceaccount_test.yaml b/braintrust/tests/ai-gateway-serviceaccount_test.yaml new file mode 100644 index 0000000..a6d2e2f --- /dev/null +++ b/braintrust/tests/ai-gateway-serviceaccount_test.yaml @@ -0,0 +1,75 @@ +suite: AI Gateway service account lifecycle +templates: + - ai-gateway-serviceaccount.yaml + - ai-gateway-deployment.yaml + - ai-gateway-configmap.yaml + - migration-gate.yaml + - brainstore-startup-gate-rbac.yaml +tests: + - it: omits Gateway identity and workload when Gateway is disabled + set: + aiGateway.enabled: false + asserts: + - hasDocuments: + count: 0 + template: ai-gateway-serviceaccount.yaml + - hasDocuments: + count: 0 + template: ai-gateway-deployment.yaml + + - it: keeps Gateway identity without granting ordering permissions when ordering is disabled + release: + namespace: install-namespace + set: + aiGateway.enabled: true + aiGateway.serviceAccount.name: gateway-identity + deploymentOrdering.enabled: false + global.createNamespace: true + global.namespace: workload-namespace + asserts: + - equal: + path: metadata.name + value: gateway-identity + template: ai-gateway-serviceaccount.yaml + - equal: + path: metadata.namespace + value: workload-namespace + template: ai-gateway-serviceaccount.yaml + - equal: + path: spec.template.spec.serviceAccountName + value: gateway-identity + template: ai-gateway-deployment.yaml + - isNull: + path: spec.template.spec.initContainers + template: ai-gateway-deployment.yaml + - hasDocuments: + count: 0 + template: migration-gate.yaml + - hasDocuments: + count: 0 + template: brainstore-startup-gate-rbac.yaml + + - it: grants both ordered waits to the workload identity rather than the namespace default + release: + namespace: workload-namespace + set: + aiGateway.enabled: true + aiGateway.serviceAccount.name: gateway-identity + api.serviceAccount.name: api-identity + brainstore.serviceAccount.name: brainstore-identity + asserts: + - equal: + path: subjects + value: + - {kind: ServiceAccount, name: api-identity, namespace: workload-namespace} + - {kind: ServiceAccount, name: brainstore-identity, namespace: workload-namespace} + - {kind: ServiceAccount, name: gateway-identity, namespace: workload-namespace} + template: migration-gate.yaml + documentIndex: 2 + - equal: + path: subjects + value: + - {kind: ServiceAccount, name: api-identity, namespace: workload-namespace} + - {kind: ServiceAccount, name: gateway-identity, namespace: workload-namespace} + template: brainstore-startup-gate-rbac.yaml + documentIndex: 1 diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index 18a750e..a23ce5d 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -294,10 +294,11 @@ tests: namespace: workload set: aiGateway.enabled: true + aiGateway.serviceAccount.name: gateway-account asserts: - contains: path: subjects - content: {kind: ServiceAccount, name: test-migration-gate, namespace: workload} + content: {kind: ServiceAccount, name: gateway-account, namespace: workload} - contains: path: subjects content: {kind: ServiceAccount, name: braintrust-loop-runtime, namespace: workload} diff --git a/braintrust/tests/labels-merge-serviceaccounts_test.yaml b/braintrust/tests/labels-merge-serviceaccounts_test.yaml index 8e661f9..435a65f 100644 --- a/braintrust/tests/labels-merge-serviceaccounts_test.yaml +++ b/braintrust/tests/labels-merge-serviceaccounts_test.yaml @@ -3,6 +3,7 @@ templates: - api-serviceaccount.yaml - loop-runtime-serviceaccount.yaml - brainstore-serviceaccount.yaml + - ai-gateway-serviceaccount.yaml tests: # These tests verify the deepCopy fix for service accounts. # Without deepCopy, labels would bleed between components due to merge mutation. @@ -65,3 +66,41 @@ tests: value: shared: global-sa loop-specific: loop-sa + + - it: keeps Gateway account labels and annotations scoped to that account + template: ai-gateway-serviceaccount.yaml + set: &gatewayIdentityLabels + aiGateway.enabled: true + global.labels.shared: global-sa + global.labels.tier: global-tier + aiGateway.labels.tier: gateway-tier + aiGateway.labels.gateway-only: gateway-sa + api.labels.api-only: api-sa + aiGateway.annotations.serviceaccount: + example.com/identity: gateway + aiGateway.annotations.pod: + example.com/pod-only: gateway-pod + asserts: + - equal: + path: metadata.labels + value: + shared: global-sa + tier: gateway-tier + gateway-only: gateway-sa + - equal: + path: metadata.annotations + value: + example.com/identity: gateway + + - it: does not leak Gateway identity metadata into the API account + template: api-serviceaccount.yaml + set: *gatewayIdentityLabels + asserts: + - equal: + path: metadata.labels + value: + shared: global-sa + tier: global-tier + api-only: api-sa + - isNull: + path: metadata.annotations["example.com/identity"] diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index a806639..5a290d5 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -335,7 +335,6 @@ tests: set: {"aiGateway.enabled": true} asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"equal": {"path": "spec.template.spec.serviceAccountName", "value": "test-migration-gate"}} - {"equal": {"path": "spec.template.spec.automountServiceAccountToken", "value": false}} - {"equal": {"path": "spec.template.spec.initContainers[1].name", "value": "wait-for-brainstore"}} - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 3000}} diff --git a/braintrust/values.yaml b/braintrust/values.yaml index eb70d44..0de494f 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -387,6 +387,7 @@ aiGateway: deployment: {} service: {} pod: {} + serviceaccount: {} # With autoscaling, the default minReplicas. replicas: 2 # Needs metrics-server; without it the HPA holds minReplicas. @@ -409,6 +410,8 @@ aiGateway: type: ClusterIP port: 8080 portName: http + serviceAccount: + name: "braintrust-ai-gateway" # Optional cloud region identifier (e.g. us-east-1). Sets GATEWAY_REGION when provided. region: "" braintrustAppUrl: "https://www.braintrust.dev" From 22d756a0aa4bc742d9d4d10a240386d78b91d71e Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Thu, 12 Nov 2026 17:46:41 -0500 Subject: [PATCH 10/13] Require automation writers in Brainstore deployment ordering --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 2315542..efe19e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,7 +110,7 @@ The four brainstore configmaps (`brainstore-reader-configmap.yaml`, `brainstore- ### Brainstore Automation Writer loop config -The optional Automation Writer pool (`brainstore.automationwriter`, default `replicas: 0`) isolates the automations writer loop via `BRAINSTORE_WRITER_LOOP_CONFIG`. That variable exists starting in brainstore image `v2.16.0`. Enabling the pool (`replicas > 0`) fails the render when `brainstore.image.tag` is a release tag (`vX.Y.Z`) older than `v2.16.0`. Commit SHAs and other custom tags must keep rendering, so `braintrust.automationWriter.validate` only calls `semverCompare` after the tag matches the release-tag regex (`semverCompare` errors on a SHA). Do not bump the chart-wide default image for this. `brainstore-automationwriter-configmap.yaml` sets `include:automations`, and `brainstore-writer-configmap.yaml` sets `exclude:automations` only when `brainstore.automationwriter.replicas > 0`. A null or missing replica count is 0: an empty `replicas` field would make Kubernetes run 1 pod while the writer still handled automations. A missing `brainstore.automationwriter` map — what `helm upgrade --reuse-values` retains from a chart that predates the pool — is disabled: the writer does not exclude automations, and the automation writer manifests are omitted. A partial map, such as `--set brainstore.automationwriter.replicas=1` on those retained values, is merged with the template defaults in `braintrust.automationWriter.defaults` before any field is read. On Azure with the Container Storage driver, `brainstore.automationwriter.volume.size` is required only when `replicas > 0`; a zero-replica pool renders with `emptyDir` so an upgrade does not demand disk for a pool that is off. These two must stay in sync: the automation writer runs automations and the regular writer must exclude them whenever the pool is enabled. An enabled pool has an internal Service and publishes `BRAINSTORE_AUTOMATION_WRITER_URL` to all API pools whenever replicas are positive. A compatible API release decides which queries use the endpoint. The API startup gate and its RBAC deployment names cover readers, writers, and fast readers only. Resolve names and Service fields through `braintrust.automationWriter.config` so partial reused values keep working. Use `brainstore.automationwriter.service.port` wherever an automation writer port is required. Coordinate API routing changes before scaling the pool to zero; old API processes retain their URL until restarted. +The optional Automation Writer pool (`brainstore.automationwriter`, default `replicas: 0`) isolates the automations writer loop via `BRAINSTORE_WRITER_LOOP_CONFIG`. That variable exists starting in brainstore image `v2.16.0`. Enabling the pool (`replicas > 0`) fails the render when `brainstore.image.tag` is a release tag (`vX.Y.Z`) older than `v2.16.0`. Commit SHAs and other custom tags must keep rendering, so `braintrust.automationWriter.validate` only calls `semverCompare` after the tag matches the release-tag regex (`semverCompare` errors on a SHA). Do not bump the chart-wide default image for this. `brainstore-automationwriter-configmap.yaml` sets `include:automations`, and `brainstore-writer-configmap.yaml` sets `exclude:automations` only when `brainstore.automationwriter.replicas > 0`. A null or missing replica count is 0: an empty `replicas` field would make Kubernetes run 1 pod while the writer still handled automations. A missing `brainstore.automationwriter` map — what `helm upgrade --reuse-values` retains from a chart that predates the pool — is disabled: the writer does not exclude automations, and the automation writer manifests are omitted. A partial map, such as `--set brainstore.automationwriter.replicas=1` on those retained values, is merged with the template defaults in `braintrust.automationWriter.defaults` before any field is read. On Azure with the Container Storage driver, `brainstore.automationwriter.volume.size` is required only when `replicas > 0`; a zero-replica pool renders with `emptyDir` so an upgrade does not demand disk for a pool that is off. These two must stay in sync: the automation writer runs automations and the regular writer must exclude them whenever the pool is enabled. An enabled pool has an internal Service and publishes `BRAINSTORE_AUTOMATION_WRITER_URL` to all API pools whenever replicas are positive. A compatible API release decides which queries use the endpoint. Deployment ordering and its RBAC must cover all rendered Brainstore roles: readers, writers, fast readers, and automation writers. Include zero-replica Deployments until their old pods have drained; omit automation writers only when their Deployment is not rendered. Resolve names and Service fields through `braintrust.automationWriter.config` so partial reused values keep working. Use `brainstore.automationwriter.service.port` wherever an automation writer port is required. Coordinate API routing changes before scaling the pool to zero; old API processes retain their URL until restarted. ### Disruption budgets and GKE Standard From 01273ba08264507e379b733361acc6b2b8bc11a6 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Fri, 13 Nov 2026 16:49:37 -0500 Subject: [PATCH 11/13] Fix ordered upgrades for pinned images and legacy gates --- braintrust/README.md | 22 +- braintrust/files/brainstore-startup-gate.py | 48 ++- .../templates/_brainstore-startup-gate.tpl | 15 +- braintrust/templates/_deployment-ordering.tpl | 78 ++++ braintrust/templates/_helpers.tpl | 1 + .../brainstore-startup-gate-configmap.yaml | 25 ++ braintrust/templates/migration-job.yaml | 1 + .../tests/brainstore-startup-gate_test.yaml | 37 +- .../tests/legacy-startup-gate_test.yaml | 172 ++++++++ braintrust/tests/migration-image_test.yaml | 373 ++++++++++++++++++ braintrust/tests/migrations_test.yaml | 28 +- braintrust/values.yaml | 9 +- tests/test_brainstore_startup_gate.py | 190 ++++++++- 13 files changed, 966 insertions(+), 33 deletions(-) create mode 100644 braintrust/templates/brainstore-startup-gate-configmap.yaml create mode 100644 braintrust/tests/legacy-startup-gate_test.yaml create mode 100644 braintrust/tests/migration-image_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 00f2bd2..52a3101 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -561,8 +561,13 @@ Example values files for different cloud providers and configurations are locate `deploymentOrdering.enabled` (default `true`) runs one migration Job per Helm revision before Brainstore starts. All API pools, enabled AI Gateway, and enabled Loop Runtime then wait for the full compatible Brainstore rollout and start in -parallel. Set it to `false` for unordered startup. This replaces -`api.brainstoreStartupGate`; migrate existing opt-outs to the new setting. +parallel. Each ordered upgrade deliberately restarts Brainstore, even with +unchanged images: the revision-specific migration wait and release marker prevent +an older Ready rollout from admitting new applications before migrations finish. + +**Upgrade:** `api.brainstoreStartupGate` is no longer read. Existing opt-outs must +set `deploymentOrdering.enabled: false`; the old `false` value does not disable +the new ordering. Gateway has its own Kubernetes ServiceAccount regardless of ordering; configure its name with `aiGateway.serviceAccount.name`. @@ -570,14 +575,21 @@ its name with `aiGateway.serviceAccount.name`. Ordering gates container startup, not Kubernetes object updates. Older replacement pods also wait during an unfinished Brainstore rollout. -`deploymentOrdering.timeoutSeconds` bounds each phase; `minimumVersion` supplies -a stable compatibility floor for opaque API image tags. See [values.yaml](values.yaml) -for settings. Use `--wait --wait-for-jobs` with a sufficient Helm timeout. +`deploymentOrdering.timeoutSeconds` bounds each phase. Stable tags select a +migration bundle covering the newest API pool or Brainstore release. Opaque +API/Brainstore pins require both a stable `minimumVersion` compatibility assertion +and an explicit standalone-API `migrationImage` containing all required migrations. +Known stable images still undergo version checks; opaque Brainstore pods must +match their Deployment's exact image and declared release. See [values.yaml](values.yaml). +Use `--wait --wait-for-jobs` with a sufficient Helm timeout. Migration records are retained for replacement pods. Retry failed migrations with a new Helm upgrade. Rollback requires hooks and does not reverse SQL changes; migrations must remain backward-compatible. +The legacy startup-gate ConfigMap is preserved unchanged while old API replicas +may still need it; new pods use their revision-specific helper ConfigMap. + After uninstall, before reusing a release name, delete Jobs labeled `braintrust.dev/migration-release=` with foreground deletion and wait for completion, then delete labeled ConfigMaps, Roles, and RoleBindings. Never delete diff --git a/braintrust/files/brainstore-startup-gate.py b/braintrust/files/brainstore-startup-gate.py index 6ca344a..a504695 100644 --- a/braintrust/files/brainstore-startup-gate.py +++ b/braintrust/files/brainstore-startup-gate.py @@ -43,7 +43,7 @@ def compatible(image, minimum): return False # This assumes Brainstore preserves compatibility with older APIs within # a major release; version numbers alone cannot establish that contract. - # Unknown tags and major-version migrations must not pass this gate. + # Unknown tags cannot establish compatibility without a template assertion. return version[0] == minimum[0] and version >= minimum @@ -84,10 +84,10 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision): must belong to that rollout, and terminating Pods must disappear because connections may remain usable until their process exits. - Deployment rollouts replace Pods, so use each Pod's declared release tag. - Container status can report a different alias or only a digest. Neither is - a portable release identifier; manual in-place Pod image changes are outside - this check's contract. + Stable image tags establish compatibility directly. Opaque image pins + instead require the template's declared version and exact image/annotation + matches on every live Pod. Container status may report only an alias or + digest; manual in-place Pod image changes are outside this check's contract. """ metadata = deployment.get("metadata", {}) spec = deployment.get("spec", {}) @@ -103,7 +103,8 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision): desired = spec.get("replicas", 1) template = spec.get("template", {}) revision_key = "braintrust.dev/release-revision" - revision = template.get("metadata", {}).get("annotations", {}).get(revision_key) + template_annotations = template.get("metadata", {}).get("annotations", {}) + revision = template_annotations.get(revision_key) try: current_revision = int(revision) except (TypeError, ValueError): @@ -111,20 +112,41 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision): if current_revision < minimum_revision: return f"{name}: waiting for release revision {minimum_revision} or later" live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] + opaque_image = None if desired == 0: if live: return f"{name}: waiting for scaled-down Pods to disappear" else: - # Compatible Pods cannot compensate for an incompatible desired image. - image = container_image(template.get("spec", {}), container) - if not compatible(image, minimum): - return f"{name}: desired image {image!r} is below or outside the required release" + # An assertion must never override a known incompatible stable image. + desired_image = container_image(template.get("spec", {}), container) + if not desired_image: + return f"{name}: desired Brainstore image is missing" + try: + image_version(desired_image) + except ValueError: + version_key = "braintrust.dev/brainstore-version" + declared_version = template_annotations.get(version_key) + try: + version = release_version(declared_version) + except (TypeError, ValueError): + return f"{name}: opaque desired image requires a stable declared Brainstore version" + if version[0] != minimum[0] or version < minimum: + return f"{name}: declared Brainstore version is below or outside the required release" + opaque_image = desired_image + else: + if not compatible(desired_image, minimum): + return f"{name}: desired image {desired_image!r} is below or outside the required release" for pod in live: image = container_image(pod.get("spec", {}), container) - if not compatible(image, minimum): - pod_name = pod.get("metadata", {}).get("name", "unknown") - return f"{name}: incompatible Pod {pod_name} still exists ({image!r})" pod_metadata = pod.get("metadata", {}) + if opaque_image is not None: + if image != opaque_image: + return f"{name}: waiting for Pods with the exact desired opaque image" + if pod_metadata.get("annotations", {}).get(version_key) != declared_version: + return f"{name}: waiting for Pods with the declared Brainstore version" + elif not compatible(image, minimum): + pod_name = pod_metadata.get("name", "unknown") + return f"{name}: incompatible Pod {pod_name} still exists ({image!r})" if pod_metadata.get("annotations", {}).get(revision_key) != revision: return f"{name}: waiting for Pods from the current release revision" if pod_metadata.get("deletionTimestamp"): diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index 0da9ee2..3e9a537 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -2,13 +2,26 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. */}} {{- define "braintrust.brainstoreStartupGate.minimumVersion" -}} -{{- $minimum := .gate.minimumVersion | default (first (splitList "@" .api.image.tag)) -}} +{{- $minimum := .gate.minimumVersion | default (first (splitList "@" (toString .api.image.tag))) -}} {{- if not (regexMatch "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" $minimum) -}} {{- fail "deploymentOrdering requires a stable API release tag or an explicit stable minimumVersion" -}} {{- end -}} {{- trimPrefix "v" $minimum -}} {{- end -}} +{{/* Opaque builds need an operator-supplied release assertion, not a version guess. */}} +{{- define "braintrust.brainstoreStartupGate.brainstoreVersion" -}} +{{- $version := first (splitList "@" (toString .Values.brainstore.image.tag)) -}} +{{- if not (regexMatch "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" $version) -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} +{{- $version = required "deploymentOrdering.minimumVersion is required for opaque Brainstore image tags" $config.minimumVersion -}} +{{- end -}} +{{- if not (regexMatch "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" $version) -}} +{{- fail "deploymentOrdering.minimumVersion must be a stable release version" -}} +{{- end -}} +{{- trimPrefix "v" $version -}} +{{- end -}} + {{- define "braintrust.brainstoreStartupGate.name" -}} {{- printf "%s-brainstore-startup" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/braintrust/templates/_deployment-ordering.tpl b/braintrust/templates/_deployment-ordering.tpl index d7a3ccf..a0e15e6 100644 --- a/braintrust/templates/_deployment-ordering.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -3,6 +3,7 @@ {{- $defaults := dict "enabled" true "minimumVersion" "" + "migrationImage" "" "timeoutSeconds" 1200 "pollIntervalSeconds" 15 "backoffLimit" 2 @@ -32,6 +33,83 @@ {{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} {{- end -}} +{{/* Migration scripts ship in standalone-api, never in the Brainstore or gate +helper image. Preserve API pins, but translate a newer Brainstore release to +the shared API repository without carrying over the Brainstore digest. */}} +{{- define "braintrust.migrations.image" -}} +{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} +{{- if $config.enabled -}} +{{- $stableRelease := "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" -}} +{{- $images := list -}} +{{- range $pool := include "braintrust.apiPools" . | fromYamlArray -}} +{{- $images = append $images (dict "repository" $pool.config.image.repository "tag" (toString $pool.config.image.tag) "brainstore" false) -}} +{{- end -}} +{{- $images = append $images (dict "repository" .Values.api.image.repository "tag" (toString .Values.brainstore.image.tag) "brainstore" true) -}} +{{- $selectedImage := "" -}} +{{- $selectedVersion := "" -}} +{{- $major := "" -}} +{{- $opaque := false -}} +{{- range $image := $images -}} +{{- $tag := first (splitList "@" $image.tag) -}} +{{- if regexMatch $stableRelease $tag -}} +{{- $version := trimPrefix "v" $tag -}} +{{- $imageMajor := first (splitList "." $version) -}} +{{- if and $major (ne $major $imageMajor) -}} +{{- fail "deploymentOrdering cannot migrate API and Brainstore service images from different major releases" -}} +{{- end -}} +{{- $major = $imageMajor -}} +{{- if or (not $selectedVersion) (semverCompare (printf ">%s" $selectedVersion) $version) -}} +{{- $selectedVersion = $version -}} +{{- $selectedTag := $image.tag -}} +{{- if $image.brainstore -}} +{{- $selectedTag = $tag -}} +{{- end -}} +{{- $selectedImage = printf "%s:%s" $image.repository $selectedTag -}} +{{- end -}} +{{- else -}} +{{- $opaque = true -}} +{{- end -}} +{{- end -}} +{{- $override := $config.migrationImage -}} +{{- if kindIs "invalid" $override -}} +{{- $override = "" -}} +{{- end -}} +{{- if not (kindIs "string" $override) -}} +{{- fail "deploymentOrdering.migrationImage must be a full standalone-API image reference" -}} +{{- end -}} +{{- if and $opaque (not $override) -}} +{{- fail "deploymentOrdering.migrationImage is required for opaque API or Brainstore service image tags; minimumVersion cannot select a migration bundle" -}} +{{- end -}} +{{- $minimum := $config.minimumVersion | default "" | toString -}} +{{- if and $opaque (not $minimum) -}} +{{- fail "deploymentOrdering.minimumVersion is required for opaque API or Brainstore service image tags" -}} +{{- end -}} +{{- if and $minimum (not (regexMatch $stableRelease $minimum)) -}} +{{- fail "deploymentOrdering.minimumVersion must be a stable release version" -}} +{{- end -}} +{{- if $override -}} +{{- $reference := first (splitList "@" $override) -}} +{{- $lastComponent := last (splitList "/" $reference) -}} +{{- $overrideTag := "" -}} +{{- if contains ":" $lastComponent -}} +{{- $overrideTag = last (splitList ":" $lastComponent) -}} +{{- end -}} +{{- if regexMatch $stableRelease $overrideTag -}} +{{- $requiredVersion := $selectedVersion -}} +{{- if and $minimum (or (not $requiredVersion) (semverCompare (printf ">%s" $requiredVersion) $minimum)) -}} +{{- $requiredVersion = trimPrefix "v" $minimum -}} +{{- end -}} +{{- if and $requiredVersion (semverCompare (printf "<%s" $requiredVersion) $overrideTag) -}} +{{- fail (printf "deploymentOrdering.migrationImage release %s is older than required release %s" $overrideTag $requiredVersion) -}} +{{- end -}} +{{- end -}} +{{- $override -}} +{{- else -}} +{{- $selectedImage -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{- define "braintrust.migrations.initContainer" -}} {{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} - name: wait-for-migrations diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index f16d6f4..aceb1e3 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -306,6 +306,7 @@ and normal writable-layer/log overhead. {{- if $ordering.enabled -}} {{/* Release identity rejects a previous Ready generation before Helm updates it. */}} {{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} +{{- $_ := set $annotations "braintrust.dev/brainstore-version" (include "braintrust.brainstoreStartupGate.brainstoreVersion" $root) -}} {{- end -}} {{- toYaml $annotations -}} {{- end -}} diff --git a/braintrust/templates/brainstore-startup-gate-configmap.yaml b/braintrust/templates/brainstore-startup-gate-configmap.yaml new file mode 100644 index 0000000..4b761da --- /dev/null +++ b/braintrust/templates/brainstore-startup-gate-configmap.yaml @@ -0,0 +1,25 @@ +{{- $namespace := include "braintrust.namespace" . -}} +{{- $name := include "braintrust.brainstoreStartupGate.name" . -}} +{{- $existing := lookup "v1" "ConfigMap" $namespace $name -}} +{{- if $existing }} +{{/* Old ReplicaSets require their original script and environment contract. */}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + namespace: {{ $namespace }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.api.labels)) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- mergeOverwrite (deepCopy (.Values.api.annotations.configmap | default dict)) (dict "helm.sh/resource-policy" "keep") | toYaml | nindent 4 }} +{{- if hasKey $existing "data" }} +data: + {{- toYaml $existing.data | nindent 2 }} +{{- end }} +{{- if hasKey $existing "binaryData" }} +binaryData: + {{- toYaml $existing.binaryData | nindent 2 }} +{{- end }} +{{- end }} diff --git a/braintrust/templates/migration-job.yaml b/braintrust/templates/migration-job.yaml index 4291ac5..71fe42f 100644 --- a/braintrust/templates/migration-job.yaml +++ b/braintrust/templates/migration-job.yaml @@ -12,6 +12,7 @@ container is retained: sidecars and HTTP probes must not keep this Job alive. */ {{- $_ := unset $container $key -}} {{- end -}} {{- $_ := set $container "name" "migrate" -}} +{{- $_ := set $container "image" (include "braintrust.migrations.image" .) -}} {{- $_ := set $container "command" (list "/braintrust/venv/bin/python" "/braintrust/api-schema/lambda_function.py" "--execute") -}} {{- $_ := set $container "envFrom" (list (dict "configMapRef" (dict "name" (printf "%s-config" $name)))) -}} {{- $_ := set $pod.spec "containers" (list $container) -}} diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index a23ce5d..29c9759 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -189,9 +189,9 @@ tests: set: deploymentOrdering.enabled: true api.image.tag: deadbeef + deploymentOrdering.migrationImage: public.ecr.aws/braintrust/standalone-api:v2.23.0 asserts: - - failedTemplate: - errorMessage: deploymentOrdering requires a stable API release tag or an explicit stable minimumVersion + - failedTemplate: {} - it: should protect an API-only image bump while Brainstore is still old template: api-deployment.yaml @@ -302,3 +302,36 @@ tests: - contains: path: subjects content: {kind: ServiceAccount, name: braintrust-loop-runtime, namespace: workload} + + - it: requires an explicit compatibility assertion for opaque Brainstore images + template: brainstore-reader-deployment.yaml + set: + brainstore.image.tag: deadbeef + deploymentOrdering.migrationImage: public.ecr.aws/braintrust/standalone-api:v2.23.0 + asserts: + - failedTemplate: {} + + - it: reserves the actual stable Brainstore version against annotation overrides + template: brainstore-reader-deployment.yaml + set: + brainstore.image.tag: v2.22.0@sha256:abc + brainstore.reader.annotations.pod: + braintrust.dev/brainstore-version: "9.0.0" + deploymentOrdering.minimumVersion: "2.21.0" + asserts: + - equal: + path: spec.template.metadata.annotations["braintrust.dev/brainstore-version"] + value: "2.22.0" + + - it: reserves the explicit compatibility assertion for opaque Brainstore builds + template: brainstore-reader-deployment.yaml + set: + brainstore.image.tag: deadbeef + deploymentOrdering.minimumVersion: "2.22.0" + deploymentOrdering.migrationImage: public.ecr.aws/braintrust/standalone-api:release-bundle + brainstore.reader.annotations.pod: + braintrust.dev/brainstore-version: "9.0.0" + asserts: + - equal: + path: spec.template.metadata.annotations["braintrust.dev/brainstore-version"] + value: "2.22.0" diff --git a/braintrust/tests/legacy-startup-gate_test.yaml b/braintrust/tests/legacy-startup-gate_test.yaml new file mode 100644 index 0000000..a90c4b7 --- /dev/null +++ b/braintrust/tests/legacy-startup-gate_test.yaml @@ -0,0 +1,172 @@ +suite: retain existing legacy Brainstore startup gate ConfigMaps +templates: + - brainstore-startup-gate-configmap.yaml + - api-configmap.yaml +release: + name: legacy + namespace: release-ns +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + # Keep the fake provider active even when the requested legacy gate is absent. + - apiVersion: v1 + kind: ConfigMap + metadata: + name: unrelated-configmap + namespace: release-ns +tests: + - it: preserves every original text and binary entry instead of substituting the new gate + template: brainstore-startup-gate-configmap.yaml + kubernetesProvider: &existingGate + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: legacy-brainstore-startup + namespace: release-ns + data: &legacyData + gate.py: | + # Original legacy script; it has no rollout-revision arguments. + print("keep original script: {{ do not evaluate }}") + other-key: "literal: value\nwith a trailing newline\n" + empty-key: "" + binaryData: &legacyBinaryData + original.bin: AAEC/w== + empty.bin: "" + set: + global.labels.shared: global + global.labels.global-only: retained + api.labels.shared: api + api.annotations.configmap: + operator-note: retained + helm.sh/resource-policy: delete + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: legacy-brainstore-startup + - equal: + path: metadata.namespace + value: release-ns + - equal: + path: metadata.labels + value: + shared: api + global-only: retained + - equal: + path: metadata.annotations + value: + operator-note: retained + helm.sh/resource-policy: keep + - equal: + path: data + value: *legacyData + - equal: + path: binaryData + value: *legacyBinaryData + + - it: retains the original gate even when deployment ordering is disabled + template: brainstore-startup-gate-configmap.yaml + kubernetesProvider: *existingGate + set: + deploymentOrdering.enabled: false + asserts: + - hasDocuments: + count: 1 + - equal: + path: data + value: *legacyData + - equal: + path: binaryData + value: *legacyBinaryData + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + + - it: looks up the actual workload namespace rather than the release namespace + template: brainstore-startup-gate-configmap.yaml + set: + global.createNamespace: true + global.namespace: workload-ns + kubernetesProvider: + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: legacy-brainstore-startup + namespace: workload-ns + data: + gate.py: original workload namespace script + - apiVersion: v1 + kind: ConfigMap + metadata: + name: legacy-brainstore-startup + namespace: release-ns + data: + gate.py: wrong release namespace script + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: workload-ns + - equal: + path: data + value: + gate.py: original workload namespace script + + - it: does not create a gate on fresh installs without an existing object + template: brainstore-startup-gate-configmap.yaml + asserts: + - hasDocuments: + count: 0 + + - it: does not retain a same-named object outside the workload namespace + template: brainstore-startup-gate-configmap.yaml + kubernetesProvider: *existingGate + set: + global.createNamespace: true + global.namespace: workload-ns + asserts: + - hasDocuments: + count: 0 + + - it: preserves binary-only objects without inventing text data + template: brainstore-startup-gate-configmap.yaml + kubernetesProvider: + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: legacy-brainstore-startup + namespace: release-ns + binaryData: *legacyBinaryData + asserts: + - hasDocuments: + count: 1 + - isNull: + path: data + - equal: + path: binaryData + value: *legacyBinaryData + + - it: does not leak retention annotations into application ConfigMaps + template: api-configmap.yaml + kubernetesProvider: *existingGate + set: + api.annotations.configmap: + operator-note: preserved + asserts: + - equal: + path: metadata.annotations.operator-note + value: preserved + - isNull: + path: metadata.annotations["helm.sh/resource-policy"] diff --git a/braintrust/tests/migration-image_test.yaml b/braintrust/tests/migration-image_test.yaml new file mode 100644 index 0000000..bb70cef --- /dev/null +++ b/braintrust/tests/migration-image_test.yaml @@ -0,0 +1,373 @@ +suite: migration bundle selection across serving images +templates: + - migration-job.yaml + - api-configmap.yaml +release: + name: test + namespace: workload + revision: 7 +set: + api.image.repository: registry.example/standalone-api + api.image.tag: v2.21.0 + brainstore.image.repository: registry.example/brainstore + brainstore.image.tag: v2.22.0 +tests: + - it: uses standalone API scripts at the newer Brainstore release + template: migration-job.yaml + documentIndex: 0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + + - it: preserves the repository and full pin of a newer API image + template: migration-job.yaml + documentIndex: 0 + set: + api.image.repository: mirror.example:5000/custom/api + api.image.tag: v2.24.0@sha256:api + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example:5000/custom/api:v2.24.0@sha256:api + + - it: accepts stable API tags without a v prefix + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: 2.24.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:2.24.0 + + - it: drops the Brainstore digest when selecting its release in the API repository + template: migration-job.yaml + documentIndex: 0 + set: + brainstore.image.tag: v2.24.0@sha256:brainstore + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.24.0 + + - it: preserves a Brainstore stable tag without its v prefix or digest + template: migration-job.yaml + documentIndex: 0 + set: + brainstore.image.tag: 2.24.0@sha256:brainstore + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:2.24.0 + + - it: compares release components numerically rather than lexicographically + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: v2.9.0 + brainstore.image.tag: v2.22.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + + - it: keeps the API digest when API and Brainstore releases are equal + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: v2.22.0@sha256:api + brainstore.image.tag: 2.22.0@sha256:brainstore + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0@sha256:api + + - it: selects a newer isolated ingest pool including its custom repository and digest + template: migration-job.yaml + documentIndex: 0 + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.image.repository: mirror.example:5000/ingest-api + api.workloadIsolation.ingest.image.tag: v2.24.0@sha256:ingest + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example:5000/ingest-api:v2.24.0@sha256:ingest + + - it: considers the background pool even when ingest is also newer than the shared API + template: migration-job.yaml + documentIndex: 0 + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.image.tag: v2.24.0 + api.workloadIsolation.background.image.repository: registry.example/background-api + api.workloadIsolation.background.image.tag: 2.25.0@sha256:background + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/background-api:2.25.0@sha256:background + + - it: uses the shared API repository when Brainstore is newer than every isolated pool + template: migration-job.yaml + documentIndex: 0 + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.background.image.repository: registry.example/background-api + api.workloadIsolation.background.image.tag: v2.25.0 + brainstore.image.tag: v2.26.0@sha256:brainstore + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.26.0 + + - it: ignores image overrides for API pools that are not rendered + template: migration-job.yaml + documentIndex: 0 + set: + api.workloadIsolation.enabled: false + api.workloadIsolation.ingest.image.tag: v3.0.0 + api.workloadIsolation.background.image.tag: deadbeef + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + + - it: blocks a newer API major before executing migrations + template: migration-job.yaml + set: + api.image.tag: v3.0.0 + asserts: + - failedTemplate: &mixed-major + errorMessage: deploymentOrdering cannot migrate API and Brainstore service images from different major releases + + - it: blocks a newer Brainstore major before executing migrations + template: migration-job.yaml + set: + brainstore.image.tag: v3.0.0@sha256:brainstore + asserts: + - failedTemplate: *mixed-major + + - it: does not let an explicit opaque bundle bypass incompatible API pool majors + template: migration-job.yaml + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.background.image.tag: v3.0.0 + deploymentOrdering.migrationImage: registry.example/migration-api:custom + asserts: + - failedTemplate: *mixed-major + + - it: requires a bundle for a SHA API even with an explicit release floor + template: migration-job.yaml + set: + api.image.tag: deadbeef + deploymentOrdering.minimumVersion: v2.22.0 + asserts: + - failedTemplate: &opaque-bundle + errorMessage: deploymentOrdering.migrationImage is required for opaque API or Brainstore service image tags; minimumVersion cannot select a migration bundle + + - it: requires a bundle for a SHA Brainstore instead of borrowing the floor's release + template: migration-job.yaml + set: + brainstore.image.tag: 0123456789abcdef0123456789abcdef0123456789 + deploymentOrdering.minimumVersion: v2.25.0 + asserts: + - failedTemplate: *opaque-bundle + + - it: handles an all-digit numeric API SHA without a semver parsing error + template: migration-job.yaml + set: + api.image.tag: 1234567 + deploymentOrdering.minimumVersion: v2.22.0 + asserts: + - failedTemplate: *opaque-bundle + + - it: requires a bundle for prerelease serving tags + template: migration-job.yaml + set: + api.image.tag: v2.24.0-rc.1 + deploymentOrdering.minimumVersion: v2.22.0 + asserts: + - failedTemplate: *opaque-bundle + + - it: requires a bundle for custom serving tags with a digest + template: migration-job.yaml + set: + brainstore.image.tag: customer-build@sha256:brainstore + deploymentOrdering.minimumVersion: v2.22.0 + asserts: + - failedTemplate: *opaque-bundle + + - it: requires a bundle if any isolated API pool is opaque + template: migration-job.yaml + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.ingest.image.tag: deadbeef + api.workloadIsolation.background.image.tag: v2.25.0 + deploymentOrdering.minimumVersion: v2.22.0 + asserts: + - failedTemplate: *opaque-bundle + + - it: accepts an explicit digest-only standalone API bundle for opaque services + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: 0123456789abcdef0123456789abcdef0123456789 + brainstore.image.tag: deadbeef + deploymentOrdering.minimumVersion: v2.22.0 + deploymentOrdering.migrationImage: mirror.example:5000/standalone-api@sha256:bundle + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example:5000/standalone-api@sha256:bundle + + - it: accepts an explicit prerelease bundle as the operator's compatibility assertion + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: 1234567 + brainstore.image.tag: customer-build + deploymentOrdering.minimumVersion: v2.22.0 + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.24.0-rc.1@sha256:bundle + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.24.0-rc.1@sha256:bundle + + - it: accepts an opaque bundle for stable services without inventing a floor requirement + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering.migrationImage: registry.example/standalone-api:customer-migrations + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:customer-migrations + + - it: preserves a known explicit bundle tag and digest at the required release + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering.migrationImage: mirror.example:5000/standalone-api:2.22.0@sha256:bundle + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: mirror.example:5000/standalone-api:2.22.0@sha256:bundle + + - it: rejects a known explicit bundle older than Brainstore + template: migration-job.yaml + set: + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.21.0@sha256:bundle + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.migrationImage release v2.21.0 is older than required release 2.22.0 + + - it: rejects a known explicit bundle older than the newest isolated API pool + template: migration-job.yaml + set: + api.workloadIsolation.enabled: true + api.workloadIsolation.background.image.tag: v2.25.0 + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.24.0 + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.migrationImage release v2.24.0 is older than required release 2.25.0 + + - it: rejects a known explicit bundle older than the explicit floor for opaque services + template: migration-job.yaml + set: + api.image.tag: deadbeef + deploymentOrdering.minimumVersion: v2.25.0 + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.24.0 + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.migrationImage release v2.24.0 is older than required release 2.25.0 + + - it: accepts a known explicit bundle at the floor for opaque services + template: migration-job.yaml + documentIndex: 0 + set: + api.image.tag: deadbeef + brainstore.image.tag: customer-build + deploymentOrdering.minimumVersion: v2.25.0 + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.25.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.25.0 + + - it: still requires a release floor for opaque services when a bundle is explicit + template: migration-job.yaml + set: + brainstore.image.tag: deadbeef + deploymentOrdering.migrationImage: registry.example/standalone-api:customer-migrations + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.minimumVersion is required for opaque API or Brainstore service image tags + + - it: validates an explicit floor before comparing a stable bundle against it + template: migration-job.yaml + set: + deploymentOrdering.minimumVersion: not-a-version + deploymentOrdering.migrationImage: registry.example/standalone-api:v2.24.0 + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.minimumVersion must be a stable release version + + - it: leaves an explicitly untagged standalone API reference to the operator's assertion + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering.migrationImage: registry.example/standalone-api + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api + + - it: rejects a non-string migration image instead of silently ignoring zero + template: migration-job.yaml + set: + deploymentOrdering.migrationImage: 0 + asserts: + - failedTemplate: + errorMessage: deploymentOrdering.migrationImage must be a full standalone-API image reference + + - it: treats a null migration image as automatic selection + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering.migrationImage: null + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + + - it: selects a bundle when reused values omit the ordering map + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering: null + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + + - it: ignores unused opaque images invalid overrides and floors when ordering is disabled + template: migration-job.yaml + set: + deploymentOrdering.enabled: false + deploymentOrdering.minimumVersion: not-a-version + deploymentOrdering.migrationImage: 0 + api.image.tag: deadbeef + brainstore.image.tag: customer-build + asserts: + - hasDocuments: + count: 0 + + - it: does not reject unused cross-major combinations when ordering is disabled + template: migration-job.yaml + set: + deploymentOrdering.enabled: false + api.image.tag: v3.0.0 + asserts: + - hasDocuments: + count: 0 diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index 5a290d5..d182dba 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -23,7 +23,7 @@ release: namespace: workload revision: 7 tests: - - it: runs a regular bounded migration Job using the target API image + - it: runs a regular bounded migration Job using the newest standalone API bundle template: migration-job.yaml documentIndex: 0 set: @@ -56,7 +56,7 @@ tests: value: [/braintrust/venv/bin/python, /braintrust/api-schema/lambda_function.py, --execute] - equal: path: spec.template.spec.containers[0].image - value: registry.example/custom-api:v2.15.0 + value: registry.example/custom-api:v2.23.0 - isNull: path: spec.template.spec.initContainers - isNull: @@ -278,6 +278,29 @@ tests: - isNull: path: metadata.labels["reader-only"] + - it: keeps the serving API image unchanged when Brainstore selects a newer bundle + template: api-deployment.yaml + set: + api.image.repository: registry.example/custom-api + api.image.tag: v2.21.0 + brainstore.image.tag: v2.22.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/custom-api:v2.21.0 + + - it: keeps the Brainstore image and digest unchanged when selecting an API bundle + template: brainstore-reader-deployment.yaml + set: + api.image.repository: registry.example/custom-api + api.image.tag: v2.21.0 + brainstore.image.repository: registry.example/custom-brainstore + brainstore.image.tag: v2.22.0@sha256:brainstore + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/custom-brainstore:v2.22.0@sha256:brainstore + - it: gates the reader pool template: brainstore-reader-deployment.yaml asserts: @@ -617,6 +640,7 @@ tests: set: api.image.tag: deadbeef deploymentOrdering.minimumVersion: v2.15.0 + deploymentOrdering.migrationImage: public.ecr.aws/braintrust/standalone-api:v2.23.0 asserts: - contains: path: spec.template.spec.initContainers[1].env diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 0de494f..193a781 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -820,14 +820,17 @@ virtualService: # See README.md#release-ordering for configuration and operational limits. deploymentOrdering: enabled: true - # Empty derives the compatible Brainstore floor from each API pool's image tag. - # Opaque API tags require an explicit stable minimumVersion. + # Empty derives the API compatibility floor from each pool's stable image tag. + # Required for opaque API/Brainstore tags; asserts the opaque Brainstore release. minimumVersion: "" + # Full standalone-API image containing migrations for all selected service images. + # Empty selects the newest stable API/Brainstore release; required for opaque pins. + migrationImage: "" # Separate budget for each phase (migration Job/wait, then Brainstore rollout). timeoutSeconds: 1200 pollIntervalSeconds: 15 backoffLimit: 2 - # Small read-only init-container helper; migration itself uses api.image. + # Small read-only init-container helper; separate from the migration bundle above. image: "python:3.13-alpine@sha256:79e7a9b9ff1cbceff819f856fb374477792a5967759d94df266de7b7b4120e6f" resources: requests: diff --git a/tests/test_brainstore_startup_gate.py b/tests/test_brainstore_startup_gate.py index 40a2ae9..9d8892d 100644 --- a/tests/test_brainstore_startup_gate.py +++ b/tests/test_brainstore_startup_gate.py @@ -15,27 +15,33 @@ def pod(version="v2.15.0", uid="new", is_ready=True, revision=8, - container="brainstore-reader", **metadata): + container="brainstore-reader", image=None, declared_version=None, **metadata): + image = image if image is not None else "registry/brainstore:" + version if revision is not None: - metadata["annotations"] = {"braintrust.dev/release-revision": str(revision)} + metadata.setdefault("annotations", {})["braintrust.dev/release-revision"] = str(revision) + if declared_version is not None: + metadata.setdefault("annotations", {})["braintrust.dev/brainstore-version"] = declared_version return { "metadata": {"name": uid, "uid": uid, **metadata}, - "spec": {"containers": [{"name": container, "image": "registry/brainstore:" + version}]}, + "spec": {"containers": [{"name": container, "image": image}]}, "status": { "phase": "Running", "conditions": [{"type": "Ready", "status": "True" if is_ready else "False"}], - "containerStatuses": [{"name": container, "image": "registry/brainstore:" + version, + "containerStatuses": [{"name": container, "image": image, "ready": is_ready, "state": {"running": {}}}], }, } -def deployment(version="v2.15.0", replicas=1, revision=8, container="brainstore-reader"): +def deployment(version="v2.15.0", replicas=1, revision=8, container="brainstore-reader", + image=None, declared_version=None): return { "metadata": {"name": "reader", "uid": "reader-deployment", "generation": 2}, "spec": {"replicas": replicas, "template": { - "metadata": {"annotations": pod(revision=revision)["metadata"].get("annotations", {})}, - "spec": pod(version, container=container)["spec"], + "metadata": {"annotations": pod( + revision=revision, declared_version=declared_version, + )["metadata"].get("annotations", {})}, + "spec": pod(version, container=container, image=image)["spec"], }}, "status": {"observedGeneration": 2, "updatedReplicas": replicas, "readyReplicas": replicas, "availableReplicas": replicas, "replicas": replicas}, @@ -106,6 +112,176 @@ def test_runtime_alias_does_not_override_an_incompatible_pod_spec(self): self.assertIsNotNone(self.check(pods=[current])) +class OpaqueImageTests(unittest.TestCase): + def rollout(self, image="registry/brainstore:deadbeef", declared_version="2.15.0", + revision=8): + return ( + deployment(image=image, declared_version=declared_version, revision=revision), + pod(image=image, declared_version=declared_version, revision=revision), + ) + + def check(self, resource, pods, minimum="2.15.0"): + return gate.check_deployment( + resource, pods, "brainstore-reader", gate.release_version(minimum), 8, + ) + + def test_sha_custom_prerelease_and_digest_pins_accept_explicit_assertions(self): + for image in ( + "registry/brainstore:deadbeef", "registry/brainstore:" + "a" * 40, + "registry/brainstore:0123456", "registry/brainstore:" + "0" * 40, + "registry/brainstore:custom-build", "registry/brainstore:latest", + "registry/brainstore:v2.15.0-rc.1", + "localhost:5000/brainstore@sha256:abc", + "registry/brainstore:deadbeef@sha256:abc", + ): + for declared_version in ("2.15.0", "v2.16.0"): + with self.subTest(image=image, declared_version=declared_version): + resource, current = self.rollout(image, declared_version) + self.assertIsNone(self.check(resource, [current])) + + def test_missing_or_invalid_template_assertion_blocks_matching_pods(self): + for declared_version in (None, "", "unknown", "2.015.0", "2.15", + "2.15.0-rc.1", "2.15.0+build", "2.15.0\n", 215, True): + with self.subTest(declared_version=declared_version): + resource, current = self.rollout(declared_version=declared_version) + self.assertIsNotNone(self.check(resource, [current])) + + def test_lower_or_other_major_template_assertions_cannot_be_overridden_by_pods(self): + for declared_version in ("2.14.99", "1.99.0", "3.0.0"): + with self.subTest(declared_version=declared_version): + resource, current = self.rollout(declared_version=declared_version) + self.assertIsNotNone(self.check(resource, [current])) + current["metadata"]["annotations"]["braintrust.dev/brainstore-version"] = "2.15.0" + self.assertIsNotNone(self.check(resource, [current])) + + def test_every_live_pod_must_have_the_exact_template_assertion(self): + for declared_version in (None, "", "unknown", "2.14.0", "3.0.0", + "2.16.0", "v2.15.0", 215): + with self.subTest(declared_version=declared_version): + resource, _ = self.rollout() + current = pod(image="registry/brainstore:deadbeef", + declared_version=declared_version) + self.assertIsNotNone(self.check(resource, [current])) + resource["spec"]["replicas"] = 2 + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + resource["status"][counter] = 2 + stray = pod(uid="stray", image="registry/brainstore:deadbeef", + declared_version="2.15.0") + self.assertIsNotNone(self.check(resource, [stray, current])) + + def test_matching_assertion_does_not_allow_a_different_image_or_stray_backend(self): + for desired_image, other_image in ( + ("registry/brainstore:deadbeef", "registry/brainstore:feedface"), + ("registry/brainstore:deadbeef", "other/brainstore:deadbeef"), + ("registry/brainstore:deadbeef", "registry/brainstore:v2.15.0"), + ("registry/brainstore:deadbeef@sha256:abc", "registry/brainstore:deadbeef"), + ("registry/brainstore@sha256:abc", "registry/brainstore@sha256:def"), + ): + with self.subTest(desired_image=desired_image, other_image=other_image): + resource, _ = self.rollout(desired_image) + stray = pod(uid="stray", image=other_image, declared_version="2.15.0") + self.assertIsNotNone(self.check(resource, [stray])) + resource["spec"]["replicas"] = 2 + for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + resource["status"][counter] = 2 + current = pod(image=desired_image, declared_version="2.15.0") + self.assertIsNotNone(self.check(resource, [current, stray])) + + def test_assertion_does_not_replace_a_missing_desired_or_pod_container(self): + resource, current = self.rollout() + current["spec"]["containers"] = [] + self.assertIsNotNone(self.check(resource, [current])) + resource, current = self.rollout() + resource["spec"]["template"]["spec"]["containers"] = [] + self.assertIsNotNone(self.check(resource, [current])) + + def test_forged_assertions_cannot_override_stable_image_versions(self): + for image in ("registry/brainstore:v2.14.0", "registry/brainstore:v3.0.0", + "registry/brainstore:v2.14.0@sha256:abc", + "registry/brainstore:v3.0.0@sha256:abc"): + with self.subTest(image=image): + resource, current = self.rollout(image) + self.assertIsNotNone(self.check(resource, [current])) + compatible_pod = pod(declared_version="2.15.0") + self.assertIsNotNone(self.check(resource, [compatible_pod])) + resource = deployment(declared_version="2.15.0") + self.assertIsNotNone(self.check(resource, [current])) + + def test_stable_images_use_actual_versions_not_assertions(self): + for declared_version in (None, "invalid", "1.0.0", "3.0.0"): + with self.subTest(declared_version=declared_version): + resource, current = self.rollout("registry/brainstore:v2.16.0", + declared_version) + self.assertIsNone(self.check(resource, [current])) + resource = deployment() + _, opaque = self.rollout() + self.assertIsNotNone(self.check(resource, [opaque])) + + def test_current_assertion_still_requires_a_completed_observed_rollout(self): + for change in ("generation", "revision", "pod_revision", "terminating", + "unready", "missing_pod", "surge", + "updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): + with self.subTest(change=change): + resource, current = self.rollout() + pods = [current] + if change == "generation": + resource["status"]["observedGeneration"] = 1 + elif change == "revision": + resource["spec"]["template"]["metadata"]["annotations"][ + "braintrust.dev/release-revision"] = "7" + elif change == "pod_revision": + current["metadata"]["annotations"]["braintrust.dev/release-revision"] = "7" + elif change == "terminating": + current["metadata"]["deletionTimestamp"] = "2026-09-30T12:00:00Z" + elif change == "unready": + current["status"]["containerStatuses"][0]["ready"] = False + elif change == "missing_pod": + pods = [] + elif change == "surge": + pods.append(pod(uid="surge", image="registry/brainstore:deadbeef", + declared_version="2.15.0")) + else: + resource["status"][change] = 0 + self.assertIsNotNone(self.check(resource, pods)) + + def test_older_api_uses_later_opaque_rollouts_own_assertion(self): + resource, current = self.rollout("registry/brainstore:later-sha", "2.16.0", revision=9) + self.assertIsNone(self.check(resource, [current])) + self.assertIsNotNone(self.check(resource, [current], minimum="2.17.0")) + current["metadata"]["annotations"]["braintrust.dev/brainstore-version"] = "2.15.0" + self.assertIsNotNone(self.check(resource, [current])) + resource, current = self.rollout("registry/brainstore:major-three", "3.0.0", revision=10) + self.assertIsNotNone(self.check(resource, [current])) + self.assertIsNone(self.check(resource, [current], minimum="3.0.0")) + + + def test_later_opaque_major_rollout_resets_an_older_apis_confirmation(self): + first, first_pod = self.rollout() + later, later_pod = self.rollout("registry/brainstore:major-three", "3.0.0", revision=9) + client = Mock() + client.deployment.side_effect = [first, later, later] + client.pods.side_effect = [[first_pod], [later_pod], [later_pod]] + clock = FakeClock() + with self.assertRaisesRegex(TimeoutError, "declared Brainstore version"): + gate.wait_for_fleet( + client, [{"deployment": "reader", "container": "brainstore-reader"}], + (2, 15, 0), 8, 45, 15, clock.clock, clock.sleep, Mock(), + ) + self.assertEqual(client.deployment.call_count, 3) + + def test_opaque_zero_role_needs_drain_not_a_compatible_assertion(self): + for declared_version in (None, "invalid", "1.0.0", "3.0.0"): + with self.subTest(declared_version=declared_version): + resource = deployment(replicas=0, image="registry/brainstore@sha256:abc", + declared_version=declared_version) + self.assertIsNone(self.check(resource, [])) + current = pod(image="registry/brainstore@sha256:abc", + declared_version=declared_version) + self.assertIsNotNone(self.check(resource, [current])) + resource["status"]["replicas"] = 1 + self.assertIsNotNone(self.check(resource, [])) + + class RolloutTests(unittest.TestCase): def check(self, resource=None, pods=None, minimum_revision=8): return gate.check_deployment( From 83d51dcd3f362411d0808f44b113dc4b1b473839 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Fri, 13 Nov 2026 17:27:38 -0500 Subject: [PATCH 12/13] Order changed workloads without revision-driven restarts --- braintrust/README.md | 28 +- braintrust/files/brainstore-startup-gate.py | 120 +++- braintrust/files/job-gate.py | 42 +- braintrust/files/migration-rollback.py | 100 --- .../templates/_brainstore-startup-gate.tpl | 29 +- braintrust/templates/_deployment-ordering.tpl | 47 +- braintrust/templates/_helpers.tpl | 10 +- braintrust/templates/migration-gate.yaml | 17 +- braintrust/templates/migration-job.yaml | 36 +- braintrust/templates/migration-rollback.yaml | 100 --- .../tests/ai-gateway-serviceaccount_test.yaml | 9 + .../tests/brainstore-startup-gate_test.yaml | 24 - braintrust/tests/migration-image_test.yaml | 40 ++ braintrust/tests/migrations_test.yaml | 176 +++--- braintrust/values.yaml | 1 + tests/test_brainstore_startup_gate.py | 577 ++++++++++++------ tests/test_job_gate.py | 368 +++++------ 17 files changed, 903 insertions(+), 821 deletions(-) delete mode 100644 braintrust/files/migration-rollback.py delete mode 100644 braintrust/templates/migration-rollback.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 52a3101..3655aa2 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -558,12 +558,10 @@ Example values files for different cloud providers and configurations are locate ## Release ordering -`deploymentOrdering.enabled` (default `true`) runs one migration Job per Helm -revision before Brainstore starts. All API pools, enabled AI Gateway, and enabled -Loop Runtime then wait for the full compatible Brainstore rollout and start in -parallel. Each ordered upgrade deliberately restarts Brainstore, even with -unchanged images: the revision-specific migration wait and release marker prevent -an older Ready rollout from admitting new applications before migrations finish. +`deploymentOrdering.enabled` (default `true`) orders startup as migrations → +Brainstore → API pools, enabled AI Gateway, and enabled Loop Runtime. Only changed +workload pod templates roll out; unchanged workloads keep serving. A no-op Helm +upgrade creates no new migration Job or pod rollout. **Upgrade:** `api.brainstoreStartupGate` is no longer read. Existing opt-outs must set `deploymentOrdering.enabled: false`; the old `false` value does not disable @@ -572,8 +570,10 @@ the new ordering. Gateway has its own Kubernetes ServiceAccount regardless of ordering; configure its name with `aiGateway.serviceAccount.name`. -Ordering gates container startup, not Kubernetes object updates. Older replacement -pods also wait during an unfinished Brainstore rollout. +Ordering gates container startup, not Kubernetes object updates. Gates read the +latest desired state from a stable ConfigMap and verify each Brainstore role's +expected pod-template identity, observed generation, and complete readiness. +Old Ready pods cannot satisfy a changed role's gate; unchanged healthy roles can. `deploymentOrdering.timeoutSeconds` bounds each phase. Stable tags select a migration bundle covering the newest API pool or Brainstore release. Opaque @@ -583,12 +583,16 @@ Known stable images still undergo version checks; opaque Brainstore pods must match their Deployment's exact image and declared release. See [values.yaml](values.yaml). Use `--wait --wait-for-jobs` with a sufficient Helm timeout. -Migration records are retained for replacement pods. Retry failed migrations with -a new Helm upgrade. Rollback requires hooks and does not reverse SQL changes; -migrations must remain backward-compatible. +Migration Jobs are keyed by their image and execution configuration; successful +Jobs and immutable configuration snapshots are retained and reused, including on +rollback. After diagnosing a failed migration, delete its failed Job before +retrying the same inputs with Helm. Rollback does not reverse SQL changes; +migrations must remain backward-compatible. External Secret value changes and +replaced images behind unchanged tags are not detected; use immutable image pins +and explicitly restart affected workloads when changing external secrets. The legacy startup-gate ConfigMap is preserved unchanged while old API replicas -may still need it; new pods use their revision-specific helper ConfigMap. +may still need it; new pods use the stable ordering ConfigMap. After uninstall, before reusing a release name, delete Jobs labeled `braintrust.dev/migration-release=` with foreground deletion and wait for diff --git a/braintrust/files/brainstore-startup-gate.py b/braintrust/files/brainstore-startup-gate.py index a504695..6d42898 100644 --- a/braintrust/files/brainstore-startup-gate.py +++ b/braintrust/files/brainstore-startup-gate.py @@ -1,8 +1,8 @@ """Wait for compatible Brainstore rollouts before application startup. Uses only Python's standard library and read-only Kubernetes API requests. -The required version is fixed in the application's Pod environment, not a -ConfigMap that changes underneath an older Pod when the next release is installed. +The required version is fixed in the application's Pod environment. The expected +fleet is read from the stable ordering ConfigMap on every observation. """ import json @@ -19,6 +19,8 @@ VERSION = re.compile(r"v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\Z") TERMINAL_PHASES = {"Failed", "Succeeded"} +TEMPLATE_ANNOTATION = "braintrust.dev/pod-template" +TEMPLATE_HASH = re.compile(r"[0-9a-f]{64}\Z") def release_version(value): @@ -76,7 +78,7 @@ def ready(pod, container): ) -def check_deployment(deployment, pods, container, minimum, minimum_revision): +def check_deployment(deployment, pods, target, minimum): """Return a wait reason, or None when the compatible rollout is complete. The current Deployment rollout must finish, even for an older application @@ -93,24 +95,26 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision): spec = deployment.get("spec", {}) status = deployment.get("status", {}) name = metadata.get("name", "unknown") - if minimum_revision <= 0: - return f"{name}: minimum release revision must be positive" + container = target["container"] if metadata.get("deletionTimestamp"): return f"{name}: deployment is being deleted" # Do not trust availability status from before the latest spec update. - if status.get("observedGeneration", 0) < metadata.get("generation", 1): + generation = metadata.get("generation") + observed = status.get("observedGeneration", 0) + if (type(generation) is not int or generation <= 0 + or type(observed) is not int or observed < generation): return f"{name}: controller has not observed the desired generation" - desired = spec.get("replicas", 1) + desired = target["replicas"] + if type(spec.get("replicas", 1)) is not int or spec.get("replicas", 1) != desired: + return f"{name}: waiting for the expected {desired} replicas in the Deployment spec" + min_ready = target["minReadySeconds"] + if (type(spec.get("minReadySeconds", 0)) is not int + or spec.get("minReadySeconds", 0) != min_ready): + return f"{name}: waiting for the expected minReadySeconds in the Deployment spec" template = spec.get("template", {}) - revision_key = "braintrust.dev/release-revision" template_annotations = template.get("metadata", {}).get("annotations", {}) - revision = template_annotations.get(revision_key) - try: - current_revision = int(revision) - except (TypeError, ValueError): - return f"{name}: desired release revision is missing or invalid" - if current_revision < minimum_revision: - return f"{name}: waiting for release revision {minimum_revision} or later" + if template_annotations.get(TEMPLATE_ANNOTATION) != target["template"]: + return f"{name}: waiting for the expected Pod template in the Deployment spec" live = [pod for pod in pods if pod.get("status", {}).get("phase") not in TERMINAL_PHASES] opaque_image = None if desired == 0: @@ -147,15 +151,16 @@ def check_deployment(deployment, pods, container, minimum, minimum_revision): elif not compatible(image, minimum): pod_name = pod_metadata.get("name", "unknown") return f"{name}: incompatible Pod {pod_name} still exists ({image!r})" - if pod_metadata.get("annotations", {}).get(revision_key) != revision: - return f"{name}: waiting for Pods from the current release revision" + if pod_metadata.get("annotations", {}).get(TEMPLATE_ANNOTATION) != target["template"]: + return f"{name}: waiting for Pods with the expected Pod template" if pod_metadata.get("deletionTimestamp"): return f"{name}: waiting for terminating Pods to disappear" # Require both current Pod readiness and Deployment availability; the # latter also accounts for the Deployment's minReadySeconds setting. ready_count = sum(ready(pod, container) for pod in live) counters = ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas") - if any(status.get(counter, 0) != desired for counter in counters): + if any(type(status.get(counter, 0)) is not int or status.get(counter, 0) != desired + for counter in counters): return f"{name}: waiting for all {desired} replicas to complete the current rollout" if len(live) != desired or ready_count != desired: return f"{name}: waiting for exactly {desired} current Ready Pods" @@ -217,29 +222,91 @@ def pods(self, selector, deadline): return items -def check_fleet(client, targets, minimum, minimum_revision, deadline): +def load_ordering_config(client, name, deadline): + name = urllib.parse.quote(name, safe="") + resource = client.get( + f"/api/v1/namespaces/{client.namespace}/configmaps/{name}", deadline, + ) + if not isinstance(resource, dict) or not isinstance(resource.get("data"), dict): + raise ValueError("ordering ConfigMap data is missing") + config = json.loads(resource["data"]["ordering.json"]) + if not isinstance(config, dict): + raise ValueError("ordering configuration must be an object") + if not isinstance(config.get("job"), str) or not config["job"]: + raise ValueError("ordering configuration requires a Job name") + targets = config.get("targets") + if not isinstance(targets, list) or not targets: + raise ValueError("ordering configuration requires nonempty targets") + for target in targets: + if not isinstance(target, dict): + raise ValueError("ordering targets must be objects") + for field in ("deployment", "container"): + if not isinstance(target.get(field), str) or not target[field]: + raise ValueError(f"ordering target requires a {field} name") + fingerprint = target.get("template") + if not isinstance(fingerprint, str) or not TEMPLATE_HASH.fullmatch(fingerprint): + raise ValueError("ordering target requires a SHA256 Pod template") + for field in ("replicas", "minReadySeconds"): + if type(target.get(field)) is not int or target[field] < 0: + raise ValueError(f"ordering target requires nonnegative integer {field}") + return config + + +def job_complete(job): + if not isinstance(job, dict): + raise ValueError("Job observation must be an object") + metadata = job.get("metadata", {}) + status = job.get("status", {}) + if not isinstance(metadata, dict) or not isinstance(status, dict): + raise ValueError("Job metadata and status must be objects") + conditions = status.get("conditions", []) + if not isinstance(conditions, list) or any(not isinstance(c, dict) for c in conditions): + raise ValueError("Job conditions must be objects") + if any(c.get("type") in {"Failed", "FailureTarget"} and c.get("status") == "True" + for c in conditions): + raise RuntimeError("Migration Job failed; inspect its logs before retrying") + if metadata.get("deletionTimestamp"): + return False + return any(c.get("type") == "Complete" and c.get("status") == "True" for c in conditions) + + +def migration_complete(client, name, deadline): + name = urllib.parse.quote(name, safe="") + path = f"/apis/batch/v1/namespaces/{client.namespace}/jobs/{name}" + return job_complete(client.get(path, deadline)) + + +def check_fleet(client, targets, minimum, deadline): for target in targets: deployment = client.deployment(target["deployment"], deadline) # The chart's Services select app=. Include stray Pods # with that label too; they can receive traffic regardless of ownership. pods = client.pods("app=" + target["deployment"], deadline) reason = check_deployment( - deployment, pods, target["container"], minimum, minimum_revision, + deployment, pods, target, minimum, ) if reason: return reason return None -def wait_for_fleet(client, targets, minimum, minimum_revision, timeout, interval, +def wait_for_fleet(client, configmap, minimum, timeout, interval, clock=time.monotonic, sleep=time.sleep, log=print): # All roles and API requests share one budget, unaffected by wall-clock changes. deadline = clock() + timeout successes = 0 last_reason = "no observation yet" + expected = None while clock() < deadline: try: - reason = check_fleet(client, targets, minimum, minimum_revision, deadline) + ordering = load_ordering_config(client, configmap, deadline) + if ordering != expected: + successes = 0 + expected = ordering + if migration_complete(client, ordering["job"], deadline): + reason = check_fleet(client, ordering["targets"], minimum, deadline) + else: + reason = f"migration Job {ordering['job']} has not completed" except urllib.error.HTTPError as error: # RBAC and Deployments may be applied after the application Pod is created. path = urllib.parse.urlsplit(error.url).path @@ -268,20 +335,19 @@ def wait_for_fleet(client, targets, minimum, minimum_revision, timeout, interval def main(): minimum = release_version(os.environ["BRAINSTORE_MINIMUM_VERSION"]) - minimum_revision = int(os.environ["BRAINSTORE_MINIMUM_REVISION"]) - targets = json.loads(os.environ["BRAINSTORE_DEPLOYMENTS"]) timeout = int(os.environ["BRAINSTORE_GATE_TIMEOUT_SECONDS"]) interval = int(os.environ["BRAINSTORE_GATE_POLL_SECONDS"]) - if not targets or minimum_revision <= 0 or timeout <= 0 or interval <= 0 or interval >= timeout: + configmap = os.environ["ORDERING_CONFIGMAP"] + if not configmap or timeout <= 0 or interval <= 0 or interval >= timeout: raise ValueError("invalid startup gate configuration") client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/brainstore-gate") - wait_for_fleet(client, targets, minimum, minimum_revision, timeout, interval) + wait_for_fleet(client, configmap, minimum, timeout, interval) if __name__ == "__main__": try: main() - except (OSError, ValueError, KeyError, TypeError) as error: + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: print(str(error), file=sys.stderr, flush=True) # Fail closed: Kubernetes retries this init container; the application stays stopped. sys.exit(1) diff --git a/braintrust/files/job-gate.py b/braintrust/files/job-gate.py index 99af236..6438e8e 100644 --- a/braintrust/files/job-gate.py +++ b/braintrust/files/job-gate.py @@ -1,55 +1,47 @@ -"""Wait for the exact release Job; never infer success from absence.""" +"""Wait for the current content-addressed migration Job; fail closed.""" import os import sys import time import urllib.error -import urllib.parse -from brainstore_gate import KubernetesClient +from brainstore_gate import KubernetesClient, load_ordering_config, migration_complete -def job_complete(job): - conditions = job.get("status", {}).get("conditions", []) - if any(c.get("type") in {"Failed", "FailureTarget"} and c.get("status") == "True" - for c in conditions): - raise RuntimeError("Job failed; inspect its logs and retry with a new Helm upgrade") - if job.get("metadata", {}).get("deletionTimestamp"): - return False - return any(c.get("type") == "Complete" and c.get("status") == "True" for c in conditions) - - -def wait_for_job(client, name, timeout, interval, +def wait_for_job(client, configmap, timeout, interval, clock=time.monotonic, sleep=time.sleep, log=print): deadline = clock() + timeout - path = f"/apis/batch/v1/namespaces/{client.namespace}/jobs/{urllib.parse.quote(name, safe='')}" - reason = "Job has not completed" + reason = "no observation yet" while clock() < deadline: + stage = "ordering configuration" try: - complete = job_complete(client.get(path, deadline)) + ordering = load_ordering_config(client, configmap, deadline) + name = ordering["job"] + stage = f"Job {name}" + complete = migration_complete(client, name, deadline) if complete and clock() < deadline: log(f"Job {name} completed successfully", flush=True) return - reason = "Job has not completed" + reason = f"Job {name} has not completed" except urllib.error.HTTPError as error: # Job and RBAC may be applied after the Deployment, including 403/404. - reason = f"Kubernetes API returned HTTP {error.code}" + reason = f"{stage}: Kubernetes API returned HTTP {error.code}" error.close() except (OSError, ValueError, KeyError, TypeError) as error: - reason = f"Kubernetes observation failed: {type(error).__name__}" - log(f"Waiting for Job {name}: {reason}", flush=True) + reason = f"{stage}: Kubernetes observation failed: {type(error).__name__}" + log(f"Waiting for migration Job: {reason}", flush=True) sleep(min(interval, max(0, deadline - clock()))) - raise TimeoutError(f"Job {name} wait timed out after {timeout}s: {reason}") + raise TimeoutError(f"Migration Job wait timed out after {timeout}s: {reason}") def main(): timeout = int(os.environ["JOB_TIMEOUT_SECONDS"]) interval = int(os.environ["JOB_POLL_SECONDS"]) - name = os.environ["JOB_NAME"] - if not name or interval <= 0 or timeout <= interval: + configmap = os.environ["ORDERING_CONFIGMAP"] + if not configmap or interval <= 0 or timeout <= interval: raise ValueError("invalid Job gate configuration") client = KubernetesClient(os.environ["POD_NAMESPACE"], "/var/run/migration-gate") - wait_for_job(client, name, timeout, interval) + wait_for_job(client, configmap, timeout, interval) if __name__ == "__main__": diff --git a/braintrust/files/migration-rollback.py b/braintrust/files/migration-rollback.py deleted file mode 100644 index 27363f8..0000000 --- a/braintrust/files/migration-rollback.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Remove the target revision's retained resources before Helm restores them. - -Helm 3 cannot update a retained object absent from the current release manifest. -Delete only the named target revision; foreground Job deletion waits for all of -its migration pods before rollback can recreate the Job. -""" -import json -import os -from pathlib import Path -import ssl -import time -import urllib.error -import urllib.parse -import urllib.request - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, request, response, code, message, headers, new_url): - raise urllib.error.HTTPError(request.full_url, code, message, headers, response) - - -class Client: - def __init__(self): - host = os.environ["KUBERNETES_SERVICE_HOST"] - if ":" in host and not host.startswith("["): - host = f"[{host}]" - self.base = f"https://{host}:{os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443')}" - self.credentials = Path("/var/run/secrets/kubernetes.io/serviceaccount") - self.opener = urllib.request.build_opener( - urllib.request.ProxyHandler({}), NoRedirect(), - urllib.request.HTTPSHandler(context=ssl.create_default_context( - cafile=str(self.credentials / "ca.crt"))), - ) - - def request(self, method, path, deadline): - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("rollback release cleanup timed out") - token = (self.credentials / "token").read_text().strip() - payload = json.dumps({"propagationPolicy": "Foreground"}).encode() if method == "DELETE" else None - request = urllib.request.Request(self.base + path, method=method, data=payload, - headers={"Authorization": f"Bearer {token}", - "Content-Type": "application/json"}) - try: - with self.opener.open(request, timeout=min(5, remaining)): - return True - except urllib.error.HTTPError as error: - missing = error.code == 404 - error.close() - if missing: - return False - # Do not log response bodies or credentials. - raise RuntimeError(f"Kubernetes cleanup {method} returned HTTP {error.code}") from None - - -def resource_paths(namespace, name): - namespace = urllib.parse.quote(namespace, safe="") - name = urllib.parse.quote(name, safe="") - return [ - f"/apis/batch/v1/namespaces/{namespace}/jobs/{name}", - f"/api/v1/namespaces/{namespace}/configmaps/{name}-config", - f"/api/v1/namespaces/{namespace}/configmaps/{name}-gate", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings/{name}", - f"/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles/{name}", - ] - - -def cleanup(client, namespace, name, timeout, clock=time.monotonic, sleep=time.sleep): - deadline = clock() + timeout - paths = resource_paths(namespace, name) - - def check_deadline(): - if clock() >= deadline: - raise TimeoutError("rollback release cleanup timed out") - - def request(method, path): - check_deadline() - exists = client.request(method, path, deadline) - check_deadline() - return exists - - def wait_until_deleted(path): - while request("GET", path): - sleep(min(2, max(0, deadline - clock()))) - - # Stop the migration before removing any config or permissions it relies on. - for path in paths: - request("DELETE", path) - wait_until_deleted(path) - - -def main(): - timeout = int(os.environ["JOB_TIMEOUT_SECONDS"]) - if timeout <= 0: - raise ValueError("JOB_TIMEOUT_SECONDS must be positive") - cleanup(Client(), os.environ["POD_NAMESPACE"], os.environ["JOB_NAME"], timeout) - - -if __name__ == "__main__": - main() diff --git a/braintrust/templates/_brainstore-startup-gate.tpl b/braintrust/templates/_brainstore-startup-gate.tpl index 3e9a537..1325dcd 100644 --- a/braintrust/templates/_brainstore-startup-gate.tpl +++ b/braintrust/templates/_brainstore-startup-gate.tpl @@ -39,6 +39,29 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. {{- toJson $targets -}} {{- end -}} +{{/* Keep desired identities outside application pod templates: dependency-only +changes must not restart otherwise unchanged applications. */}} +{{- define "braintrust.brainstoreStartupGate.expectedTargets" -}} +{{- $targets := list -}} +{{- range $role := list "reader" "writer" "fastreader" "automationwriter" -}} +{{- if or (ne $role "automationwriter") $.Values.brainstore.automationwriter -}} +{{- $deployment := include (printf "%s/brainstore-%s-deployment.yaml" $.Template.BasePath $role) $ | fromYaml -}} +{{- $replicas := 1 -}} +{{- if not (kindIs "invalid" $deployment.spec.replicas) -}} +{{- $replicas = int $deployment.spec.replicas -}} +{{- end -}} +{{- $targets = append $targets (dict + "deployment" $deployment.metadata.name + "container" (printf "brainstore-%s" $role) + "template" (index $deployment.spec.template.metadata.annotations "braintrust.dev/pod-template") + "replicas" $replicas + "minReadySeconds" (int $deployment.spec.minReadySeconds) +) -}} +{{- end -}} +{{- end -}} +{{- toJson $targets -}} +{{- end -}} + {{- define "braintrust.brainstoreStartupGate.validateApi" -}} {{- $rolling := .api.strategy.rollingUpdate | default dict -}} {{- if or (ne .api.strategy.type "RollingUpdate") (not (has (toString $rolling.maxUnavailable) (list "0" "0%"))) -}} @@ -74,14 +97,12 @@ An API-only image bump must not silently accept the still-old Brainstore fleet. fieldPath: metadata.namespace - name: BRAINSTORE_MINIMUM_VERSION value: {{ include "braintrust.brainstoreStartupGate.minimumVersion" (dict "gate" $config "api" .api) | quote }} - - name: BRAINSTORE_DEPLOYMENTS - value: {{ include "braintrust.brainstoreStartupGate.targets" .root | quote }} + - name: ORDERING_CONFIGMAP + value: {{ include "braintrust.deploymentOrdering.name" .root | quote }} - name: BRAINSTORE_GATE_TIMEOUT_SECONDS value: {{ $config.timeoutSeconds | quote }} - name: BRAINSTORE_GATE_POLL_SECONDS value: {{ $config.pollIntervalSeconds | quote }} - - name: BRAINSTORE_MINIMUM_REVISION - value: {{ .root.Release.Revision | quote }} volumeMounts: - name: migration-gate-code mountPath: /opt/migration-gate diff --git a/braintrust/templates/_deployment-ordering.tpl b/braintrust/templates/_deployment-ordering.tpl index a0e15e6..f11677d 100644 --- a/braintrust/templates/_deployment-ordering.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -28,9 +28,46 @@ {{- toYaml $config -}} {{- end -}} -{{/* Preserve the revision suffix even for long release names. */}} +{{/* Workloads reference only this stable desired-state ConfigMap, never a Job. */}} +{{- define "braintrust.deploymentOrdering.name" -}} +{{- printf "%s-%s-ordering" (.Release.Name | trunc 40 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) -}} +{{- end -}} + +{{/* One execution contract supplies both the content hash and the actual Job. +Use a fixed name when hashing to avoid self-referential labels and snapshot refs. */}} +{{- define "braintrust.migrations.execution" -}} +{{- $root := .root -}} +{{- $name := .name -}} +{{- $config := include "braintrust.deploymentOrdering.config" $root | fromYaml -}} +{{- $apiConfig := include (print $root.Template.BasePath "/api-configmap.yaml") $root | fromYaml -}} +{{- $api := include "braintrust.apiPoolConfig" (dict "root" $root "overrides" (dict)) | fromYaml -}} +{{- $deployment := include "braintrust.apiDeployment" (dict "root" $root "api" $api "role" "migration" "migrationJob" true) | fromYaml -}} +{{- $pod := $deployment.spec.template -}} +{{- $container := first $pod.spec.containers -}} +{{- range $key := list "ports" "livenessProbe" "readinessProbe" "startupProbe" -}} +{{- $_ := unset $container $key -}} +{{- end -}} +{{- $_ := set $container "name" "migrate" -}} +{{- $_ := set $container "image" (include "braintrust.migrations.image" $root) -}} +{{- $_ := set $container "command" (list "/braintrust/venv/bin/python" "/braintrust/api-schema/lambda_function.py" "--execute") -}} +{{- $_ := set $container "envFrom" (list (dict "configMapRef" (dict "name" (printf "%s-config" $name)))) -}} +{{- $_ := set $pod.spec "containers" (list $container) -}} +{{- $_ := set $pod.spec "restartPolicy" "Never" -}} +{{- $_ := set $pod.spec "automountServiceAccountToken" false -}} +{{- $_ := unset $pod.spec "topologySpreadConstraints" -}} +{{/* The snapshot's data, not mutable API ConfigMap metadata, drives execution. */}} +{{- $_ := set $pod.metadata.annotations "checksum/config" ($apiConfig.data | toJson | sha256sum) -}} +{{/* A migration pod must never match API Service / Deployment selectors. */}} +{{- $_ := set $pod.metadata.labels "app" $name -}} +{{- $_ := set $pod.metadata.labels "braintrust.dev/api-pool" "migration" -}} +{{- dict "pod" $pod "apiConfig" $apiConfig.data "settings" (dict "backoffLimit" (int $config.backoffLimit) "activeDeadlineSeconds" (int $config.timeoutSeconds)) | toYaml -}} +{{- end -}} + +{{/* Leave room for the immutable configuration snapshot suffix, even at the +maximum release-name length. Release revisions are deliberately not inputs. */}} {{- define "braintrust.migrations.name" -}} -{{- printf "%s-%s-migrate-r%d" (.Release.Name | trunc 24 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) (int .Release.Revision) -}} +{{- $execution := include "braintrust.migrations.execution" (dict "root" . "name" "migration-content") | fromYaml -}} +{{- printf "%s-%s-migrate-%s" (.Release.Name | trunc 20 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) ($execution | toJson | sha256sum | trunc 16) -}} {{- end -}} {{/* Migration scripts ship in standalone-api, never in the Brainstore or gate @@ -133,8 +170,8 @@ the shared API repository without carrying over the Brainstore digest. */}} valueFrom: fieldRef: fieldPath: metadata.namespace - - name: JOB_NAME - value: {{ include "braintrust.migrations.name" . | quote }} + - name: ORDERING_CONFIGMAP + value: {{ include "braintrust.deploymentOrdering.name" . | quote }} - name: JOB_TIMEOUT_SECONDS value: {{ $config.timeoutSeconds | quote }} - name: JOB_POLL_SECONDS @@ -151,7 +188,7 @@ the shared API repository without carrying over the Brainstore digest. */}} {{- define "braintrust.migrations.volumes" -}} - name: migration-gate-code configMap: - name: {{ include "braintrust.migrations.name" . }}-gate + name: {{ include "braintrust.deploymentOrdering.name" . }} - name: migration-gate-credentials projected: sources: diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index aceb1e3..6b35d51 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -294,7 +294,7 @@ and normal writable-layer/log overhead. {{- toYaml $resources -}} {{- end -}} -{{/* Merge Brainstore pod annotations, reserving release identity for rollout checks. */}} +{{/* Merge pod annotations; content identity rejects stale Ready rollouts. */}} {{- define "braintrust.brainstore.podAnnotations" -}} {{- $root := .root -}} {{- $annotations := dict "checksum/config" (include (printf "%s/brainstore-%s-configmap.yaml" $root.Template.BasePath .role) $root | sha256sum) -}} @@ -304,10 +304,14 @@ and normal writable-layer/log overhead. {{- $annotations = mergeOverwrite $annotations (deepCopy (.config.annotations.pod | default dict)) -}} {{- $ordering := include "braintrust.deploymentOrdering.config" $root | fromYaml -}} {{- if $ordering.enabled -}} -{{/* Release identity rejects a previous Ready generation before Helm updates it. */}} -{{- $_ := set $annotations "braintrust.dev/release-revision" (toString $root.Release.Revision) -}} +{{- $_ := unset $annotations "braintrust.dev/pod-template" -}} {{- $_ := set $annotations "braintrust.dev/brainstore-version" (include "braintrust.brainstoreStartupGate.brainstoreVersion" $root) -}} +{{- if not ($root.brainstoreFingerprint | default false) -}} +{{- $context := merge (dict "brainstoreFingerprint" true) $root -}} +{{- $deployment := include (printf "%s/brainstore-%s-deployment.yaml" $root.Template.BasePath .role) $context | fromYaml -}} +{{- $_ := set $annotations "braintrust.dev/pod-template" ($deployment.spec.template | toJson | sha256sum) -}} {{- end -}} +{{- end }}{{/* ordering.enabled */}} {{- toYaml $annotations -}} {{- end -}} diff --git a/braintrust/templates/migration-gate.yaml b/braintrust/templates/migration-gate.yaml index 5ffe7a5..8c9cdf4 100644 --- a/braintrust/templates/migration-gate.yaml +++ b/braintrust/templates/migration-gate.yaml @@ -1,16 +1,15 @@ {{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $config.enabled -}} -{{- $name := include "braintrust.migrations.name" . -}} +{{- $name := include "braintrust.deploymentOrdering.name" . -}} apiVersion: v1 kind: ConfigMap metadata: - name: {{ $name }}-gate + name: {{ $name }} namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep labels: braintrust.dev/migration-release: {{ .Release.Name | quote }} data: + ordering.json: {{ dict "job" (include "braintrust.migrations.name" .) "targets" (include "braintrust.brainstoreStartupGate.expectedTargets" . | fromJsonArray) | toJson | quote }} gate.py: | {{- .Files.Get "files/job-gate.py" | nindent 4 }} brainstore_gate.py: | @@ -21,15 +20,17 @@ kind: Role metadata: name: {{ $name }} namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep labels: braintrust.dev/migration-release: {{ .Release.Name | quote }} rules: - apiGroups: ["batch"] resources: ["jobs"] resourceNames: - - {{ $name | quote }} + - {{ include "braintrust.migrations.name" . | quote }} + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{ $name | quote }}] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -37,8 +38,6 @@ kind: RoleBinding metadata: name: {{ $name }} namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/resource-policy: keep labels: braintrust.dev/migration-release: {{ .Release.Name | quote }} roleRef: diff --git a/braintrust/templates/migration-job.yaml b/braintrust/templates/migration-job.yaml index 71fe42f..a9bc86c 100644 --- a/braintrust/templates/migration-job.yaml +++ b/braintrust/templates/migration-job.yaml @@ -1,27 +1,8 @@ {{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} {{- if $config.enabled -}} {{- $name := include "braintrust.migrations.name" . -}} -{{- $api := include "braintrust.apiPoolConfig" (dict "root" . "overrides" (dict)) | fromYaml -}} -{{/* Reuse the API pod contract so identity, TLS, secrets, security and scheduling -stay in sync. The migrationJob flag suppresses release ordering. Only the main -container is retained: sidecars and HTTP probes must not keep this Job alive. */}} -{{- $deployment := include "braintrust.apiDeployment" (dict "root" . "api" $api "role" "migration" "migrationJob" true) | fromYaml -}} -{{- $pod := $deployment.spec.template -}} -{{- $container := first $pod.spec.containers -}} -{{- range $key := list "ports" "livenessProbe" "readinessProbe" "startupProbe" -}} -{{- $_ := unset $container $key -}} -{{- end -}} -{{- $_ := set $container "name" "migrate" -}} -{{- $_ := set $container "image" (include "braintrust.migrations.image" .) -}} -{{- $_ := set $container "command" (list "/braintrust/venv/bin/python" "/braintrust/api-schema/lambda_function.py" "--execute") -}} -{{- $_ := set $container "envFrom" (list (dict "configMapRef" (dict "name" (printf "%s-config" $name)))) -}} -{{- $_ := set $pod.spec "containers" (list $container) -}} -{{- $_ := set $pod.spec "restartPolicy" "Never" -}} -{{- $_ := set $pod.spec "automountServiceAccountToken" false -}} -{{- $_ := unset $pod.spec "topologySpreadConstraints" -}} -{{/* A migration pod must never match API Service / Deployment selectors. */}} -{{- $_ := set $pod.metadata.labels "app" $name -}} -{{- $_ := set $pod.metadata.labels "braintrust.dev/api-pool" "migration" -}} +{{/* Reuse exactly the execution contract used to content-address this Job. */}} +{{- $execution := include "braintrust.migrations.execution" (dict "root" . "name" $name) | fromYaml -}} apiVersion: batch/v1 kind: Job metadata: @@ -32,14 +13,12 @@ metadata: labels: {{- mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.api.labels) (dict "braintrust.dev/migration-release" .Release.Name) | toYaml | nindent 4 }} spec: - backoffLimit: {{ $config.backoffLimit }} - activeDeadlineSeconds: {{ $config.timeoutSeconds }} + {{- toYaml $execution.settings | nindent 2 }} template: - {{- toYaml $pod | nindent 4 }} + {{- toYaml $execution.pod | nindent 4 }} --- -{{/* Snapshot non-secret configuration for this attempt rather than reading a -mutable API ConfigMap after a subsequent upgrade. Secrets remain Secret refs. */}} -{{- $apiConfig := include (print .Template.BasePath "/api-configmap.yaml") . | fromYaml }} +{{/* Successful content-addressed Jobs and their configuration snapshots are +reused across upgrades and rollbacks. Secrets remain Secret references. */}} apiVersion: v1 kind: ConfigMap metadata: @@ -49,6 +28,7 @@ metadata: helm.sh/resource-policy: keep labels: braintrust.dev/migration-release: {{ .Release.Name | quote }} +immutable: true data: - {{- toYaml $apiConfig.data | nindent 2 }} + {{- toYaml $execution.apiConfig | nindent 2 }} {{- end }} diff --git a/braintrust/templates/migration-rollback.yaml b/braintrust/templates/migration-rollback.yaml deleted file mode 100644 index 20e536d..0000000 --- a/braintrust/templates/migration-rollback.yaml +++ /dev/null @@ -1,100 +0,0 @@ -{{- $config := include "braintrust.deploymentOrdering.config" . | fromYaml -}} -{{- if $config.enabled -}} -{{- $name := include "braintrust.migrations.name" . -}} -{{- $hook := printf "%s-reset" $name -}} -{{/* Hook manifests belong to the target release, so the name is the original -revision's name, not the new rollback revision. No hook runs during install or -upgrade. Helm executes these by weight, then deletes successful hooks in reverse. */}} -{{- range $kind := list "ServiceAccount" "Role" "RoleBinding" }} ---- -apiVersion: {{ ternary "v1" "rbac.authorization.k8s.io/v1" (eq $kind "ServiceAccount") }} -kind: {{ $kind }} -metadata: - name: {{ $hook }} - namespace: {{ include "braintrust.namespace" $ }} - annotations: - helm.sh/hook: pre-rollback - helm.sh/hook-weight: {{ index (dict "ServiceAccount" "-3" "Role" "-2" "RoleBinding" "-1") $kind | quote }} - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -{{- if eq $kind "Role" }} -rules: - - apiGroups: ["batch"] - resources: ["jobs"] - resourceNames: [{{ $name | quote }}] - verbs: ["get", "delete"] - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: [{{ printf "%s-config" $name | quote }}, {{ printf "%s-gate" $name | quote }}] - verbs: ["get", "delete"] - - apiGroups: ["rbac.authorization.k8s.io"] - resources: ["roles", "rolebindings"] - resourceNames: [{{ $name | quote }}] - verbs: ["get", "delete"] -{{- else if eq $kind "RoleBinding" }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $hook }} -subjects: - - kind: ServiceAccount - name: {{ $hook }} - namespace: {{ include "braintrust.namespace" $ }} -{{- end }} -{{- end }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $hook }} - namespace: {{ include "braintrust.namespace" . }} - annotations: - helm.sh/hook: pre-rollback - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: {{ $config.backoffLimit }} - activeDeadlineSeconds: {{ $config.timeoutSeconds }} - template: - spec: - serviceAccountName: {{ $hook }} - restartPolicy: Never - {{- with .Values.api.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.api.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: cleanup - image: {{ $config.image | quote }} - command: - - python3 - - -B - - -u - - -c - - | - {{- .Files.Get "files/migration-rollback.py" | nindent 14 }} - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - resources: - {{- toYaml $config.resources | nindent 12 }} - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: JOB_NAME - value: {{ $name | quote }} - - name: JOB_TIMEOUT_SECONDS - value: {{ $config.timeoutSeconds | quote }} -{{- end }} diff --git a/braintrust/tests/ai-gateway-serviceaccount_test.yaml b/braintrust/tests/ai-gateway-serviceaccount_test.yaml index a6d2e2f..5efffcd 100644 --- a/braintrust/tests/ai-gateway-serviceaccount_test.yaml +++ b/braintrust/tests/ai-gateway-serviceaccount_test.yaml @@ -5,6 +5,15 @@ templates: - ai-gateway-configmap.yaml - migration-gate.yaml - brainstore-startup-gate-rbac.yaml + - api-configmap.yaml + - brainstore-reader-deployment.yaml + - brainstore-reader-configmap.yaml + - brainstore-writer-deployment.yaml + - brainstore-writer-configmap.yaml + - brainstore-fastreader-deployment.yaml + - brainstore-fastreader-configmap.yaml + - brainstore-automationwriter-deployment.yaml + - brainstore-automationwriter-configmap.yaml tests: - it: omits Gateway identity and workload when Gateway is disabled set: diff --git a/braintrust/tests/brainstore-startup-gate_test.yaml b/braintrust/tests/brainstore-startup-gate_test.yaml index 29c9759..8a70f0f 100644 --- a/braintrust/tests/brainstore-startup-gate_test.yaml +++ b/braintrust/tests/brainstore-startup-gate_test.yaml @@ -41,30 +41,6 @@ tests: content: name: BRAINSTORE_MINIMUM_VERSION value: "2.15.0" - - - it: should retain zero-replica roles in the rollout drain check - template: api-deployment.yaml - release: - revision: 7 - set: - brainstore.reader.name: drain-reader - brainstore.reader.replicas: 0 - brainstore.writer.name: drain-writer - brainstore.writer.replicas: 0 - brainstore.fastreader.name: drain-fastreader - brainstore.fastreader.replicas: 0 - brainstore.automationwriter.name: drain-automationwriter - brainstore.automationwriter.replicas: 0 - asserts: - - equal: - path: spec.template.spec.initContainers[1].env[2].value - value: '[{"container":"brainstore-reader","deployment":"drain-reader"},{"container":"brainstore-writer","deployment":"drain-writer"},{"container":"brainstore-fastreader","deployment":"drain-fastreader"},{"container":"brainstore-automationwriter","deployment":"drain-automationwriter"}]' - - contains: - path: spec.template.spec.initContainers[1].env - content: - name: BRAINSTORE_MINIMUM_REVISION - value: "7" - - it: should mount gate credentials only into the init container template: api-deployment.yaml set: diff --git a/braintrust/tests/migration-image_test.yaml b/braintrust/tests/migration-image_test.yaml index bb70cef..49d655b 100644 --- a/braintrust/tests/migration-image_test.yaml +++ b/braintrust/tests/migration-image_test.yaml @@ -19,6 +19,12 @@ tests: - equal: path: spec.template.spec.containers[0].image value: registry.example/standalone-api:v2.22.0 + - matchRegex: + path: metadata.name + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' + - matchRegex: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' - it: preserves the repository and full pin of a newer API image template: migration-job.yaml @@ -220,6 +226,9 @@ tests: - equal: path: spec.template.spec.containers[0].image value: mirror.example:5000/standalone-api@sha256:bundle + - matchRegex: + path: metadata.name + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' - it: accepts an explicit prerelease bundle as the operator's compatibility assertion template: migration-job.yaml @@ -371,3 +380,34 @@ tests: asserts: - hasDocuments: count: 0 + + - it: retains an immutable configuration snapshot for an opaque pinned migration bundle + template: migration-job.yaml + documentIndex: 1 + set: + api.image.tag: customer-api + brainstore.image.tag: customer-brainstore + deploymentOrdering.minimumVersion: v2.22.0 + deploymentOrdering.migrationImage: mirror.example:5000/standalone-api@sha256:bundle + asserts: + - equal: + path: immutable + value: true + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - matchRegex: + path: metadata.name + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' + + - it: keeps migration execution independent of the waiter helper image + template: migration-job.yaml + documentIndex: 0 + set: + deploymentOrdering.image: mirror.example/python:3.13 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/standalone-api:v2.22.0 + - isNull: + path: spec.template.spec.initContainers diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index d182dba..ba45e02 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -1,6 +1,5 @@ -suite: release migration Job and Deployment dependencies +suite: content-addressed migration Job and stable Deployment dependencies templates: - - migration-rollback.yaml - migration-job.yaml - migration-gate.yaml - api-deployment.yaml @@ -34,7 +33,7 @@ tests: of: Job - matchRegex: path: metadata.name - pattern: '^test-[a-f0-9]{8}-migrate-r7$' + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' - equal: path: metadata.annotations["helm.sh/resource-policy"] value: keep @@ -71,8 +70,14 @@ tests: - equal: path: spec.template.spec.automountServiceAccountToken value: false + - matchRegex: + path: spec.template.spec.containers[0].envFrom[0].configMapRef.name + pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' + - matchRegex: + path: spec.template.metadata.annotations["checksum/config"] + pattern: '^[a-f0-9]{64}$' - - it: snapshots configuration for the attempt without copying Secrets + - it: retains an immutable configuration snapshot without copying Secrets template: migration-job.yaml documentIndex: 1 set: @@ -82,13 +87,16 @@ tests: of: ConfigMap - matchRegex: path: metadata.name - pattern: '-migrate-r7-config$' + pattern: '-migrate-[a-f0-9]{16}-config$' - equal: path: data.ORG_NAME value: migration-org - equal: path: metadata.annotations["helm.sh/resource-policy"] value: keep + - equal: + path: immutable + value: true - isNull: path: data.PG_URL @@ -101,14 +109,17 @@ tests: - equal: path: spec.template.spec.initContainers[1].name value: wait-for-brainstore - - matchRegex: - path: spec.template.spec.initContainers[0].env[1].value - pattern: '-migrate-r7$' + - equal: + path: spec.template.spec.initContainers[0].env[1] + value: {name: ORDERING_CONFIGMAP, value: test-9f86d081-ordering} + - contains: + path: spec.template.spec.volumes + content: {name: migration-gate-code, configMap: {name: test-9f86d081-ordering}} - equal: path: spec.template.spec.automountServiceAccountToken value: false - - it: restricts workload gates to reading only the exact migration Job + - it: restricts workload gates to reading the desired-state ConfigMap and exact Job template: migration-gate.yaml documentIndex: 1 asserts: @@ -125,16 +136,25 @@ tests: value: [get] - lengthEqual: path: rules - count: 1 + count: 2 - matchRegex: path: rules[0].resourceNames[0] - pattern: '-migrate-r7$' + pattern: '-migrate-[a-f0-9]{16}$' - lengthEqual: path: rules[0].resourceNames count: 1 - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep + path: rules[1].apiGroups + value: [""] + - equal: + path: rules[1].resources + value: [configmaps] + - equal: + path: rules[1].verbs + value: [get] + - equal: + path: rules[1].resourceNames + value: [test-9f86d081-ordering] - it: binds all pool identities and the Brainstore account in the actual namespace template: migration-gate.yaml @@ -149,9 +169,6 @@ tests: - equal: path: metadata.namespace value: actual-namespace - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - contains: path: subjects content: {kind: ServiceAccount, name: ingest, namespace: actual-namespace} @@ -288,6 +305,9 @@ tests: - equal: path: spec.template.spec.containers[0].image value: registry.example/custom-api:v2.21.0 + - equal: + path: spec.template.spec.initContainers[0].env[1] + value: {name: ORDERING_CONFIGMAP, value: test-9f86d081-ordering} - it: keeps the Brainstore image and digest unchanged when selecting an API bundle template: brainstore-reader-deployment.yaml @@ -305,28 +325,28 @@ tests: template: brainstore-reader-deployment.yaml asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[1]", "value": {"name": "ORDERING_CONFIGMAP", "value": "test-9f86d081-ordering"}}} - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} - it: gates the writer pool template: brainstore-writer-deployment.yaml asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[1]", "value": {"name": "ORDERING_CONFIGMAP", "value": "test-9f86d081-ordering"}}} - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} - it: gates the fastreader pool template: brainstore-fastreader-deployment.yaml asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[1]", "value": {"name": "ORDERING_CONFIGMAP", "value": "test-9f86d081-ordering"}}} - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} - it: gates the automationwriter pool template: brainstore-automationwriter-deployment.yaml asserts: - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"matchRegex": {"path": "spec.template.spec.initContainers[0].env[1].value", "pattern": "-migrate-r7$"}} + - {"equal": {"path": "spec.template.spec.initContainers[0].env[1]", "value": {"name": "ORDERING_CONFIGMAP", "value": "test-9f86d081-ordering"}}} - {"equal": {"path": "spec.progressDeadlineSeconds", "value": 1800}} - it: gates isolated API pool 0 @@ -432,7 +452,9 @@ tests: - it: merges partial reused values with defaults template: migration-job.yaml documentIndex: 0 - set: {"deploymentOrdering": null, "deploymentOrdering.backoffLimit": 0} + set: + deploymentOrdering: + backoffLimit: 0 asserts: - {"equal": {"path": "spec.backoffLimit", "value": 0}} - {"equal": {"path": "spec.activeDeadlineSeconds", "value": 1200}} @@ -447,14 +469,14 @@ tests: - it: rejects invalid timeoutSeconds 0 - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.timeoutSeconds": 0} asserts: - failedTemplate: errorMessage: deploymentOrdering timing values must be positive integers - it: rejects invalid timeoutSeconds - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.timeoutSeconds": ""} asserts: - failedTemplate: {} @@ -469,38 +491,38 @@ tests: value: 1200 - it: rejects invalid timeoutSeconds -1 - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.timeoutSeconds": -1} asserts: - failedTemplate: {} - it: rejects invalid pollIntervalSeconds 0 - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.pollIntervalSeconds": 0} asserts: - failedTemplate: {} - it: rejects invalid pollIntervalSeconds - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.pollIntervalSeconds": ""} asserts: - failedTemplate: {} - it: rejects invalid pollIntervalSeconds -1 - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.pollIntervalSeconds": -1} asserts: - failedTemplate: {} - it: rejects a poll interval outside the budget - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.pollIntervalSeconds": 1200} asserts: - failedTemplate: errorMessage: deploymentOrdering.timeoutSeconds must exceed pollIntervalSeconds - it: rejects an empty helper image - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.image": ""} asserts: - failedTemplate: @@ -513,93 +535,63 @@ tests: - {"hasDocuments": {"count": 0}} - it: rejects a negative retry count - template: migration-rollback.yaml + template: migration-job.yaml set: {"deploymentOrdering.backoffLimit": -1} asserts: - failedTemplate: errorMessage: deploymentOrdering.backoffLimit must be a nonnegative integer - - it: preserves revision suffix with long release names + - it: preserves the content suffix with long release names template: migration-job.yaml documentIndex: 0 release: {"name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision": 101} asserts: - - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,56}$"}} - - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-r101$"}} - - - it: gates a partially specified automation writer - template: brainstore-automationwriter-deployment.yaml - set: {"brainstore.automationwriter": {"replicas": 1}} - asserts: - - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} - - {"equal": {"path": "spec.replicas", "value": 1}} + - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,54}$"}} + - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-[a-f0-9]{16}$"}} - - it: prepares target revision rollback cleanup without install or upgrade hooks - template: migration-rollback.yaml - documentIndex: 3 + - it: bounds immutable snapshot names with long release names + template: migration-job.yaml + documentIndex: 1 + release: {"name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision": 101} asserts: - - isKind: - of: Job - - equal: - path: metadata.annotations["helm.sh/hook"] - value: pre-rollback - - equal: - path: metadata.annotations["helm.sh/hook-delete-policy"] - value: before-hook-creation,hook-succeeded - matchRegex: - path: spec.template.spec.containers[0].env[1].value - pattern: '^test-9f86d081-migrate-r7$' + path: metadata.name + pattern: '^.{1,61}$' - matchRegex: - path: spec.template.spec.serviceAccountName - pattern: '-migrate-r7-reset$' - - equal: - path: spec.activeDeadlineSeconds - value: 1200 + path: metadata.name + pattern: '-migrate-[a-f0-9]{16}-config$' - - it: limits rollback cleanup to the exact target revision - template: migration-rollback.yaml - documentIndex: 1 + - it: uses a bounded stable ordering name with long release names + template: api-deployment.yaml + release: {"name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision": 101} asserts: - equal: - path: metadata.annotations["helm.sh/hook-weight"] - value: "-2" - - equal: - path: rules[0].verbs - value: [get, delete] - - equal: - path: rules[1].verbs - value: [get, delete] - - equal: - path: rules[2].verbs - value: [get, delete] - - equal: - path: rules[0].resourceNames - value: [test-9f86d081-migrate-r7] - - equal: - path: rules[1].resourceNames - value: [test-9f86d081-migrate-r7-config, test-9f86d081-migrate-r7-gate] - - equal: - path: rules[2].resourceNames - value: [test-9f86d081-migrate-r7] + path: spec.template.spec.initContainers[0].env[1].name + value: ORDERING_CONFIGMAP + - matchRegex: + path: spec.template.spec.initContainers[0].env[1].value + pattern: '^.{1,58}$' + - matchRegex: + path: spec.template.spec.initContainers[0].env[1].value + pattern: '-[a-f0-9]{8}-ordering$' - - it: does not render rollback hooks when migrations are disabled - template: migration-rollback.yaml - set: - deploymentOrdering.enabled: false + - it: gates a partially specified automation writer + template: brainstore-automationwriter-deployment.yaml + set: {"brainstore.automationwriter": {"replicas": 1}} asserts: - - hasDocuments: - count: 0 + - {"equal": {"path": "spec.template.spec.initContainers[0].name", "value": "wait-for-migrations"}} + - {"equal": {"path": "spec.replicas", "value": 1}} - - it: reserves the release marker without discarding custom pod annotations + - it: reserves the desired pod fingerprint without discarding custom annotations template: brainstore-reader-deployment.yaml set: brainstore.reader.annotations.pod: - braintrust.dev/release-revision: forged + braintrust.dev/pod-template: forged custom: preserved asserts: - - equal: - path: spec.template.metadata.annotations["braintrust.dev/release-revision"] - value: "7" + - matchRegex: + path: spec.template.metadata.annotations["braintrust.dev/pod-template"] + pattern: "^[a-f0-9]{64}$" - equal: path: spec.template.metadata.annotations.custom value: preserved diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 193a781..1d3bbbe 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -817,6 +817,7 @@ virtualService: # Order startup as migrations -> completed Brainstore rollout -> all other apps. # Disable to deploy everything without chart-managed startup ordering. +# Unchanged workloads keep running; successful migration inputs are reused. # See README.md#release-ordering for configuration and operational limits. deploymentOrdering: enabled: true diff --git a/tests/test_brainstore_startup_gate.py b/tests/test_brainstore_startup_gate.py index 9d8892d..cb7f272 100644 --- a/tests/test_brainstore_startup_gate.py +++ b/tests/test_brainstore_startup_gate.py @@ -1,5 +1,6 @@ import importlib.util import io +import json import os from pathlib import Path import tempfile @@ -14,11 +15,28 @@ spec.loader.exec_module(gate) -def pod(version="v2.15.0", uid="new", is_ready=True, revision=8, +TEMPLATE = "a" * 64 +OLD_TEMPLATE = "b" * 64 +LATER_TEMPLATE = "c" * 64 + + +def target(template=TEMPLATE, replicas=1, min_ready=0, name="reader", + container="brainstore-reader"): + return {"deployment": name, "container": container, "template": template, + "replicas": replicas, "minReadySeconds": min_ready} + + +def ordering(*targets, job_name="release-migrate-" + "d" * 32): + return {"data": {"ordering.json": json.dumps({ + "job": job_name, "targets": list(targets) if targets else [target()], + })}} + + +def pod(version="v2.15.0", uid="new", is_ready=True, template=TEMPLATE, container="brainstore-reader", image=None, declared_version=None, **metadata): image = image if image is not None else "registry/brainstore:" + version - if revision is not None: - metadata.setdefault("annotations", {})["braintrust.dev/release-revision"] = str(revision) + if template is not None: + metadata.setdefault("annotations", {})["braintrust.dev/pod-template"] = template if declared_version is not None: metadata.setdefault("annotations", {})["braintrust.dev/brainstore-version"] = declared_version return { @@ -33,13 +51,13 @@ def pod(version="v2.15.0", uid="new", is_ready=True, revision=8, } -def deployment(version="v2.15.0", replicas=1, revision=8, container="brainstore-reader", - image=None, declared_version=None): +def deployment(version="v2.15.0", replicas=1, template=TEMPLATE, container="brainstore-reader", + image=None, declared_version=None, min_ready=0): return { "metadata": {"name": "reader", "uid": "reader-deployment", "generation": 2}, - "spec": {"replicas": replicas, "template": { + "spec": {"replicas": replicas, "minReadySeconds": min_ready, "template": { "metadata": {"annotations": pod( - revision=revision, declared_version=declared_version, + template=template, declared_version=declared_version, )["metadata"].get("annotations", {})}, "spec": pod(version, container=container, image=image)["spec"], }}, @@ -53,7 +71,7 @@ def check(self, resource=None, pods=None, minimum="2.15.0"): return gate.check_deployment( resource if resource is not None else deployment(), pods if pods is not None else [pod()], - "brainstore-reader", gate.release_version(minimum), 8, + target(), gate.release_version(minimum), ) def test_new_api_waits_when_helm_has_not_updated_brainstore_spec(self): @@ -113,16 +131,16 @@ def test_runtime_alias_does_not_override_an_incompatible_pod_spec(self): class OpaqueImageTests(unittest.TestCase): - def rollout(self, image="registry/brainstore:deadbeef", declared_version="2.15.0", - revision=8): + def rollout(self, image="registry/brainstore:deadbeef", declared_version="2.15.0"): return ( - deployment(image=image, declared_version=declared_version, revision=revision), - pod(image=image, declared_version=declared_version, revision=revision), + deployment(image=image, declared_version=declared_version), + pod(image=image, declared_version=declared_version), ) def check(self, resource, pods, minimum="2.15.0"): return gate.check_deployment( - resource, pods, "brainstore-reader", gate.release_version(minimum), 8, + resource, pods, target(replicas=resource["spec"]["replicas"]), + gate.release_version(minimum), ) def test_sha_custom_prerelease_and_digest_pins_accept_explicit_assertions(self): @@ -218,7 +236,7 @@ def test_stable_images_use_actual_versions_not_assertions(self): self.assertIsNotNone(self.check(resource, [opaque])) def test_current_assertion_still_requires_a_completed_observed_rollout(self): - for change in ("generation", "revision", "pod_revision", "terminating", + for change in ("generation", "template", "pod_template", "terminating", "unready", "missing_pod", "surge", "updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): with self.subTest(change=change): @@ -226,11 +244,11 @@ def test_current_assertion_still_requires_a_completed_observed_rollout(self): pods = [current] if change == "generation": resource["status"]["observedGeneration"] = 1 - elif change == "revision": + elif change == "template": resource["spec"]["template"]["metadata"]["annotations"][ - "braintrust.dev/release-revision"] = "7" - elif change == "pod_revision": - current["metadata"]["annotations"]["braintrust.dev/release-revision"] = "7" + "braintrust.dev/pod-template"] = OLD_TEMPLATE + elif change == "pod_template": + current["metadata"]["annotations"]["braintrust.dev/pod-template"] = OLD_TEMPLATE elif change == "terminating": current["metadata"]["deletionTimestamp"] = "2026-09-30T12:00:00Z" elif change == "unready": @@ -245,27 +263,27 @@ def test_current_assertion_still_requires_a_completed_observed_rollout(self): self.assertIsNotNone(self.check(resource, pods)) def test_older_api_uses_later_opaque_rollouts_own_assertion(self): - resource, current = self.rollout("registry/brainstore:later-sha", "2.16.0", revision=9) + resource, current = self.rollout("registry/brainstore:later-sha", "2.16.0") self.assertIsNone(self.check(resource, [current])) self.assertIsNotNone(self.check(resource, [current], minimum="2.17.0")) current["metadata"]["annotations"]["braintrust.dev/brainstore-version"] = "2.15.0" self.assertIsNotNone(self.check(resource, [current])) - resource, current = self.rollout("registry/brainstore:major-three", "3.0.0", revision=10) + resource, current = self.rollout("registry/brainstore:major-three", "3.0.0") self.assertIsNotNone(self.check(resource, [current])) self.assertIsNone(self.check(resource, [current], minimum="3.0.0")) def test_later_opaque_major_rollout_resets_an_older_apis_confirmation(self): first, first_pod = self.rollout() - later, later_pod = self.rollout("registry/brainstore:major-three", "3.0.0", revision=9) - client = Mock() + later, later_pod = self.rollout("registry/brainstore:major-three", "3.0.0") + client = api_client() client.deployment.side_effect = [first, later, later] client.pods.side_effect = [[first_pod], [later_pod], [later_pod]] clock = FakeClock() with self.assertRaisesRegex(TimeoutError, "declared Brainstore version"): gate.wait_for_fleet( - client, [{"deployment": "reader", "container": "brainstore-reader"}], - (2, 15, 0), 8, 45, 15, clock.clock, clock.sleep, Mock(), + client, "release-ordering", (2, 15, 0), 45, 15, + clock.clock, clock.sleep, Mock(), ) self.assertEqual(client.deployment.call_count, 3) @@ -282,24 +300,48 @@ def test_opaque_zero_role_needs_drain_not_a_compatible_assertion(self): self.assertIsNotNone(self.check(resource, [])) +def migration_job(*conditions): + return {"status": {"conditions": [ + {"type": condition, "status": "True"} for condition in conditions + ]}} + + +def api_client(configs=None, jobs=None): + configs = iter(configs) if configs is not None else None + jobs = iter(jobs) if jobs is not None else None + + def get(path, deadline): + if "/configmaps/" in path: + resource = next(configs) if configs is not None else ordering() + elif "/jobs/" in path: + resource = next(jobs) if jobs is not None else migration_job("Complete") + else: + raise AssertionError(f"unexpected API path: {path}") + if isinstance(resource, BaseException): + raise resource + return resource + + client = Mock(namespace="ns") + client.get.side_effect = get + return client + + class RolloutTests(unittest.TestCase): - def check(self, resource=None, pods=None, minimum_revision=8): + def check(self, resource=None, pods=None, expected=None): return gate.check_deployment( - resource if resource is not None else deployment(revision=8), - pods if pods is not None else [pod(revision=8)], - "brainstore-reader", (2, 15, 0), minimum_revision, + resource if resource is not None else deployment(), + pods if pods is not None else [pod()], + expected if expected is not None else target(), (2, 15, 0), ) - def test_same_image_release_waits_for_current_pods(self): - old = pod(revision=7) - current = pod(revision=8, is_ready=False) - self.assertIsNotNone(self.check(pods=[old, current])) + def test_same_image_content_change_waits_for_current_pods(self): + old = pod(template=OLD_TEMPLATE) + self.assertIsNotNone(self.check(pods=[old, pod(is_ready=False)])) self.assertIsNotNone(self.check(pods=[old])) - current = pod(revision=8) - self.assertIsNone(self.check(pods=[current])) + self.assertIsNone(self.check(pods=[pod()])) - def test_same_release_env_change_waits_for_updated_replicas(self): - resource = deployment(revision=8) + def test_env_change_waits_for_updated_replicas(self): + resource = deployment() resource["metadata"]["generation"] = 3 resource["status"]["observedGeneration"] = 3 resource["spec"]["template"]["spec"]["containers"][0]["env"] = [ @@ -308,40 +350,67 @@ def test_same_release_env_change_waits_for_updated_replicas(self): resource["status"]["updatedReplicas"] = 0 self.assertIsNotNone(self.check(resource)) resource["status"]["updatedReplicas"] = 1 - current = pod(revision=8) + current = pod() current["spec"] = resource["spec"]["template"]["spec"] self.assertIsNone(self.check(resource, [current])) - def test_old_complete_generation_cannot_pass_before_helm_update(self): - self.assertIsNotNone(self.check(deployment(revision=7), [pod(revision=7)])) - - def test_revision_floor_rejects_missing_malformed_and_old_markers(self): - for revision in (None, "", "invalid", "8.0", 7): - with self.subTest(revision=revision): - self.assertIsNotNone(self.check(deployment(revision=revision))) - - def test_revision_floor_is_required_and_cannot_be_disabled(self): - with self.assertRaises(TypeError): - gate.check_deployment(deployment(), [pod()], "brainstore-reader", (2, 15, 0)) - for minimum_revision in (0, -1): - with self.subTest(minimum_revision=minimum_revision): - self.assertIsNotNone(self.check(minimum_revision=minimum_revision)) + def test_old_complete_generation_cannot_pass_before_helm_updates_content(self): + self.assertIsNotNone(self.check(deployment(template=OLD_TEMPLATE), + [pod(template=OLD_TEMPLATE)])) + + def test_exact_expected_content_is_required_on_deployment_and_live_pods(self): + for template in (None, "", "invalid", "a" * 63, OLD_TEMPLATE, LATER_TEMPLATE): + with self.subTest(template=template): + self.assertIsNotNone(self.check(deployment(template=template))) + self.assertIsNotNone(self.check(pods=[pod(template=template)])) + self.assertIsNone(self.check(deployment(template=LATER_TEMPLATE), + [pod(template=LATER_TEMPLATE)], + target(template=LATER_TEMPLATE))) + + def test_expected_replica_count_blocks_stale_healthy_deployment(self): + for replicas in (0, 2): + with self.subTest(replicas=replicas): + current = deployment(replicas=replicas) + pods = [pod(uid=str(index)) for index in range(replicas)] + self.assertIsNotNone(self.check(current, pods)) + self.assertIsNone(self.check(current, pods, target(replicas=replicas))) + self.assertIsNotNone(self.check(expected=target(replicas=0))) + + def test_expected_min_ready_seconds_blocks_stale_healthy_deployment(self): + self.assertIsNotNone(self.check(expected=target(min_ready=60))) + self.assertIsNone(self.check(deployment(min_ready=60), [pod()], target(min_ready=60))) + self.assertIsNotNone(self.check(deployment(min_ready=60))) + + def test_malformed_deployment_counts_fail_closed(self): + for field in ("replicas", "minReadySeconds"): + for value in (True, False, None, -1, "1", 1.0): + with self.subTest(field=field, value=value): + resource = deployment() + resource["spec"][field] = value + self.assertIsNotNone(self.check(resource)) - def test_unobserved_generation_does_not_pass(self): - resource = deployment(revision=8) - resource["status"]["observedGeneration"] = 1 - self.assertIsNotNone(self.check(resource)) + def test_unobserved_or_invalid_generation_does_not_pass(self): + for generation in (None, 0, -1, True, "2", 2.0): + with self.subTest(generation=generation): + resource = deployment() + resource["metadata"]["generation"] = generation + self.assertIsNotNone(self.check(resource)) + for observed in (None, 0, 1, -1, True, "2", 2.0): + with self.subTest(observed=observed): + resource = deployment() + resource["status"]["observedGeneration"] = observed + self.assertIsNotNone(self.check(resource)) def test_deleted_deployment_does_not_pass(self): resource = deployment() resource["metadata"]["deletionTimestamp"] = "2026-09-30T12:00:00Z" self.assertIsNotNone(self.check(resource)) - def test_every_replica_counter_must_equal_current_desired_count(self): + def test_every_replica_counter_must_equal_expected_count(self): for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): - for count in (None, 0, 2): + for count in (None, 0, 2, True, False, "1", 1.0, -1): with self.subTest(counter=counter, count=count): - resource = deployment(revision=8) + resource = deployment() if count is None: del resource["status"][counter] else: @@ -349,31 +418,30 @@ def test_every_replica_counter_must_equal_current_desired_count(self): self.assertIsNotNone(self.check(resource)) def test_requires_exactly_desired_live_ready_current_pods(self): - for pods in ([], [pod(revision=8, is_ready=False)], - [pod(revision=8), pod(uid="surge", revision=8)], - [pod(revision=8, deletionTimestamp="2026-09-30T12:00:00Z")], - [pod(revision=8), pod(uid="terminating", revision=8, is_ready=False, - deletionTimestamp="2026-09-30T12:00:00Z")]): + for pods in ([], [pod(is_ready=False)], [pod(), pod(uid="surge")], + [pod(deletionTimestamp="2026-09-30T12:00:00Z")], + [pod(), pod(uid="terminating", is_ready=False, + deletionTimestamp="2026-09-30T12:00:00Z")]): with self.subTest(pods=pods): self.assertIsNotNone(self.check(pods=pods)) - for status_change in ("missing", "not_running", "not_ready", "pod_not_ready", "pending"): - with self.subTest(status_change=status_change): + for change in ("missing", "not_running", "not_ready", "pod_not_ready", "pending"): + with self.subTest(change=change): current = pod() status = current["status"] - if status_change == "missing": + if change == "missing": status["containerStatuses"] = [] - elif status_change == "not_running": + elif change == "not_running": status["containerStatuses"][0]["state"] = {"waiting": {}} - elif status_change == "not_ready": + elif change == "not_ready": status["containerStatuses"][0]["ready"] = False - elif status_change == "pod_not_ready": + elif change == "pod_not_ready": status["conditions"] = [] else: status["phase"] = "Pending" self.assertIsNotNone(self.check(pods=[current])) - def test_stray_service_backend_is_not_exempt_from_revision_check(self): - stray = pod(uid="stray", revision=7) + def test_stray_service_backend_also_requires_expected_content(self): + stray = pod(uid="stray", template=OLD_TEMPLATE) self.assertIsNotNone(self.check(pods=[stray])) del stray["metadata"]["annotations"] self.assertIsNotNone(self.check(pods=[stray])) @@ -381,60 +449,128 @@ def test_stray_service_backend_is_not_exempt_from_revision_check(self): def test_terminal_pods_do_not_block(self): for phase in ("Succeeded", "Failed"): with self.subTest(phase=phase): - old = pod("v2.14.0", revision=7) + old = pod("v2.14.0", template=OLD_TEMPLATE) old["status"]["phase"] = phase - self.assertIsNone(self.check(pods=[pod(revision=8), old])) + self.assertIsNone(self.check(pods=[pod(), old])) def test_honors_deployment_min_ready_seconds_availability(self): - resource = deployment(revision=8) - resource["spec"]["minReadySeconds"] = 60 + resource = deployment(min_ready=60) resource["status"]["availableReplicas"] = 0 - self.assertIsNotNone(self.check(resource)) + self.assertIsNotNone(self.check(resource, expected=target(min_ready=60))) resource["status"]["availableReplicas"] = 1 - self.assertIsNone(self.check(resource)) + self.assertIsNone(self.check(resource, expected=target(min_ready=60))) - def test_zero_role_requires_release_identity_and_completed_scale_down(self): - resource = deployment(replicas=0, revision=8) - self.assertIsNone(self.check(resource, [])) - self.assertIsNotNone(self.check(resource, [pod(revision=8)])) + def test_zero_role_requires_expected_content_and_completed_scale_down(self): + expected = target(replicas=0) + resource = deployment(replicas=0) + self.assertIsNone(self.check(resource, [], expected)) + self.assertIsNotNone(self.check(resource, [pod()], expected)) for counter in ("updatedReplicas", "readyReplicas", "availableReplicas", "replicas"): - with self.subTest(counter=counter): - resource["status"][counter] = 1 - self.assertIsNotNone(self.check(resource, [])) - del resource["status"][counter] - self.assertIsNone(self.check(resource, [])) - for revision in (None, 7): - with self.subTest(revision=revision): - self.assertIsNotNone(self.check(deployment(replicas=0, revision=revision), [])) + for count in (1, -1, True, False, "0", 0.0): + with self.subTest(counter=counter, count=count): + resource["status"][counter] = count + self.assertIsNotNone(self.check(resource, [], expected)) + del resource["status"][counter] + self.assertIsNone(self.check(resource, [], expected)) + for template in (None, OLD_TEMPLATE): + with self.subTest(template=template): + self.assertIsNotNone(self.check( + deployment(replicas=0, template=template), [], expected, + )) resource["status"]["observedGeneration"] = 1 - self.assertIsNotNone(self.check(resource, [])) + self.assertIsNotNone(self.check(resource, [], expected)) def test_zero_role_does_not_require_an_inactive_image_or_container(self): + expected = target(replicas=0) for version in ("v2.14.0", "v3.0.0", "deadbeef", "v2.15.0-rc.1"): with self.subTest(version=version): - self.assertIsNone(self.check(deployment(version, replicas=0), [])) + self.assertIsNone(self.check(deployment(version, replicas=0), [], expected)) resource = deployment(replicas=0) resource["spec"]["template"]["spec"]["containers"] = [] - self.assertIsNone(self.check(resource, [])) + self.assertIsNone(self.check(resource, [], expected)) terminating = pod(is_ready=False, deletionTimestamp="2026-09-30T12:00:00Z") - self.assertIsNotNone(self.check(resource, [terminating])) + self.assertIsNotNone(self.check(resource, [terminating], expected)) for phase in ("Succeeded", "Failed"): with self.subTest(phase=phase): - terminal = pod("v2.14.0", revision=7) + terminal = pod("v2.14.0", template=OLD_TEMPLATE) terminal["status"]["phase"] = phase - self.assertIsNone(self.check(resource, [terminal])) - - def test_uses_current_replica_count_after_scaling(self): - resource = deployment(replicas=2, revision=8) - self.assertIsNotNone(self.check(resource, [pod(revision=8)])) - self.assertIsNone(self.check(resource, [pod(revision=8), pod(uid="second", revision=8)])) - - def test_older_application_accepts_later_completed_but_not_unfinished_rollout(self): - resource = deployment("v2.16.0", revision=9) - self.assertIsNone(self.check(resource, [pod("v2.16.0", revision=9)])) - self.assertIsNotNone(self.check(resource, [pod(revision=8)])) + self.assertIsNone(self.check(resource, [terminal], expected)) + + def test_older_application_accepts_latest_expected_compatible_rollout_only(self): + resource = deployment("v2.16.0", template=LATER_TEMPLATE) + expected = target(template=LATER_TEMPLATE) + later = pod("v2.16.0", template=LATER_TEMPLATE) + self.assertIsNone(self.check(resource, [later], expected)) + self.assertIsNotNone(self.check(resource, [pod()], expected)) resource["status"]["updatedReplicas"] = 0 - self.assertIsNotNone(self.check(resource, [pod("v2.16.0", revision=9)])) + self.assertIsNotNone(self.check(resource, [later], expected)) + + +class OrderingConfigTests(unittest.TestCase): + def load(self, resource): + client = Mock(namespace="custom%20namespace") + client.get.return_value = resource + return gate.load_ordering_config(client, "release-ordering", 100) + + def test_loads_latest_config_from_api_and_preserves_zero_targets(self): + client = Mock(namespace="custom%20namespace") + client.get.side_effect = [ordering(), ordering(target(replicas=0, min_ready=60))] + first = gate.load_ordering_config(client, "release/ordering", 100) + second = gate.load_ordering_config(client, "release/ordering", 100) + self.assertEqual(first["targets"], [target()]) + self.assertEqual(second["targets"], [target(replicas=0, min_ready=60)]) + self.assertEqual([call.args for call in client.get.call_args_list], [ + ("/api/v1/namespaces/custom%20namespace/configmaps/release%2Fordering", 100), + ] * 2) + + def test_missing_or_malformed_configmap_data_fails_closed(self): + for resource in (None, [], {}, {"data": None}, {"data": []}, {"data": {}}, + {"data": {"ordering.json": None}}, + {"data": {"ordering.json": ""}}, + {"data": {"ordering.json": "not JSON"}}): + with self.subTest(resource=resource), self.assertRaises((KeyError, TypeError, ValueError)): + self.load(resource) + + def test_job_and_nonempty_target_list_are_required(self): + for config in (None, [], {}, {"job": "job"}, {"targets": [target()]}, + {"job": "job", "targets": []}, + {"job": "job", "targets": {}}, + {"job": "job", "targets": [None]}, + {"job": "job", "targets": [{}]}): + with self.subTest(config=config), self.assertRaises(ValueError): + self.load({"data": {"ordering.json": json.dumps(config)}}) + for job_name in (None, "", True, 123, [], {}): + with self.subTest(job_name=job_name), self.assertRaises(ValueError): + self.load(ordering(job_name=job_name)) + + def test_every_target_requires_names_and_sha256_content(self): + invalid = { + "deployment": (None, "", True, 1, [], {}), + "container": (None, "", True, 1, [], {}), + "template": (None, "", True, 1, "a" * 63, "a" * 65, "A" * 64, "z" * 64, + TEMPLATE + "\n"), + } + for field, values in invalid.items(): + for value in values: + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + current = target() + current[field] = value + self.load(ordering(target(), current)) + for field in target(): + with self.subTest(missing=field), self.assertRaises(ValueError): + current = target() + del current[field] + self.load(ordering(current)) + + def test_replica_and_min_ready_shapes_are_nonnegative_integers_not_booleans(self): + for field in ("replicas", "minReadySeconds"): + for value in (None, -1, True, False, 1.0, "1", [], {}): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + current = target() + current[field] = value + self.load(ordering(current)) + self.assertEqual(self.load(ordering(target(replicas=0, min_ready=0)))["targets"], + [target(replicas=0, min_ready=0)]) class FakeClock: @@ -449,62 +585,110 @@ def sleep(self, seconds): class WaitTests(unittest.TestCase): - def wait(self, observations, timeout=60): + def wait(self, observations, timeout=60, configs=None, jobs=None): clock = FakeClock() log = Mock() + client = api_client(configs, jobs) with patch.object(gate, "check_fleet", side_effect=observations) as check: - gate.wait_for_fleet(Mock(), [{"deployment": "reader", "container": "brainstore-reader"}], - (2, 15, 0), 8, timeout, 15, clock.clock, clock.sleep, log) - return clock, log, check + gate.wait_for_fleet(client, "release-ordering", (2, 15, 0), timeout, 15, + clock.clock, clock.sleep, log) + return clock, log, check, client - def test_requires_two_successful_observations(self): - clock, _, check = self.wait([None, None]) + def test_requires_two_successful_observations_and_reads_config_and_job_each_time(self): + clock, _, check, client = self.wait([None, None]) self.assertEqual(check.call_count, 2) + self.assertEqual(client.get.call_count, 4) self.assertEqual(clock.now, 15) def test_failed_check_resets_confirmation(self): - _, _, check = self.wait([None, "old pod remains", None, None]) + _, _, check, _ = self.wait([None, "old pod remains", None, None]) self.assertEqual(check.call_count, 4) + def test_every_expected_state_change_resets_confirmation(self): + for changed in (ordering(target(template=LATER_TEMPLATE)), ordering(target(replicas=2)), + ordering(target(min_ready=60)), ordering(target(name="new-reader")), + ordering(job_name="new-job")): + with self.subTest(changed=changed): + clock, _, check, _ = self.wait([None, None, None], + configs=[ordering(), changed, changed]) + self.assertEqual(check.call_count, 3) + self.assertEqual(clock.now, 30) + def test_transient_missing_deployment_and_rbac_are_retried(self): - errors = [urllib.error.HTTPError("https://api", code, "synthetic", None, None) + errors = [urllib.error.HTTPError("https://api", code, "synthetic", {}, io.BytesIO()) for code in (404, 403)] - _, _, check = self.wait(errors + [None, None]) + _, _, check, _ = self.wait(errors + [None, None]) self.assertEqual(check.call_count, 4) def test_failed_api_reads_reset_successful_observations(self): for error in (OSError("synthetic"), ValueError("synthetic"), KeyError("synthetic"), TypeError("synthetic")): with self.subTest(error=type(error).__name__): - clock, _, check = self.wait([None, error, None, None]) + clock, _, check, _ = self.wait([None, error, None, None]) self.assertEqual(check.call_count, 4) self.assertEqual(clock.now, 45) + def test_invalid_config_resets_confirmation_without_observing_deployments(self): + clock, _, check, client = self.wait( + [None, None, None], configs=[ordering(), {"data": {}}, ordering(), ordering()], + ) + self.assertEqual(client.get.call_count, 7) + self.assertEqual(check.call_count, 3) + self.assertEqual(clock.now, 45) + + def test_current_migration_job_must_be_complete_on_every_observation(self): + clock, _, check, client = self.wait( + [None, None, None], timeout=75, + configs=[ordering(), ordering(job_name="new-job"), + ordering(job_name="new-job"), ordering(job_name="new-job")], + jobs=[migration_job("Complete"), migration_job(), + migration_job("Complete"), migration_job("Complete")], + ) + self.assertEqual(check.call_count, 3) + self.assertEqual(clock.now, 45) + job_paths = [call.args[0] for call in client.get.call_args_list if "/jobs/" in call.args[0]] + self.assertEqual(job_paths[1:], ["/apis/batch/v1/namespaces/ns/jobs/new-job"] * 3) + + def test_pending_missing_or_failed_migration_never_releases_api(self): + for observation in (migration_job(), + urllib.error.HTTPError("https://api/jobs/job", 404, "", {}, io.BytesIO())): + with self.subTest(observation=observation), self.assertRaises(TimeoutError): + self.wait([], timeout=30, jobs=[observation, observation]) + with self.assertRaises(RuntimeError): + self.wait([], jobs=[migration_job("Failed", "Complete")]) + def test_timeout_never_releases_api(self): with self.assertRaisesRegex(TimeoutError, "old pod remains"): self.wait(["old pod remains"] * 3, timeout=30) def test_kubernetes_api_failure_never_releases_api(self): with self.assertRaisesRegex(TimeoutError, "HTTP 429"): - self.wait([urllib.error.HTTPError("https://api", 429, "synthetic", None, None)] * 3, 30) + self.wait([urllib.error.HTTPError("https://api", 429, "synthetic", {}, io.BytesIO())] * 3, 30) + + def test_missing_or_malformed_ordering_config_never_releases_api(self): + for resource in ({}, ordering(target(template="invalid")), ordering(job_name=""), + urllib.error.HTTPError("https://api/configmaps/gate", 404, "", {}, io.BytesIO())): + with self.subTest(resource=resource), self.assertRaises(TimeoutError): + self.wait([], timeout=30, configs=[resource, resource]) class FleetTests(unittest.TestCase): - def test_compatible_deployment_and_pod_churn_do_not_delay_an_old_api(self): - client = Mock() - first = deployment("v2.15.0", revision=8) - later = deployment("v2.16.0", revision=9) + def test_latest_compatible_content_requires_two_matching_observations(self): + client = api_client([ordering(), ordering(target(template=LATER_TEMPLATE)), + ordering(target(template=LATER_TEMPLATE))]) + first = deployment("v2.15.0") + later = deployment("v2.16.0", template=LATER_TEMPLATE) later["metadata"]["generation"] = 3 later["status"]["observedGeneration"] = 3 - client.deployment.side_effect = [first, later] - client.pods.side_effect = [[pod("v2.15.0", "old", revision=8)], - [pod("v2.16.0", "new", revision=9)]] - targets = [{"deployment": "reader", "container": "brainstore-reader"}] + client.deployment.side_effect = [first, later, later] + client.pods.side_effect = [[pod("v2.15.0", "old")], + [pod("v2.16.0", "new", template=LATER_TEMPLATE)], + [pod("v2.16.0", "new", template=LATER_TEMPLATE)]] clock = FakeClock() - gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 60, 15, + gate.wait_for_fleet(client, "release-ordering", (2, 15, 0), 60, 15, clock.clock, clock.sleep, Mock()) - self.assertEqual(client.deployment.call_count, 2) - self.assertEqual(clock.now, 15) + self.assertEqual(client.deployment.call_count, 3) + self.assertEqual(clock.now, 30) def test_checks_all_roles_and_includes_stray_service_backends(self): client = Mock() @@ -514,83 +698,92 @@ def test_checks_all_roles_and_includes_stray_service_backends(self): for role in ("reader", "writer", "fastreader", "automationwriter"): name = "custom-" + role container = "brainstore-" + role - targets.append({"deployment": name, "container": container}) - resource = deployment() + targets.append(target(name=name, container=container)) + resource = deployment(container=container) resource["metadata"]["name"] = name resource["metadata"]["uid"] = name - resource["spec"]["template"]["spec"]["containers"][0]["name"] = container - current = pod(uid=role) - current["spec"]["containers"][0]["name"] = container - current["status"]["containerStatuses"][0]["name"] = container deployments.append(resource) - pod_sets.append([current]) + pod_sets.append([pod(uid=role, container=container)]) client.deployment.side_effect = deployments client.pods.side_effect = pod_sets - reason = gate.check_fleet(client, targets, (2, 15, 0), 8, 100) - self.assertIsNone(reason) + self.assertIsNone(gate.check_fleet(client, targets, (2, 15, 0), 100)) self.assertEqual(client.deployment.call_count, 4) self.assertEqual([call.args[0] for call in client.pods.call_args_list], ["app=custom-reader", "app=custom-writer", "app=custom-fastreader", "app=custom-automationwriter"]) self.assertTrue(all(call.args[1] == 100 for call in client.deployment.call_args_list)) self.assertTrue(all(call.args[1] == 100 for call in client.pods.call_args_list)) - # A manually labelled Pod can receive Service traffic without belonging - # to a ReplicaSet, so it must not be filtered out by owner references. - stray = pod("v2.14.0", "stray") + # Service-labelled backends are not exempt based on owner references. client.deployment.side_effect = deployments - client.pods.side_effect = [[pod(), stray]] - reason = gate.check_fleet(client, targets, (2, 15, 0), 8, 100) - self.assertIsNotNone(reason) - - def test_nonpositive_revision_floor_cannot_release_application(self): - for revision in (0, -1): - with self.subTest(revision=revision): - target = {"deployment": "reader", "container": "brainstore-reader"} - client = Mock() - client.deployment.return_value = deployment() - client.pods.return_value = [pod()] - clock = FakeClock() - with self.assertRaises(TimeoutError): - gate.wait_for_fleet(client, [target], (2, 15, 0), revision, 30, 15, - clock.clock, clock.sleep, Mock()) + client.pods.side_effect = [[pod(), pod("v2.14.0", "stray")]] + self.assertIsNotNone(gate.check_fleet(client, targets, (2, 15, 0), 100)) + + def test_each_role_uses_its_own_expected_counts_and_zero_role_must_drain(self): + client = Mock() + targets = [target(replicas=2, min_ready=60), + target(replicas=0, name="automationwriter", + container="brainstore-automationwriter")] + reader = deployment(replicas=2, min_ready=60) + writer = deployment(replicas=0, container="brainstore-automationwriter") + writer["metadata"]["name"] = "automationwriter" + client.deployment.side_effect = [reader, writer] + client.pods.side_effect = [[pod(), pod(uid="second")], []] + self.assertIsNone(gate.check_fleet(client, targets, (2, 15, 0), 100)) + client.deployment.side_effect = [reader, writer] + client.pods.side_effect = [[pod(), pod(uid="second")], + [pod(container="brainstore-automationwriter")]] + self.assertIsNotNone(gate.check_fleet(client, targets, (2, 15, 0), 100)) def test_writer_waits_across_same_image_and_later_rollouts(self): container = "brainstore-writer" - resources = [deployment(revision=revision, container=container) - for revision in (7, 8, 8, 9, 9, 9)] + templates = (OLD_TEMPLATE, TEMPLATE, TEMPLATE, + LATER_TEMPLATE, LATER_TEMPLATE, LATER_TEMPLATE) + resources = [deployment(template=template, container=container) for template in templates] for resource in resources: resource["metadata"]["name"] = "writer" resources[1]["status"]["updatedReplicas"] = 0 resources[3]["status"]["updatedReplicas"] = 0 - old = pod(revision=7, container=container) - current = pod(revision=8, container=container) - later = pod(revision=9, container=container) - client = Mock() + old = pod(template=OLD_TEMPLATE, container=container) + current = pod(container=container) + later = pod(template=LATER_TEMPLATE, container=container) + initial = ordering(target(name="writer", container=container)) + latest = ordering(target(template=LATER_TEMPLATE, name="writer", container=container)) + client = api_client([initial, initial, initial, latest, latest, latest]) client.deployment.side_effect = resources client.pods.side_effect = [ - [old], [old, pod(revision=8, is_ready=False, container=container)], - [current], [current, pod(revision=9, is_ready=False, container=container)], + [old], [old, pod(is_ready=False, container=container)], + [current], [current, pod(template=LATER_TEMPLATE, is_ready=False, container=container)], [later], [later], ] - targets = [{"deployment": "writer", "container": container}] clock = FakeClock() - gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 120, 15, + gate.wait_for_fleet(client, "release-ordering", (2, 15, 0), 120, 15, clock.clock, clock.sleep, Mock()) self.assertEqual(clock.now, 75) + def test_new_config_does_not_accept_old_healthy_deployment_before_helm_update(self): + for expected in (target(template=LATER_TEMPLATE), target(replicas=2), + target(replicas=0), target(min_ready=60)): + with self.subTest(expected=expected): + client = api_client([ordering(expected), ordering(expected)]) + client.deployment.return_value = deployment() + client.pods.return_value = [pod()] + clock = FakeClock() + with self.assertRaises(TimeoutError): + gate.wait_for_fleet(client, "release-ordering", (2, 15, 0), 30, 15, + clock.clock, clock.sleep, Mock()) + def test_completed_observation_cannot_release_application_after_deadline(self): clock = FakeClock() def observe(name, deadline): clock.sleep(30) - return deployment(revision=8) + return deployment() - client = Mock() + client = api_client() client.deployment.side_effect = observe - client.pods.return_value = [pod(revision=8)] - targets = [{"deployment": "reader", "container": "brainstore-reader"}] + client.pods.return_value = [pod()] with self.assertRaises(TimeoutError): - gate.wait_for_fleet(client, targets, (2, 15, 0), 8, 60, 15, + gate.wait_for_fleet(client, "release-ordering", (2, 15, 0), 60, 15, clock.clock, clock.sleep, Mock()) @@ -598,26 +791,40 @@ class MainTests(unittest.TestCase): def environment(self): return { "BRAINSTORE_MINIMUM_VERSION": "2.15.0", - "BRAINSTORE_MINIMUM_REVISION": "8", - "BRAINSTORE_DEPLOYMENTS": '[{"deployment":"reader","container":"brainstore-reader"}]', + "ORDERING_CONFIGMAP": "release-ordering", "BRAINSTORE_GATE_TIMEOUT_SECONDS": "60", "BRAINSTORE_GATE_POLL_SECONDS": "15", "POD_NAMESPACE": "custom", } - def test_missing_invalid_or_nonpositive_revision_floor_fails_closed(self): - for revision in (None, "", "invalid", "8.0", "0", "-1"): - with self.subTest(revision=revision): - environment = self.environment() - if revision is None: - del environment["BRAINSTORE_MINIMUM_REVISION"] - else: - environment["BRAINSTORE_MINIMUM_REVISION"] = revision - with patch.dict(os.environ, environment, clear=True), \ - patch.object(gate, "wait_for_fleet") as wait, \ - self.assertRaises((KeyError, ValueError)): - gate.main() - wait.assert_not_called() + def test_uses_stable_configmap_and_fixed_minimum_version(self): + with patch.dict(os.environ, self.environment(), clear=True), \ + patch.object(gate, "KubernetesClient") as client, \ + patch.object(gate, "wait_for_fleet") as wait: + gate.main() + client.assert_called_once_with("custom", "/var/run/brainstore-gate") + wait.assert_called_once_with(client.return_value, "release-ordering", (2, 15, 0), 60, 15) + + def test_missing_or_invalid_environment_fails_before_polling(self): + invalid = { + "ORDERING_CONFIGMAP": (None, ""), + "BRAINSTORE_MINIMUM_VERSION": (None, "", "latest", "2.15.0-rc.1"), + "BRAINSTORE_GATE_TIMEOUT_SECONDS": (None, "", "invalid", "0", "-1", "15"), + "BRAINSTORE_GATE_POLL_SECONDS": (None, "", "invalid", "0", "-1", "60"), + } + for field, values in invalid.items(): + for value in values: + with self.subTest(field=field, value=value): + environment = self.environment() + if value is None: + del environment[field] + else: + environment[field] = value + with patch.dict(os.environ, environment, clear=True), \ + patch.object(gate, "wait_for_fleet") as wait, \ + self.assertRaises((KeyError, ValueError)): + gate.main() + wait.assert_not_called() class ClientTests(unittest.TestCase): diff --git a/tests/test_job_gate.py b/tests/test_job_gate.py index 25bc52a..ba11b3d 100644 --- a/tests/test_job_gate.py +++ b/tests/test_job_gate.py @@ -1,10 +1,12 @@ import importlib.util +import io import itertools import json +import os from pathlib import Path import sys import unittest -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import Mock, patch import urllib.error FILES = Path(__file__).resolve().parents[1] / "braintrust/files" @@ -12,14 +14,13 @@ for name, file in [ ("brainstore_gate", "brainstore-startup-gate.py"), ("job_gate", "job-gate.py"), - ("migration_rollback", "migration-rollback.py"), ]: spec = importlib.util.spec_from_file_location(name, FILES / file) module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) gate = sys.modules["job_gate"] -rollback = sys.modules["migration_rollback"] +shared = sys.modules["brainstore_gate"] def job(*conditions, **metadata): @@ -28,6 +29,14 @@ def job(*conditions, **metadata): ]}} +def ordering(name="release-migrate-" + "a" * 32): + return {"data": {"ordering.json": json.dumps({ + "job": name, + "targets": [{"deployment": "reader", "container": "brainstore-reader", + "template": "b" * 64, "replicas": 1, "minReadySeconds": 0}], + })}} + + class Clock: def __init__(self): self.now = 0 @@ -40,11 +49,29 @@ def sleep(self, seconds): class JobGateTests(unittest.TestCase): - def wait(self, observations, name="release-migrate-r2", timeout=20): - clock = Clock() + def client(self, observations, configs=None): + observations = iter(observations) + configs = iter(configs) if configs is not None else itertools.repeat(ordering()) + + def get(path, deadline): + if "/configmaps/" in path: + resource = next(configs) + elif "/jobs/" in path: + resource = next(observations) + else: + raise AssertionError(f"unexpected API path: {path}") + if isinstance(resource, BaseException): + raise resource + return resource + client = Mock(namespace="test-namespace") - client.get.side_effect = observations - gate.wait_for_job(client, name, timeout, 5, + client.get.side_effect = get + return client + + def wait(self, observations, timeout=20, configs=None): + clock = Clock() + client = self.client(observations, configs) + gate.wait_for_job(client, "release-ordering", timeout, 5, clock=clock, sleep=clock.sleep, log=Mock()) return client, clock.now @@ -54,45 +81,98 @@ def test_only_complete_condition_counts_as_success(self): {"status": {"conditions": [{"type": "Complete", "status": "False"}]}}, job("Complete", deletionTimestamp="now")): with self.subTest(resource=resource): - self.assertFalse(gate.job_complete(resource)) - self.assertTrue(gate.job_complete(job("Complete"))) + self.assertFalse(shared.job_complete(resource)) + self.assertTrue(shared.job_complete(job("Complete"))) def test_failure_takes_precedence_over_completion_and_deletion(self): for condition in ("Failed", "FailureTarget"): for metadata in ({}, {"deletionTimestamp": "now"}): with self.subTest(condition=condition, metadata=metadata): with self.assertRaises(RuntimeError): - gate.job_complete(job(condition, "Complete", **metadata)) + shared.job_complete(job(condition, "Complete", **metadata)) + + def test_malformed_job_observations_fail_closed(self): + for resource in (None, [], {"metadata": None}, {"status": None}, + {"status": {"conditions": None}}, + {"status": {"conditions": {}}}, + {"status": {"conditions": [None]}}, + {"status": {"conditions": ["Complete"]}}): + with self.subTest(resource=resource), self.assertRaises(ValueError): + shared.job_complete(resource) def test_exact_job_wait_retries_missing_job_rbac_and_server_errors(self): - errors = [urllib.error.HTTPError("url", code, "error", {}, None) + errors = [urllib.error.HTTPError("url", code, "error", {}, io.BytesIO()) for code in (404, 403, 500)] client, elapsed = self.wait(errors + [job("Complete")]) self.assertEqual(elapsed, 15) - self.assertEqual( - {call.args for call in client.get.call_args_list}, - {("/apis/batch/v1/namespaces/test-namespace/jobs/release-migrate-r2", 20)}, - ) + self.assertEqual([call.args for call in client.get.call_args_list], [ + ("/api/v1/namespaces/test-namespace/configmaps/release-ordering", 20), + ("/apis/batch/v1/namespaces/test-namespace/jobs/release-migrate-" + "a" * 32, 20), + ] * 4) def test_retained_success_does_not_wait_for_another_observation(self): - client = Mock(namespace="ns") - client.get.return_value = job("Complete") - sleep = Mock() - gate.wait_for_job(client, "release-migrate-r1", 20, 5, clock=lambda: 0, - sleep=sleep, log=Mock()) - self.assertEqual(client.get.call_count, 1) - sleep.assert_not_called() + client, elapsed = self.wait([job("Complete")]) + self.assertEqual(client.get.call_count, 2) + self.assertEqual(elapsed, 0) def test_running_job_then_success(self): _, elapsed = self.wait([job(), job("Complete")]) self.assertEqual(elapsed, 5) - def test_running_missing_or_forbidden_job_never_allows_startup(self): - for observation in (job(), urllib.error.HTTPError("url", 404, "", {}, None), - urllib.error.HTTPError("url", 403, "", {}, None)): - with self.subTest(observation=observation): + def test_current_config_job_is_selected_on_every_observation(self): + client, elapsed = self.wait([job(), job("Complete")], configs=[ + ordering("old-content-job"), ordering("new-content-job"), + ]) + self.assertEqual(elapsed, 5) + paths = [call.args[0] for call in client.get.call_args_list] + self.assertEqual(paths, [ + "/api/v1/namespaces/test-namespace/configmaps/release-ordering", + "/apis/batch/v1/namespaces/test-namespace/jobs/old-content-job", + "/api/v1/namespaces/test-namespace/configmaps/release-ordering", + "/apis/batch/v1/namespaces/test-namespace/jobs/new-content-job", + ]) + + def test_missing_config_does_not_reuse_previously_selected_job(self): + client, elapsed = self.wait([job(), job("Complete")], configs=[ + ordering("old-content-job"), {"data": {}}, ordering("new-content-job"), + ]) + self.assertEqual(elapsed, 10) + paths = [call.args[0] for call in client.get.call_args_list if "/jobs/" in call.args[0]] + self.assertEqual(paths, [ + "/apis/batch/v1/namespaces/test-namespace/jobs/old-content-job", + "/apis/batch/v1/namespaces/test-namespace/jobs/new-content-job", + ]) + + def test_missing_or_forbidden_ordering_config_retries_without_reading_job(self): + errors = [urllib.error.HTTPError("url", code, "error", {}, io.BytesIO()) + for code in (404, 403)] + client, elapsed = self.wait([job("Complete")], configs=errors + [ordering()]) + self.assertEqual(elapsed, 10) + self.assertEqual(client.get.call_count, 4) + self.assertEqual(sum("/jobs/" in call.args[0] for call in client.get.call_args_list), 1) + + def test_missing_or_malformed_config_never_allows_startup(self): + invalid_target = ordering() + config = json.loads(invalid_target["data"]["ordering.json"]) + config["targets"][0]["replicas"] = True + invalid_target["data"]["ordering.json"] = json.dumps(config) + for resource in ({}, {"data": {}}, {"data": {"ordering.json": "not JSON"}}, + ordering(""), invalid_target, + {"data": {"ordering.json": '{"job":"job","targets":[]}'}}): + with self.subTest(resource=resource): + client = self.client([], itertools.repeat(resource)) + clock = Clock() with self.assertRaises(TimeoutError): - self.wait(itertools.repeat(observation)) + gate.wait_for_job(client, "release-ordering", 20, 5, + clock=clock, sleep=clock.sleep, log=Mock()) + self.assertTrue(all("/configmaps/" in call.args[0] + for call in client.get.call_args_list)) + + def test_running_missing_or_forbidden_job_never_allows_startup(self): + for observation in (job(), urllib.error.HTTPError("url", 404, "", {}, io.BytesIO()), + urllib.error.HTTPError("url", 403, "", {}, io.BytesIO())): + with self.subTest(observation=observation), self.assertRaises(TimeoutError): + self.wait(itertools.repeat(observation)) def test_failed_job_never_allows_startup(self): with self.assertRaises(RuntimeError): @@ -101,201 +181,75 @@ def test_failed_job_never_allows_startup(self): def test_transient_observation_errors_retry(self): client, elapsed = self.wait([OSError("network"), ValueError("bad JSON"), job("Complete")]) - self.assertEqual(client.get.call_count, 3) + self.assertEqual(client.get.call_count, 6) self.assertEqual(elapsed, 10) def test_observation_errors_do_not_disclose_response_details(self): - clock = Clock() - client = Mock(namespace="ns") - client.get.side_effect = ValueError("secret response body") - log = Mock() - with self.assertRaises(TimeoutError) as raised: - gate.wait_for_job(client, "job", 6, 5, clock=clock, - sleep=clock.sleep, log=log) - self.assertNotIn("secret response body", str(raised.exception)) - self.assertNotIn("secret response body", str(log.call_args_list)) + for config_failure in (False, True): + with self.subTest(config_failure=config_failure): + clock = Clock() + errors = itertools.repeat(ValueError("secret response body")) + client = self.client([], errors) if config_failure else self.client(errors) + log = Mock() + with self.assertRaises(TimeoutError) as raised: + gate.wait_for_job(client, "release-ordering", 6, 5, clock=clock, + sleep=clock.sleep, log=log) + self.assertNotIn("secret response body", str(raised.exception)) + self.assertNotIn("secret response body", str(log.call_args_list)) + + def test_job_path_escapes_current_job_name(self): + client, _ = self.wait([job("Complete")], configs=[ordering("name/other")]) + self.assertEqual(client.get.call_args.args[0], + "/apis/batch/v1/namespaces/test-namespace/jobs/name%2Fother") def test_late_success_does_not_escape_deadline(self): clock = Clock() client = Mock(namespace="ns") - def late(*args): + + def late(path, deadline): + if "/configmaps/" in path: + return ordering() clock.now = 21 return job("Complete") + client.get.side_effect = late with self.assertRaises(TimeoutError): - gate.wait_for_job(client, "job", 20, 5, clock=clock, + gate.wait_for_job(client, "release-ordering", 20, 5, clock=clock, sleep=clock.sleep, log=Mock()) -class RollbackCleanupTests(unittest.TestCase): - name = "release-migrate-r7" - - def test_resource_paths_escape_and_limit_cleanup_to_the_target_migration(self): - self.assertEqual(rollback.resource_paths("ns/other", "name/other"), [ - "/apis/batch/v1/namespaces/ns%2Fother/jobs/name%2Fother", - "/api/v1/namespaces/ns%2Fother/configmaps/name%2Fother-config", - "/api/v1/namespaces/ns%2Fother/configmaps/name%2Fother-gate", - "/apis/rbac.authorization.k8s.io/v1/namespaces/ns%2Fother/rolebindings/name%2Fother", - "/apis/rbac.authorization.k8s.io/v1/namespaces/ns%2Fother/roles/name%2Fother", - ]) - - def test_job_is_absent_before_any_config_or_rbac_cleanup(self): - paths = rollback.resource_paths("ns", self.name) - job_path = paths[0] - observations = iter([True, False]) - deleted = absent = False - clock = Clock() - def request(method, path, deadline): - nonlocal deleted, absent - self.assertIn(path, paths) - self.assertEqual(deadline, 10) - if path == job_path: - if method == "DELETE": - deleted = True - return True - self.assertTrue(deleted) - exists = next(observations) - absent = not exists - return exists - self.assertTrue(absent) - return method == "DELETE" - client = Mock() - client.request.side_effect = request - rollback.cleanup(client, "ns", self.name, 10, clock=clock, sleep=clock.sleep) - expected = [("DELETE", job_path, 10), ("GET", job_path, 10), - ("GET", job_path, 10)] - expected.extend((method, path, 10) for path in paths[1:] - for method in ("DELETE", "GET")) - self.assertEqual([call.args for call in client.request.call_args_list], expected) - self.assertEqual(clock.now, 2) - - def test_cleanup_of_absent_resources_is_idempotent(self): - client = Mock() - client.request.return_value = False - sleep = Mock() - rollback.cleanup(client, "ns", self.name, 10, clock=lambda: 0, sleep=sleep) - self.assertEqual( - [call.args for call in client.request.call_args_list], - [(method, path, 10) for path in rollback.resource_paths("ns", self.name) - for method in ("DELETE", "GET")], - ) - sleep.assert_not_called() - - def test_stuck_job_blocks_all_config_and_rbac_cleanup(self): - job_path = rollback.resource_paths("ns", self.name)[0] - client = Mock() - client.request.return_value = True - clock = Clock() - with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", self.name, 3, - clock=clock, sleep=clock.sleep) - self.assertEqual(clock.now, 3) - self.assertEqual( - [call.args for call in client.request.call_args_list], - [("DELETE", job_path, 3), ("GET", job_path, 3), ("GET", job_path, 3)], - ) - - def test_config_cleanup_uses_remaining_job_deadline(self): - paths = rollback.resource_paths("ns", self.name) - observations = iter([True, False]) - client = Mock() - clock = Clock() - def request(method, path, deadline): - self.assertEqual(deadline, 5) - if method == "GET" and path == paths[0]: - return next(observations) - return True - client.request.side_effect = request - with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", self.name, 5, - clock=clock, sleep=clock.sleep) - self.assertEqual(clock.now, 5) - self.assertEqual( - [call.args[1] for call in client.request.call_args_list - if call.args[0] == "DELETE"], - paths[:2], - ) - self.assertEqual(client.request.call_args.args[:2], ("GET", paths[1])) - - def test_late_absence_does_not_escape_shared_deadline(self): - paths = rollback.resource_paths("ns", self.name) - for late_path in paths: - with self.subTest(late_path=late_path): - clock = Clock() - client = Mock() - def request(method, path, deadline): - self.assertEqual(deadline, 3) - if method == "GET": - clock.now = 3 if path == late_path else clock.now + 0.25 - return False - client.request.side_effect = request - with self.assertRaises(TimeoutError): - rollback.cleanup(client, "ns", self.name, 3, - clock=clock, sleep=clock.sleep) - self.assertEqual(client.request.call_args.args[:2], ("GET", late_path)) - - def test_permission_or_transport_failure_stops_cleanup(self): - paths = rollback.resource_paths("ns", self.name) - for failure in (RuntimeError("forbidden"), OSError("network")): - for failing_path in paths: - for failing_method in ("DELETE", "GET"): - with self.subTest(failure=failure, path=failing_path, - method=failing_method): - client = Mock() - def request(method, path, deadline): - if (method, path) == (failing_method, failing_path): - raise failure - return False - client.request.side_effect = request - with self.assertRaises(type(failure)): - rollback.cleanup(client, "ns", self.name, 10, - clock=lambda: 0, sleep=Mock()) - self.assertEqual(client.request.call_args.args[:2], - (failing_method, failing_path)) - self.assertTrue(all(call.args[1] in paths[:paths.index(failing_path) + 1] - for call in client.request.call_args_list)) - - -class RollbackClientTests(unittest.TestCase): - def client(self): - client = rollback.Client.__new__(rollback.Client) - client.base = "https://kubernetes:443" - client.credentials = MagicMock() - (client.credentials / "token").read_text.return_value = "private-token\n" - client.opener = MagicMock() - return client - - def test_delete_uses_foreground_propagation_and_shared_request_budget(self): - client = self.client() - with patch.object(rollback.time, "monotonic", return_value=8): - self.assertTrue(client.request("DELETE", "/exact/path", 10)) - request = client.opener.open.call_args.args[0] - self.assertEqual(request.full_url, "https://kubernetes:443/exact/path") - self.assertEqual(json.loads(request.data), {"propagationPolicy": "Foreground"}) - self.assertEqual(request.get_header("Authorization"), "Bearer private-token") - self.assertEqual(client.opener.open.call_args.kwargs["timeout"], 2) - - def test_only_not_found_is_absence_and_error_bodies_are_not_disclosed(self): - for status in (404, 403, 500): - with self.subTest(status=status): - client = self.client() - body = Mock() - client.opener.open.side_effect = urllib.error.HTTPError( - "https://kubernetes", status, "private response", {}, body) - with patch.object(rollback.time, "monotonic", return_value=0): - if status == 404: - self.assertFalse(client.request("GET", "/exact/path", 10)) +class MainTests(unittest.TestCase): + def environment(self): + return {"ORDERING_CONFIGMAP": "release-ordering", "JOB_TIMEOUT_SECONDS": "20", + "JOB_POLL_SECONDS": "5", "POD_NAMESPACE": "custom"} + + def test_uses_stable_ordering_configmap_not_fixed_job_environment(self): + with patch.dict(os.environ, self.environment(), clear=True), \ + patch.object(gate, "KubernetesClient") as client, \ + patch.object(gate, "wait_for_job") as wait: + gate.main() + client.assert_called_once_with("custom", "/var/run/migration-gate") + wait.assert_called_once_with(client.return_value, "release-ordering", 20, 5) + + def test_missing_or_invalid_environment_fails_before_polling(self): + invalid = { + "ORDERING_CONFIGMAP": (None, ""), + "JOB_TIMEOUT_SECONDS": (None, "", "invalid", "0", "-1", "5"), + "JOB_POLL_SECONDS": (None, "", "invalid", "0", "-1", "20"), + } + for field, values in invalid.items(): + for value in values: + with self.subTest(field=field, value=value): + environment = self.environment() + if value is None: + del environment[field] else: - with self.assertRaises(RuntimeError) as raised: - client.request("GET", "/exact/path", 10) - self.assertNotIn("private", str(raised.exception)) - body.close.assert_called_once_with() - - def test_redirects_never_forward_credentials(self): - handler = rollback.NoRedirect() - with self.assertRaises(urllib.error.HTTPError): - handler.redirect_request(Mock(full_url="https://kubernetes"), Mock(), - 302, "redirect", {}, "https://elsewhere") + environment[field] = value + with patch.dict(os.environ, environment, clear=True), \ + patch.object(gate, "wait_for_job") as wait, \ + self.assertRaises((KeyError, ValueError)): + gate.main() + wait.assert_not_called() if __name__ == "__main__": From 26ca2188305fce1e816820ea91f8e2601c775e27 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Fri, 13 Nov 2026 21:20:50 -0500 Subject: [PATCH 13/13] Simplify content-addressed migration Job names --- braintrust/README.md | 7 ++++--- braintrust/templates/_deployment-ordering.tpl | 7 ++++--- braintrust/tests/migration-image_test.yaml | 8 ++++---- braintrust/tests/migrations_test.yaml | 8 ++++---- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 3655aa2..29bb31e 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -583,9 +583,10 @@ Known stable images still undergo version checks; opaque Brainstore pods must match their Deployment's exact image and declared release. See [values.yaml](values.yaml). Use `--wait --wait-for-jobs` with a sufficient Helm timeout. -Migration Jobs are keyed by their image and execution configuration; successful -Jobs and immutable configuration snapshots are retained and reused, including on -rollback. After diagnosing a failed migration, delete its failed Job before +Migration Jobs use `-migrate-`, keyed by release identity, +image, and execution configuration. Successful Jobs and immutable configuration +snapshots are retained and reused, including on rollback. After diagnosing a failed +migration, delete its failed Job before retrying the same inputs with Helm. Rollback does not reverse SQL changes; migrations must remain backward-compatible. External Secret value changes and replaced images behind unchanged tags are not detected; use immutable image pins diff --git a/braintrust/templates/_deployment-ordering.tpl b/braintrust/templates/_deployment-ordering.tpl index f11677d..1320dff 100644 --- a/braintrust/templates/_deployment-ordering.tpl +++ b/braintrust/templates/_deployment-ordering.tpl @@ -63,11 +63,12 @@ Use a fixed name when hashing to avoid self-referential labels and snapshot refs {{- dict "pod" $pod "apiConfig" $apiConfig.data "settings" (dict "backoffLimit" (int $config.backoffLimit) "activeDeadlineSeconds" (int $config.timeoutSeconds)) | toYaml -}} {{- end -}} -{{/* Leave room for the immutable configuration snapshot suffix, even at the -maximum release-name length. Release revisions are deliberately not inputs. */}} +{{/* Reserve space for the immutable snapshot suffix; hash full release identity +with execution inputs so truncated prefixes cannot collide across releases. */}} {{- define "braintrust.migrations.name" -}} {{- $execution := include "braintrust.migrations.execution" (dict "root" . "name" "migration-content") | fromYaml -}} -{{- printf "%s-%s-migrate-%s" (.Release.Name | trunc 20 | trimSuffix "-") (.Release.Name | sha256sum | trunc 8) ($execution | toJson | sha256sum | trunc 16) -}} +{{- $identity := dict "release" .Release.Name "namespace" .Release.Namespace "execution" $execution -}} +{{- printf "%s-migrate-%s" (.Release.Name | trunc 31 | trimSuffix "-") ($identity | toJson | sha256sum | trunc 16) -}} {{- end -}} {{/* Migration scripts ship in standalone-api, never in the Brainstore or gate diff --git a/braintrust/tests/migration-image_test.yaml b/braintrust/tests/migration-image_test.yaml index 49d655b..ffc10ac 100644 --- a/braintrust/tests/migration-image_test.yaml +++ b/braintrust/tests/migration-image_test.yaml @@ -21,10 +21,10 @@ tests: value: registry.example/standalone-api:v2.22.0 - matchRegex: path: metadata.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' + pattern: '^test-migrate-[a-f0-9]{16}$' - matchRegex: path: spec.template.spec.containers[0].envFrom[0].configMapRef.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' + pattern: '^test-migrate-[a-f0-9]{16}-config$' - it: preserves the repository and full pin of a newer API image template: migration-job.yaml @@ -228,7 +228,7 @@ tests: value: mirror.example:5000/standalone-api@sha256:bundle - matchRegex: path: metadata.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' + pattern: '^test-migrate-[a-f0-9]{16}$' - it: accepts an explicit prerelease bundle as the operator's compatibility assertion template: migration-job.yaml @@ -398,7 +398,7 @@ tests: value: keep - matchRegex: path: metadata.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' + pattern: '^test-migrate-[a-f0-9]{16}-config$' - it: keeps migration execution independent of the waiter helper image template: migration-job.yaml diff --git a/braintrust/tests/migrations_test.yaml b/braintrust/tests/migrations_test.yaml index ba45e02..6d50b6d 100644 --- a/braintrust/tests/migrations_test.yaml +++ b/braintrust/tests/migrations_test.yaml @@ -33,7 +33,7 @@ tests: of: Job - matchRegex: path: metadata.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}$' + pattern: '^test-migrate-[a-f0-9]{16}$' - equal: path: metadata.annotations["helm.sh/resource-policy"] value: keep @@ -72,7 +72,7 @@ tests: value: false - matchRegex: path: spec.template.spec.containers[0].envFrom[0].configMapRef.name - pattern: '^test-[a-f0-9]{8}-migrate-[a-f0-9]{16}-config$' + pattern: '^test-migrate-[a-f0-9]{16}-config$' - matchRegex: path: spec.template.metadata.annotations["checksum/config"] pattern: '^[a-f0-9]{64}$' @@ -546,7 +546,7 @@ tests: documentIndex: 0 release: {"name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision": 101} asserts: - - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,54}$"}} + - {"matchRegex": {"path": "metadata.name", "pattern": "^.{1,56}$"}} - {"matchRegex": {"path": "metadata.name", "pattern": "-migrate-[a-f0-9]{16}$"}} - it: bounds immutable snapshot names with long release names @@ -556,7 +556,7 @@ tests: asserts: - matchRegex: path: metadata.name - pattern: '^.{1,61}$' + pattern: '^.{1,63}$' - matchRegex: path: metadata.name pattern: '-migrate-[a-f0-9]{16}-config$'