Fix Helm PostgreSQL deployment and add E2E workflow

.github/workflows/helm-e2e.yaml

@@ -0,0 +1,114 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: helm-e2e
+on:
+  pull_request:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+    - name: Setup Go
+      uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+      with:
+        go-version-file: go.mod
+    - name: Setup Helm
+      uses: azure/setup-helm@v4
+    - name: Cache micro-VM assets
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: bin/microvm-assets/amd64
+        key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }}
+    - name: Enable KVM
+      run: |
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+          | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger --name-match=kvm
+    - name: Create cluster
+      run: hack/create-kind-cluster.sh
+    - name: Install observability fixtures
+      run: |
+        kubectl apply -f manifests/ate-install/kind/otel-collector.yaml
+        kubectl apply -f manifests/ate-install/kind/prometheus.yaml
+    - name: Build chart images
+      run: |
+        for component in ateapi atecontroller atelet podcertcontroller atenet; do
+          KO_DOCKER_REPO="localhost:5001/${component}" \
+            ./hack/run-tool.sh ko build --bare --tags helm-e2e \
+              --platform linux/amd64 "./cmd/${component}"
+        done
+    - name: Install Agent Substrate with Helm
+      run: |
+        helm upgrade --install substrate-crds charts/substrate-crds
+        helm upgrade --install substrate charts/substrate \
+          --namespace ate-system \
+          --create-namespace \
+          --set image.registry=localhost:5001 \
+          --set image.tag=helm-e2e \
+          --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \
+          --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \
+          --set postgres.resources.requests.cpu=500m
+    - name: Bootstrap mTLS authorities
+      run: |
+        hack/install-ate-kind.sh --create-podcertificate-controller-cas
+        hack/install-ate-kind.sh --create-jwt-authority-pool-secret
+        hack/install-ate-kind.sh --create-actor-id-ca-pool-secret
+        hack/install-ate-kind.sh --create-actor-id-ca-certs-secret
+        hack/install-ate-kind.sh --create-api-authentication-config
+    - name: Wait for Helm install
+      run: |
+        helm upgrade substrate charts/substrate \
+          --namespace ate-system \
+          --reuse-values \
+          --wait --timeout=10m
+    - name: Deploy egress gateway fixture
+      env:
+        KO_DOCKER_REPO: localhost:5001
+        KO_DEFAULTPLATFORMS: linux/amd64
+      run: |
+        ./hack/run-tool.sh ko apply -f manifests/ate-install/atenet-egress.yaml -- --context=kind-kind
+        kubectl --context kind-kind rollout status deployment/atenet-egress -n ate-system --timeout=120s
+    - name: Deploy micro-VM counter demo
+      run: hack/run-microvm-demo-kind.sh --skip-control-plane
+    - name: Deploy gVisor counter demo
+      run: hack/install-ate-kind.sh --deploy-demo-counter
+    - name: Deploy egress demo
+      run: hack/install-ate-kind.sh --deploy-demo-egress
+    - name: Wait for micro-VM golden snapshot
+      run: |
+        kubectl --context kind-kind wait --for=condition=Ready \
+          actortemplate/counter-microvm -n ate-demo-counter-microvm --timeout=600s
+    - name: Run E2E tests (gVisor)
+      run: hack/run-e2e-kind.sh -v -args --no-color
+    - name: Run E2E tests (micro-VM)
+      env:
+        E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm
+        E2E_TEMPLATE_NAME: counter-microvm
+        E2E_TEMPLATE_READY_TIMEOUT: 600s
+      run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color
+    - name: Dump diagnostics on failure
+      if: failure()
+      run: |
+        kubectl --context kind-kind get actortemplate,workerpool,pods -A -o wide || true
+        for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do
+          echo "=== logs: ate-system/${p} ==="
+          kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true
+        done

charts/substrate/README.md

@@ -2,47 +2,27 @@
 
 Helm chart for installing Agent Substrate.
 
-## Install modes
-
-| Mode | Default? | Cluster requirements | Trade-off |
-|------|----------|----------------------|-----------|
-| `jwt`  | yes | none beyond stock K8s | Server certs and actor signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. |
-| `mtls` |     | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. |
+The chart uses mTLS and PostgreSQL by default. It requires the
+`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and
+`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1`
+API.
 
 ```bash
 # CRDs
 helm upgrade --install substrate-crds ./charts/substrate-crds
 
-# JWT mode (default; no off-by-default feature gates)
+# Install Substrate
 helm upgrade --install substrate ./charts/substrate
-
-# mTLS mode (requires off-by-default feature gates)
-helm upgrade --install substrate ./charts/substrate \
-  --set auth.mode=mtls
 ```
 
 By default, component images are pulled from `ghcr.io/kagent-dev/substrate`
 using the chart `appVersion` as the tag. Override `image.registry` and
 `image.tag` to install from a different image repository or tag.
 
-## JWT-mode bootstrap
-
-JWT mode is standalone by default. The chart generates:
-
-- `Secret/ateapi-tls`
-- `ConfigMap/ateapi-ca`
-- `Secret/actor-id-jwt-pool`
-- `Secret/actor-id-ca-pool`
-
-Existing generated data is reused on upgrade so key material does not rotate
-during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring
-your own resources with those names.
-
 ## Render manifests without applying
 
 ```bash
-helm template substrate ./charts/substrate                       # jwt
-helm template substrate ./charts/substrate --set auth.mode=mtls
+helm template substrate ./charts/substrate
 ```
 
 `manifests/ate-install/` in the repo is the rendered mTLS output and is
@@ -55,17 +35,9 @@ See `values.yaml` for the full set; the important keys:
 
 | Key | Default | Notes |
 |-----|---------|-------|
-| `auth.mode` | `jwt` | `jwt` or `mtls` |
-| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers |
-| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience |
-| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and actor signing material |
-| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name |
-| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name |
-| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey |
-| `valkey.replicas` | `6` | StatefulSet size |
+| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL |
+| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size |
 | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
 | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
-| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis |
-| `redis.useIAMAuth` | `false` | Google IAM auth |
 | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth |
 | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics |

charts/substrate/templates/NOTES.txt

@@ -1,21 +1,7 @@
-substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }}
+substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL
 
-{{ if eq .Values.auth.mode "mtls" -}}
-NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled:
+REQUIRED Kubernetes feature gates:
   - ClusterTrustBundle
   - ClusterTrustBundleProjection
   - PodCertificateRequest
-plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you
-must enable these explicitly. To install without them, pick auth.mode=jwt.
-{{- else }}
-JWT mode is active.
-
-{{- if .Values.auth.jwt.bootstrap.enabled }}
-JWT bootstrap resources are managed by this chart. Existing key material is
-reused on upgrade.
-{{- else }}
-JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }},
-{{ .Values.auth.jwt.caBundleConfigMap }}, actor-id-jwt-pool, and
-actor-id-ca-pool before pods become healthy.
-{{- end }}
-{{- end }}
+The certificates.k8s.io/v1beta1 API must also be enabled.

charts/substrate/templates/_helpers.tpl

@@ -103,17 +103,3 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time.
 {{- printf "%s/%s" $registry $name -}}
 {{- end -}}
 {{- end -}}
-
-{{/*
-Validate auth.mode at template time.
-*/}}
-{{- define "substrate.validateAuthMode" -}}
-{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}}
-{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}}
-{{- end -}}
-{{- if eq .Values.auth.mode "jwt" -}}
-{{- if not .Values.auth.jwt.issuer -}}
-{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}

charts/substrate/templates/ate-api-server-envvars.yaml

@@ -20,8 +20,5 @@ metadata:
   name: {{ .Values.ateApiServerEnvVarsConfigMap }}
   namespace: {{ .Release.Namespace }}
 data:
-  ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }}
-  ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }}
-  ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }}
-  ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }}
-  ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }}
+  ATE_API_STORE_BACKEND: "postgres"
+  ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }}

charts/substrate/templates/ate-api-server.yaml

@@ -23,7 +23,10 @@ rules:
   resources: ["pods"]
   verbs: ["get", "watch", "list"]
 - apiGroups: ["ate.dev"]
-  resources: ["actortemplates", "workerpools", "sandboxconfigs"]
+  resources: ["actortemplates", "workerpools", "sandboxconfigs", "csidriverconfigs"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
   verbs: ["get", "watch", "list"]
 # Secret reads for env source resolution are intentionally NOT granted
 # cluster-wide here. Each demo / tenant is responsible for granting
@@ -74,48 +77,19 @@ spec:
     spec:
       serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }}
       terminationGracePeriodSeconds: 40
-{{- if eq .Values.auth.mode "jwt" }}
-      initContainers:
-      - name: assemble-cred-bundle
-        image: {{ .Values.images.busybox }}
-        command:
-        - sh
-        - -c
-        - cat /run/ateapi-tls-src/tls.crt /run/ateapi-tls-src/tls.key > /run/ateapi-tls/credential-bundle.pem
-        volumeMounts:
-        - { name: ateapi-tls-src, mountPath: /run/ateapi-tls-src, readOnly: true }
-        - { name: ateapi-tls,     mountPath: /run/ateapi-tls }
-{{- end }}
       containers:
       - name: ate-api-server
         image: {{ include "substrate.componentImage" (list "ateapi" .) }}
         args:
         - "--grpc-listen-addr=0.0.0.0:443"
-{{- if eq .Values.auth.mode "mtls" }}
         - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem"
-        - "--redis-cluster-address=@env"
-        - "--redis-ca-certs=/etc/valkey-ca/ca.crt"
-        - "--redis-use-iam-auth=@env"
-        - "--redis-tls-server-name=@env"
-        - "--redis-client-cert=@env"
-        - "--client-jwt-issuer=@env"
-        - "--client-jwt-audience={{ .Values.auth.jwt.audience }}"
+        - "--authentication-config=/etc/ateapi/authentication/authentication.yaml"
+        - "--store-backend=@env"
+        - "--postgres-connection-string=@env"
         - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
         - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"
         - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
         - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem"
-{{- else }}
-        - "--grpc-server-cred-bundle=/run/ateapi-tls/credential-bundle.pem"
-        - "--atelet-insecure=true"
-        - "--redis-cluster-address=@env"
-        - "--redis-no-tls=true"
-        - "--redis-use-iam-auth=@env"
-        - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}"
-        - "--client-jwt-audience={{ .Values.auth.jwt.audience }}"
-        - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
-        - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"
-        - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-{{- end }}
         - "--drain-delay=13s"
         - "--drain-timeout=15s"
         env:
@@ -142,17 +116,11 @@ spec:
             name: {{ .Values.ateApiServerEnvVarsConfigMap }}
             optional: true
         volumeMounts:
-{{- if eq .Values.auth.mode "mtls" }}
         - { name: servicedns,          mountPath: /run/servicedns.podcert.ate.dev }
         - { name: actor-id-jwt-pool,   mountPath: /run/actor-id-jwt-pool }
-        - { name: valkey-ca-certs,     mountPath: /etc/valkey-ca, readOnly: true }
         - { name: actor-id-ca-pool,    mountPath: /run/actor-id-ca-pool, readOnly: true }
         - { name: podidentity,         mountPath: /run/podidentity.podcert.ate.dev, readOnly: true }
-{{- else }}
-        - { name: ateapi-tls,          mountPath: /run/ateapi-tls, readOnly: true }
-        - { name: actor-id-jwt-pool,   mountPath: /run/actor-id-jwt-pool }
-        - { name: actor-id-ca-pool,    mountPath: /run/actor-id-ca-pool, readOnly: true }
-{{- end }}
+        - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true }
         ports:
         - containerPort: 443
         - name: prometheus
@@ -171,35 +139,36 @@ spec:
           initialDelaySeconds: 10
           periodSeconds: 10
       volumes:
-{{- if eq .Values.auth.mode "mtls" }}
       - name: servicedns
         projected:
           sources:
           - podCertificate:
               signerName: servicedns.podcert.ate.dev/identity
               keyType: ECDSAP256
               credentialBundlePath: credential-bundle.pem
+          - clusterTrustBundle:
+              signerName: servicedns.podcert.ate.dev/identity
+              labelSelector:
+                matchLabels:
+                  podcert.ate.dev/canarying: live
+              path: trust-bundle.pem
       - name: actor-id-jwt-pool
         projected:
           sources:
           - secret:
               name: actor-id-jwt-pool
               items:
               - { key: pool, path: pool.json }
-      - name: valkey-ca-certs
-        projected:
-          sources:
-          - secret:
-              name: valkey-ca-certs
-              items:
-              - { key: ca.crt, path: ca.crt }
       - name: actor-id-ca-pool
         projected:
           sources:
           - secret:
               name: actor-id-ca-pool
               items:
               - { key: pool, path: pool.json }
+      - name: authentication-config
+        configMap:
+          name: ate-api-authentication
       - name: podidentity
         projected:
           sources:
@@ -213,27 +182,6 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
-{{- else }}
-      - name: ateapi-tls-src
-        secret:
-          secretName: {{ .Values.auth.jwt.serverCertSecret }}
-      - name: ateapi-tls
-        emptyDir: {}
-      - name: actor-id-jwt-pool
-        projected:
-          sources:
-          - secret:
-              name: actor-id-jwt-pool
-              items:
-              - { key: pool, path: pool.json }
-      - name: actor-id-ca-pool
-        projected:
-          sources:
-          - secret:
-              name: actor-id-ca-pool
-              items:
-              - { key: pool, path: pool.json }
-{{- end }}
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget