diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 000000000..122e3f445 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Deploy egress gateway fixture + env: + KO_DOCKER_REPO: localhost:5001 + KO_DEFAULTPLATFORMS: linux/amd64 + run: | + ./hack/run-tool.sh ko apply -f manifests/ate-install/atenet-egress.yaml -- --context=kind-kind + kubectl --context kind-kind rollout status deployment/atenet-egress -n ate-system --timeout=120s + - name: Deploy micro-VM counter demo + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Wait for micro-VM golden snapshot + run: | + kubectl --context kind-kind wait --for=condition=Ready \ + actortemplate/counter-microvm -n ate-demo-counter-microvm --timeout=600s + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm + E2E_TEMPLATE_NAME: counter-microvm + E2E_TEMPLATE_READY_TIMEOUT: 600s + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get actortemplate,workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 6cc98069b..e7364f4e3 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -2,47 +2,27 @@ Helm chart for installing Agent Substrate. -## Install modes - -| Mode | Default? | Cluster requirements | Trade-off | -|------|----------|----------------------|-----------| -| `jwt` | yes | none beyond stock K8s | Server certs and actor signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. | -| `mtls` | | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. | +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. ```bash # CRDs helm upgrade --install substrate-crds ./charts/substrate-crds -# JWT mode (default; no off-by-default feature gates) +# Install Substrate helm upgrade --install substrate ./charts/substrate - -# mTLS mode (requires off-by-default feature gates) -helm upgrade --install substrate ./charts/substrate \ - --set auth.mode=mtls ``` By default, component images are pulled from `ghcr.io/kagent-dev/substrate` using the chart `appVersion` as the tag. Override `image.registry` and `image.tag` to install from a different image repository or tag. -## JWT-mode bootstrap - -JWT mode is standalone by default. The chart generates: - -- `Secret/ateapi-tls` -- `ConfigMap/ateapi-ca` -- `Secret/actor-id-jwt-pool` -- `Secret/actor-id-ca-pool` - -Existing generated data is reused on upgrade so key material does not rotate -during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring -your own resources with those names. - ## Render manifests without applying ```bash -helm template substrate ./charts/substrate # jwt -helm template substrate ./charts/substrate --set auth.mode=mtls +helm template substrate ./charts/substrate ``` `manifests/ate-install/` in the repo is the rendered mTLS output and is @@ -55,17 +35,9 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| -| `auth.mode` | `jwt` | `jwt` or `mtls` | -| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers | -| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience | -| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and actor signing material | -| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name | -| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name | -| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey | -| `valkey.replicas` | `6` | StatefulSet size | +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | -| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis | -| `redis.useIAMAuth` | `false` | Google IAM auth | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt index abad3a8d1..c0e9875a4 100644 --- a/charts/substrate/templates/NOTES.txt +++ b/charts/substrate/templates/NOTES.txt @@ -1,21 +1,7 @@ -substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }} +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL -{{ if eq .Values.auth.mode "mtls" -}} -NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled: +REQUIRED Kubernetes feature gates: - ClusterTrustBundle - ClusterTrustBundleProjection - PodCertificateRequest -plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you -must enable these explicitly. To install without them, pick auth.mode=jwt. -{{- else }} -JWT mode is active. - -{{- if .Values.auth.jwt.bootstrap.enabled }} -JWT bootstrap resources are managed by this chart. Existing key material is -reused on upgrade. -{{- else }} -JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }}, -{{ .Values.auth.jwt.caBundleConfigMap }}, actor-id-jwt-pool, and -actor-id-ca-pool before pods become healthy. -{{- end }} -{{- end }} +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 36b9d3ef9..32ae08733 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -103,17 +103,3 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time. {{- printf "%s/%s" $registry $name -}} {{- end -}} {{- end -}} - -{{/* -Validate auth.mode at template time. -*/}} -{{- define "substrate.validateAuthMode" -}} -{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}} -{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}} -{{- end -}} -{{- if eq .Values.auth.mode "jwt" -}} -{{- if not .Values.auth.jwt.issuer -}} -{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 754ff847a..c0b8323b2 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -20,8 +20,5 @@ metadata: name: {{ .Values.ateApiServerEnvVarsConfigMap }} namespace: {{ .Release.Namespace }} data: - ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }} - ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }} - ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }} - ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }} - ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }} + ATE_API_STORE_BACKEND: "postgres" + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 03a3c6b89..73869cee0 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -23,7 +23,10 @@ rules: resources: ["pods"] verbs: ["get", "watch", "list"] - apiGroups: ["ate.dev"] - resources: ["actortemplates", "workerpools", "sandboxconfigs"] + resources: ["actortemplates", "workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] verbs: ["get", "watch", "list"] # Secret reads for env source resolution are intentionally NOT granted # cluster-wide here. Each demo / tenant is responsible for granting @@ -74,48 +77,19 @@ spec: spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 -{{- if eq .Values.auth.mode "jwt" }} - initContainers: - - name: assemble-cred-bundle - image: {{ .Values.images.busybox }} - command: - - sh - - -c - - cat /run/ateapi-tls-src/tls.crt /run/ateapi-tls-src/tls.key > /run/ateapi-tls/credential-bundle.pem - volumeMounts: - - { name: ateapi-tls-src, mountPath: /run/ateapi-tls-src, readOnly: true } - - { name: ateapi-tls, mountPath: /run/ateapi-tls } -{{- end }} containers: - name: ate-api-server image: {{ include "substrate.componentImage" (list "ateapi" .) }} args: - "--grpc-listen-addr=0.0.0.0:443" -{{- if eq .Values.auth.mode "mtls" }} - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - - "--redis-cluster-address=@env" - - "--redis-ca-certs=/etc/valkey-ca/ca.crt" - - "--redis-use-iam-auth=@env" - - "--redis-tls-server-name=@env" - - "--redis-client-cert=@env" - - "--client-jwt-issuer=@env" - - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--store-backend=@env" + - "--postgres-connection-string=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" -{{- else }} - - "--grpc-server-cred-bundle=/run/ateapi-tls/credential-bundle.pem" - - "--atelet-insecure=true" - - "--redis-cluster-address=@env" - - "--redis-no-tls=true" - - "--redis-use-iam-auth=@env" - - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}" - - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" - - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" -{{- end }} - "--drain-delay=13s" - "--drain-timeout=15s" env: @@ -142,17 +116,11 @@ spec: name: {{ .Values.ateApiServerEnvVarsConfigMap }} optional: true volumeMounts: -{{- if eq .Values.auth.mode "mtls" }} - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } -{{- else }} - - { name: ateapi-tls, mountPath: /run/ateapi-tls, readOnly: true } - - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } -{{- end }} + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } ports: - containerPort: 443 - name: prometheus @@ -171,7 +139,6 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: -{{- if eq .Values.auth.mode "mtls" }} - name: servicedns projected: sources: @@ -179,6 +146,12 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem - name: actor-id-jwt-pool projected: sources: @@ -186,13 +159,6 @@ spec: name: actor-id-jwt-pool items: - { key: pool, path: pool.json } - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - { key: ca.crt, path: ca.crt } - name: actor-id-ca-pool projected: sources: @@ -200,6 +166,9 @@ spec: name: actor-id-ca-pool items: - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication - name: podidentity projected: sources: @@ -213,27 +182,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- else }} - - name: ateapi-tls-src - secret: - secretName: {{ .Values.auth.jwt.serverCertSecret }} - - name: ateapi-tls - emptyDir: {} - - name: actor-id-jwt-pool - projected: - sources: - - secret: - name: actor-id-jwt-pool - items: - - { key: pool, path: pool.json } - - name: actor-id-ca-pool - projected: - sources: - - secret: - name: actor-id-ca-pool - items: - - { key: pool, path: pool.json } -{{- end }} --- apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 010c425ee..31c83b906 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -77,15 +77,8 @@ spec: # "ate-system"). Pass the chart-resolved Service so the controller # dials the right backend when substrate is installed as a subchart. - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" -{{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- else }} - - "--ateapi-use-token-auth=true" - - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" - - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" - - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" -{{- end }} {{- if .Values.otel.endpoint }} env: - name: OTEL_EXPORTER_OTLP_ENDPOINT @@ -98,7 +91,6 @@ spec: - name: healthz containerPort: 8081 protocol: TCP -{{- if eq .Values.auth.mode "mtls" }} volumeMounts: - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } @@ -119,19 +111,3 @@ spec: signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem -{{- else }} - volumeMounts: - - { name: ateapi-ca, mountPath: /run/ateapi-ca, readOnly: true } - - { name: ateapi-token, mountPath: /var/run/secrets/tokens/ateapi, readOnly: true } - volumes: - - name: ateapi-ca - configMap: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - - name: ateapi-token - projected: - sources: - - serviceAccountToken: - audience: {{ .Values.auth.jwt.audience }} - expirationSeconds: 3600 - path: token -{{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 6b2eebba4..fab67b025 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# atelet — identical across auth modes (does not dial ateapi). +# atelet apiVersion: v1 kind: ServiceAccount metadata: @@ -29,6 +29,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -68,12 +71,9 @@ spec: image: {{ include "substrate.componentImage" (list "atelet" .) }} args: - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} -{{- if eq .Values.auth.mode "mtls" }} - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem -{{- else }} - - --grpc-insecure=true -{{- end }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem {{- with .Values.atelet.extraArgs }} {{ toYaml . | indent 8 }} {{- end }} @@ -116,17 +116,17 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ateom-gvisor -{{- if eq .Values.auth.mode "mtls" }} - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true -{{- end }} + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true volumes: - name: run-ateom hostPath: path: /var/lib/ateom-gvisor type: DirectoryOrCreate -{{- if eq .Values.auth.mode "mtls" }} - name: podidentity projected: sources: @@ -140,4 +140,12 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- end }} + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml index ace864570..fc6f77030 100644 --- a/charts/substrate/templates/atenet-dns.yaml +++ b/charts/substrate/templates/atenet-dns.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# atenet-dns — identical across auth modes (does not dial ateapi). +# atenet-dns apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 9168c86fe..8810e1bc8 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -79,13 +79,8 @@ data: port: 8443 protocol: HTTPS tls: -{{- if eq .Values.auth.mode "mtls" }} cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem -{{- else }} - cert: /run/agentgateway-tls/tls.crt - key: /run/agentgateway-tls/tls.key -{{- end }} routes: - name: substrate-actors @@ -106,16 +101,62 @@ data: responseBodyMode: none requestTrailerMode: skip responseTrailerMode: skip + requestAttributes: + "filter_state['dev.ate.authority']": request.host backends: - - dynamic: {} -{{- if eq .Values.auth.mode "mtls" }} + - dynamic: + target: extproc["envoy.filters.listener.original_dst"]["local"] policies: backendTLS: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/podidentity.podcert.ate.dev/trust-bundle.pem insecureHost: true -{{- end }} + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + extProc: + host: 127.0.0.1:50051 + failureMode: failClosed + processingOptions: + requestHeaderMode: send + responseHeaderMode: skip + requestBodyMode: none + responseBodyMode: none + requestTrailerMode: skip + responseTrailerMode: skip + requestAttributes: + "filter_state['dev.ate.authority']": source.connectHeaders["host"] + backends: + - dynamic: + target: extproc["envoy.filters.listener.original_dst"]["local"] + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true --- apiVersion: apps/v1 kind: Deployment @@ -143,24 +184,19 @@ spec: image: {{ include "substrate.componentImage" (list "atenet" .) }} args: - "router" - - "--standalone" + - "--mode=ingress" - "--atenet-router=agentgateway" - "--namespace={{ .Release.Namespace }}" - "--port-http=8080" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" -{{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- else }} - - "--ateapi-use-token-auth=true" - - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" - - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" - - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" -{{- end }} - "--status-port=4040" - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" env: - name: POD_NAME valueFrom: @@ -187,19 +223,9 @@ spec: containerPort: 4040 - name: metrics containerPort: 9090 -{{- if eq .Values.auth.mode "mtls" }} volumeMounts: - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } -{{- else }} - volumeMounts: - - name: ateapi-ca - mountPath: /run/ateapi-ca - readOnly: true - - name: ateapi-token - mountPath: /var/run/secrets/tokens/ateapi - readOnly: true -{{- end }} - name: agentgateway image: {{ .Values.images.agentgateway }} args: @@ -215,6 +241,10 @@ spec: containerPort: 8080 - name: https containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 - name: readiness containerPort: 15021 - name: gw-metrics @@ -222,17 +252,11 @@ spec: volumeMounts: - name: agentgateway-config mountPath: /etc/agentgateway -{{- if eq .Values.auth.mode "mtls" }} - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true -{{- else }} - - name: agentgateway-tls - mountPath: /run/agentgateway-tls - readOnly: true -{{- end }} readinessProbe: httpGet: path: /healthz/ready @@ -242,13 +266,13 @@ spec: - name: agentgateway-config configMap: name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} -{{- if eq .Values.auth.mode "mtls" }} - name: "servicedns" projected: sources: - podCertificate: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem certificateChainPath: cert.pem keyPath: key.pem - name: servicedns-ca @@ -275,21 +299,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- else }} - - name: agentgateway-tls - secret: - secretName: {{ .Values.auth.jwt.serverCertSecret }} - - name: ateapi-ca - configMap: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - - name: ateapi-token - projected: - sources: - - serviceAccountToken: - audience: {{ .Values.auth.jwt.audience }} - expirationSeconds: 3600 - path: token -{{- end }} --- apiVersion: v1 kind: Service @@ -309,6 +318,14 @@ spec: port: 443 targetPort: 8443 protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP - name: status port: 4040 targetPort: status diff --git a/charts/substrate/templates/jwt-bootstrap.yaml b/charts/substrate/templates/jwt-bootstrap.yaml deleted file mode 100644 index cb31763fb..000000000 --- a/charts/substrate/templates/jwt-bootstrap.yaml +++ /dev/null @@ -1,73 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and (eq .Values.auth.mode "jwt") .Values.auth.jwt.bootstrap.enabled }} -{{- $apiName := include "substrate.fullname" (list "api" .) }} -{{- $routerName := include "substrate.fullname" (list "atenet-router" .) }} -{{- $apiHost := printf "%s.%s.svc" $apiName .Release.Namespace }} -{{- $ca := genCA (printf "%s-ca" $apiName) 3650 }} -{{- $serverCert := genSignedCert $apiHost nil (list $apiHost (printf "%s.%s.svc.cluster.local" $apiName .Release.Namespace) (printf "%s.%s.svc" $routerName .Release.Namespace)) 365 $ca }} -{{- $actorJWTKey := genPrivateKey "ecdsa" }} -{{- $actorCA := genCA "actor-id-ca" 3650 }} -{{- if .Values.auth.jwt.bootstrap.serverCert.enabled }} -{{- $existingTLS := lookup "v1" "Secret" .Release.Namespace .Values.auth.jwt.serverCertSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.auth.jwt.serverCertSecret }} - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: {{ if $existingTLS }}{{ index $existingTLS.data "tls.crt" }}{{ else }}{{ $serverCert.Cert | b64enc }}{{ end }} - tls.key: {{ if $existingTLS }}{{ index $existingTLS.data "tls.key" }}{{ else }}{{ $serverCert.Key | b64enc }}{{ end }} ---- -{{- $existingCA := lookup "v1" "ConfigMap" .Release.Namespace .Values.auth.jwt.caBundleConfigMap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - namespace: {{ .Release.Namespace }} -data: - ca.crt: | -{{- if $existingCA }} -{{ index $existingCA.data "ca.crt" | nindent 4 }} -{{- else }} -{{ $ca.Cert | nindent 4 }} -{{- end }} -{{- end }} -{{- if .Values.auth.jwt.bootstrap.sessionPools.enabled }} ---- -{{- $existingJWTSecret := lookup "v1" "Secret" .Release.Namespace "actor-id-jwt-pool" }} -apiVersion: v1 -kind: Secret -metadata: - name: actor-id-jwt-pool - namespace: {{ .Release.Namespace }} -type: Opaque -data: - pool: {{ if $existingJWTSecret }}{{ index $existingJWTSecret.data "pool" }}{{ else }}{{ dict "Authorities" (list (dict "ID" "1" "Algorithm" "ES256" "SigningKeyPEM" $actorJWTKey)) | toJson | b64enc }}{{ end }} ---- -{{- $existingCASecret := lookup "v1" "Secret" .Release.Namespace "actor-id-ca-pool" }} -apiVersion: v1 -kind: Secret -metadata: - name: actor-id-ca-pool - namespace: {{ .Release.Namespace }} -type: Opaque -data: - pool: {{ if $existingCASecret }}{{ index $existingCASecret.data "pool" }}{{ else }}{{ dict "CAs" (list (dict "ID" "1" "SigningKeyPEM" $actorCA.Key "RootCertificatePEM" $actorCA.Cert)) | toJson | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/charts/substrate/templates/jwt-oidc-rbac.yaml b/charts/substrate/templates/jwt-oidc-rbac.yaml deleted file mode 100644 index a9fd499e9..000000000 --- a/charts/substrate/templates/jwt-oidc-rbac.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if eq .Values.auth.mode "jwt" }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} -rules: -- nonResourceURLs: - - /.well-known/openid-configuration - - /openid/v1/jwks - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} -subjects: -- kind: ServiceAccount - name: {{ include "substrate.fullname" (list "ate-api-server" .) }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml index 63401c00d..073291828 100644 --- a/charts/substrate/templates/namespace.yaml +++ b/charts/substrate/templates/namespace.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- include "substrate.validateAuthMode" . -}} {{- if .Values.createNamespace }} apiVersion: v1 kind: Namespace diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml index 3aaaa9df9..86fc23b4a 100644 --- a/charts/substrate/templates/pod-certificate-controller.yaml +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if eq .Values.auth.mode "mtls" -}} apiVersion: v1 kind: Namespace metadata: @@ -197,4 +196,3 @@ spec: securityContext: {} serviceAccountName: default terminationGracePeriodSeconds: 30 -{{- end }} diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 000000000..feee69610 --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,161 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/tls/credential-bundle.pem' + ssl_key_file = '/run/tls/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + local all all trust + hostssl all all all trust clientcert=verify-ca +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + initContainers: + - name: fix-tls-perms + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - -c + - | + set -e + cp /run/servicedns.podcert.ate.dev/credential-bundle.pem /run/tls/credential-bundle.pem + chmod 600 /run/tls/credential-bundle.pem + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: tls + mountPath: /run/tls + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: tls + mountPath: /run/tls + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: tls + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 3dc4e9d16..36af4296f 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -26,12 +26,13 @@ metadata: spec: sandboxClass: gvisor default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: - runsc: - url: "gs://gvisor/releases/release/20260622/x86_64/runsc" - sha256: "f18a948bf9c8bbb54eb998549a3a8d719a1c7de2efbe8fdd2ff0ee5fecd06f19" + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" arm64: - runsc: - url: "gs://gvisor/releases/release/20260622/aarch64/runsc" - sha256: "62eee121f8c188e347c428acc96f111568ede3be37b906046b6f28bbe2cc40c0" + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml index 48302d2fd..f25d43409 100644 --- a/charts/substrate/templates/sandboxconfig-validation.yaml +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -31,12 +31,13 @@ spec: operations: ["CREATE", "UPDATE"] resources: ["sandboxconfigs"] validations: - # gVisor needs a "runsc" asset for every architecture it advertises. + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. - expression: >- object.spec.sandboxClass != 'gvisor' || (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, 'runsc' in object.spec.assets[arch])) - message: "a gvisor SandboxConfig must define a 'runsc' asset for every architecture under spec.assets" + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- diff --git a/charts/substrate/templates/valkey.yaml b/charts/substrate/templates/valkey.yaml deleted file mode 100644 index ca164e769..000000000 --- a/charts/substrate/templates/valkey.yaml +++ /dev/null @@ -1,269 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.valkey.enabled -}} -{{- $sts := include "substrate.fullname" (list "valkey-cluster" .) -}} -{{- $headless := include "substrate.fullname" (list "valkey-cluster-service" .) -}} -{{- $ns := .Release.Namespace -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "substrate.fullname" (list "valkey-config" .) }} - namespace: {{ .Release.Namespace }} -data: - valkey.conf: | -{{- if eq .Values.auth.mode "mtls" }} - # Enforce TLS and disable standard port - port 0 - tls-port 6379 - tls-cluster yes - tls-replication yes - - # Load certificates from projected volume - tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem - tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem - tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem - tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem - tls-ca-cert-file /etc/valkey-ca/ca.crt - tls-auth-clients yes - - # Reload every 10 minutes. - # The interval should be less than the 30m headroom (notAfter - beginRefreshAt) - # set by cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go. - tls-auto-reload-interval 600 - - # Enable cluster mode -{{- else }} - # Plaintext: serve on the standard port, no TLS. - port 6379 - -{{- end }} - cluster-enabled yes - cluster-config-file nodes.conf - cluster-node-timeout 5000 - appendonly yes - protected-mode no ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $headless }} - namespace: {{ .Release.Namespace }} -spec: - clusterIP: None - selector: - app: valkey-cluster - ports: - - name: valkey - port: 6379 - targetPort: 6379 - - name: bus - port: 16379 - targetPort: 16379 ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $sts }} - namespace: {{ .Release.Namespace }} -spec: - selector: - app: valkey-cluster - ports: - - name: valkey - port: 6379 - targetPort: 6379 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ $sts }} - namespace: {{ .Release.Namespace }} -spec: - serviceName: {{ $headless }} - replicas: {{ .Values.valkey.replicas }} - podManagementPolicy: Parallel - selector: - matchLabels: - app: valkey-cluster - template: - metadata: - labels: - app: valkey-cluster - spec: - containers: - - name: valkey - image: {{ .Values.images.valkey }} - command: ["valkey-server", "/etc/valkey/valkey.conf"] - ports: - - name: valkey - containerPort: 6379 - - name: bus - containerPort: 16379 - volumeMounts: - - name: config - mountPath: /etc/valkey -{{- if eq .Values.auth.mode "mtls" }} - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - - name: valkey-ca-certs - mountPath: /etc/valkey-ca - readOnly: true -{{- end }} - - name: data - mountPath: /data - volumes: - - name: config - configMap: - name: {{ include "substrate.fullname" (list "valkey-config" .) }} -{{- if eq .Values.auth.mode "mtls" }} - - name: servicedns - projected: - sources: - - podCertificate: - signerName: servicedns.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: podidentity - projected: - sources: - - podCertificate: - signerName: podidentity.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - key: ca.crt - path: ca.crt -{{- end }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: {{ .Values.valkey.storageSize }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "substrate.fullname" (list "valkey-cluster-init" .) }} - namespace: {{ .Release.Namespace }} -spec: - template: - metadata: - labels: - app: valkey-cluster-init - spec: - restartPolicy: OnFailure - containers: - - name: init - image: {{ .Values.images.valkey }} -{{- if eq .Values.auth.mode "mtls" }} - volumeMounts: - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - - name: valkey-ca-certs - mountPath: /etc/valkey-ca - readOnly: true -{{- end }} - command: - - /bin/sh - - -c - - | - set -e - echo "Waiting for all Valkey pods to resolve..." - for i in 0 1 2 3 4 5; do - until getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-${i} DNS..." - sleep 2 - done - done - - echo "All pods resolved. Getting IPs..." - POD_IPS="" - for i in 0 1 2 3 4 5; do - ip=$(getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc | awk '{print $1}') - POD_IPS="${POD_IPS} ${ip}:6379" - done - - echo "Checking if Valkey cluster is already initialized..." -{{- if eq .Values.auth.mode "mtls" }} - until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc ping >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-0 to respond to ping..." - sleep 2 - done - - INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc cluster info 2>/dev/null | grep cluster_state || true) - - if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then - echo "Initializing Valkey cluster..." - valkey-cli --tls \ - --cacert /etc/valkey-ca/ca.crt \ - --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ - --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ - --cluster create ${POD_IPS} \ - --cluster-replicas 1 \ - --cluster-yes - echo "Cluster initialization complete!" - else - echo "Cluster already initialized." - fi -{{- else }} - until valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 ping >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-0 to respond to ping..." - sleep 2 - done - - INIT_STATUS=$(valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 cluster info 2>/dev/null | grep cluster_state || true) - - if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then - echo "Initializing Valkey cluster..." - valkey-cli \ - --cluster create ${POD_IPS} \ - --cluster-replicas 1 \ - --cluster-yes - echo "Cluster initialization complete!" - else - echo "Cluster already initialized." - fi -{{- end }} -{{- if eq .Values.auth.mode "mtls" }} - volumes: - - name: podidentity - projected: - sources: - - podCertificate: - signerName: podidentity.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - key: ca.crt - path: ca.crt -{{- end }} -{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index e27f77527..78f34b12a 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -14,56 +14,8 @@ # Default values for the substrate chart. # -# The chart supports two installation modes via `auth.mode`: -# -# - "jwt" (default): No PodCertificateRequest / ClusterTrustBundle usage. Server -# certs and actor signing pools are generated by the chart by default, -# and can be disabled when you want to provide your own key material. -# Clients authenticate to ateapi with a projected Kubernetes ServiceAccount -# token. Valkey runs plaintext. -# -# - "mtls": Server certs are issued by the in-cluster podcertcontroller via -# PodCertificateRequest + projected into pods via the ClusterTrustBundle / -# podCertificate projection sources. Valkey runs with full TLS + -# client-cert verification. REQUIRES the off-by-default Kubernetes feature -# gates: -# ClusterTrustBundle, ClusterTrustBundleProjection, PodCertificateRequest -# and the v1beta1 certificates API. - -auth: - mode: jwt # jwt | mtls - - jwt: - # OIDC issuer URL the cluster uses to mint SA tokens. The default matches - # stock kind/kubeadm-style clusters. Override this for managed clusters - # whose service account issuer is provider-specific. Examples: - # GKE: https://container.googleapis.com/v1/projects//locations//clusters/ - # kind: https://kubernetes.default.svc.cluster.local - # EKS: https://oidc.eks..amazonaws.com/id/ - issuer: https://kubernetes.default.svc.cluster.local - - # Audience SA tokens are minted for, and that ateapi expects. - audience: api.ate-system.svc - - bootstrap: - # Generate JWT-mode TLS and actor-signing key material with Helm. - # Existing generated resources are reused on upgrade via lookup. - enabled: true - serverCert: - enabled: true - sessionPools: - enabled: true - - # Name of a kubernetes.io/tls Secret in the release namespace, with keys - # tls.crt and tls.key. Created by the chart when - # auth.jwt.bootstrap.serverCert.enabled=true. - serverCertSecret: ateapi-tls - - # Name of a ConfigMap in the release namespace with key "ca.crt" holding - # the CA(s) that signed serverCertSecret. Clients mount it to verify the - # ateapi server certificate. Created by the chart when - # auth.jwt.bootstrap.serverCert.enabled=true. - caBundleConfigMap: ateapi-ca +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. # Set to true to have the chart create the release namespace. # Off by default — most helm workflows expect the namespace to already exist @@ -71,10 +23,16 @@ auth: # manifests/ate-install/ install path (kubectl apply). createNamespace: false -valkey: - enabled: true - replicas: 6 +postgres: storageSize: 1Gi + connectionString: "" + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi rustfs: enabled: true @@ -94,19 +52,8 @@ atelet: extraArgs: [] extraEnv: [] -redis: - # Override the cluster address. Empty -> derived from valkey.enabled - # (defaults to "valkey-cluster.ate-system.svc:6379"). - clusterAddress: "" - # Google IAM auth (for managed Memorystore / cloud Valkey). - useIAMAuth: false - # Override TLS server name for Redis hostname verification (mtls mode). - tlsServerName: "" - # File path for Redis client TLS credential bundle (mtls mode). - clientCert: "" - # Name of a ConfigMap in the release namespace that supplies per-environment -# overrides for ate-api-server (ATE_API_REDIS_*, ATE_API_K8SJWT_ISSUER, ...). +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). # Mounted via envFrom with optional=true. Created by the chart from these values. ateApiServerEnvVarsConfigMap: ate-api-server-envvars @@ -118,9 +65,9 @@ image: tag: "" images: - valkey: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: cr.agentgateway.dev/agentgateway:v1.4.1 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.34610af2@sha256:c471d7a835d9d318353d531fe3a6195aa164cf44825c867c4cb69feae09d4a86 coredns: coredns/coredns:1.11.1 busybox: busybox:1.36 diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 5975bdce1..c3cd29cff 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -56,6 +56,7 @@ KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" ATE_ATEAPI_CLIENT_AUTH="${ATE_ATEAPI_CLIENT_AUTH:-cert}" +SKIP_CONTROL_PLANE=false while [[ $# -gt 0 ]]; do case "$1" in @@ -68,6 +69,7 @@ while [[ $# -gt 0 ]]; do shift ATE_ATEAPI_CLIENT_AUTH="$1" ;; + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; *) echo "Error: unknown argument $1" >&2 exit 1 @@ -99,13 +101,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index d1cfddc96..f674a49cd 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -55,7 +55,7 @@ patches: path: /spec/template/spec/containers/1 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v1.4.1 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.34610af2@sha256:c471d7a835d9d318353d531fe3a6195aa164cf44825c867c4cb69feae09d4a86 args: - -f - /etc/agentgateway/config.yaml