From 8ffad22f05c8b2d2c2a48c1dac42d666a2ee9b97 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Thu, 4 Jun 2026 10:46:30 -0400 Subject: [PATCH 1/9] release workflow (#6) * wip - publish release * release yaml --- .github/workflows/release.yaml | 102 +++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..e87842781f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,102 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atelet ateom-gvisor podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true From 7d7a5ff4368c5d3a1992b9b9461ca1825d2fd6db Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Wed, 10 Jun 2026 12:56:16 -0400 Subject: [PATCH 2/9] feat: allow running with vanilla k8s (#3) * enable websockets (#4) Signed-off-by: Peter Jausovec Co-authored-by: Peter Jausovec * feat: allow running with vanilla k8s - add a helm chart - allow JWT auth instead of mTLS * update helm chart images * fix rbac. note that JWT verification is not cached and might not work on some k8s distributions that not expose the JWKS * fix: add chart boilerplate headers * fix: support jwt helm install on plain kind * feat: add substrate crds helm chart * feat: make jwt helm installs standalone * fix: make helm defaults cloud-neutral * fix: sync crd chart templates * fix: use agentgateway in helm chart * fix: update agentgateway install overlays * fix: project agentgateway tls key separately --------- Signed-off-by: Peter Jausovec Co-authored-by: Eitan Yarmush Co-authored-by: Peter Jausovec --- .github/workflows/release.yaml | 35 +- Makefile | 19 +- README.md | 21 +- charts/substrate-crds/Chart.yaml | 28 ++ charts/substrate-crds/README.md | 13 + .../templates/ate.dev_actortemplates.yaml | 448 +++++++++++++++++ .../templates/ate.dev_sandboxconfigs.yaml | 146 ++++++ .../templates/ate.dev_workerpools.yaml | 459 ++++++++++++++++++ charts/substrate/Chart.yaml | 27 ++ charts/substrate/README.md | 71 +++ charts/substrate/templates/NOTES.txt | 21 + charts/substrate/templates/_helpers.tpl | 119 +++++ .../templates/ate-api-server-envvars.yaml | 27 ++ .../substrate/templates/ate-api-server.yaml | 262 ++++++++++ charts/substrate/templates/ate-client.yaml | 25 + .../substrate/templates/ate-controller.yaml | 126 +++++ charts/substrate/templates/atelet.yaml | 143 ++++++ charts/substrate/templates/atenet-dns.yaml | 177 +++++++ charts/substrate/templates/atenet-router.yaml | 293 +++++++++++ charts/substrate/templates/jwt-bootstrap.yaml | 73 +++ charts/substrate/templates/jwt-oidc-rbac.yaml | 42 ++ charts/substrate/templates/namespace.yaml | 23 + .../templates/pod-certificate-controller.yaml | 200 ++++++++ charts/substrate/templates/role.yaml | 109 +++++ charts/substrate/templates/rustfs.yaml | 137 ++++++ .../templates/sandboxconfig-gvisor.yaml | 37 ++ .../templates/sandboxconfig-validation.yaml | 56 +++ charts/substrate/templates/valkey.yaml | 269 ++++++++++ charts/substrate/values.yaml | 126 +++++ cmd/ateapi/internal/controlapi/dialer.go | 9 +- cmd/ateapi/internal/controlapi/dialer_test.go | 16 + cmd/ateapi/main.go | 27 +- cmd/atecontroller/internal/controllers/gen.go | 2 +- cmd/atelet/main.go | 117 ++--- hack/create-kind-cluster.sh | 15 +- hack/gen-rbac.sh | 37 ++ hack/install-ate-kind-jwt.sh | 151 ++++++ hack/render-manifests.sh | 120 +++++ hack/values-kind-jwt.yaml | 41 ++ hack/verify/crd-chart.sh | 47 ++ internal/credbundle/credbundle.go | 20 +- manifests/ate-install/ate-api-server.yaml | 82 ++-- manifests/ate-install/atenet-dns.yaml | 88 ++-- manifests/ate-install/kind/kustomization.yaml | 3 +- 44 files changed, 4157 insertions(+), 150 deletions(-) create mode 100644 charts/substrate-crds/Chart.yaml create mode 100644 charts/substrate-crds/README.md create mode 100644 charts/substrate-crds/templates/ate.dev_actortemplates.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_workerpools.yaml create mode 100644 charts/substrate/Chart.yaml create mode 100644 charts/substrate/README.md create mode 100644 charts/substrate/templates/NOTES.txt create mode 100644 charts/substrate/templates/_helpers.tpl create mode 100644 charts/substrate/templates/ate-api-server-envvars.yaml create mode 100644 charts/substrate/templates/ate-api-server.yaml create mode 100644 charts/substrate/templates/ate-client.yaml create mode 100644 charts/substrate/templates/ate-controller.yaml create mode 100644 charts/substrate/templates/atelet.yaml create mode 100644 charts/substrate/templates/atenet-dns.yaml create mode 100644 charts/substrate/templates/atenet-router.yaml create mode 100644 charts/substrate/templates/jwt-bootstrap.yaml create mode 100644 charts/substrate/templates/jwt-oidc-rbac.yaml create mode 100644 charts/substrate/templates/namespace.yaml create mode 100644 charts/substrate/templates/pod-certificate-controller.yaml create mode 100644 charts/substrate/templates/role.yaml create mode 100644 charts/substrate/templates/rustfs.yaml create mode 100644 charts/substrate/templates/sandboxconfig-gvisor.yaml create mode 100644 charts/substrate/templates/sandboxconfig-validation.yaml create mode 100644 charts/substrate/templates/valkey.yaml create mode 100644 charts/substrate/values.yaml create mode 100755 hack/gen-rbac.sh create mode 100755 hack/install-ate-kind-jwt.sh create mode 100755 hack/render-manifests.sh create mode 100644 hack/values-kind-jwt.yaml create mode 100755 hack/verify/crd-chart.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e87842781f..2688d5e375 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,6 +67,9 @@ jobs: - name: Install ko uses: ko-build/setup-ko@v0.7 + - name: Install Helm + uses: azure/setup-helm@v4 + - name: Log in to GHCR uses: docker/login-action@v3 with: @@ -85,7 +88,7 @@ jobs: run: | set -o errexit -o nounset -o pipefail - for component in ateapi atelet ateom-gvisor podcertcontroller atenet; do + for component in ateapi atecontroller atelet ateom-gvisor podcertcontroller atenet; do KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ ./hack/run-tool.sh ko build \ --tags "${IMAGE_TAGS}" \ @@ -94,6 +97,36 @@ jobs: "./cmd/${component}" done + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + - name: Create GitHub Release if: inputs.create_release uses: softprops/action-gh-release@v2 diff --git a/Makefile b/Makefile index dc788d3ac3..8aa4f24f90 100644 --- a/Makefile +++ b/Makefile @@ -40,9 +40,10 @@ build: build-images build-atectl .PHONY: build-images build-images: - $(KO) build \ + $(KO) build --base-import-paths \ --ldflags="$(LDFLAGS)" \ ./cmd/ateapi \ + ./cmd/atecontroller \ ./cmd/atelet \ ./cmd/podcertcontroller \ ./cmd/atenet @@ -96,3 +97,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/README.md b/README.md index 4ab8ed56ee..18c8c60afb 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,10 @@ To quickly set up the complete environment: 2. Run the following steps: ```shell # create cluster and local registry (IPv4; IP_FAMILY=dual|ipv6 overrides) -hack/create-kind-cluster.sh +KIND_ENABLE_PODCERT=false hack/create-kind-cluster.sh -# install ate, valkey, rustfs -hack/install-ate-kind.sh --deploy-ate-system +# install ate, valkey, rustfs using Helm in JWT mode +hack/install-ate-kind-jwt.sh # install counter demo hack/install-ate-kind.sh --deploy-demo-counter @@ -113,6 +113,21 @@ kubectl port-forward -n ate-system svc/atenet-router 8000:80 curl -X POST -H "Host: my-counter-1.demo.actors.resources.substrate.ate.dev" -i http://localhost:8000/ ``` +#### mTLS mode + +JWT mode is the default install path and does not require pod certificate +feature gates. To test the older mTLS path, create kind with the +`ClusterTrustBundle` / `PodCertificateRequest` feature gates enabled and use the +mTLS install helper. + +```shell +# create cluster WITH podcert feature gates +hack/create-kind-cluster.sh + +# install ate using the mTLS manifests path +hack/install-ate-kind.sh --deploy-ate-system +``` + ### GKE Quickstart (Development) 1. Create and configure your environment file: diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml new file mode 100644 index 0000000000..2de7d898e7 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml @@ -0,0 +1,448 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: actortemplates.ate.dev +spec: + group: ate.dev + names: + kind: ActorTemplate + listKind: ActorTemplateList + plural: actortemplates + shortNames: + - actortemplate + singular: actortemplate + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of ActorTemplate. This field + is immutable. + properties: + containers: + description: Containers is the workload definition. + items: + description: A single application container that you want to run + within a WorkerPool. + properties: + command: + description: Entrypoint array. Not executed within a shell. + items: + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: atomic + env: + description: Environment variables to set in the worker replicas. + items: + description: |- + EnvVar represents an environment variable supplied to a container in an + ActorTemplate. It models only a subset of Kubernetes Pod env behavior: + literal values are not expanded with Kubernetes-style $(VAR) references, + and envFrom and valueFrom are not supported. + properties: + name: + description: |- + Name is the name of the environment variable. May be any printable ASCII + character except '='. + minLength: 1 + pattern: ^[ -<>-~]+$ + type: string + value: + description: |- + Value is the literal value of the environment variable. Unlike in + Kubernetes pods, this value is not interpolated, and $(VAR) + references are not expanded. + minLength: 0 + type: string + required: + - name + - value + type: object + maxItems: 32 + type: array + image: + description: Image to use for the worker replicas. + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + name: + description: Name of the container. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + readyz: + description: |- + Readyz is an optional HTTP readiness probe. When set, the actor is not + considered ready (and Run/Restore RPCs do not return success) until the + container's HTTP endpoint returns 200. + properties: + httpGet: + description: HTTPGet specifies the HTTP request to perform + against the container. + properties: + path: + default: /readyz + description: |- + Path to access on the HTTP server. Defaults to "/readyz". + Must be a valid URL path starting with "/". Only characters permitted + by RFC 3986 path segments are accepted; percent-escapes must be a + literal "%" followed by exactly two hex digits. Query strings ("?") + and fragments ("#") must be omitted. + maxLength: 1024 + pattern: ^/([A-Za-z0-9\-._~!$&'()*+,;=:@/]|%[0-9A-Fa-f]{2})*$ + type: string + port: + description: Port to access on the container. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - port + type: object + required: + - httpGet + type: object + volumeMounts: + description: volumeMounts define the volumes to mount into this + container. + items: + description: VolumeMount describes a mounting of a Volume + within a actor. + properties: + mountPath: + description: |- + Path within the actor at which the volume should be mounted. Must be a + clean absolute Unix path: must start with '/', not be '/', and contain + no ':', '..', '.', '//', trailing '/', or control characters. + maxLength: 4096 + type: string + x-kubernetes-validations: + - message: 'MountPath must be a clean absolute Unix path: + must start with ''/'', not be ''/'', and contain no + '':'', ''..'', ''.'', ''//'', trailing ''/'', or control + characters' + rule: self.startsWith('/') && size(self) > 1 && !self.endsWith('/') + && !self.contains('//') && !self.contains(':') && + !self.matches('[\x00-\x1f\x7f]') && !self.matches('(^|/)[.][.]?(/|$)') + name: + description: This must match the Name of a Volume. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + required: + - mountPath + - name + type: object + maxItems: 32 + type: array + required: + - image + - name + type: object + maxItems: 10 + type: array + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family this template's actors run + on. Only worker pools whose SandboxClass matches are eligible. Snapshots are + not portable across classes, so this is a hard gate, AND'd with WorkerSelector + and the actor's worker_selector. Defaults to gvisor. + + + 1) How does someone discover what classes are available, or what they mean? + 2) How does someone define a new sandbox class? + 3) Does a class mean the specific type of sandbox tech or does it include some aspect of config (e.g. can we have 2 different classes which both use gVisor with different config, or 2 classes which use different microvms) + 4) How does the default get set and who sets it? + + See Also: WorkerPool SandboxClass + enum: + - gvisor + - microvm + type: string + snapshotsConfig: + description: Snapshots configuration for the actor. + properties: + location: + description: |- + Location is the base object-storage URI snapshots of this template's + actors are stored under. + minLength: 1 + type: string + onCommit: + default: Full + description: |- + OnCommit specifies what to include in the snapshot when a commit is requested. + If not provided, the "Full" behavior is used by default. + onCommit must be a subset of the onPause content. + + For example: + - if onPause is "Full", then onCommit can be "Full" or "Data". + - if onPause is "Data", then onCommit must be "Data". + enum: + - Full + - Data + type: string + onPause: + default: Full + description: |- + OnPause specifies what to include in the snapshot when the actor is paused. + If not provided, the "Full" behavior is used by default. + enum: + - Full + - Data + type: string + required: + - location + type: object + x-kubernetes-validations: + - message: onCommit must be a subset of onPause + rule: '(has(self.onPause) ? self.onPause : ''Full'') == ''Full'' + || (has(self.onCommit) ? self.onCommit : ''Full'') == (has(self.onPause) + ? self.onPause : ''Full'')' + volumes: + description: Volumes defines the volumes to mount into all containers + in the actor. + items: + properties: + durableDir: + description: |- + durableDir represents a durable directory on rootfs that persists across + resumes and participates in snapshots. + type: object + externalVolumeTemplate: + description: |- + externalVolumeTemplate represents an external volume dynamically provisioned + for each actor. The volume only lives as long as the actor and is deleted + when the actor is deleted. + properties: + capacity: + anyOf: + - type: integer + - type: string + description: capacity specifies the size of the volume to + create. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClassName: + description: storageClassName refers to the StorageClass + to create the volume from. + type: string + required: + - capacity + - storageClassName + type: object + name: + description: name of the volume. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + required: + - name + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [durableDir externalVolumeTemplate] + must be set + rule: '[has(self.durableDir),has(self.externalVolumeTemplate)].filter(x,x==true).size() + == 1' + maxItems: 32 + type: array + workerSelector: + description: |- + WorkerSelector restricts which worker pools actors from this template may + use. The scheduler only considers pools whose labels match this selector. + If nil, all pools are eligible (subject to the actor's own worker_selector). + Acts as a gate: the actor's worker_selector can only narrow this set further, + never expand it. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - snapshotsConfig + type: object + x-kubernetes-validations: + - message: Spec is immutable + rule: self == oldSelf + - message: At most one DurableDir-typed volume is supported per ActorTemplate + rule: '!has(self.volumes) || self.volumes.filter(v, has(v.durableDir)).size() + <= 1' + - message: A container may mount at most one DurableDir-typed volume + rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) + || c.volumeMounts.filter(vm, has(self.volumes) && self.volumes.exists(v, + v.name == vm.name && has(v.durableDir))).size() <= 1)' + - message: All volumes defined in spec.volumes must be mounted by at least + one container + rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) + && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, + vm.name == v.name)))' + - message: ExternalVolumes are not supported when sandboxClass is 'microvm' + rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || + !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' + status: + description: status is the observed state of ActorTemplate + properties: + conditions: + description: conditions defines the status conditions array + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + goldenActorID: + type: string + goldenSnapshot: + type: string + phase: + description: Phase of the actor template. + type: string + takeGoldenSnapshotAt: + format: date-time + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..119df9bdbf --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,146 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .spec.default + name: Default + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools + and decouples sandbox binary selection from ActorTemplate. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "runsc" asset; a micro-VM backend expects several (e.g. "cloud-hypervisor", + "kata-kernel", "kata-image"). The schema is intentionally generic; + per-class requirements are enforced by a ValidatingAdmissionPolicy. + type: object + default: + description: |- + Default marks this SandboxConfig as the cluster-wide default for its + SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to + the default config for its SandboxClass. At most one default is expected + per SandboxClass. + type: boolean + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. A + WorkerPool only uses SandboxConfigs whose SandboxClass matches its own. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..669705004a --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,459 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + ateomImage: + description: AteomImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement) and which + SandboxConfigs are eligible. The concrete binary is still selected by + AteomImage. Defaults to gvisor. + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + sandboxConfigName: + description: |- + SandboxConfigName names a cluster-scoped SandboxConfig to use for fetching + sandbox binaries. It overrides the cluster-wide default SandboxConfig for + this pool's SandboxClass. The referenced config's SandboxClass must match + this pool's SandboxClass. If empty, the default SandboxConfig for the + SandboxClass is used. + type: string + template: + description: Template holds optional pod scheduling and resource settings + for worker pods. + properties: + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + terminationGracePeriodSeconds: + default: 300 + description: |- + TerminationGracePeriodSeconds is the termination grace period applied to + this pool's worker pods. On eviction, ateom traps SIGTERM and forwards it + to the actor so it can save state and exit cleanly before the kubelet + sends SIGKILL. Tune this to the maximum time your actors need to shut + down gracefully. Defaults to 300 (5 minutes). + format: int32 + minimum: 1 + type: integer + required: + - ateomImage + - replicas + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..6cc98069b0 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,71 @@ +# substrate + +Helm chart for installing Agent Substrate. + +## Install modes + +| Mode | Default? | Cluster requirements | Trade-off | +|------|----------|----------------------|-----------| +| `jwt` | yes | none beyond stock K8s | Server certs and actor signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. | +| `mtls` | | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. | + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# JWT mode (default; no off-by-default feature gates) +helm upgrade --install substrate ./charts/substrate + +# mTLS mode (requires off-by-default feature gates) +helm upgrade --install substrate ./charts/substrate \ + --set auth.mode=mtls +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## JWT-mode bootstrap + +JWT mode is standalone by default. The chart generates: + +- `Secret/ateapi-tls` +- `ConfigMap/ateapi-ca` +- `Secret/actor-id-jwt-pool` +- `Secret/actor-id-ca-pool` + +Existing generated data is reused on upgrade so key material does not rotate +during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring +your own resources with those names. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate # jwt +helm template substrate ./charts/substrate --set auth.mode=mtls +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `auth.mode` | `jwt` | `jwt` or `mtls` | +| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers | +| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience | +| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and actor signing material | +| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name | +| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name | +| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey | +| `valkey.replicas` | `6` | StatefulSet size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis | +| `redis.useIAMAuth` | `false` | Google IAM auth | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..abad3a8d1e --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,21 @@ +substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }} + +{{ if eq .Values.auth.mode "mtls" -}} +NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you +must enable these explicitly. To install without them, pick auth.mode=jwt. +{{- else }} +JWT mode is active. + +{{- if .Values.auth.jwt.bootstrap.enabled }} +JWT bootstrap resources are managed by this chart. Existing key material is +reused on upgrade. +{{- else }} +JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }}, +{{ .Values.auth.jwt.caBundleConfigMap }}, actor-id-jwt-pool, and +actor-id-ca-pool before pods become healthy. +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..36b9d3ef9b --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,119 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} + +{{/* +Validate auth.mode at template time. +*/}} +{{- define "substrate.validateAuthMode" -}} +{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}} +{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}} +{{- end -}} +{{- if eq .Values.auth.mode "jwt" -}} +{{- if not .Values.auth.jwt.issuer -}} +{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..754ff847a7 --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }} + ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }} + ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }} + ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }} + ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..088cbd286a --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,262 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["actortemplates", "workerpools", "sandboxconfigs"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-deployment" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 +{{- if eq .Values.auth.mode "jwt" }} + initContainers: + - name: assemble-cred-bundle + image: {{ .Values.images.busybox }} + command: + - sh + - -c + - cat /run/ateapi-tls-src/tls.crt /run/ateapi-tls-src/tls.key > /run/ateapi-tls/credential-bundle.pem + volumeMounts: + - { name: ateapi-tls-src, mountPath: /run/ateapi-tls-src, readOnly: true } + - { name: ateapi-tls, mountPath: /run/ateapi-tls } +{{- end }} + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" +{{- if eq .Values.auth.mode "mtls" }} + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--redis-cluster-address=@env" + - "--redis-ca-certs=/etc/valkey-ca/ca.crt" + - "--redis-use-iam-auth=@env" + - "--redis-tls-server-name=@env" + - "--redis-client-cert=@env" + - "--client-jwt-issuer=@env" + - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" + - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" +{{- else }} + - "--grpc-server-cred-bundle=/run/ateapi-tls/credential-bundle.pem" + - "--atelet-insecure=true" + - "--redis-cluster-address=@env" + - "--redis-no-tls=true" + - "--redis-use-iam-auth=@env" + - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}" + - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" + - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +{{- end }} + - "--drain-delay=13s" + - "--drain-timeout=15s" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: +{{- if eq .Values.auth.mode "mtls" }} + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } + - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } +{{- else }} + - { name: ateapi-tls, mountPath: /run/ateapi-tls, readOnly: true } + - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } +{{- end }} + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: session-id-jwt-pool + projected: + sources: + - secret: + name: session-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - { key: ca.crt, path: ca.crt } + - name: session-id-ca-pool + projected: + sources: + - secret: + name: session-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +{{- else }} + - name: ateapi-tls-src + secret: + secretName: {{ .Values.auth.jwt.serverCertSecret }} + - name: ateapi-tls + emptyDir: {} + - name: session-id-jwt-pool + projected: + sources: + - secret: + name: session-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: session-id-ca-pool + projected: + sources: + - secret: + name: session-id-ca-pool + items: + - { key: pool, path: pool.json } +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..3de466a04f --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.auth.mode "jwt" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client +{{- end }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..9cd93d4a0f --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,126 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: +{{- if eq .Values.auth.mode "mtls" }} + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- else }} + - "--ateapi-use-token-auth=true" + - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" + - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" + - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP +{{- if eq .Values.auth.mode "mtls" }} + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem +{{- else }} + volumeMounts: + - { name: ateapi-ca, mountPath: /run/ateapi-ca, readOnly: true } + - { name: ateapi-token, mountPath: /var/run/secrets/tokens/ateapi, readOnly: true } + volumes: + - name: ateapi-ca + configMap: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + - name: ateapi-token + projected: + sources: + - serviceAccountToken: + audience: {{ .Values.auth.jwt.audience }} + expirationSeconds: 3600 + path: token +{{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..6b2eebba47 --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,143 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet — identical across auth modes (does not dial ateapi). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} +{{- if eq .Values.auth.mode "mtls" }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem +{{- else }} + - --grpc-insecure=true +{{- end }} +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor +{{- if eq .Values.auth.mode "mtls" }} + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true +{{- end }} + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate +{{- if eq .Values.auth.mode "mtls" }} + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +{{- end }} diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml new file mode 100644 index 0000000000..0838d2c0b8 --- /dev/null +++ b/charts/substrate/templates/atenet-dns.yaml @@ -0,0 +1,177 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atenet-dns — identical across auth modes (does not dial ateapi). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + replicas: 1 + selector: + matchLabels: + app: dns + template: + metadata: + labels: + app: dns + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-dns" .) }} + shareProcessNamespace: true + initContainers: + - name: init-dns + image: {{ .Values.images.busybox }} + command: ["sh", "-c"] + args: + - | + cat <<'EOF' > /etc/coredns/Corefile + .:53 { + errors + health :8080 + ready :8181 + reload + } + EOF + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + containers: + - name: coredns + image: {{ .Values.images.coredns }} + imagePullPolicy: IfNotPresent + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + ports: + - name: dns + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + - name: dns-controller + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "dns" + - "--log-level=debug" + - "--interval=10s" + - "--corefile-path=/etc/coredns/Corefile" + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + volumes: + - name: dns-config-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..e95e5bd49f --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,293 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} +rules: +- apiGroups: + - "ate.dev" + resources: + - actortemplates + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + adminAddr: "127.0.0.1:15000" + readinessAddr: "0.0.0.0:15021" + statsAddr: "0.0.0.0:15020" + binds: + - port: 8080 + listeners: + - name: http + protocol: HTTP + routes: + - name: substrate-http + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} + - port: 8443 + listeners: + - name: https + protocol: HTTPS + tls: +{{ if eq .Values.auth.mode "mtls" }} + cert: "/run/servicedns.podcert.ate.dev/cert.pem" + key: "/run/servicedns.podcert.ate.dev/key.pem" +{{ else }} + cert: "/run/agentgateway-tls/tls.crt" + key: "/run/agentgateway-tls/tls.key" +{{ end }} + routes: + - name: substrate-https + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--standalone" + - "--networking-mode=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" +{{- if eq .Values.auth.mode "mtls" }} + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- else }} + - "--ateapi-use-token-auth=true" + - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" + - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" + - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" +{{- end }} + - "--status-port=4040" + - "--port-https=8443" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 +{{- if eq .Values.auth.mode "mtls" }} + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } +{{- else }} + volumeMounts: + - name: ateapi-ca + mountPath: /run/ateapi-ca + readOnly: true + - name: ateapi-token + mountPath: /var/run/secrets/tokens/ateapi + readOnly: true +{{- end }} + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway +{{- if eq .Values.auth.mode "mtls" }} + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" +{{- else }} + - name: agentgateway-tls + mountPath: /run/agentgateway-tls + readOnly: true +{{- end }} + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} +{{- if eq .Values.auth.mode "mtls" }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem +{{- else }} + - name: agentgateway-tls + secret: + secretName: {{ .Values.auth.jwt.serverCertSecret }} + - name: ateapi-ca + configMap: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + - name: ateapi-token + projected: + sources: + - serviceAccountToken: + audience: {{ .Values.auth.jwt.audience }} + expirationSeconds: 3600 + path: token +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP diff --git a/charts/substrate/templates/jwt-bootstrap.yaml b/charts/substrate/templates/jwt-bootstrap.yaml new file mode 100644 index 0000000000..cb31763fb3 --- /dev/null +++ b/charts/substrate/templates/jwt-bootstrap.yaml @@ -0,0 +1,73 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (eq .Values.auth.mode "jwt") .Values.auth.jwt.bootstrap.enabled }} +{{- $apiName := include "substrate.fullname" (list "api" .) }} +{{- $routerName := include "substrate.fullname" (list "atenet-router" .) }} +{{- $apiHost := printf "%s.%s.svc" $apiName .Release.Namespace }} +{{- $ca := genCA (printf "%s-ca" $apiName) 3650 }} +{{- $serverCert := genSignedCert $apiHost nil (list $apiHost (printf "%s.%s.svc.cluster.local" $apiName .Release.Namespace) (printf "%s.%s.svc" $routerName .Release.Namespace)) 365 $ca }} +{{- $actorJWTKey := genPrivateKey "ecdsa" }} +{{- $actorCA := genCA "actor-id-ca" 3650 }} +{{- if .Values.auth.jwt.bootstrap.serverCert.enabled }} +{{- $existingTLS := lookup "v1" "Secret" .Release.Namespace .Values.auth.jwt.serverCertSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.auth.jwt.serverCertSecret }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: {{ if $existingTLS }}{{ index $existingTLS.data "tls.crt" }}{{ else }}{{ $serverCert.Cert | b64enc }}{{ end }} + tls.key: {{ if $existingTLS }}{{ index $existingTLS.data "tls.key" }}{{ else }}{{ $serverCert.Key | b64enc }}{{ end }} +--- +{{- $existingCA := lookup "v1" "ConfigMap" .Release.Namespace .Values.auth.jwt.caBundleConfigMap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ca.crt: | +{{- if $existingCA }} +{{ index $existingCA.data "ca.crt" | nindent 4 }} +{{- else }} +{{ $ca.Cert | nindent 4 }} +{{- end }} +{{- end }} +{{- if .Values.auth.jwt.bootstrap.sessionPools.enabled }} +--- +{{- $existingJWTSecret := lookup "v1" "Secret" .Release.Namespace "actor-id-jwt-pool" }} +apiVersion: v1 +kind: Secret +metadata: + name: actor-id-jwt-pool + namespace: {{ .Release.Namespace }} +type: Opaque +data: + pool: {{ if $existingJWTSecret }}{{ index $existingJWTSecret.data "pool" }}{{ else }}{{ dict "Authorities" (list (dict "ID" "1" "Algorithm" "ES256" "SigningKeyPEM" $actorJWTKey)) | toJson | b64enc }}{{ end }} +--- +{{- $existingCASecret := lookup "v1" "Secret" .Release.Namespace "actor-id-ca-pool" }} +apiVersion: v1 +kind: Secret +metadata: + name: actor-id-ca-pool + namespace: {{ .Release.Namespace }} +type: Opaque +data: + pool: {{ if $existingCASecret }}{{ index $existingCASecret.data "pool" }}{{ else }}{{ dict "CAs" (list (dict "ID" "1" "SigningKeyPEM" $actorCA.Key "RootCertificatePEM" $actorCA.Cert)) | toJson | b64enc }}{{ end }} +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/jwt-oidc-rbac.yaml b/charts/substrate/templates/jwt-oidc-rbac.yaml new file mode 100644 index 0000000000..a9fd499e96 --- /dev/null +++ b/charts/substrate/templates/jwt-oidc-rbac.yaml @@ -0,0 +1,42 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.auth.mode "jwt" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} +rules: +- nonResourceURLs: + - /.well-known/openid-configuration + - /openid/v1/jwks + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..63401c00d0 --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- include "substrate.validateAuthMode" . -}} +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..3aaaa9df99 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,200 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.auth.mode "mtls" -}} +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 +{{- end }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..580f225e47 --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,109 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates/finalizers + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - actortemplates/status + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..edaad3cfa8 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,137 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..3dc4e9d168 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A +# WorkerPool with sandboxClass gvisor (the default) and no explicit +# sandboxConfigName resolves to this. atelet fetches the runsc binary matching +# the worker node's architecture. To pin a different runsc, edit the assets +# below or create another SandboxConfig and name it from the WorkerPool. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + default: true + assets: + amd64: + runsc: + url: "gs://gvisor/releases/release/20260622/x86_64/runsc" + sha256: "f18a948bf9c8bbb54eb998549a3a8d719a1c7de2efbe8fdd2ff0ee5fecd06f19" + arm64: + runsc: + url: "gs://gvisor/releases/release/20260622/aarch64/runsc" + sha256: "62eee121f8c188e347c428acc96f111568ede3be37b906046b6f28bbe2cc40c0" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..48302d2fdd --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,56 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a "runsc" asset for every architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/templates/valkey.yaml b/charts/substrate/templates/valkey.yaml new file mode 100644 index 0000000000..ca164e7698 --- /dev/null +++ b/charts/substrate/templates/valkey.yaml @@ -0,0 +1,269 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.valkey.enabled -}} +{{- $sts := include "substrate.fullname" (list "valkey-cluster" .) -}} +{{- $headless := include "substrate.fullname" (list "valkey-cluster-service" .) -}} +{{- $ns := .Release.Namespace -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "valkey-config" .) }} + namespace: {{ .Release.Namespace }} +data: + valkey.conf: | +{{- if eq .Values.auth.mode "mtls" }} + # Enforce TLS and disable standard port + port 0 + tls-port 6379 + tls-cluster yes + tls-replication yes + + # Load certificates from projected volume + tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem + tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem + tls-ca-cert-file /etc/valkey-ca/ca.crt + tls-auth-clients yes + + # Reload every 10 minutes. + # The interval should be less than the 30m headroom (notAfter - beginRefreshAt) + # set by cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go. + tls-auto-reload-interval 600 + + # Enable cluster mode +{{- else }} + # Plaintext: serve on the standard port, no TLS. + port 6379 + +{{- end }} + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 5000 + appendonly yes + protected-mode no +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $headless }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: valkey-cluster + ports: + - name: valkey + port: 6379 + targetPort: 6379 + - name: bus + port: 16379 + targetPort: 16379 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $sts }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: valkey-cluster + ports: + - name: valkey + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $sts }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $headless }} + replicas: {{ .Values.valkey.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + app: valkey-cluster + template: + metadata: + labels: + app: valkey-cluster + spec: + containers: + - name: valkey + image: {{ .Values.images.valkey }} + command: ["valkey-server", "/etc/valkey/valkey.conf"] + ports: + - name: valkey + containerPort: 6379 + - name: bus + containerPort: 16379 + volumeMounts: + - name: config + mountPath: /etc/valkey +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + - name: valkey-ca-certs + mountPath: /etc/valkey-ca + readOnly: true +{{- end }} + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "valkey-config" .) }} +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - key: ca.crt + path: ca.crt +{{- end }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.valkey.storageSize }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "valkey-cluster-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + template: + metadata: + labels: + app: valkey-cluster-init + spec: + restartPolicy: OnFailure + containers: + - name: init + image: {{ .Values.images.valkey }} +{{- if eq .Values.auth.mode "mtls" }} + volumeMounts: + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + - name: valkey-ca-certs + mountPath: /etc/valkey-ca + readOnly: true +{{- end }} + command: + - /bin/sh + - -c + - | + set -e + echo "Waiting for all Valkey pods to resolve..." + for i in 0 1 2 3 4 5; do + until getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-${i} DNS..." + sleep 2 + done + done + + echo "All pods resolved. Getting IPs..." + POD_IPS="" + for i in 0 1 2 3 4 5; do + ip=$(getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc | awk '{print $1}') + POD_IPS="${POD_IPS} ${ip}:6379" + done + + echo "Checking if Valkey cluster is already initialized..." +{{- if eq .Values.auth.mode "mtls" }} + until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc ping >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-0 to respond to ping..." + sleep 2 + done + + INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc cluster info 2>/dev/null | grep cluster_state || true) + + if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then + echo "Initializing Valkey cluster..." + valkey-cli --tls \ + --cacert /etc/valkey-ca/ca.crt \ + --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --cluster create ${POD_IPS} \ + --cluster-replicas 1 \ + --cluster-yes + echo "Cluster initialization complete!" + else + echo "Cluster already initialized." + fi +{{- else }} + until valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 ping >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-0 to respond to ping..." + sleep 2 + done + + INIT_STATUS=$(valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 cluster info 2>/dev/null | grep cluster_state || true) + + if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then + echo "Initializing Valkey cluster..." + valkey-cli \ + --cluster create ${POD_IPS} \ + --cluster-replicas 1 \ + --cluster-yes + echo "Cluster initialization complete!" + else + echo "Cluster already initialized." + fi +{{- end }} +{{- if eq .Values.auth.mode "mtls" }} + volumes: + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - key: ca.crt + path: ca.crt +{{- end }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..e27f775272 --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,126 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart supports two installation modes via `auth.mode`: +# +# - "jwt" (default): No PodCertificateRequest / ClusterTrustBundle usage. Server +# certs and actor signing pools are generated by the chart by default, +# and can be disabled when you want to provide your own key material. +# Clients authenticate to ateapi with a projected Kubernetes ServiceAccount +# token. Valkey runs plaintext. +# +# - "mtls": Server certs are issued by the in-cluster podcertcontroller via +# PodCertificateRequest + projected into pods via the ClusterTrustBundle / +# podCertificate projection sources. Valkey runs with full TLS + +# client-cert verification. REQUIRES the off-by-default Kubernetes feature +# gates: +# ClusterTrustBundle, ClusterTrustBundleProjection, PodCertificateRequest +# and the v1beta1 certificates API. + +auth: + mode: jwt # jwt | mtls + + jwt: + # OIDC issuer URL the cluster uses to mint SA tokens. The default matches + # stock kind/kubeadm-style clusters. Override this for managed clusters + # whose service account issuer is provider-specific. Examples: + # GKE: https://container.googleapis.com/v1/projects//locations//clusters/ + # kind: https://kubernetes.default.svc.cluster.local + # EKS: https://oidc.eks..amazonaws.com/id/ + issuer: https://kubernetes.default.svc.cluster.local + + # Audience SA tokens are minted for, and that ateapi expects. + audience: api.ate-system.svc + + bootstrap: + # Generate JWT-mode TLS and actor-signing key material with Helm. + # Existing generated resources are reused on upgrade via lookup. + enabled: true + serverCert: + enabled: true + sessionPools: + enabled: true + + # Name of a kubernetes.io/tls Secret in the release namespace, with keys + # tls.crt and tls.key. Created by the chart when + # auth.jwt.bootstrap.serverCert.enabled=true. + serverCertSecret: ateapi-tls + + # Name of a ConfigMap in the release namespace with key "ca.crt" holding + # the CA(s) that signed serverCertSecret. Clients mount it to verify the + # ateapi server certificate. Created by the chart when + # auth.jwt.bootstrap.serverCert.enabled=true. + caBundleConfigMap: ateapi-ca + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +valkey: + enabled: true + replicas: 6 + storageSize: 1Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +redis: + # Override the cluster address. Empty -> derived from valkey.enabled + # (defaults to "valkey-cluster.ate-system.svc:6379"). + clusterAddress: "" + # Google IAM auth (for managed Memorystore / cloud Valkey). + useIAMAuth: false + # Override TLS server name for Redis hostname verification (mtls mode). + tlsServerName: "" + # File path for Redis client TLS credential bundle (mtls mode). + clientCert: "" + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_REDIS_*, ATE_API_K8SJWT_ISSUER, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + valkey: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: cr.agentgateway.dev/agentgateway:v1.4.1 + coredns: coredns/coredns:1.11.1 + busybox: busybox:1.36 diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 43d6c4b865..66127777ba 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -32,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -74,6 +76,11 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { @@ -180,7 +187,7 @@ func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) if err != nil { return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 73b9879f98..321bdee116 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -41,6 +41,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, nil, "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, nil, "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 5d1d3e0d5a..99ce86cc36 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -70,6 +70,7 @@ var ( redisUseIAMAuth = pflag.String("redis-use-iam-auth", "true", "Whether to use Google IAM authentication for Redis/Valkey.") redisTLSServerName = pflag.String("redis-tls-server-name", "", "The ServerName to use for Redis TLS hostname verification.") redisClientCert = pflag.String("redis-client-cert", "", "The file containing client TLS certificate/key credential bundle for Redis/Valkey.") + redisNoTLS = pflag.Bool("redis-no-tls", false, "If true, connect to Redis/Valkey in plaintext.") authenticationConfigFile = pflag.String("authentication-config", "", "YAML file configuring trusted JWT providers.") storeBackend = pflag.String("store-backend", "redis", "The persistence backend to use: redis|postgres.") @@ -81,6 +82,7 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") @@ -192,7 +194,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) sm := controlapi.NewService(persistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins) actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, *actorIDCAPoolFile, persistence, workerCache) @@ -307,10 +313,12 @@ func logFlagValues(ctx context.Context) { slog.String("redis-client-cert", *redisClientCert), slog.String("authentication-config", *authenticationConfigFile), slog.String("store-backend", *storeBackend), + slog.Bool("redis-no-tls", *redisNoTLS), slog.String("actor-id-jwt-pool", *actorIDJWTPoolFile), slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), ) @@ -344,14 +352,17 @@ func connectStore(ctx context.Context) (store.Interface, error) { // connectRedis builds the Redis/Valkey TLS config, plumbs IAM auth if // requested, opens the cluster client, and pings with retries. func connectRedis(ctx context.Context) (*redis.ClusterClient, error) { - tlsConfig, err := buildRedisTLSConfig(ctx) - if err != nil { - return nil, err - } - clusterOpts := &redis.ClusterOptions{ - Addrs: []string{*redisClusterAddress}, - TLSConfig: tlsConfig, + Addrs: []string{*redisClusterAddress}, + } + if *redisNoTLS { + slog.InfoContext(ctx, "Connecting to Redis/Valkey without TLS") + } else { + tlsConfig, err := buildRedisTLSConfig(ctx) + if err != nil { + return nil, err + } + clusterOpts.TLSConfig = tlsConfig } if *redisUseIAMAuth != "false" { diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index e8b03d30f6..d4a372df9f 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -14,4 +14,4 @@ package controllers -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index bd67077808..4231d2d40d 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -91,6 +91,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -274,69 +275,75 @@ func main() { volPlugins, csiDriverConfigLister, ) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - brokerTLS := tlsCfg.Clone() - brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer brokerLis.Close() - if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } - brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) - ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ - actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn), - }) - go func() { - if err := brokerServer.Serve(brokerLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) } - }() + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() + + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + brokerTLS := tlsCfg.Clone() + brokerTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.CredentialBrokerSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + brokerLis, err := net.Listen("unix", ateompath.CredentialBrokerSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer brokerLis.Close() + if err := os.Chmod(ateompath.CredentialBrokerSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + brokerServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(brokerTLS))) + ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ + actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn), + }) + go func() { + if err := brokerServer.Serve(brokerLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } + + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c953..c2ef74673e 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -19,6 +19,12 @@ set -o errexit -o nounset -o pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" +# Enable the off-by-default certificate feature gates required by the mTLS +# install path (cmd/podcertcontroller). On by default — the Quickstart's +# `hack/install-ate-kind.sh --deploy-ate-system` uses mTLS. Opt out +# (KIND_ENABLE_PODCERT=false) only when installing JWT-mode manifests, which +# do not require these gates. +KIND_ENABLE_PODCERT="${KIND_ENABLE_PODCERT:-true}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" @@ -30,6 +36,7 @@ if [[ $# -gt 0 ]]; then echo echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." + echo " KIND_ENABLE_PODCERT Enable Pod Certificate feature gates (default: true)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." exit 0 ;; @@ -84,7 +91,7 @@ else echo "/dev/kvm not available: micro-VM support disabled (gVisor still works)." fi -echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}' (ipFamily=${IP_FAMILY})..." +echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}' (ipFamily=${IP_FAMILY}, KIND_ENABLE_PODCERT=${KIND_ENABLE_PODCERT})..." cat < "${ROOT}/bin/kind-config.yaml" kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -100,6 +107,8 @@ if [ "${HAS_KVM}" = "1" ]; then containerPath: /dev/kvm EOF fi + +if [ "${KIND_ENABLE_PODCERT}" = "true" ]; then cat <> "${ROOT}/bin/kind-config.yaml" # cmd/podcertcontroller depends on ClusterTrustBundle & PodCertificateRequest. # They are not enabled by default as of Kubernetes v1.36 @@ -110,6 +119,10 @@ featureGates: PodCertificateRequest: true runtimeConfig: "certificates.k8s.io/v1beta1": "true" +EOF +fi + +cat <> "${ROOT}/bin/kind-config.yaml" networking: ipFamily: ${IP_FAMILY} EOF diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..baa22fa517 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/install-ate-kind-jwt.sh b/hack/install-ate-kind-jwt.sh new file mode 100755 index 0000000000..66e36fb257 --- /dev/null +++ b/hack/install-ate-kind-jwt.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Agent Substrate on a kind cluster in JWT auth mode. +# +# Unlike the mTLS install path (hack/install-ate-kind.sh), this works on a +# stock Kubernetes cluster — no ClusterTrustBundle / PodCertificateRequest +# feature gates required. Suitable for a kind cluster created with +# KIND_ENABLE_PODCERT=false hack/create-kind-cluster.sh. +# +# Steps: +# 1. Render the chart with auth.mode=jwt + kind-specific values, resolve +# ko:// image refs against a local registry, and apply. +# 2. Apply the kind-only OTel collector from manifests/ate-install/kind/. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NS="${NS:-ate-system}" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" +KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" +KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" +KO_DEFAULTPLATFORMS="${KO_DEFAULTPLATFORMS:-linux/$(go env GOARCH)}" +reg_name="kind-registry" +reg_port="5001" + +export KO_DOCKER_REPO KO_DEFAULTPLATFORMS + +run_kubectl() { + kubectl ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} "$@" +} + +run_helm() { + helm ${KUBECTL_CONTEXT:+--kube-context=${KUBECTL_CONTEXT}} "$@" +} + +log_step() { + echo -e "\033[1;36m[step]:\033[0m $1" +} + +ensure_namespace() { + log_step "ensure_namespace ${NS}" + run_kubectl create namespace "${NS}" --dry-run=client -o yaml | run_kubectl apply -f - +} + +ensure_kind_local_registry() { + log_step "ensure_kind_local_registry" + + if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" == "true" ]; then + if ! docker port "${reg_name}" | grep -q "${reg_port}"; then + echo "Registry exists but is not mapped to port ${reg_port}. Recreating..." + docker rm -f "${reg_name}" + fi + fi + + if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != "true" ]; then + docker run \ + -d --restart=always \ + --label created-by=agent-substrate \ + -p "127.0.0.1:${reg_port}:5000" \ + -p "[::1]:${reg_port}:5000" \ + --network bridge --name "${reg_name}" \ + registry:3 + fi + + if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = "null" ]; then + docker network connect "kind" "${reg_name}" + fi + + local registry_dir="/etc/containerd/certs.d/localhost:${reg_port}" + local node + for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + docker exec "${node}" mkdir -p "${registry_dir}" + cat <') + + # ko resolve replaces ko:// refs with built+pushed image refs. + echo "${rendered}" | bash "${ROOT}/hack/run-tool.sh" ko resolve -f - \ + | run_kubectl apply -f - +} + +apply_crds() { + log_step "apply_crds" + run_helm upgrade --install substrate-crds "${ROOT}/charts/substrate-crds" +} + +apply_sandbox_configs() { + log_step "apply_sandbox_configs" + run_kubectl apply -f "${ROOT}/manifests/ate-install/sandboxconfig-validation.yaml" + run_kubectl apply -f "${ROOT}/manifests/ate-install/sandboxconfig-gvisor.yaml" +} + +apply_kind_extras() { + log_step "apply_kind_extras (otel-collector)" + run_kubectl apply -f "${ROOT}/manifests/ate-install/kind/otel-collector.yaml" +} + +wait_rollouts() { + log_step "wait_rollouts" + run_kubectl -n "${NS}" rollout status deployment/ate-api-server-deployment --timeout=180s + run_kubectl -n "${NS}" rollout status deployment/ate-controller --timeout=180s + run_kubectl -n "${NS}" rollout status deployment/atenet-router --timeout=180s + run_kubectl -n "${NS}" rollout status daemonset/atelet --timeout=180s + run_kubectl -n "${NS}" rollout status statefulset/valkey-cluster --timeout=180s +} + +ensure_namespace +ensure_kind_local_registry +apply_crds +apply_sandbox_configs +apply_chart +apply_kind_extras +wait_rollouts + +echo "Substrate (JWT mode) installed in namespace ${NS}." diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..bbe50befb0 --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -delete +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/values-kind-jwt.yaml b/hack/values-kind-jwt.yaml new file mode 100644 index 0000000000..bbe2e387ae --- /dev/null +++ b/hack/values-kind-jwt.yaml @@ -0,0 +1,41 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helm values for installing substrate on a kind cluster in JWT mode. +# Used by hack/install-ate-kind-jwt.sh — does NOT require the off-by-default +# certificate feature gates that the mTLS install path needs. + +auth: + mode: jwt + jwt: + # Kind's default API server issuer. + issuer: https://kubernetes.default.svc.cluster.local + audience: api.ate-system.svc + serverCertSecret: ateapi-tls + caBundleConfigMap: ateapi-ca + +createNamespace: false + +# In-cluster OTel collector deployed alongside via manifests/ate-install/kind/otel-collector.yaml +otel: + endpoint: http://opentelemetry-collector.otel-system.svc:4317 + +monitoring: + gkePodMonitoring: + enabled: false + +atelet: + gcpAuthForImagePulls: false + extraArgs: + - --localhost-registry-replacement=kind-registry:5000 diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 3d0db9f047..d4b842227c 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -112,6 +113,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -124,8 +126,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -139,7 +142,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -155,3 +158,16 @@ func Parse(bundlePath string) (*tls.Certificate, error) { PrivateKey: leafKey, }, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 5cd6ed6205..8d7f8bd121 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -1,23 +1,41 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. -# Define Permissions (Read-Only for Pods) +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ate-api-server + namespace: ate-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-api-server + namespace: ate-system +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: ate-api-server + name: ate-api-server-role rules: - apiGroups: [""] resources: ["pods"] @@ -32,37 +50,42 @@ rules: resources: ["storageclasses"] verbs: ["get", "watch", "list"] --- -# Create Service Account for Workload Identity -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ate-api-server - namespace: ate-system ---- -# 4. Bind Identity to Permissions apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: ate-api-server + name: ate-api-server-binding subjects: - kind: ServiceAccount name: ate-api-server namespace: ate-system roleRef: kind: ClusterRole - name: ate-api-server + name: ate-api-server-role apiGroup: rbac.authorization.k8s.io --- -# 5. Deploy the API Server +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: ate-system +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 +--- apiVersion: apps/v1 kind: Deployment metadata: - name: ate-api-server + name: ate-api-server-deployment namespace: ate-system spec: replicas: 2 strategy: - # Update replicas one at a time, create a new one first, then delete the old one. rollingUpdate: maxUnavailable: 0 maxSurge: 1 @@ -77,11 +100,7 @@ spec: prometheus.io/scrape: "true" prometheus.io/port: "9090" spec: - # TODO: Add topologySpreadConstraints to spread replicas across nodes and zones. serviceAccountName: ate-api-server - # Budget for the full shutdown sequence: --drain-delay (sized to cover - # the readinessProbe below reaching NotReady, plus propagation) + - # --drain-timeout. terminationGracePeriodSeconds: 40 containers: - name: ate-api-server @@ -165,9 +184,6 @@ spec: initialDelaySeconds: 5 periodSeconds: 2 failureThreshold: 3 - # /healthz stays 200 while a terminating pod drains; /readyz - # turns 503, so liveness and readiness diverge correctly during - # shutdown. livenessProbe: httpGet: path: /healthz @@ -175,7 +191,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: - - name: "servicedns" + - name: servicedns projected: sources: - podCertificate: diff --git a/manifests/ate-install/atenet-dns.yaml b/manifests/ate-install/atenet-dns.yaml index 77d71bf594..7ae43b0b10 100644 --- a/manifests/ate-install/atenet-dns.yaml +++ b/manifests/ate-install/atenet-dns.yaml @@ -1,17 +1,21 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +# atenet-dns — identical across auth modes (does not dial ateapi). apiVersion: v1 kind: ServiceAccount metadata: @@ -34,6 +38,16 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: atenet-dns + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -48,16 +62,6 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -71,6 +75,27 @@ roleRef: name: atenet-dns apiGroup: rbac.authorization.k8s.io --- +apiVersion: v1 +kind: Service +metadata: + name: dns + namespace: ate-system + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -94,8 +119,6 @@ spec: - name: init-dns image: busybox:1.36 command: ["sh", "-c"] - # Initial core file is sufficient to start CoreDNS but does not contain - # any additional configuration. The controller will update the Corefile. args: - | cat <<'EOF' > /etc/coredns/Corefile @@ -155,24 +178,3 @@ spec: volumes: - name: dns-config-volume emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - selector: - app: dns - type: ClusterIP - # Prefer, not Require: Require fails Service creation on a single-stack cluster. - ipFamilyPolicy: PreferDualStack - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP \ No newline at end of file diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 54a6b0b33d..ccb2f00f32 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -30,6 +30,7 @@ resources: - ../valkey.yaml - ../pod-certificate-controller.yaml - ate-otel-config.yaml + - ../role.yaml - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml @@ -45,7 +46,7 @@ patches: apiVersion: apps/v1 kind: Deployment metadata: - name: ate-api-server + name: ate-api-server-deployment namespace: ate-system spec: template: From 6b2156601460d1215079adf7d7a7f176cf889f7c Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 17 Jun 2026 21:17:22 +0000 Subject: [PATCH 3/9] fix: repair CI and JWT e2e setup Disable VCS stamping for license verification, tolerate transient actor resume routing, fix JWT client trust configuration, and wait on the worker cache in control API functional tests. --- .../internal/actoridentity/actoridentity.go | 1 + .../controlapi/functionaltest/common_test.go | 16 +-- hack/install-microvm-deps.sh | 1 + hack/update/licenses.sh | 8 ++ internal/credbundle/credbundle_test.go | 6 +- internal/e2e/suites/demo/demo_test.go | 114 ++++++++++++------ manifests/ate-install/ate-client.yaml | 21 ++++ manifests/ate-install/kind/kustomization.yaml | 1 + 8 files changed, 122 insertions(+), 46 deletions(-) create mode 100644 manifests/ate-install/ate-client.yaml diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index b12883112b..86d9ed7374 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/actoridjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/k8sjwt" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/principal" diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index e7c1440005..8c36af580b 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -551,13 +551,13 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node t.Fatalf("failed to update worker pod status: %v", err) } - // Wait for worker to be registered via API + // Wait for worker to be visible to the scheduler. err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + workers, err := tc.workerCache.Workers() if err != nil { - return false, nil // Retry on API error + return false, nil } - for _, w := range resp.GetWorkers() { + for _, w := range workers { if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { return true, nil } @@ -660,13 +660,13 @@ func deleteWorkerPod(t *testing.T, tc *testContext, ns string, name string) { t.Fatalf("failed to delete worker pod %s: %v", name, err) } - // Wait for worker to be removed from API + // Wait for worker to be removed from the scheduler. err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + workers, err := tc.workerCache.Workers() if err != nil { - return false, nil // Retry on API error + return false, nil } - for _, w := range resp.GetWorkers() { + for _, w := range workers { if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { return false, nil // Still there } diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index a059a41364..9045750675 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -160,6 +160,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 7861e6beb2..b7e3b97656 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 579a12bbc0..171bcb5b64 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 4f482bbf87..304bce5d48 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -648,7 +651,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -673,24 +676,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplateNamespace() == nsObj.Name && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s in namespace %s, got %d actors: %v", actorName, nsObj.Name, len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -701,8 +694,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d in our namespace %s.", - len(listResp.GetActors()), len(myActors), nsObj.Name) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -730,13 +722,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -755,11 +747,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -810,11 +803,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -834,11 +828,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1098,6 +1093,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..cc6ef76c0d --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index ccb2f00f32..b1f91e3b67 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -22,6 +22,7 @@ kind: Kustomization # resource. hack/install-ate.sh applies the ConfigMap directly for the targeted # single-component redeploys. resources: + - ../ate-client.yaml - ../ate-api-server.yaml - ../ate-controller.yaml - ./atelet From e2d9cc4373ea2a9d4c48c988ccdcd4a85b1575aa Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Wed, 1 Jul 2026 13:56:06 -0400 Subject: [PATCH 4/9] Make resource names (like service + namespace) configurable (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * atenet: make system namespace and component Service names configurable The dns-controller (`atenet dns`) and router (`atenet router`) hardcoded the substrate namespace ("ate-system") and the component Service names ("atenet-router", "dns") from the canonical install manifests under `manifests/ate-install/`. Deployments that deviate from that layout — running in a different namespace, renaming the Services, or composing substrate into a larger install that rewrites resource names — silently break: the dns-controller can't find atenet-router, the router can't find itself for /statusz, and the cluster's actor DNS never gets patched. Expose the relevant names as flags on the cobra commands and as fields on `dns.Controller` / `router.RouterConfig`. Defaults match the values in `manifests/ate-install/` so existing deployments are unaffected: atenet dns: --system-namespace (default "ate-system") --router-service-name (default "atenet-router") --dns-service-name (default "dns") atenet router: --router-service-name (default "atenet-router") * ateapi: make atelet namespace configurable via --atelet-namespace The atelet pod informer hardcoded `ateletNamespace = "ate-system"`, so ate-api-server could only locate atelet pods in that namespace. Deployments that run atelet elsewhere — an alternative install layout or a larger composition that relocates substrate components — leave the informer's cache empty and ResumeActor fails with `found 0 atelet pods on node "", expected 1`. Promote the constant to an exported default and accept the namespace as a parameter to `AteletInformer`. Add an `--atelet-namespace` flag on the ateapi binary (default DefaultAteletNamespace) that callers override when needed. * chart: pass system namespace and Service names to dns-controller and router Wire the new flags added in the previous commit through the Helm templates so the canonical-render defaults are overridden when the chart is used as a subchart (e.g. the kagent-enterprise composition where substrate.fullname prefixes all component Service names). For atenet-dns the dns-controller now receives: --system-namespace={{ .Release.Namespace }} --router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }} --dns-service-name={{ include "substrate.fullname" (list "dns" .) }} For atenet-router the /statusz lookup gets: --router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }} When the release name equals the chart name ("substrate") these expand to the canonical bare names, preserving existing behavior for top-level installs. * chart: pass --atelet-namespace to ate-api-server Wire the new ateapi flag from the previous commit through the chart so the atelet pod informer watches the chart's release namespace by default. Canonical render (release name "substrate" in namespace "ate-system") still produces "--atelet-namespace=ate-system", so behavior is unchanged for top-level installs. * chart: regenerate manifests/ate-install/ from current Helm chart Re-runs `make helm-template` so the checked-in render matches the chart. Brings in rustfs.yaml, the s3-backed atelet storage envvars, the trimmed valkey manifest, and drops the no-longer-templated sandboxconfig-gvisor and sandboxconfig-validation manifests. `make verify-helm-template` now passes. * review: centralize install defaults, derive atelet namespace from POD_NAMESPACE Addresses review comments on agent-substrate/substrate#350: - New internal/installdefaults package owns SystemNamespace, RouterServiceName, DNSServiceName. dns, router, and controlapi/informer drop their duplicate Default* constants and reference installdefaults via the matching flag declarations and tests. - Drop the --atelet-namespace flag on ateapi. The namespace is now resolved at startup from the POD_NAMESPACE env var (Kubernetes' downward API), falling back to installdefaults.SystemNamespace for non-k8s invocations (tests, local dev). atelet and ateapi share a namespace in every supported deployment topology, so a separate knob was dead weight. * review: derive atenet's system namespace from POD_NAMESPACE Same rationale as the prior atelet-namespace change: atenet, atenet-router, and substrate's CoreDNS live in a single namespace in every supported deployment topology, so a separate --system-namespace flag was dead weight. Resolve from the POD_NAMESPACE env var (Kubernetes' downward API) with installdefaults.SystemNamespace as the fallback for non-k8s runs. --router-service-name and --dns-service-name stay as flags because a subchart deployment renames those Services with a release prefix, and the binary can't derive that from pod metadata. * review: NamespaceFromPodEnv helper, APIServiceName const, ateclient hardcodes Three follow-ups from the self-review: - Extract the POD_NAMESPACE-with-SystemNamespace-fallback pattern into installdefaults.NamespaceFromPodEnv() so ateapi and atenet share a single implementation (also makes a third call site one line instead of four if anyone needs one). - Add installdefaults.PodNamespaceEnv ("POD_NAMESPACE") and APIServiceName ("api") so the constant set covers every name in the canonical install layout that's referenced by Go code. - Route internal/ateclient/builder.go's previously-hardcoded "ate-system" and "api" lookups through installdefaults, so kubectl-ate's port-forward no longer bypasses the new single source of truth. ate-controller (ServiceAccount), ate-api-server-deployment (Deployment), and "api.ate-system.svc" (JWT audience) are still hardcoded but their configurability needs a real flag/discovery story and is out of scope for this PR. * chart: render ate-client ServiceAccount in every mode The JWT install overlay (manifests/ate-install/jwt) references ate-client.yaml as a top-level resource, but the chart previously guarded the SA behind {{ if eq .Values.auth.mode "jwt" }} so render-manifests.sh (mtls) never emitted it. That divergence broke verify-helm-template after merging the upstream JWT fix that added a hand-maintained manifests/ate-install/ate-client.yaml. The SA is harmless in mtls installs (unused), so render it unconditionally so the chart is the single source of truth. --- .../templates/ate.dev_actortemplates.yaml | 71 ++++++++-- .../templates/ate.dev_csidriverconfigs.yaml | 93 ++++++++++++ .../templates/ate.dev_sandboxconfigs.yaml | 9 +- .../templates/ate.dev_workerpools.yaml | 22 +-- .../substrate/templates/ate-api-server.yaml | 34 ++--- charts/substrate/templates/ate-client.yaml | 2 - .../substrate/templates/ate-controller.yaml | 11 ++ charts/substrate/templates/atenet-dns.yaml | 10 ++ charts/substrate/templates/atenet-router.yaml | 132 ++++++++++-------- .../controlapi/functionaltest/common_test.go | 3 +- cmd/ateapi/internal/controlapi/informer.go | 6 +- cmd/ateapi/main.go | 8 +- cmd/atenet/internal/dns.go | 22 ++- cmd/atenet/internal/dns/dns.go | 35 +++-- cmd/atenet/internal/dns/dns_test.go | 24 ++-- hack/install-ate-kind-jwt.sh | 2 +- internal/ateclient/builder.go | 5 +- internal/installdefaults/installdefaults.go | 47 +++++++ manifests/ate-install/ate-api-server.yaml | 82 +++++------ manifests/ate-install/ate-client.yaml | 21 --- manifests/ate-install/atenet-dns.yaml | 88 ++++++------ manifests/ate-install/kind/kustomization.yaml | 3 +- manifests/ate-install/role.yaml | 110 +++++++++++++++ 23 files changed, 593 insertions(+), 247 deletions(-) create mode 100644 charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml create mode 100644 internal/installdefaults/installdefaults.go delete mode 100644 manifests/ate-install/ate-client.yaml create mode 100644 manifests/ate-install/role.yaml diff --git a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml index 2de7d898e7..962963b500 100644 --- a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml +++ b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml @@ -64,8 +64,26 @@ spec: description: A single application container that you want to run within a WorkerPool. properties: + args: + description: |- + Arguments to the entrypoint. Not executed within a shell. The container + image's CMD is used if this is not provided (unless command is set, + which discards the image's CMD). + + Unlike Kubernetes, variable references $(VAR_NAME) are NOT expanded. + items: + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: atomic command: - description: Entrypoint array. Not executed within a shell. + description: |- + Entrypoint array. Not executed within a shell. The container image's + ENTRYPOINT is used if this is not provided; if it is provided, the + image's ENTRYPOINT and CMD are both ignored and the process argv is + command + args. + + Unlike Kubernetes, variable references $(VAR_NAME) are NOT expanded. items: type: string maxItems: 64 @@ -144,6 +162,26 @@ spec: required: - port type: object + timeoutSeconds: + default: 30 + description: |- + TimeoutSeconds is how long to keep polling HTTPGet before giving up. + Exceeding it fails the actor start rather than proceeding with a + container that never reported ready. + + How long a workload takes to become ready is a property of that workload, + which is why this is set per template rather than cluster-wide: a heavy + runtime that needs minutes should not force every other template to wait + as long before its failures surface. + + Unset defaults to 30, applied by the API server so the effective value is + visible on the stored object rather than only in the ateom. A manifest + asking for 0 is rejected: unlike a warmup delay, a zero deadline could + never be met, so it is never what a template author means. + format: int32 + maximum: 3600 + minimum: 1 + type: integer required: - httpGet type: object @@ -239,6 +277,25 @@ spec: - Full - Data type: string + onResume: + default: {} + description: |- + OnResume specifies, per snapshot situation, what supplies the guest + state at resume (see OnResumeConfig). "fromData: Golden" requires + sandboxClass "microvm". + properties: + fromData: + default: ColdBoot + description: |- + FromData applies when the resume uses a Data-scope snapshot (from + onPause or onCommit): "ColdBoot" starts fresh from the OCI image with + the durable data restored; "Golden" combines the durable data with the + template's golden snapshot. Defaults to "ColdBoot". + enum: + - ColdBoot + - Golden + type: string + type: object required: - location type: object @@ -353,13 +410,6 @@ spec: x-kubernetes-validations: - message: Spec is immutable rule: self == oldSelf - - message: At most one DurableDir-typed volume is supported per ActorTemplate - rule: '!has(self.volumes) || self.volumes.filter(v, has(v.durableDir)).size() - <= 1' - - message: A container may mount at most one DurableDir-typed volume - rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) - || c.volumeMounts.filter(vm, has(self.volumes) && self.volumes.exists(v, - v.name == vm.name && has(v.durableDir))).size() <= 1)' - message: All volumes defined in spec.volumes must be mounted by at least one container rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) @@ -368,6 +418,11 @@ spec: - message: ExternalVolumes are not supported when sandboxClass is 'microvm' rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' + - message: 'onResume.fromData: Golden is not supported when sandboxClass + is ''gvisor''' + rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') + || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) + ? self.snapshotsConfig.onResume.fromData : ''ColdBoot'') != ''Golden''' status: description: status is the observed state of ActorTemplate properties: diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..306556d783 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml index 119df9bdbf..d765333576 100644 --- a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -96,9 +96,12 @@ spec: Assets is the set of files atelet fetches for this runtime, keyed first by architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The asset names are interpreted by the sandbox backend: gVisor expects a - "runsc" asset; a micro-VM backend expects several (e.g. "cloud-hypervisor", - "kata-kernel", "kata-image"). The schema is intentionally generic; - per-class requirements are enforced by a ValidatingAdmissionPolicy. + "gvisor" asset (the release's gvisor.tar.bz2, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. type: object default: description: |- diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml index 669705004a..f17d3d7da6 100644 --- a/charts/substrate-crds/templates/ate.dev_workerpools.yaml +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -414,21 +414,21 @@ spec: type: array x-kubernetes-list-type: atomic type: object - terminationGracePeriodSeconds: - default: 300 - description: |- - TerminationGracePeriodSeconds is the termination grace period applied to - this pool's worker pods. On eviction, ateom traps SIGTERM and forwards it - to the actor so it can save state and exit cleanly before the kubelet - sends SIGKILL. Tune this to the maximum time your actors need to shut - down gracefully. Defaults to 300 (5 minutes). - format: int32 - minimum: 1 - type: integer required: - ateomImage - replicas type: object + x-kubernetes-validations: + - message: nvidia.com/gpu is only supported when sandboxClass is 'gvisor' + rule: '!has(self.sandboxClass) || self.sandboxClass == ''gvisor'' || + !has(self.template) || !has(self.template.resources) || !((has(self.template.resources.limits) + && ''nvidia.com/gpu'' in self.template.resources.limits) || (has(self.template.resources.requests) + && ''nvidia.com/gpu'' in self.template.resources.requests))' + - message: 'nvidia.com/gpu must be set in limits: Kubernetes does not + admit a request for an extended resource without a matching limit' + rule: '!has(self.template) || !has(self.template.resources) || !has(self.template.resources.requests) + || !(''nvidia.com/gpu'' in self.template.resources.requests) || (has(self.template.resources.limits) + && ''nvidia.com/gpu'' in self.template.resources.limits)' status: description: status is the observed state of WorkerPool properties: diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 088cbd286a..03a3c6b893 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -53,7 +53,7 @@ roleRef: apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "substrate.fullname" (list "ate-api-server-deployment" .) }} + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} namespace: {{ .Release.Namespace }} spec: replicas: 2 @@ -100,8 +100,8 @@ spec: - "--redis-client-cert=@env" - "--client-jwt-issuer=@env" - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" - - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" - - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" {{- else }} @@ -112,8 +112,8 @@ spec: - "--redis-use-iam-auth=@env" - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}" - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" - - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" - - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" {{- end }} - "--drain-delay=13s" @@ -144,14 +144,14 @@ spec: volumeMounts: {{- if eq .Values.auth.mode "mtls" }} - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } - - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } - - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } {{- else }} - { name: ateapi-tls, mountPath: /run/ateapi-tls, readOnly: true } - - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } - - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } {{- end }} ports: - containerPort: 443 @@ -179,11 +179,11 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem - - name: session-id-jwt-pool + - name: actor-id-jwt-pool projected: sources: - secret: - name: session-id-jwt-pool + name: actor-id-jwt-pool items: - { key: pool, path: pool.json } - name: valkey-ca-certs @@ -193,11 +193,11 @@ spec: name: valkey-ca-certs items: - { key: ca.crt, path: ca.crt } - - name: session-id-ca-pool + - name: actor-id-ca-pool projected: sources: - secret: - name: session-id-ca-pool + name: actor-id-ca-pool items: - { key: pool, path: pool.json } - name: podidentity @@ -219,18 +219,18 @@ spec: secretName: {{ .Values.auth.jwt.serverCertSecret }} - name: ateapi-tls emptyDir: {} - - name: session-id-jwt-pool + - name: actor-id-jwt-pool projected: sources: - secret: - name: session-id-jwt-pool + name: actor-id-jwt-pool items: - { key: pool, path: pool.json } - - name: session-id-ca-pool + - name: actor-id-ca-pool projected: sources: - secret: - name: session-id-ca-pool + name: actor-id-ca-pool items: - { key: pool, path: pool.json } {{- end }} diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml index 3de466a04f..dfd2fdab68 100644 --- a/charts/substrate/templates/ate-client.yaml +++ b/charts/substrate/templates/ate-client.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if eq .Values.auth.mode "jwt" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -22,4 +21,3 @@ metadata: namespace: {{ .Release.Namespace }} labels: apps: ate-client -{{- end }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 9cd93d4a0f..010c425ee0 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -71,6 +71,12 @@ spec: - name: ate-controller image: {{ include "substrate.componentImage" (list "atecontroller" .) }} args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" {{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" @@ -79,6 +85,11 @@ spec: - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" +{{- end }} +{{- if .Values.otel.endpoint }} + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} {{- end }} ports: - name: metrics diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml index 0838d2c0b8..ace864570f 100644 --- a/charts/substrate/templates/atenet-dns.yaml +++ b/charts/substrate/templates/atenet-dns.yaml @@ -150,6 +150,16 @@ spec: - "--log-level=debug" - "--interval=10s" - "--corefile-path=/etc/coredns/Corefile" + # Pass the chart-resolved Service names so the controller looks up the + # correct objects when substrate is installed as a subchart. The + # system namespace is read from POD_NAMESPACE below. + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--dns-service-name={{ include "substrate.fullname" (list "dns" .) }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: dns-config-volume mountPath: /etc/coredns diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index e95e5bd49f..9168c86fe9 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -58,62 +58,64 @@ data: config.yaml: | # yaml-language-server: $schema=https://agentgateway.dev/schema/config config: - adminAddr: "127.0.0.1:15000" - readinessAddr: "0.0.0.0:15021" - statsAddr: "0.0.0.0:15020" - binds: - - port: 8080 - listeners: - - name: http + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- if .Values.otel.endpoint }} + frontendPolicies: + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 +{{- end }} + + gateways: + http: + port: 8080 protocol: HTTP - routes: - - name: substrate-http - matches: - - path: - pathPrefix: / - policies: - extProc: - host: "127.0.0.1:50051" - failureMode: failClosed - processingOptions: - requestBodyMode: none - responseBodyMode: none - requestHeaderMode: send - responseHeaderMode: skip - requestTrailerMode: skip - responseTrailerMode: skip - backends: - - dynamic: {} - - port: 8443 - listeners: - - name: https + https: + port: 8443 protocol: HTTPS tls: -{{ if eq .Values.auth.mode "mtls" }} - cert: "/run/servicedns.podcert.ate.dev/cert.pem" - key: "/run/servicedns.podcert.ate.dev/key.pem" -{{ else }} - cert: "/run/agentgateway-tls/tls.crt" - key: "/run/agentgateway-tls/tls.key" -{{ end }} - routes: - - name: substrate-https - matches: - - path: - pathPrefix: / - policies: - extProc: - host: "127.0.0.1:50051" - failureMode: failClosed - processingOptions: - requestBodyMode: none - responseBodyMode: none - requestHeaderMode: send - responseHeaderMode: skip - requestTrailerMode: skip - responseTrailerMode: skip - backends: - - dynamic: {} +{{- if eq .Values.auth.mode "mtls" }} + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem +{{- else }} + cert: /run/agentgateway-tls/tls.crt + key: /run/agentgateway-tls/tls.key +{{- end }} + + routes: + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + extProc: + host: 127.0.0.1:50051 + failureMode: failClosed + processingOptions: + requestHeaderMode: send + responseHeaderMode: skip + requestBodyMode: none + responseBodyMode: none + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} +{{- if eq .Values.auth.mode "mtls" }} + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true +{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -142,12 +144,12 @@ spec: args: - "router" - "--standalone" - - "--networking-mode=agentgateway" + - "--atenet-router=agentgateway" - "--namespace={{ .Release.Namespace }}" - "--port-http=8080" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - - "--ateapi-address={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" {{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" @@ -203,6 +205,11 @@ spec: args: - "-f" - "/etc/agentgateway/config.yaml" +{{- if .Values.otel.endpoint }} + env: + - name: AGENTGATEWAY_OTLP_ADDRESS + value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- end }} ports: - name: http containerPort: 8080 @@ -218,6 +225,9 @@ spec: {{- if eq .Values.auth.mode "mtls" }} - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true {{- else }} - name: agentgateway-tls mountPath: /run/agentgateway-tls @@ -257,6 +267,14 @@ spec: signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem {{- else }} - name: agentgateway-tls secret: @@ -291,3 +309,7 @@ spec: port: 443 targetPort: 8443 protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 8c36af580b..ed2dce06dd 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -26,6 +26,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/volume" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -129,7 +130,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu // 3. Initialize Informers workerFactory, workerInformer := controlapi.WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient, installdefaults.SystemNamespace) scFactory := informers.NewSharedInformerFactory(k8sClient, 0) scLister := scFactory.Storage().V1().StorageClasses().Lister() diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index 1f082cdd04..fcaa6c3ecb 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -25,15 +25,15 @@ import ( ) const ( - ateletNamespace = "ate-system" byNamespaceAndName = "by-namespace-and-name" byWorkerPool = "by-worker-pool" byNode = "by-node" workerPodLabel = "ate.dev/worker-pool" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 99ce86cc36..c5a5ad6d16 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/internal/volume" @@ -161,8 +162,13 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() diff --git a/cmd/atenet/internal/dns.go b/cmd/atenet/internal/dns.go index 85798fa73f..f169ffe9d8 100644 --- a/cmd/atenet/internal/dns.go +++ b/cmd/atenet/internal/dns.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/agent-substrate/substrate/cmd/atenet/internal/dns" + "github.com/agent-substrate/substrate/internal/installdefaults" ) type DnsConfig struct { @@ -37,6 +38,8 @@ type DnsConfig struct { Kubeconfig string ReconcileInterval time.Duration CorefilePath string + RouterServiceName string + DNSServiceName string } func NewDnsCmd() *cobra.Command { @@ -78,11 +81,20 @@ func NewDnsCmd() *cobra.Command { return fmt.Errorf("failed to initialize cluster client: %w", err) } + // atenet shares its namespace with atenet-router and substrate's + // CoreDNS in every supported deployment topology, so we read it + // from Kubernetes' downward API rather than expose a flag. + systemNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved system namespace", slog.String("system-namespace", systemNamespace)) + dnsController := &dns.Controller{ - Client: k8sClient, - Interval: cfg.ReconcileInterval, - CorefilePath: cfg.CorefilePath, - Reloader: dns.NewConfigReloader(), + Client: k8sClient, + Interval: cfg.ReconcileInterval, + CorefilePath: cfg.CorefilePath, + Reloader: dns.NewConfigReloader(), + SystemNamespace: systemNamespace, + RouterServiceName: cfg.RouterServiceName, + DNSServiceName: cfg.DNSServiceName, } slog.InfoContext(ctx, "Starting DNS Controller subsystem") @@ -94,6 +106,8 @@ func NewDnsCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations") cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of the atenet-router. Override when the deployment renames the Service.") + cmd.Flags().StringVar(&cfg.DNSServiceName, "dns-service-name", installdefaults.DNSServiceName, "Service name of substrate's CoreDNS. Override when the deployment renames the Service.") return cmd } diff --git a/cmd/atenet/internal/dns/dns.go b/cmd/atenet/internal/dns/dns.go index cf2db99b69..4cfaf34ef8 100644 --- a/cmd/atenet/internal/dns/dns.go +++ b/cmd/atenet/internal/dns/dns.go @@ -33,18 +33,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // serviceName is the name of the CoreDNS service. - serviceName = "dns" - systemNamespace = "ate-system" -) - // Controller manages the DNS configuration for the ATE. type Controller struct { Client client.Client Interval time.Duration CorefilePath string Reloader ConfigReloader + + // SystemNamespace is the namespace where atenet-router and the substrate + // CoreDNS Service live. Defaults to installdefaults.SystemNamespace. + SystemNamespace string + // RouterServiceName is the Service name of the atenet-router that the + // CoreDNS Corefile forwards actor traffic to. Defaults to + // installdefaults.RouterServiceName. + RouterServiceName string + // DNSServiceName is the Service name of substrate's CoreDNS. Defaults to + // installdefaults.DNSServiceName. + DNSServiceName string } // Run the DNS orchestration loop until ctx is canceled. @@ -71,14 +76,15 @@ func (c *Controller) Run(ctx context.Context) error { func (c *Controller) reconcile(ctx context.Context) error { slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...") - // 1. Get the ClusterIP of atenet-router in ate-system namespace + // 1. Get the ClusterIP of the atenet-router Service in the substrate namespace. routerSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.RouterServiceName, Namespace: c.SystemNamespace}, routerSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available") + slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available", + slog.String("name", c.RouterServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get atenet-router service: %w", err) + return fmt.Errorf("failed to get atenet-router service %s/%s: %w", c.SystemNamespace, c.RouterServiceName, err) } routerIP := routerSvc.Spec.ClusterIP @@ -87,14 +93,15 @@ func (c *Controller) reconcile(ctx context.Context) error { return nil } - // 2. Get the ClusterIP of dns service in ate-system namespace + // 2. Get the ClusterIP of substrate's CoreDNS Service in the same namespace. dnsSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: systemNamespace}, dnsSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.DNSServiceName, Namespace: c.SystemNamespace}, dnsSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "dns service not found, skipping until it is available") + slog.WarnContext(ctx, "dns service not found, skipping until it is available", + slog.String("name", c.DNSServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get dns service: %w", err) + return fmt.Errorf("failed to get dns service %s/%s: %w", c.SystemNamespace, c.DNSServiceName, err) } dnsIP := dnsSvc.Spec.ClusterIP diff --git a/cmd/atenet/internal/dns/dns_test.go b/cmd/atenet/internal/dns/dns_test.go index 34116db284..bf27941e18 100644 --- a/cmd/atenet/internal/dns/dns_test.go +++ b/cmd/atenet/internal/dns/dns_test.go @@ -28,6 +28,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) type mockConfigReloader struct { @@ -94,10 +96,13 @@ func TestReconcile(t *testing.T) { reloader := &mockConfigReloader{} controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: reloader, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: reloader, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } // Run one reconciliation loop @@ -185,10 +190,13 @@ func TestReconcileKubeDNSNotFound(t *testing.T) { Build() controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: &mockConfigReloader{}, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: &mockConfigReloader{}, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } ctx := context.Background() diff --git a/hack/install-ate-kind-jwt.sh b/hack/install-ate-kind-jwt.sh index 66e36fb257..8055bff465 100755 --- a/hack/install-ate-kind-jwt.sh +++ b/hack/install-ate-kind-jwt.sh @@ -133,7 +133,7 @@ apply_kind_extras() { wait_rollouts() { log_step "wait_rollouts" - run_kubectl -n "${NS}" rollout status deployment/ate-api-server-deployment --timeout=180s + run_kubectl -n "${NS}" rollout status deployment/ate-api-server --timeout=180s run_kubectl -n "${NS}" rollout status deployment/ate-controller --timeout=180s run_kubectl -n "${NS}" rollout status deployment/atenet-router --timeout=180s run_kubectl -n "${NS}" rollout status daemonset/atelet --timeout=180s diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 81cf110c59..61c845c7d2 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -161,7 +162,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "api", 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) if err != nil { return nil, err } @@ -251,7 +252,7 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 0000000000..8f47d84f0b --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,47 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import "os" + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + // RouterServiceName is the Service name of atenet-router. + RouterServiceName = "atenet-router" + // DNSServiceName is the Service name of substrate's CoreDNS. + DNSServiceName = "dns" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 8d7f8bd121..5cd6ed6205 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -1,41 +1,23 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: ate-api-server - namespace: ate-system -spec: - maxUnavailable: 1 - selector: - matchLabels: - app: ate-api-server ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ate-api-server - namespace: ate-system ---- +# Define Permissions (Read-Only for Pods) apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: ate-api-server-role + name: ate-api-server rules: - apiGroups: [""] resources: ["pods"] @@ -50,42 +32,37 @@ rules: resources: ["storageclasses"] verbs: ["get", "watch", "list"] --- +# Create Service Account for Workload Identity +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-api-server + namespace: ate-system +--- +# 4. Bind Identity to Permissions apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: ate-api-server-binding + name: ate-api-server subjects: - kind: ServiceAccount name: ate-api-server namespace: ate-system roleRef: kind: ClusterRole - name: ate-api-server-role + name: ate-api-server apiGroup: rbac.authorization.k8s.io --- -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: ate-system -spec: - clusterIP: None - selector: - app: ate-api-server - ports: - - name: grpc - protocol: TCP - port: 443 - targetPort: 443 ---- +# 5. Deploy the API Server apiVersion: apps/v1 kind: Deployment metadata: - name: ate-api-server-deployment + name: ate-api-server namespace: ate-system spec: replicas: 2 strategy: + # Update replicas one at a time, create a new one first, then delete the old one. rollingUpdate: maxUnavailable: 0 maxSurge: 1 @@ -100,7 +77,11 @@ spec: prometheus.io/scrape: "true" prometheus.io/port: "9090" spec: + # TODO: Add topologySpreadConstraints to spread replicas across nodes and zones. serviceAccountName: ate-api-server + # Budget for the full shutdown sequence: --drain-delay (sized to cover + # the readinessProbe below reaching NotReady, plus propagation) + + # --drain-timeout. terminationGracePeriodSeconds: 40 containers: - name: ate-api-server @@ -184,6 +165,9 @@ spec: initialDelaySeconds: 5 periodSeconds: 2 failureThreshold: 3 + # /healthz stays 200 while a terminating pod drains; /readyz + # turns 503, so liveness and readiness diverge correctly during + # shutdown. livenessProbe: httpGet: path: /healthz @@ -191,7 +175,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: - - name: servicedns + - name: "servicedns" projected: sources: - podCertificate: diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml deleted file mode 100644 index cc6ef76c0d..0000000000 --- a/manifests/ate-install/ate-client.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ate-client - namespace: ate-system - labels: - apps: ate-client diff --git a/manifests/ate-install/atenet-dns.yaml b/manifests/ate-install/atenet-dns.yaml index 7ae43b0b10..bca571befe 100644 --- a/manifests/ate-install/atenet-dns.yaml +++ b/manifests/ate-install/atenet-dns.yaml @@ -1,21 +1,17 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. -# Run `make helm-template` to regenerate. - -# atenet-dns — identical across auth modes (does not dial ateapi). apiVersion: v1 kind: ServiceAccount metadata: @@ -38,16 +34,6 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -62,6 +48,16 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: atenet-dns + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -75,27 +71,6 @@ roleRef: name: atenet-dns apiGroup: rbac.authorization.k8s.io --- -apiVersion: v1 -kind: Service -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - selector: - app: dns - type: ClusterIP - # Prefer, not Require: Require fails Service creation on a single-stack cluster. - ipFamilyPolicy: PreferDualStack - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -119,6 +94,8 @@ spec: - name: init-dns image: busybox:1.36 command: ["sh", "-c"] + # Initial core file is sufficient to start CoreDNS but does not contain + # any additional configuration. The controller will update the Corefile. args: - | cat <<'EOF' > /etc/coredns/Corefile @@ -178,3 +155,24 @@ spec: volumes: - name: dns-config-volume emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: dns + namespace: ate-system + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index b1f91e3b67..2e05332a84 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -22,7 +22,6 @@ kind: Kustomization # resource. hack/install-ate.sh applies the ConfigMap directly for the targeted # single-component redeploys. resources: - - ../ate-client.yaml - ../ate-api-server.yaml - ../ate-controller.yaml - ./atelet @@ -47,7 +46,7 @@ patches: apiVersion: apps/v1 kind: Deployment metadata: - name: ate-api-server-deployment + name: ate-api-server namespace: ate-system spec: template: diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..f68fce36f9 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates/finalizers + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - actortemplates/status + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From f93cfcf82b25fa10c9d6fa20c1307d4a0886f662 Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 2 Jul 2026 10:16:21 -0400 Subject: [PATCH 5/9] Decouple actor lock TTL from workflow deadline via heartbeat (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ActorWorkflow.ResumeActor and SuspendActor used to derive their workflow ctx from the Redis lock TTL via acquireActorLock(ctx, id, 30s, 2s) — the workflow deadline and the lock TTL were a single 28s knob. That meant image pulls / restores that legitimately need more than 28s death-looped forever, while raising the knob also raised how long peers wait to retry an actor after a crashed ateapi replica. Split the two concerns: - Lock TTL stays short (30s constant, internal). Bounds peer failover. - Workflow deadline is a separate operator-configurable knob via the new --actor-workflow-deadline pflag (default 5m). Bounds a single Resume/Suspend. - A heartbeat goroutine refreshes the lock every lockTTL/3 (~10s) for the full workflow duration. On RefreshLock=false or any Redis error (peer stole the lock, Redis blip), the workflow ctx is cancelled with errLostActorLock as the cause so in-flight steps unwind cleanly and the mutual-exclusion invariant is preserved. - The release function stops the heartbeat (waits for goroutine exit) before best-effort ReleaseLock. Adds store.Interface.RefreshLock with a Redis CAS Lua script mirroring the existing ReleaseLock script. Signed-off-by: Eitan Yarmush --- .../templates/ate.dev_actortemplates.yaml | 79 +++++++++++++++++++ .../internal/actoridentity/actoridentity.go | 1 - cmd/ateapi/internal/controlapi/dialer_test.go | 7 +- .../controlapi/functionaltest/common_test.go | 16 ++-- cmd/ateapi/internal/controlapi/service.go | 7 +- cmd/ateapi/internal/controlapi/workflow.go | 13 ++- .../internal/controlapi/workflow_lock_test.go | 49 ++++++++++++ .../controlapi/workflow_suspend_test.go | 3 +- .../controlapi/workflow_testutil_test.go | 3 +- cmd/ateapi/main.go | 5 +- 10 files changed, 163 insertions(+), 20 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/workflow_lock_test.go diff --git a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml index 962963b500..8095f04684 100644 --- a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml +++ b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml @@ -226,6 +226,74 @@ spec: type: object maxItems: 10 type: array + resources: + description: |- + Resources declares the compute resources for each actor of this template. + Unlike a pod, an actor is sized by its Limits: the sandbox is built to the + CPU/memory limits (cgroup caps, and for the micro-VM the VM's vCPU count and + memory), the scheduler only places the actor on a worker whose capacity is + >= these limits, and the limits are supplied to the sandbox over the actor + RPCs. Because the size is baked into snapshots, it is part of the immutable + spec. Requests and claims are not supported (actors are sized by limits only). + A zero or absent limit leaves the sandbox at the runtime default (unlimited + for gVisor, the kata config for the micro-VM). + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object sandboxClass: default: gvisor description: |- @@ -423,6 +491,17 @@ spec: rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : ''ColdBoot'') != ''Golden''' + - message: spec.resources.requests is not supported; actors are sized + by spec.resources.limits only + rule: '!has(self.resources) || !has(self.resources.requests)' + - message: spec.resources.claims is not supported + rule: '!has(self.resources) || !has(self.resources.claims)' + - message: For sandboxClass 'microvm', spec.resources.limits.memory must + be at least 256Mi (128Mi VMM reserve + 128Mi guest minimum); below + this the VM cannot boot + rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || + !has(self.resources) || !has(self.resources.limits) || !(''memory'' + in self.resources.limits) || !quantity(self.resources.limits[''memory'']).isLessThan(quantity(''256Mi''))' status: description: status is the observed state of ActorTemplate properties: diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index 86d9ed7374..b12883112b 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -30,7 +30,6 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/actoridjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/k8sjwt" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/principal" diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 321bdee116..1b55fad7c0 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -214,7 +215,7 @@ func TestDialForWorkerTarget(t *testing.T) { Spec: corev1.PodSpec{NodeName: "node-1"}, } ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: tc.ateletIP}}}, } @@ -241,7 +242,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("unknown worker pod", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: "10.244.1.7"}}}, } @@ -253,7 +254,7 @@ func TestDialForWorkerErrors(t *testing.T) { t.Run("atelet without assigned IPs", func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } d := newDialerForPods(t, workerPod, ateletPod) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index ed2dce06dd..72cd54fb13 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -61,10 +61,8 @@ const ( testAtespace = "test-atespace" testActorID = "id1" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -187,7 +185,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu mockDriverName: mockPlugin, } } - service := controlapi.NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins) + service := controlapi.NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", 30*time.Second, volPlugins) // 5. Start REAL gRPC Server for ATE API grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor)) @@ -594,7 +592,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -602,7 +600,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -611,7 +609,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -628,7 +626,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index da9e140960..eb5fcadf17 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -51,7 +52,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewService creates a service. instruments may be nil; the record helpers no-op. +// NewService creates a service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments may be nil. func NewService( persistence store.Interface, workerCache *workercache.Cache, @@ -63,6 +65,7 @@ func NewService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, ) *Service { s := &Service{ @@ -76,7 +79,7 @@ func NewService( instruments: instruments, volumePlugins: volumePlugins, } - s.actorWorkflow = NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s) + s.actorWorkflow = NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 6e4fd87da2..751718b6c4 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -79,9 +80,12 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + // workflowDeadline is the maximum duration of a single actor workflow. + workflowDeadline time.Duration } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -93,6 +97,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -106,6 +111,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, } } @@ -124,14 +130,17 @@ type actorWorkflowStore interface { func (w *ActorWorkflow) acquireActorLock(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lock, error) { lockKey := "lock:actor:" + actorRef.Atespace + ":" + actorRef.Name + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) - lock, err := w.store.AcquireLock(ctx, lockKey) + lock, err := w.store.AcquireLock(workflowCtx, lockKey) if err != nil { + cancel() if errors.Is(err, store.ErrLockConflict) { return nil, nil, status.Error(grpcCodes.Aborted, "another operation is in progress for this actor") } return nil, nil, fmt.Errorf("while acquiring lock: %w", err) } + context.AfterFunc(lock.Context(), cancel) return lock.Context(), lock, nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_lock_test.go b/cmd/ateapi/internal/controlapi/workflow_lock_test.go new file mode 100644 index 0000000000..003d97b665 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lock_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func TestAcquireActorLockWorkflowDeadline(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{mr.Addr()}}) + t.Cleanup(func() { _ = rdb.Close() }) + w := &ActorWorkflow{store: ateredis.NewPersistence(rdb), workflowDeadline: 20 * time.Millisecond} + + ctx, lock, err := w.acquireActorLock(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLock: %v", err) + } + t.Cleanup(lock.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 659b4694b2..a0008b8ef8 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -18,6 +18,7 @@ import ( "context" "errors" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" @@ -695,7 +696,7 @@ func TestSuspendActor_PausedWithoutLocalSnapshotCrashes(t *testing.T) { }); err != nil { t.Fatalf("add template to indexer: %v", err) } - w := NewActorWorkflow(st, nil, nil, listersv1alpha1.NewActorTemplateLister(indexer), nil, nil, nil, nil, "", nil) + w := NewActorWorkflow(st, nil, nil, listersv1alpha1.NewActorTemplateLister(indexer), nil, nil, nil, nil, "", nil, time.Minute) seedWorkflowActor(t, ctx, st, resources.ActorRef{Atespace: "team-a", Name: "id1"}, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_PAUSED) diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index a6f02f7371..c2e7368990 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -18,6 +18,7 @@ import ( "context" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/internal/resources" @@ -42,7 +43,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplNamespace, tmplN }); err != nil { t.Fatalf("add template to indexer: %v", err) } - return NewActorWorkflow(st, nil, nil, listersv1alpha1.NewActorTemplateLister(indexer), nil, nil, nil, nil, "", nil) + return NewActorWorkflow(st, nil, nil, listersv1alpha1.NewActorTemplateLister(indexer), nil, nil, nil, nil, "", nil, time.Minute) } // seedWorkflowActor stores an actor with the given state, bound to the given diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index c5a5ad6d16..0b59182f1f 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -88,6 +88,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") + showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") ) @@ -205,7 +207,7 @@ func main() { dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) } ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) - sm := controlapi.NewService(persistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins) + sm := controlapi.NewService(persistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, *actorWorkflowDeadline, volPlugins) actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, *actorIDCAPoolFile, persistence, workerCache) debugSrv := debugapi.NewService(persistence) @@ -327,6 +329,7 @@ func logFlagValues(ctx context.Context) { slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } From b82569d70e0184b59d9c4ce7fd1b4fa713cb8eeb Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 19 Aug 2026 09:42:58 -0400 Subject: [PATCH 6/9] Fix Helm PostgreSQL deployment and add E2E workflow (#21) Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 114 ++++++++ .../templates/ate.dev_actortemplates.yaml | 14 +- charts/substrate/README.md | 44 +-- charts/substrate/templates/NOTES.txt | 20 +- charts/substrate/templates/_helpers.tpl | 14 - .../templates/ate-api-server-envvars.yaml | 7 +- .../substrate/templates/ate-api-server.yaml | 86 ++---- .../substrate/templates/ate-controller.yaml | 24 -- charts/substrate/templates/atelet.yaml | 26 +- charts/substrate/templates/atenet-dns.yaml | 2 +- charts/substrate/templates/atenet-router.yaml | 113 ++++---- charts/substrate/templates/jwt-bootstrap.yaml | 73 ----- charts/substrate/templates/jwt-oidc-rbac.yaml | 42 --- charts/substrate/templates/namespace.yaml | 1 - .../templates/pod-certificate-controller.yaml | 2 - charts/substrate/templates/postgres.yaml | 161 +++++++++++ .../templates/sandboxconfig-gvisor.yaml | 13 +- .../templates/sandboxconfig-validation.yaml | 7 +- charts/substrate/templates/valkey.yaml | 269 ------------------ charts/substrate/values.yaml | 81 +----- hack/run-microvm-demo.sh | 18 +- .../agentgateway/kustomization.yaml | 2 +- 22 files changed, 436 insertions(+), 697 deletions(-) create mode 100644 .github/workflows/helm-e2e.yaml delete mode 100644 charts/substrate/templates/jwt-bootstrap.yaml delete mode 100644 charts/substrate/templates/jwt-oidc-rbac.yaml create mode 100644 charts/substrate/templates/postgres.yaml delete mode 100644 charts/substrate/templates/valkey.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..122e3f4454 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Deploy egress gateway fixture + env: + KO_DOCKER_REPO: localhost:5001 + KO_DEFAULTPLATFORMS: linux/amd64 + run: | + ./hack/run-tool.sh ko apply -f manifests/ate-install/atenet-egress.yaml -- --context=kind-kind + kubectl --context kind-kind rollout status deployment/atenet-egress -n ate-system --timeout=120s + - name: Deploy micro-VM counter demo + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Wait for micro-VM golden snapshot + run: | + kubectl --context kind-kind wait --for=condition=Ready \ + actortemplate/counter-microvm -n ate-demo-counter-microvm --timeout=600s + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm + E2E_TEMPLATE_NAME: counter-microvm + E2E_TEMPLATE_READY_TIMEOUT: 600s + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get actortemplate,workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml index 8095f04684..f87af91c37 100644 --- a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml +++ b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml @@ -328,6 +328,8 @@ spec: OnCommit specifies what to include in the snapshot when a commit is requested. If not provided, the "Full" behavior is used by default. onCommit must be a subset of the onPause content. + Note: Data scope only captures DurableDir-typed volumes; external/CSI + volumes are not snapshotted as they persist independently. For example: - if onPause is "Full", then onCommit can be "Full" or "Data". @@ -341,6 +343,8 @@ spec: description: |- OnPause specifies what to include in the snapshot when the actor is paused. If not provided, the "Full" behavior is used by default. + Note: Data scope only captures DurableDir-typed volumes; external/CSI + volumes are not snapshotted as they persist independently. enum: - Full - Data @@ -421,6 +425,9 @@ spec: == 1' maxItems: 32 type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map workerSelector: description: |- WorkerSelector restricts which worker pools actors from this template may @@ -483,9 +490,6 @@ spec: rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))' - - message: ExternalVolumes are not supported when sandboxClass is 'microvm' - rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || - !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' - message: 'onResume.fromData: Golden is not supported when sandboxClass is ''gvisor''' rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') @@ -502,6 +506,10 @@ spec: rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.resources) || !has(self.resources.limits) || !(''memory'' in self.resources.limits) || !quantity(self.resources.limits[''memory'']).isLessThan(quantity(''256Mi''))' + - message: All volume mounts must refer to a volume defined in spec.volumes + rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) + || c.volumeMounts.all(vm, has(self.volumes) && self.volumes.exists(v, + v.name == vm.name)))' status: description: status is the observed state of ActorTemplate properties: diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 6cc98069b0..e7364f4e37 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -2,47 +2,27 @@ Helm chart for installing Agent Substrate. -## Install modes - -| Mode | Default? | Cluster requirements | Trade-off | -|------|----------|----------------------|-----------| -| `jwt` | yes | none beyond stock K8s | Server certs and actor signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. | -| `mtls` | | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. | +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. ```bash # CRDs helm upgrade --install substrate-crds ./charts/substrate-crds -# JWT mode (default; no off-by-default feature gates) +# Install Substrate helm upgrade --install substrate ./charts/substrate - -# mTLS mode (requires off-by-default feature gates) -helm upgrade --install substrate ./charts/substrate \ - --set auth.mode=mtls ``` By default, component images are pulled from `ghcr.io/kagent-dev/substrate` using the chart `appVersion` as the tag. Override `image.registry` and `image.tag` to install from a different image repository or tag. -## JWT-mode bootstrap - -JWT mode is standalone by default. The chart generates: - -- `Secret/ateapi-tls` -- `ConfigMap/ateapi-ca` -- `Secret/actor-id-jwt-pool` -- `Secret/actor-id-ca-pool` - -Existing generated data is reused on upgrade so key material does not rotate -during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring -your own resources with those names. - ## Render manifests without applying ```bash -helm template substrate ./charts/substrate # jwt -helm template substrate ./charts/substrate --set auth.mode=mtls +helm template substrate ./charts/substrate ``` `manifests/ate-install/` in the repo is the rendered mTLS output and is @@ -55,17 +35,9 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| -| `auth.mode` | `jwt` | `jwt` or `mtls` | -| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers | -| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience | -| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and actor signing material | -| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name | -| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name | -| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey | -| `valkey.replicas` | `6` | StatefulSet size | +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | -| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis | -| `redis.useIAMAuth` | `false` | Google IAM auth | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt index abad3a8d1e..c0e9875a45 100644 --- a/charts/substrate/templates/NOTES.txt +++ b/charts/substrate/templates/NOTES.txt @@ -1,21 +1,7 @@ -substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }} +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL -{{ if eq .Values.auth.mode "mtls" -}} -NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled: +REQUIRED Kubernetes feature gates: - ClusterTrustBundle - ClusterTrustBundleProjection - PodCertificateRequest -plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you -must enable these explicitly. To install without them, pick auth.mode=jwt. -{{- else }} -JWT mode is active. - -{{- if .Values.auth.jwt.bootstrap.enabled }} -JWT bootstrap resources are managed by this chart. Existing key material is -reused on upgrade. -{{- else }} -JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }}, -{{ .Values.auth.jwt.caBundleConfigMap }}, actor-id-jwt-pool, and -actor-id-ca-pool before pods become healthy. -{{- end }} -{{- end }} +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 36b9d3ef9b..32ae087336 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -103,17 +103,3 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time. {{- printf "%s/%s" $registry $name -}} {{- end -}} {{- end -}} - -{{/* -Validate auth.mode at template time. -*/}} -{{- define "substrate.validateAuthMode" -}} -{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}} -{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}} -{{- end -}} -{{- if eq .Values.auth.mode "jwt" -}} -{{- if not .Values.auth.jwt.issuer -}} -{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 754ff847a7..c0b8323b27 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -20,8 +20,5 @@ metadata: name: {{ .Values.ateApiServerEnvVarsConfigMap }} namespace: {{ .Release.Namespace }} data: - ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }} - ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }} - ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }} - ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }} - ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }} + ATE_API_STORE_BACKEND: "postgres" + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 03a3c6b893..73869cee0e 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -23,7 +23,10 @@ rules: resources: ["pods"] verbs: ["get", "watch", "list"] - apiGroups: ["ate.dev"] - resources: ["actortemplates", "workerpools", "sandboxconfigs"] + resources: ["actortemplates", "workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] verbs: ["get", "watch", "list"] # Secret reads for env source resolution are intentionally NOT granted # cluster-wide here. Each demo / tenant is responsible for granting @@ -74,48 +77,19 @@ spec: spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 -{{- if eq .Values.auth.mode "jwt" }} - initContainers: - - name: assemble-cred-bundle - image: {{ .Values.images.busybox }} - command: - - sh - - -c - - cat /run/ateapi-tls-src/tls.crt /run/ateapi-tls-src/tls.key > /run/ateapi-tls/credential-bundle.pem - volumeMounts: - - { name: ateapi-tls-src, mountPath: /run/ateapi-tls-src, readOnly: true } - - { name: ateapi-tls, mountPath: /run/ateapi-tls } -{{- end }} containers: - name: ate-api-server image: {{ include "substrate.componentImage" (list "ateapi" .) }} args: - "--grpc-listen-addr=0.0.0.0:443" -{{- if eq .Values.auth.mode "mtls" }} - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - - "--redis-cluster-address=@env" - - "--redis-ca-certs=/etc/valkey-ca/ca.crt" - - "--redis-use-iam-auth=@env" - - "--redis-tls-server-name=@env" - - "--redis-client-cert=@env" - - "--client-jwt-issuer=@env" - - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--store-backend=@env" + - "--postgres-connection-string=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" -{{- else }} - - "--grpc-server-cred-bundle=/run/ateapi-tls/credential-bundle.pem" - - "--atelet-insecure=true" - - "--redis-cluster-address=@env" - - "--redis-no-tls=true" - - "--redis-use-iam-auth=@env" - - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}" - - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" - - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" -{{- end }} - "--drain-delay=13s" - "--drain-timeout=15s" env: @@ -142,17 +116,11 @@ spec: name: {{ .Values.ateApiServerEnvVarsConfigMap }} optional: true volumeMounts: -{{- if eq .Values.auth.mode "mtls" }} - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } -{{- else }} - - { name: ateapi-tls, mountPath: /run/ateapi-tls, readOnly: true } - - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } - - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } -{{- end }} + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } ports: - containerPort: 443 - name: prometheus @@ -171,7 +139,6 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 volumes: -{{- if eq .Values.auth.mode "mtls" }} - name: servicedns projected: sources: @@ -179,6 +146,12 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem - name: actor-id-jwt-pool projected: sources: @@ -186,13 +159,6 @@ spec: name: actor-id-jwt-pool items: - { key: pool, path: pool.json } - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - { key: ca.crt, path: ca.crt } - name: actor-id-ca-pool projected: sources: @@ -200,6 +166,9 @@ spec: name: actor-id-ca-pool items: - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication - name: podidentity projected: sources: @@ -213,27 +182,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- else }} - - name: ateapi-tls-src - secret: - secretName: {{ .Values.auth.jwt.serverCertSecret }} - - name: ateapi-tls - emptyDir: {} - - name: actor-id-jwt-pool - projected: - sources: - - secret: - name: actor-id-jwt-pool - items: - - { key: pool, path: pool.json } - - name: actor-id-ca-pool - projected: - sources: - - secret: - name: actor-id-ca-pool - items: - - { key: pool, path: pool.json } -{{- end }} --- apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 010c425ee0..31c83b9066 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -77,15 +77,8 @@ spec: # "ate-system"). Pass the chart-resolved Service so the controller # dials the right backend when substrate is installed as a subchart. - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" -{{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- else }} - - "--ateapi-use-token-auth=true" - - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" - - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" - - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" -{{- end }} {{- if .Values.otel.endpoint }} env: - name: OTEL_EXPORTER_OTLP_ENDPOINT @@ -98,7 +91,6 @@ spec: - name: healthz containerPort: 8081 protocol: TCP -{{- if eq .Values.auth.mode "mtls" }} volumeMounts: - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } @@ -119,19 +111,3 @@ spec: signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem -{{- else }} - volumeMounts: - - { name: ateapi-ca, mountPath: /run/ateapi-ca, readOnly: true } - - { name: ateapi-token, mountPath: /var/run/secrets/tokens/ateapi, readOnly: true } - volumes: - - name: ateapi-ca - configMap: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - - name: ateapi-token - projected: - sources: - - serviceAccountToken: - audience: {{ .Values.auth.jwt.audience }} - expirationSeconds: 3600 - path: token -{{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index 6b2eebba47..fab67b025b 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# atelet — identical across auth modes (does not dial ateapi). +# atelet apiVersion: v1 kind: ServiceAccount metadata: @@ -29,6 +29,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -68,12 +71,9 @@ spec: image: {{ include "substrate.componentImage" (list "atelet" .) }} args: - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} -{{- if eq .Values.auth.mode "mtls" }} - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem -{{- else }} - - --grpc-insecure=true -{{- end }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem {{- with .Values.atelet.extraArgs }} {{ toYaml . | indent 8 }} {{- end }} @@ -116,17 +116,17 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ateom-gvisor -{{- if eq .Values.auth.mode "mtls" }} - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true -{{- end }} + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true volumes: - name: run-ateom hostPath: path: /var/lib/ateom-gvisor type: DirectoryOrCreate -{{- if eq .Values.auth.mode "mtls" }} - name: podidentity projected: sources: @@ -140,4 +140,12 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- end }} + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml index ace864570f..fc6f770306 100644 --- a/charts/substrate/templates/atenet-dns.yaml +++ b/charts/substrate/templates/atenet-dns.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# atenet-dns — identical across auth modes (does not dial ateapi). +# atenet-dns apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 9168c86fe9..8810e1bc83 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -79,13 +79,8 @@ data: port: 8443 protocol: HTTPS tls: -{{- if eq .Values.auth.mode "mtls" }} cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem key: /run/servicedns.podcert.ate.dev/credential-bundle.pem -{{- else }} - cert: /run/agentgateway-tls/tls.crt - key: /run/agentgateway-tls/tls.key -{{- end }} routes: - name: substrate-actors @@ -106,16 +101,62 @@ data: responseBodyMode: none requestTrailerMode: skip responseTrailerMode: skip + requestAttributes: + "filter_state['dev.ate.authority']": request.host backends: - - dynamic: {} -{{- if eq .Values.auth.mode "mtls" }} + - dynamic: + target: extproc["envoy.filters.listener.original_dst"]["local"] policies: backendTLS: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/podidentity.podcert.ate.dev/trust-bundle.pem insecureHost: true -{{- end }} + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + extProc: + host: 127.0.0.1:50051 + failureMode: failClosed + processingOptions: + requestHeaderMode: send + responseHeaderMode: skip + requestBodyMode: none + responseBodyMode: none + requestTrailerMode: skip + responseTrailerMode: skip + requestAttributes: + "filter_state['dev.ate.authority']": source.connectHeaders["host"] + backends: + - dynamic: + target: extproc["envoy.filters.listener.original_dst"]["local"] + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true --- apiVersion: apps/v1 kind: Deployment @@ -143,24 +184,19 @@ spec: image: {{ include "substrate.componentImage" (list "atenet" .) }} args: - "router" - - "--standalone" + - "--mode=ingress" - "--atenet-router=agentgateway" - "--namespace={{ .Release.Namespace }}" - "--port-http=8080" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" -{{- if eq .Values.auth.mode "mtls" }} - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- else }} - - "--ateapi-use-token-auth=true" - - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" - - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" - - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" -{{- end }} - "--status-port=4040" - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" env: - name: POD_NAME valueFrom: @@ -187,19 +223,9 @@ spec: containerPort: 4040 - name: metrics containerPort: 9090 -{{- if eq .Values.auth.mode "mtls" }} volumeMounts: - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } -{{- else }} - volumeMounts: - - name: ateapi-ca - mountPath: /run/ateapi-ca - readOnly: true - - name: ateapi-token - mountPath: /var/run/secrets/tokens/ateapi - readOnly: true -{{- end }} - name: agentgateway image: {{ .Values.images.agentgateway }} args: @@ -215,6 +241,10 @@ spec: containerPort: 8080 - name: https containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 - name: readiness containerPort: 15021 - name: gw-metrics @@ -222,17 +252,11 @@ spec: volumeMounts: - name: agentgateway-config mountPath: /etc/agentgateway -{{- if eq .Values.auth.mode "mtls" }} - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" - name: podidentity mountPath: /run/podidentity.podcert.ate.dev readOnly: true -{{- else }} - - name: agentgateway-tls - mountPath: /run/agentgateway-tls - readOnly: true -{{- end }} readinessProbe: httpGet: path: /healthz/ready @@ -242,13 +266,13 @@ spec: - name: agentgateway-config configMap: name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} -{{- if eq .Values.auth.mode "mtls" }} - name: "servicedns" projected: sources: - podCertificate: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem certificateChainPath: cert.pem keyPath: key.pem - name: servicedns-ca @@ -275,21 +299,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem -{{- else }} - - name: agentgateway-tls - secret: - secretName: {{ .Values.auth.jwt.serverCertSecret }} - - name: ateapi-ca - configMap: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - - name: ateapi-token - projected: - sources: - - serviceAccountToken: - audience: {{ .Values.auth.jwt.audience }} - expirationSeconds: 3600 - path: token -{{- end }} --- apiVersion: v1 kind: Service @@ -309,6 +318,14 @@ spec: port: 443 targetPort: 8443 protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP - name: status port: 4040 targetPort: status diff --git a/charts/substrate/templates/jwt-bootstrap.yaml b/charts/substrate/templates/jwt-bootstrap.yaml deleted file mode 100644 index cb31763fb3..0000000000 --- a/charts/substrate/templates/jwt-bootstrap.yaml +++ /dev/null @@ -1,73 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and (eq .Values.auth.mode "jwt") .Values.auth.jwt.bootstrap.enabled }} -{{- $apiName := include "substrate.fullname" (list "api" .) }} -{{- $routerName := include "substrate.fullname" (list "atenet-router" .) }} -{{- $apiHost := printf "%s.%s.svc" $apiName .Release.Namespace }} -{{- $ca := genCA (printf "%s-ca" $apiName) 3650 }} -{{- $serverCert := genSignedCert $apiHost nil (list $apiHost (printf "%s.%s.svc.cluster.local" $apiName .Release.Namespace) (printf "%s.%s.svc" $routerName .Release.Namespace)) 365 $ca }} -{{- $actorJWTKey := genPrivateKey "ecdsa" }} -{{- $actorCA := genCA "actor-id-ca" 3650 }} -{{- if .Values.auth.jwt.bootstrap.serverCert.enabled }} -{{- $existingTLS := lookup "v1" "Secret" .Release.Namespace .Values.auth.jwt.serverCertSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.auth.jwt.serverCertSecret }} - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: {{ if $existingTLS }}{{ index $existingTLS.data "tls.crt" }}{{ else }}{{ $serverCert.Cert | b64enc }}{{ end }} - tls.key: {{ if $existingTLS }}{{ index $existingTLS.data "tls.key" }}{{ else }}{{ $serverCert.Key | b64enc }}{{ end }} ---- -{{- $existingCA := lookup "v1" "ConfigMap" .Release.Namespace .Values.auth.jwt.caBundleConfigMap }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.auth.jwt.caBundleConfigMap }} - namespace: {{ .Release.Namespace }} -data: - ca.crt: | -{{- if $existingCA }} -{{ index $existingCA.data "ca.crt" | nindent 4 }} -{{- else }} -{{ $ca.Cert | nindent 4 }} -{{- end }} -{{- end }} -{{- if .Values.auth.jwt.bootstrap.sessionPools.enabled }} ---- -{{- $existingJWTSecret := lookup "v1" "Secret" .Release.Namespace "actor-id-jwt-pool" }} -apiVersion: v1 -kind: Secret -metadata: - name: actor-id-jwt-pool - namespace: {{ .Release.Namespace }} -type: Opaque -data: - pool: {{ if $existingJWTSecret }}{{ index $existingJWTSecret.data "pool" }}{{ else }}{{ dict "Authorities" (list (dict "ID" "1" "Algorithm" "ES256" "SigningKeyPEM" $actorJWTKey)) | toJson | b64enc }}{{ end }} ---- -{{- $existingCASecret := lookup "v1" "Secret" .Release.Namespace "actor-id-ca-pool" }} -apiVersion: v1 -kind: Secret -metadata: - name: actor-id-ca-pool - namespace: {{ .Release.Namespace }} -type: Opaque -data: - pool: {{ if $existingCASecret }}{{ index $existingCASecret.data "pool" }}{{ else }}{{ dict "CAs" (list (dict "ID" "1" "SigningKeyPEM" $actorCA.Key "RootCertificatePEM" $actorCA.Cert)) | toJson | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/charts/substrate/templates/jwt-oidc-rbac.yaml b/charts/substrate/templates/jwt-oidc-rbac.yaml deleted file mode 100644 index a9fd499e96..0000000000 --- a/charts/substrate/templates/jwt-oidc-rbac.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if eq .Values.auth.mode "jwt" }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} -rules: -- nonResourceURLs: - - /.well-known/openid-configuration - - /openid/v1/jwks - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} -subjects: -- kind: ServiceAccount - name: {{ include "substrate.fullname" (list "ate-api-server" .) }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml index 63401c00d0..073291828b 100644 --- a/charts/substrate/templates/namespace.yaml +++ b/charts/substrate/templates/namespace.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- include "substrate.validateAuthMode" . -}} {{- if .Values.createNamespace }} apiVersion: v1 kind: Namespace diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml index 3aaaa9df99..86fc23b4a9 100644 --- a/charts/substrate/templates/pod-certificate-controller.yaml +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if eq .Values.auth.mode "mtls" -}} apiVersion: v1 kind: Namespace metadata: @@ -197,4 +196,3 @@ spec: securityContext: {} serviceAccountName: default terminationGracePeriodSeconds: 30 -{{- end }} diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..feee69610d --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,161 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/tls/credential-bundle.pem' + ssl_key_file = '/run/tls/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + local all all trust + hostssl all all all trust clientcert=verify-ca +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + initContainers: + - name: fix-tls-perms + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - -c + - | + set -e + cp /run/servicedns.podcert.ate.dev/credential-bundle.pem /run/tls/credential-bundle.pem + chmod 600 /run/tls/credential-bundle.pem + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: tls + mountPath: /run/tls + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: tls + mountPath: /run/tls + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: tls + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 3dc4e9d168..36af4296f3 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -26,12 +26,13 @@ metadata: spec: sandboxClass: gvisor default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: - runsc: - url: "gs://gvisor/releases/release/20260622/x86_64/runsc" - sha256: "f18a948bf9c8bbb54eb998549a3a8d719a1c7de2efbe8fdd2ff0ee5fecd06f19" + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" arm64: - runsc: - url: "gs://gvisor/releases/release/20260622/aarch64/runsc" - sha256: "62eee121f8c188e347c428acc96f111568ede3be37b906046b6f28bbe2cc40c0" + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml index 48302d2fdd..f25d43409b 100644 --- a/charts/substrate/templates/sandboxconfig-validation.yaml +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -31,12 +31,13 @@ spec: operations: ["CREATE", "UPDATE"] resources: ["sandboxconfigs"] validations: - # gVisor needs a "runsc" asset for every architecture it advertises. + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. - expression: >- object.spec.sandboxClass != 'gvisor' || (has(object.spec.assets) && size(object.spec.assets) > 0 && - object.spec.assets.all(arch, 'runsc' in object.spec.assets[arch])) - message: "a gvisor SandboxConfig must define a 'runsc' asset for every architecture under spec.assets" + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" # The micro-VM (cloud-hypervisor) runtime needs its asset set for every # architecture it advertises. - expression: >- diff --git a/charts/substrate/templates/valkey.yaml b/charts/substrate/templates/valkey.yaml deleted file mode 100644 index ca164e7698..0000000000 --- a/charts/substrate/templates/valkey.yaml +++ /dev/null @@ -1,269 +0,0 @@ -{{/* -Copyright 2026 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.valkey.enabled -}} -{{- $sts := include "substrate.fullname" (list "valkey-cluster" .) -}} -{{- $headless := include "substrate.fullname" (list "valkey-cluster-service" .) -}} -{{- $ns := .Release.Namespace -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "substrate.fullname" (list "valkey-config" .) }} - namespace: {{ .Release.Namespace }} -data: - valkey.conf: | -{{- if eq .Values.auth.mode "mtls" }} - # Enforce TLS and disable standard port - port 0 - tls-port 6379 - tls-cluster yes - tls-replication yes - - # Load certificates from projected volume - tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem - tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem - tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem - tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem - tls-ca-cert-file /etc/valkey-ca/ca.crt - tls-auth-clients yes - - # Reload every 10 minutes. - # The interval should be less than the 30m headroom (notAfter - beginRefreshAt) - # set by cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go. - tls-auto-reload-interval 600 - - # Enable cluster mode -{{- else }} - # Plaintext: serve on the standard port, no TLS. - port 6379 - -{{- end }} - cluster-enabled yes - cluster-config-file nodes.conf - cluster-node-timeout 5000 - appendonly yes - protected-mode no ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $headless }} - namespace: {{ .Release.Namespace }} -spec: - clusterIP: None - selector: - app: valkey-cluster - ports: - - name: valkey - port: 6379 - targetPort: 6379 - - name: bus - port: 16379 - targetPort: 16379 ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $sts }} - namespace: {{ .Release.Namespace }} -spec: - selector: - app: valkey-cluster - ports: - - name: valkey - port: 6379 - targetPort: 6379 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ $sts }} - namespace: {{ .Release.Namespace }} -spec: - serviceName: {{ $headless }} - replicas: {{ .Values.valkey.replicas }} - podManagementPolicy: Parallel - selector: - matchLabels: - app: valkey-cluster - template: - metadata: - labels: - app: valkey-cluster - spec: - containers: - - name: valkey - image: {{ .Values.images.valkey }} - command: ["valkey-server", "/etc/valkey/valkey.conf"] - ports: - - name: valkey - containerPort: 6379 - - name: bus - containerPort: 16379 - volumeMounts: - - name: config - mountPath: /etc/valkey -{{- if eq .Values.auth.mode "mtls" }} - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - - name: valkey-ca-certs - mountPath: /etc/valkey-ca - readOnly: true -{{- end }} - - name: data - mountPath: /data - volumes: - - name: config - configMap: - name: {{ include "substrate.fullname" (list "valkey-config" .) }} -{{- if eq .Values.auth.mode "mtls" }} - - name: servicedns - projected: - sources: - - podCertificate: - signerName: servicedns.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: podidentity - projected: - sources: - - podCertificate: - signerName: podidentity.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - key: ca.crt - path: ca.crt -{{- end }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: {{ .Values.valkey.storageSize }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "substrate.fullname" (list "valkey-cluster-init" .) }} - namespace: {{ .Release.Namespace }} -spec: - template: - metadata: - labels: - app: valkey-cluster-init - spec: - restartPolicy: OnFailure - containers: - - name: init - image: {{ .Values.images.valkey }} -{{- if eq .Values.auth.mode "mtls" }} - volumeMounts: - - name: podidentity - mountPath: /run/podidentity.podcert.ate.dev - - name: valkey-ca-certs - mountPath: /etc/valkey-ca - readOnly: true -{{- end }} - command: - - /bin/sh - - -c - - | - set -e - echo "Waiting for all Valkey pods to resolve..." - for i in 0 1 2 3 4 5; do - until getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-${i} DNS..." - sleep 2 - done - done - - echo "All pods resolved. Getting IPs..." - POD_IPS="" - for i in 0 1 2 3 4 5; do - ip=$(getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc | awk '{print $1}') - POD_IPS="${POD_IPS} ${ip}:6379" - done - - echo "Checking if Valkey cluster is already initialized..." -{{- if eq .Values.auth.mode "mtls" }} - until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc ping >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-0 to respond to ping..." - sleep 2 - done - - INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc cluster info 2>/dev/null | grep cluster_state || true) - - if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then - echo "Initializing Valkey cluster..." - valkey-cli --tls \ - --cacert /etc/valkey-ca/ca.crt \ - --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ - --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ - --cluster create ${POD_IPS} \ - --cluster-replicas 1 \ - --cluster-yes - echo "Cluster initialization complete!" - else - echo "Cluster already initialized." - fi -{{- else }} - until valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 ping >/dev/null 2>&1; do - echo "Waiting for {{ $sts }}-0 to respond to ping..." - sleep 2 - done - - INIT_STATUS=$(valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 cluster info 2>/dev/null | grep cluster_state || true) - - if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then - echo "Initializing Valkey cluster..." - valkey-cli \ - --cluster create ${POD_IPS} \ - --cluster-replicas 1 \ - --cluster-yes - echo "Cluster initialization complete!" - else - echo "Cluster already initialized." - fi -{{- end }} -{{- if eq .Values.auth.mode "mtls" }} - volumes: - - name: podidentity - projected: - sources: - - podCertificate: - signerName: podidentity.podcert.ate.dev/identity - keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem - - name: valkey-ca-certs - projected: - sources: - - secret: - name: valkey-ca-certs - items: - - key: ca.crt - path: ca.crt -{{- end }} -{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index e27f775272..78f34b12ab 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -14,56 +14,8 @@ # Default values for the substrate chart. # -# The chart supports two installation modes via `auth.mode`: -# -# - "jwt" (default): No PodCertificateRequest / ClusterTrustBundle usage. Server -# certs and actor signing pools are generated by the chart by default, -# and can be disabled when you want to provide your own key material. -# Clients authenticate to ateapi with a projected Kubernetes ServiceAccount -# token. Valkey runs plaintext. -# -# - "mtls": Server certs are issued by the in-cluster podcertcontroller via -# PodCertificateRequest + projected into pods via the ClusterTrustBundle / -# podCertificate projection sources. Valkey runs with full TLS + -# client-cert verification. REQUIRES the off-by-default Kubernetes feature -# gates: -# ClusterTrustBundle, ClusterTrustBundleProjection, PodCertificateRequest -# and the v1beta1 certificates API. - -auth: - mode: jwt # jwt | mtls - - jwt: - # OIDC issuer URL the cluster uses to mint SA tokens. The default matches - # stock kind/kubeadm-style clusters. Override this for managed clusters - # whose service account issuer is provider-specific. Examples: - # GKE: https://container.googleapis.com/v1/projects//locations//clusters/ - # kind: https://kubernetes.default.svc.cluster.local - # EKS: https://oidc.eks..amazonaws.com/id/ - issuer: https://kubernetes.default.svc.cluster.local - - # Audience SA tokens are minted for, and that ateapi expects. - audience: api.ate-system.svc - - bootstrap: - # Generate JWT-mode TLS and actor-signing key material with Helm. - # Existing generated resources are reused on upgrade via lookup. - enabled: true - serverCert: - enabled: true - sessionPools: - enabled: true - - # Name of a kubernetes.io/tls Secret in the release namespace, with keys - # tls.crt and tls.key. Created by the chart when - # auth.jwt.bootstrap.serverCert.enabled=true. - serverCertSecret: ateapi-tls - - # Name of a ConfigMap in the release namespace with key "ca.crt" holding - # the CA(s) that signed serverCertSecret. Clients mount it to verify the - # ateapi server certificate. Created by the chart when - # auth.jwt.bootstrap.serverCert.enabled=true. - caBundleConfigMap: ateapi-ca +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. # Set to true to have the chart create the release namespace. # Off by default — most helm workflows expect the namespace to already exist @@ -71,10 +23,16 @@ auth: # manifests/ate-install/ install path (kubectl apply). createNamespace: false -valkey: - enabled: true - replicas: 6 +postgres: storageSize: 1Gi + connectionString: "" + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi rustfs: enabled: true @@ -94,19 +52,8 @@ atelet: extraArgs: [] extraEnv: [] -redis: - # Override the cluster address. Empty -> derived from valkey.enabled - # (defaults to "valkey-cluster.ate-system.svc:6379"). - clusterAddress: "" - # Google IAM auth (for managed Memorystore / cloud Valkey). - useIAMAuth: false - # Override TLS server name for Redis hostname verification (mtls mode). - tlsServerName: "" - # File path for Redis client TLS credential bundle (mtls mode). - clientCert: "" - # Name of a ConfigMap in the release namespace that supplies per-environment -# overrides for ate-api-server (ATE_API_REDIS_*, ATE_API_K8SJWT_ISSUER, ...). +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). # Mounted via envFrom with optional=true. Created by the chart from these values. ateApiServerEnvVarsConfigMap: ate-api-server-envvars @@ -118,9 +65,9 @@ image: tag: "" images: - valkey: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: cr.agentgateway.dev/agentgateway:v1.4.1 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.34610af2@sha256:c471d7a835d9d318353d531fe3a6195aa164cf44825c867c4cb69feae09d4a86 coredns: coredns/coredns:1.11.1 busybox: busybox:1.36 diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 5975bdce1a..c3cd29cff5 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -56,6 +56,7 @@ KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" ATE_ATEAPI_CLIENT_AUTH="${ATE_ATEAPI_CLIENT_AUTH:-cert}" +SKIP_CONTROL_PLANE=false while [[ $# -gt 0 ]]; do case "$1" in @@ -68,6 +69,7 @@ while [[ $# -gt 0 ]]; do shift ATE_ATEAPI_CLIENT_AUTH="$1" ;; + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; *) echo "Error: unknown argument $1" >&2 exit 1 @@ -99,13 +101,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 8511f195e0..cd18da609c 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -60,7 +60,7 @@ patches: path: /spec/template/spec/containers/1 value: name: agentgateway - image: cr.agentgateway.dev/agentgateway:v1.4.1 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.34610af2@sha256:c471d7a835d9d318353d531fe3a6195aa164cf44825c867c4cb69feae09d4a86 args: - -f - /etc/agentgateway/config.yaml From bf58fb5e41bf90ed690594a983b64ea1e7f86f9f Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 19 Aug 2026 15:30:17 +0000 Subject: [PATCH 7/9] release ateom-microvm image Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2688d5e375..fb93fdbe31 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,7 +88,7 @@ jobs: run: | set -o errexit -o nounset -o pipefail - for component in ateapi atecontroller atelet ateom-gvisor podcertcontroller atenet; do + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ ./hack/run-tool.sh ko build \ --tags "${IMAGE_TAGS}" \ From 8a3a204ea97cf7a5d203bc805c04a9ac14e9c507 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 19 Aug 2026 15:43:56 +0000 Subject: [PATCH 8/9] release kubectl-ate binaries Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fb93fdbe31..4a26a5cbe9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -127,9 +127,28 @@ jobs: helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + - name: Create GitHub Release if: inputs.create_release uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.tag.outputs.value }} generate_release_notes: true + files: dist/kubectl-ate-* From 05e4579f3a5629b802ee194397cc006631dcdef7 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Thu, 20 Aug 2026 10:36:36 +0700 Subject: [PATCH 9/9] feat(helm): Add configurable pause image for gVisor SandboxConfig --- charts/substrate/README.md | 1 + charts/substrate/templates/sandboxconfig-gvisor.yaml | 2 +- charts/substrate/values.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index e7364f4e37..db7e1089b7 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -41,3 +41,4 @@ See `values.yaml` for the full set; the important keys: | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | +| `images.pause` | `registry.k8s.io/pause:3.10.2@sha256:f548e...` | Root sandbox container image for the `gvisor-default` SandboxConfig; must stay digest-pinned. Override for platforms with their own pause image (e.g. Rancher) or air-gapped/proxy registries | diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 36af4296f3..c9717a3a5b 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -26,7 +26,7 @@ metadata: spec: sandboxClass: gvisor default: true - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + pauseImage: {{ .Values.images.pause | quote }} assets: amd64: gvisor: diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 78f34b12ab..28d24ad469 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -65,6 +65,7 @@ image: tag: "" images: + pause: registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73