ate-setup: add a render subcommand for kustomize generation

cmd/ate-setup/commands.md

@@ -122,6 +122,74 @@ apiserver, the controller, atenet, and atelet. It creates every `create`
 resource below on the way, so those subcommands are only needed to redo one on
 a running cluster.
 
+## Render
+
+| `ate-setup` | `hack/install-ate.sh` |
+|---|---|
+| `render ate-system` | (no shell equivalent) |
+
+`render ate-system` writes the manifests `deploy ate-system` applies to a fresh
+cluster into `--output-dir`, a new or empty directory, without contacting a
+cluster. It needs `--image-repo` and `--image-tag`, and pins every image to a
+digest:
+
+```bash
+ate-setup render ate-system --kind --image-repo REPO --image-tag TAG \
+  --output-dir DIR
+```
+
+Other clusters store snapshots on GCS today, so they also need the snapshot
+location and the snapshot transfer service account:
+
+```bash
+ate-setup render ate-system --image-repo REPO --image-tag TAG \
+  --snapshot-location gs://BUCKET/ \
+  --gcp-project-id PROJECT --snapshot-transfer-gsa-name NAME \
+  --output-dir DIR
+```
+
+The files are numbered in apply order. Create the key pools after the
+namespace, and apply the CRDs and wait for them before the rest, because a
+later file holds a SandboxConfig:
+
+```bash
+kubectl apply -f DIR/00-namespace.yaml
+ate-setup create jwt-authority-pool
+ate-setup create actor-id-ca-pool
+ate-setup create actor-id-ca-certs
+ate-setup create egress-mitm-ca-pool
+ate-setup create podcertificate-controller-cas
+kubectl apply -f DIR/01-crds.yaml
+kubectl wait --for=condition=Established crd --all
+kubectl apply -f DIR
+```
+
+The CA and JWT pool Secrets are not written: `actor-id-jwt-pool`,
+`actor-id-ca-pool`, `egress-mitm-ca-pool`, and the podcertificate controller's
+`service-dns-ca-pool`, `pod-identity-ca-pool` and `postgres-ca-pool`. Each holds
+a private key generated for one cluster. Rendered, the keys would end up
+wherever the output goes, such as a release or a GitOps repository, and every
+apply would replace the pools and invalidate every certificate and token issued
+from them. The `create` commands above, like `deploy ate-system`, create each
+pool only when it is absent. `actor-id-ca-certs` is the root certificate copied
+from the `actor-id-ca-pool` Secret in the cluster, so it is created there too.
+
+The `k8s-credential-provider` namespace policy is not written either. Operators
+edit it, and reapplying it would reset their policy, so create it once with
+`kubectl apply -f manifests/egress-credential-injection/namespace-policy.yaml`.
+The bundled PostgreSQL identities are not created: `deploy ate-system` and
+`deploy postgres` create them by running SQL in the postgres pod, which no
+manifest does yet. The CSI drivers are not written. None of deploy's checks
+against an existing install run, so do not apply the output over a running
+install. The service account issuer is `EXPECTED_JWT_ISSUER`,
+else the GKE issuer when the `--gcp-*` cluster settings are set, else the
+in-cluster default. Without `--credential-provider`, the egress gateway uses
+the bundled Kubernetes Secrets provider, `{"name":"k8s.io"}`, as kind installs
+do. With an external database, `ate-api-server-secret-envvars`
+is not written, so its connection strings stay out of the directory; create it
+with the keys `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`,
+`ATE_API_POSTGRES_OWNER_CONNECTION_STRING` and `ATE_API_POSTGRES_SCHEMA`.
+
 ## Publish
 
 | `ate-setup` | `hack/install-ate.sh` |

cmd/ate-setup/internal/cmd/record_test.go

@@ -44,6 +44,7 @@ func TestOnlySystemDeploysRecordTheirRun(t *testing.T) {
 		{path: "delete benchmarks", want: false},
 		{path: "setup csi", want: false},
 		{path: "publish", want: false},
+		{path: "render ate-system", want: false},
 	} {
 		t.Run(tc.path, func(t *testing.T) {
 			target, _, err := rootCmd.Find(strings.Fields(tc.path))
@@ -60,16 +61,17 @@ func TestOnlySystemDeploysRecordTheirRun(t *testing.T) {
 	}
 }
 
-// The publish commands build and push without a cluster, so the release job
+// The publish and render commands run without a cluster, so the release job
 // runs them with no kubeconfig. Every other command connects to one.
-func TestOnlyPublishSkipsTheCluster(t *testing.T) {
+func TestOnlyPublishAndRenderSkipTheCluster(t *testing.T) {
 	for _, tc := range []struct {
 		path string
 		want bool
 	}{
 		{path: "publish release-images", want: false},
 		{path: "publish release-binaries", want: false},
 		{path: "publish worker-images", want: false},
+		{path: "render ate-system", want: false},
 		{path: "deploy ate-system", want: true},
 		{path: "delete ate-system", want: true},
 		{path: "setup csi", want: true},

cmd/ate-setup/internal/cmd/render.go

@@ -0,0 +1,112 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"go.ate.dev/substrate/cmd/ate-setup/internal/config"
+)
+
+// renderAteSystemPath is the command path the render settings bind to.
+const renderAteSystemPath = "render ate-system"
+
+var renderCmd = &cobra.Command{
+	Use:   "render",
+	Short: "Write the manifests a deploy applies, without contacting a cluster",
+	// Rendering reads the checkout and the image registry and never a cluster,
+	// so it runs where there is no kubeconfig, such as a release job.
+	Annotations: map[string]string{clusterAnnotation: "no"},
+}
+
+var renderAteSystemCmd = &cobra.Command{
+	Use:   "ate-system",
+	Short: "Write the manifests deploy ate-system applies into a directory",
+	Long: `Write the manifests "deploy ate-system" applies to a fresh cluster into
+--output-dir, one file per apply step, with every image pinned to a digest.
+
+  ate-setup render ate-system --kind --image-repo REPO --image-tag TAG \
+    --output-dir DIR
+
+Other clusters store snapshots on GCS today, so they also need the snapshot
+location and the snapshot transfer service account:
+
+  ate-setup render ate-system --image-repo REPO --image-tag TAG \
+    --snapshot-location gs://BUCKET/ \
+    --gcp-project-id PROJECT --snapshot-transfer-gsa-name NAME \
+    --output-dir DIR
+
+The files are numbered in apply order. Create the key pools after the
+namespace, and apply the CRDs and wait for them before the rest, because a
+later file holds a SandboxConfig:
+
+  kubectl apply -f DIR/00-namespace.yaml
+  ate-setup create jwt-authority-pool
+  ate-setup create actor-id-ca-pool
+  ate-setup create actor-id-ca-certs
+  ate-setup create egress-mitm-ca-pool
+  ate-setup create podcertificate-controller-cas
+  kubectl apply -f DIR/01-crds.yaml
+  kubectl wait --for=condition=Established crd --all
+  kubectl apply -f DIR
+
+The same settings shape the output as shape a deploy, except that nothing is
+read from a cluster:
+
+- The service account issuer is EXPECTED_JWT_ISSUER, else the GKE issuer when
+  --gcp-project-id, --gcp-cluster-location and --gcp-cluster-name are set,
+  else https://kubernetes.default.svc.
+- Without --credential-provider (ATE_CREDENTIAL_PROVIDER), the egress gateway
+  uses the bundled Kubernetes Secrets provider, {"name":"k8s.io"}, as kind
+  installs do. It runs in the cluster and needs nothing outside it.
+- None of deploy's checks against an existing install run, and the settings
+  ConfigMaps are written whole. Do not apply the output over a running
+  install.
+- The CA and JWT pool Secrets are not written. Each holds a private key
+  generated for one cluster, which would end up wherever the output goes,
+  and reapplying a rendered pool would replace it and invalidate every
+  certificate and token issued from it. The "ate-setup create" commands
+  above create each pool only when it is absent, as deploy ate-system does.
+  actor-id-ca-certs is copied from the actor-id-ca-pool Secret in the
+  cluster, so it is created there too.
+- The k8s-credential-provider namespace policy is not written. Operators
+  edit it, and reapplying it would reset their policy. Create it once with
+  kubectl apply -f manifests/egress-credential-injection/namespace-policy.yaml.
+- The bundled PostgreSQL identities are not created: deploy ate-system and
+  deploy postgres create them by running SQL in the postgres pod, which no
+  manifest does yet. The CSI drivers are not written either.
+- With an external database (ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING),
+  ate-api-server-secret-envvars is not written, so its connection strings stay
+  out of the directory. Create it before applying, with keys
+  ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING,
+  ATE_API_POSTGRES_OWNER_CONNECTION_STRING and ATE_API_POSTGRES_SCHEMA.`,
+	Args: cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return env.RenderAteSystem(cmd.Context(), env.Cfg.Resolved().String("render.outputDir"))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(renderCmd)
+	renderCmd.AddCommand(renderAteSystemCmd)
+
+	config.RegisterCommand(config.Setting{
+		Key: "render.outputDir", Env: "ATE_RENDER_OUTPUT_DIR", Flag: "output-dir",
+		Kind:     config.KindString,
+		Commands: []string{renderAteSystemPath},
+		Usage:    "Directory the rendered manifests are written to; it must be new or empty",
+	})
+	config.BindCommandFlags(renderAteSystemPath, renderAteSystemCmd.Flags())
+}

cmd/ate-setup/internal/kube/manifest.go

@@ -25,6 +25,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
+	sigsyaml "sigs.k8s.io/yaml"
 )
 
 // manifestExtensions are the file suffixes kubectl treats as manifests when
@@ -66,6 +67,20 @@ func DecodeManifestBytes(data []byte) ([]*unstructured.Unstructured, error) {
 	return DecodeManifest(bytes.NewReader(data))
 }
 
+// EncodeManifest writes objs as a multi-document YAML stream, in order.
+func EncodeManifest(objs []*unstructured.Unstructured) ([]byte, error) {
+	var out bytes.Buffer
+	for _, obj := range objs {
+		doc, err := sigsyaml.Marshal(obj.Object)
+		if err != nil {
+			return nil, fmt.Errorf("while encoding %s: %w", Describe(obj), err)
+		}
+		out.WriteString("---\n")
+		out.Write(doc)
+	}
+	return out.Bytes(), nil
+}
+
 // LoadPath reads a manifest file, or every manifest directly inside a
 // directory.
 //

cmd/ate-setup/internal/steps/apienv.go

@@ -131,7 +131,23 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error {
 			}
 		}
 	}
-	configVars := cloudSQLEnvVars(cloudsql)
+	configVars := e.apiServerConfigVars(readWriteRole, ownerRole, poolMaxConns, snapshotLocation)
+	maps.Copy(configVars, cloudSQLEnvVars(cloudsql))
+	if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil {
+		return err
+	}
+	if err := e.Kube.ApplySecret(ctx, e.Namespace(), SecretAPIEnvVars,
+		buildAPIServerEnvVars(readWriteDSN, ownerDSN, schema)); err != nil {
+		return err
+	}
+	return e.applyPostgresServerCA(ctx)
+}
+
+// apiServerConfigVars is the ate-api-server-envvars ConfigMap payload without
+// the Cloud SQL settings, which only CreateAPIServerEnvVars adds. render ate-system
+// writes this payload, and Cloud SQL is not part of the core install.
+func (e *Env) apiServerConfigVars(readWriteRole, ownerRole, poolMaxConns, snapshotLocation string) map[string]string {
+	configVars := map[string]string{}
 	configVars["ATE_API_POSTGRES_READ_WRITE_ROLE"] = readWriteRole
 	configVars["ATE_API_POSTGRES_OWNER_ROLE"] = ownerRole
 	if poolMaxConns != "" {
@@ -144,14 +160,7 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error {
 	if snapshotLocation != "" {
 		configVars["ATE_API_SNAPSHOT_LOCATION"] = snapshotLocation
 	}
-	if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil {
-		return err
-	}
-	if err := e.Kube.ApplySecret(ctx, e.Namespace(), SecretAPIEnvVars,
-		buildAPIServerEnvVars(readWriteDSN, ownerDSN, schema)); err != nil {
-		return err
-	}
-	return e.applyPostgresServerCA(ctx)
+	return configVars
 }
 
 // UpdateAPIServerEnvVars is CreateAPIServerEnvVars for an install that is not
@@ -349,6 +358,15 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er
 	if err != nil {
 		return err
 	}
+	if err := e.setAPIServerDeploymentFields(objs, hash); err != nil {
+		return err
+	}
+	return e.Kube.Apply(ctx, objs)
+}
+
+// setAPIServerDeploymentFields stamps the environment digest and sets the
+// authorization flags on the ate-api-server Deployment in objs.
+func (e *Env) setAPIServerDeploymentFields(objs []*unstructured.Unstructured, hash string) error {
 	if err := stampAPIServerEnvHash(objs, hash); err != nil {
 		return err
 	}
@@ -358,7 +376,7 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er
 	if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" {
 		log.Warnf("%s", msg)
 	}
-	return e.Kube.Apply(ctx, objs)
+	return nil
 }
 
 // apiServerAuthzFlag is the ate-api-server flag setAPIServerAuthzArgs owns.

cmd/ate-setup/internal/steps/create.go

@@ -141,11 +141,7 @@ func (e *Env) CreateAPIConfig(ctx context.Context) error {
 		return err
 	}
 
-	logLevel := e.Cfg.APILogLevel
-	if logLevel == "" {
-		logLevel = defaultAPILogLevel
-	}
-	apiConfig, err := buildAPIConfig(logLevel)
+	apiConfig, err := e.apiConfig()
 	if err != nil {
 		return err
 	}
@@ -158,6 +154,15 @@ func (e *Env) CreateAPIConfig(ctx context.Context) error {
 	})
 }
 
+// apiConfig renders apiconfig.textproto at the configured log level.
+func (e *Env) apiConfig() (string, error) {
+	logLevel := e.Cfg.APILogLevel
+	if logLevel == "" {
+		logLevel = defaultAPILogLevel
+	}
+	return buildAPIConfig(logLevel)
+}
+
 // buildAPIConfig renders apiconfig.textproto.
 func buildAPIConfig(logLevel string) (string, error) {
 	b, err := prototext.MarshalOptions{Multiline: true}.Marshal(&ateconfigpb.APIConfig{
@@ -197,6 +202,18 @@ func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error {
 // coordinates on GKE, otherwise read from the cluster's OpenID discovery
 // document, falling back to the in-cluster default.
 func (e *Env) jwtIssuer(ctx context.Context) string {
+	if issuer := e.configuredJWTIssuer(); issuer != "" {
+		return issuer
+	}
+	if issuer := e.Kube.OIDCIssuer(ctx); issuer != "" {
+		return issuer
+	}
+	return inClusterIssuer
+}
+
+// configuredJWTIssuer is the issuer the settings name, from EXPECTED_JWT_ISSUER
+// or the GKE cluster coordinates, or "" when they name none.
+func (e *Env) configuredJWTIssuer() string {
 	cfg := e.Cfg
 	if cfg.ExpectedJWTIssuer != "" {
 		return cfg.ExpectedJWTIssuer
@@ -205,10 +222,7 @@ func (e *Env) jwtIssuer(ctx context.Context) string {
 		return fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s",
 			cfg.ProjectID, cfg.ClusterLocation, cfg.ClusterName)
 	}
-	if issuer := e.Kube.OIDCIssuer(ctx); issuer != "" {
-		return issuer
-	}
-	return inClusterIssuer
+	return ""
 }
 
 // inClusterIssuer is the default issuer for a cluster that does not publish a