diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index b10ce64d40..3f3a63d615 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -122,6 +122,74 @@ apiserver, the controller, atenet, and atelet. It creates every `create` resource below on the way, so those subcommands are only needed to redo one on a running cluster. +## Render + +| `ate-setup` | `hack/install-ate.sh` | +|---|---| +| `render ate-system` | (no shell equivalent) | + +`render ate-system` writes the manifests `deploy ate-system` applies to a fresh +cluster into `--output-dir`, a new or empty directory, without contacting a +cluster. It needs `--image-repo` and `--image-tag`, and pins every image to a +digest: + +```bash +ate-setup render ate-system --kind --image-repo REPO --image-tag TAG \ + --output-dir DIR +``` + +Other clusters store snapshots on GCS today, so they also need the snapshot +location and the snapshot transfer service account: + +```bash +ate-setup render ate-system --image-repo REPO --image-tag TAG \ + --snapshot-location gs://BUCKET/ \ + --gcp-project-id PROJECT --snapshot-transfer-gsa-name NAME \ + --output-dir DIR +``` + +The files are numbered in apply order. Create the key pools after the +namespace, and apply the CRDs and wait for them before the rest, because a +later file holds a SandboxConfig: + +```bash +kubectl apply -f DIR/00-namespace.yaml +ate-setup create jwt-authority-pool +ate-setup create actor-id-ca-pool +ate-setup create actor-id-ca-certs +ate-setup create egress-mitm-ca-pool +ate-setup create podcertificate-controller-cas +kubectl apply -f DIR/01-crds.yaml +kubectl wait --for=condition=Established crd --all +kubectl apply -f DIR +``` + +The CA and JWT pool Secrets are not written: `actor-id-jwt-pool`, +`actor-id-ca-pool`, `egress-mitm-ca-pool`, and the podcertificate controller's +`service-dns-ca-pool`, `pod-identity-ca-pool` and `postgres-ca-pool`. Each holds +a private key generated for one cluster. Rendered, the keys would end up +wherever the output goes, such as a release or a GitOps repository, and every +apply would replace the pools and invalidate every certificate and token issued +from them. The `create` commands above, like `deploy ate-system`, create each +pool only when it is absent. `actor-id-ca-certs` is the root certificate copied +from the `actor-id-ca-pool` Secret in the cluster, so it is created there too. + +The `k8s-credential-provider` namespace policy is not written either. Operators +edit it, and reapplying it would reset their policy, so create it once with +`kubectl apply -f manifests/egress-credential-injection/namespace-policy.yaml`. +The bundled PostgreSQL identities are not created: `deploy ate-system` and +`deploy postgres` create them by running SQL in the postgres pod, which no +manifest does yet. The CSI drivers are not written. None of deploy's checks +against an existing install run, so do not apply the output over a running +install. The service account issuer is `EXPECTED_JWT_ISSUER`, +else the GKE issuer when the `--gcp-*` cluster settings are set, else the +in-cluster default. Without `--credential-provider`, the egress gateway uses +the bundled Kubernetes Secrets provider, `{"name":"k8s.io"}`, as kind installs +do. With an external database, `ate-api-server-secret-envvars` +is not written, so its connection strings stay out of the directory; create it +with the keys `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`, +`ATE_API_POSTGRES_OWNER_CONNECTION_STRING` and `ATE_API_POSTGRES_SCHEMA`. + ## Publish | `ate-setup` | `hack/install-ate.sh` | diff --git a/cmd/ate-setup/internal/cmd/record_test.go b/cmd/ate-setup/internal/cmd/record_test.go index 838509a5c9..36593762ba 100644 --- a/cmd/ate-setup/internal/cmd/record_test.go +++ b/cmd/ate-setup/internal/cmd/record_test.go @@ -44,6 +44,7 @@ func TestOnlySystemDeploysRecordTheirRun(t *testing.T) { {path: "delete benchmarks", want: false}, {path: "setup csi", want: false}, {path: "publish", want: false}, + {path: "render ate-system", want: false}, } { t.Run(tc.path, func(t *testing.T) { target, _, err := rootCmd.Find(strings.Fields(tc.path)) @@ -60,9 +61,9 @@ func TestOnlySystemDeploysRecordTheirRun(t *testing.T) { } } -// The publish commands build and push without a cluster, so the release job +// The publish and render commands run without a cluster, so the release job // runs them with no kubeconfig. Every other command connects to one. -func TestOnlyPublishSkipsTheCluster(t *testing.T) { +func TestOnlyPublishAndRenderSkipTheCluster(t *testing.T) { for _, tc := range []struct { path string want bool @@ -70,6 +71,7 @@ func TestOnlyPublishSkipsTheCluster(t *testing.T) { {path: "publish release-images", want: false}, {path: "publish release-binaries", want: false}, {path: "publish worker-images", want: false}, + {path: "render ate-system", want: false}, {path: "deploy ate-system", want: true}, {path: "delete ate-system", want: true}, {path: "setup csi", want: true}, diff --git a/cmd/ate-setup/internal/cmd/render.go b/cmd/ate-setup/internal/cmd/render.go new file mode 100644 index 0000000000..f7e7228f06 --- /dev/null +++ b/cmd/ate-setup/internal/cmd/render.go @@ -0,0 +1,112 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/spf13/cobra" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" +) + +// renderAteSystemPath is the command path the render settings bind to. +const renderAteSystemPath = "render ate-system" + +var renderCmd = &cobra.Command{ + Use: "render", + Short: "Write the manifests a deploy applies, without contacting a cluster", + // Rendering reads the checkout and the image registry and never a cluster, + // so it runs where there is no kubeconfig, such as a release job. + Annotations: map[string]string{clusterAnnotation: "no"}, +} + +var renderAteSystemCmd = &cobra.Command{ + Use: "ate-system", + Short: "Write the manifests deploy ate-system applies into a directory", + Long: `Write the manifests "deploy ate-system" applies to a fresh cluster into +--output-dir, one file per apply step, with every image pinned to a digest. + + ate-setup render ate-system --kind --image-repo REPO --image-tag TAG \ + --output-dir DIR + +Other clusters store snapshots on GCS today, so they also need the snapshot +location and the snapshot transfer service account: + + ate-setup render ate-system --image-repo REPO --image-tag TAG \ + --snapshot-location gs://BUCKET/ \ + --gcp-project-id PROJECT --snapshot-transfer-gsa-name NAME \ + --output-dir DIR + +The files are numbered in apply order. Create the key pools after the +namespace, and apply the CRDs and wait for them before the rest, because a +later file holds a SandboxConfig: + + kubectl apply -f DIR/00-namespace.yaml + ate-setup create jwt-authority-pool + ate-setup create actor-id-ca-pool + ate-setup create actor-id-ca-certs + ate-setup create egress-mitm-ca-pool + ate-setup create podcertificate-controller-cas + kubectl apply -f DIR/01-crds.yaml + kubectl wait --for=condition=Established crd --all + kubectl apply -f DIR + +The same settings shape the output as shape a deploy, except that nothing is +read from a cluster: + +- The service account issuer is EXPECTED_JWT_ISSUER, else the GKE issuer when + --gcp-project-id, --gcp-cluster-location and --gcp-cluster-name are set, + else https://kubernetes.default.svc. +- Without --credential-provider (ATE_CREDENTIAL_PROVIDER), the egress gateway + uses the bundled Kubernetes Secrets provider, {"name":"k8s.io"}, as kind + installs do. It runs in the cluster and needs nothing outside it. +- None of deploy's checks against an existing install run, and the settings + ConfigMaps are written whole. Do not apply the output over a running + install. +- The CA and JWT pool Secrets are not written. Each holds a private key + generated for one cluster, which would end up wherever the output goes, + and reapplying a rendered pool would replace it and invalidate every + certificate and token issued from it. The "ate-setup create" commands + above create each pool only when it is absent, as deploy ate-system does. + actor-id-ca-certs is copied from the actor-id-ca-pool Secret in the + cluster, so it is created there too. +- The k8s-credential-provider namespace policy is not written. Operators + edit it, and reapplying it would reset their policy. Create it once with + kubectl apply -f manifests/egress-credential-injection/namespace-policy.yaml. +- The bundled PostgreSQL identities are not created: deploy ate-system and + deploy postgres create them by running SQL in the postgres pod, which no + manifest does yet. The CSI drivers are not written either. +- With an external database (ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING), + ate-api-server-secret-envvars is not written, so its connection strings stay + out of the directory. Create it before applying, with keys + ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING, + ATE_API_POSTGRES_OWNER_CONNECTION_STRING and ATE_API_POSTGRES_SCHEMA.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return env.RenderAteSystem(cmd.Context(), env.Cfg.Resolved().String("render.outputDir")) + }, +} + +func init() { + rootCmd.AddCommand(renderCmd) + renderCmd.AddCommand(renderAteSystemCmd) + + config.RegisterCommand(config.Setting{ + Key: "render.outputDir", Env: "ATE_RENDER_OUTPUT_DIR", Flag: "output-dir", + Kind: config.KindString, + Commands: []string{renderAteSystemPath}, + Usage: "Directory the rendered manifests are written to; it must be new or empty", + }) + config.BindCommandFlags(renderAteSystemPath, renderAteSystemCmd.Flags()) +} diff --git a/cmd/ate-setup/internal/kube/manifest.go b/cmd/ate-setup/internal/kube/manifest.go index f79b2f70b0..c7820fde2d 100644 --- a/cmd/ate-setup/internal/kube/manifest.go +++ b/cmd/ate-setup/internal/kube/manifest.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" + sigsyaml "sigs.k8s.io/yaml" ) // manifestExtensions are the file suffixes kubectl treats as manifests when @@ -66,6 +67,20 @@ func DecodeManifestBytes(data []byte) ([]*unstructured.Unstructured, error) { return DecodeManifest(bytes.NewReader(data)) } +// EncodeManifest writes objs as a multi-document YAML stream, in order. +func EncodeManifest(objs []*unstructured.Unstructured) ([]byte, error) { + var out bytes.Buffer + for _, obj := range objs { + doc, err := sigsyaml.Marshal(obj.Object) + if err != nil { + return nil, fmt.Errorf("while encoding %s: %w", Describe(obj), err) + } + out.WriteString("---\n") + out.Write(doc) + } + return out.Bytes(), nil +} + // LoadPath reads a manifest file, or every manifest directly inside a // directory. // diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 481f72de1e..40088b39af 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -131,7 +131,23 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { } } } - configVars := cloudSQLEnvVars(cloudsql) + configVars := e.apiServerConfigVars(readWriteRole, ownerRole, poolMaxConns, snapshotLocation) + maps.Copy(configVars, cloudSQLEnvVars(cloudsql)) + if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil { + return err + } + if err := e.Kube.ApplySecret(ctx, e.Namespace(), SecretAPIEnvVars, + buildAPIServerEnvVars(readWriteDSN, ownerDSN, schema)); err != nil { + return err + } + return e.applyPostgresServerCA(ctx) +} + +// apiServerConfigVars is the ate-api-server-envvars ConfigMap payload without +// the Cloud SQL settings, which only CreateAPIServerEnvVars adds. render ate-system +// writes this payload, and Cloud SQL is not part of the core install. +func (e *Env) apiServerConfigVars(readWriteRole, ownerRole, poolMaxConns, snapshotLocation string) map[string]string { + configVars := map[string]string{} configVars["ATE_API_POSTGRES_READ_WRITE_ROLE"] = readWriteRole configVars["ATE_API_POSTGRES_OWNER_ROLE"] = ownerRole if poolMaxConns != "" { @@ -144,14 +160,7 @@ func (e *Env) CreateAPIServerEnvVars(ctx context.Context) error { if snapshotLocation != "" { configVars["ATE_API_SNAPSHOT_LOCATION"] = snapshotLocation } - if err := e.Kube.ApplyConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars, configVars); err != nil { - return err - } - if err := e.Kube.ApplySecret(ctx, e.Namespace(), SecretAPIEnvVars, - buildAPIServerEnvVars(readWriteDSN, ownerDSN, schema)); err != nil { - return err - } - return e.applyPostgresServerCA(ctx) + return configVars } // UpdateAPIServerEnvVars is CreateAPIServerEnvVars for an install that is not @@ -349,6 +358,15 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err != nil { return err } + if err := e.setAPIServerDeploymentFields(objs, hash); err != nil { + return err + } + return e.Kube.Apply(ctx, objs) +} + +// setAPIServerDeploymentFields stamps the environment digest and sets the +// authorization flags on the ate-api-server Deployment in objs. +func (e *Env) setAPIServerDeploymentFields(objs []*unstructured.Unstructured, hash string) error { if err := stampAPIServerEnvHash(objs, hash); err != nil { return err } @@ -358,7 +376,7 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } - return e.Kube.Apply(ctx, objs) + return nil } // apiServerAuthzFlag is the ate-api-server flag setAPIServerAuthzArgs owns. diff --git a/cmd/ate-setup/internal/steps/create.go b/cmd/ate-setup/internal/steps/create.go index 296a450cb5..7811e24eed 100644 --- a/cmd/ate-setup/internal/steps/create.go +++ b/cmd/ate-setup/internal/steps/create.go @@ -141,11 +141,7 @@ func (e *Env) CreateAPIConfig(ctx context.Context) error { return err } - logLevel := e.Cfg.APILogLevel - if logLevel == "" { - logLevel = defaultAPILogLevel - } - apiConfig, err := buildAPIConfig(logLevel) + apiConfig, err := e.apiConfig() if err != nil { return err } @@ -158,6 +154,15 @@ func (e *Env) CreateAPIConfig(ctx context.Context) error { }) } +// apiConfig renders apiconfig.textproto at the configured log level. +func (e *Env) apiConfig() (string, error) { + logLevel := e.Cfg.APILogLevel + if logLevel == "" { + logLevel = defaultAPILogLevel + } + return buildAPIConfig(logLevel) +} + // buildAPIConfig renders apiconfig.textproto. func buildAPIConfig(logLevel string) (string, error) { b, err := prototext.MarshalOptions{Multiline: true}.Marshal(&ateconfigpb.APIConfig{ @@ -197,6 +202,18 @@ func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error { // coordinates on GKE, otherwise read from the cluster's OpenID discovery // document, falling back to the in-cluster default. func (e *Env) jwtIssuer(ctx context.Context) string { + if issuer := e.configuredJWTIssuer(); issuer != "" { + return issuer + } + if issuer := e.Kube.OIDCIssuer(ctx); issuer != "" { + return issuer + } + return inClusterIssuer +} + +// configuredJWTIssuer is the issuer the settings name, from EXPECTED_JWT_ISSUER +// or the GKE cluster coordinates, or "" when they name none. +func (e *Env) configuredJWTIssuer() string { cfg := e.Cfg if cfg.ExpectedJWTIssuer != "" { return cfg.ExpectedJWTIssuer @@ -205,10 +222,7 @@ func (e *Env) jwtIssuer(ctx context.Context) string { return fmt.Sprintf("https://container.googleapis.com/v1/projects/%s/locations/%s/clusters/%s", cfg.ProjectID, cfg.ClusterLocation, cfg.ClusterName) } - if issuer := e.Kube.OIDCIssuer(ctx); issuer != "" { - return issuer - } - return inClusterIssuer + return "" } // inClusterIssuer is the default issuer for a cluster that does not publish a diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index bacf30a4e5..3b96322dfa 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -216,31 +216,41 @@ func (e *Env) DeployPodCertificateController(ctx context.Context) error { if err := e.EnsurePodCertificateCAs(ctx); err != nil { return err } + objs, err := e.podCertificateControllerObjects(ctx) + if err != nil { + return err + } + if err := e.Kube.Apply(ctx, objs); err != nil { + return err + } + if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, NamespacePodCert, "podcertificate-controller", e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { + return err + } + return e.WaitForPodCertificateTrustBundles(ctx) +} + +// podCertificateControllerObjects renders the podcertificate controller with +// its images resolved and WORKERS_PER_SIGNER set. +func (e *Env) podCertificateControllerObjects(ctx context.Context) ([]*unstructured.Unstructured, error) { path := e.Cfg.Manifest("pod-certificate-controller.yaml") if e.Cfg.Size10() { path = e.Cfg.Manifest("podcert-size10") } manifest, err := e.renderResolve(ctx, path) if err != nil { - return err + return nil, err } objs, err := kube.DecodeManifestBytes(manifest) if err != nil { - return err + return nil, err } if e.Cfg.PodcertWorkersPerSigner > 0 { log.Infof("Setting WORKERS_PER_SIGNER to %d", e.Cfg.PodcertWorkersPerSigner) if err := setPodcertWorkersPerSigner(objs, e.Cfg.PodcertWorkersPerSigner); err != nil { - return err + return nil, err } } - if err := e.Kube.Apply(ctx, objs); err != nil { - return err - } - if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, NamespacePodCert, "podcertificate-controller", e.Cfg.WaitTimeout(BootstrapTimeout)); err != nil { - return err - } - return e.WaitForPodCertificateTrustBundles(ctx) + return objs, nil } // setPodcertWorkersPerSigner sets WORKERS_PER_SIGNER on the diff --git a/cmd/ate-setup/internal/steps/render.go b/cmd/ate-setup/internal/steps/render.go new file mode 100644 index 0000000000..74a4b074d2 --- /dev/null +++ b/cmd/ate-setup/internal/steps/render.go @@ -0,0 +1,348 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" +) + +// renderedFile is one file RenderAteSystem writes. +type renderedFile struct { + name string + manifest []byte +} + +// RenderAteSystem writes the manifests deploy ate-system applies to a fresh +// cluster into dir, one file per apply step. The file names sort in apply +// order. The CRDs in 01-crds.yaml must be established before the rest is +// applied, because 03-sandboxconfig.yaml holds a SandboxConfig. +// +// It contacts no cluster, so it leaves out what deploy ate-system generates or +// reads there: +// +// - The CA and JWT pool Secrets: actor-id-jwt-pool, actor-id-ca-pool and +// egress-mitm-ca-pool in the install namespace, and service-dns-ca-pool, +// pod-identity-ca-pool and postgres-ca-pool in +// podcertificate-controller-system. Each holds a private key generated for +// one cluster. Written here, the keys would end up wherever the output +// goes, and every apply would replace the pools, invalidating every +// certificate and token issued from them. deploy creates each pool only +// when it is absent, which a manifest cannot express. +// - actor-id-ca-certs, the root certificate copied from the actor-id-ca-pool +// Secret in the cluster. +// - The k8s-credential-provider namespace policy. Operators edit it, so +// deploy creates it only when it is absent. +// - The bundled PostgreSQL identities, which deploy creates by running SQL +// in the postgres pod. +// - The CSI drivers. +// +// It also skips deploy's checks against an existing install. The service +// account issuer is the one configuredJWTIssuer names, or the in-cluster +// default. An external database's connection strings stay out of dir: +// ate-api-server-secret-envvars is written only for the bundled database, +// whose connection strings hold no credentials. +func (e *Env) RenderAteSystem(ctx context.Context, dir string) error { + log.Step("render_ate_system") + + if err := e.RequireCanonicalNamespace("render ate-system"); err != nil { + return err + } + if !e.Cfg.Images.IsPrebuilt() { + return errors.New("render ate-system pins published images; set --image-repo and --image-tag (ATE_IMAGE_REPO, ATE_IMAGE_TAG)") + } + provider, err := e.renderCredentialProvider() + if err != nil { + return err + } + if err := e.Cfg.RequireTransferServiceAccount(); err != nil { + return err + } + snapshotLocation, err := clusterSnapshotLocation(e.Cfg.SnapshotLocation, e.Cfg.PreviewGates) + if err != nil { + return err + } + if err := requireEmptyDir(dir); err != nil { + return err + } + + var files []renderedFile + add := func(name string, manifest []byte) { + files = append(files, renderedFile{name: name, manifest: manifest}) + } + + namespace, err := kube.ReadPath(e.Cfg.Manifest("ate-system-namespace.yaml")) + if err != nil { + return err + } + add("00-namespace.yaml", namespace) + + crds, err := e.ResolveManifest(ctx, e.Cfg.Manifest("generated")) + if err != nil { + return err + } + add("01-crds.yaml", crds) + + podcert, err := e.podCertificateControllerObjects(ctx) + if err != nil { + return err + } + if err := addObjects(add, "02-podcertificate-controller.yaml", podcert); err != nil { + return err + } + + sandboxConfig, err := e.sandboxConfigManifest() + if err != nil { + return err + } + add("03-sandboxconfig.yaml", sandboxConfig) + + system, err := e.renderSystemManifests(ctx) + if err != nil { + return err + } + systemObjs, err := kube.DecodeManifestBytes(system) + if err != nil { + return err + } + // Every workload reads ate-otel-config through a required envFrom, so it + // goes in with the settings, ahead of them, as applyOtelConfig does. + otelConfig, systemObjs, err := e.takeOtelConfig(systemObjs) + if err != nil { + return err + } + + settings, hash, err := e.apiServerSettingsObjects(snapshotLocation) + if err != nil { + return err + } + if err := addObjects(add, "04-settings.yaml", append([]*unstructured.Unstructured{otelConfig}, settings...)); err != nil { + return err + } + + if e.Cfg.PostgresReadWriteConnectionString == "" { + postgres, err := e.postgresObjects() + if err != nil { + return err + } + if err := addObjects(add, "05-postgres.yaml", postgres); err != nil { + return err + } + } + + if err := e.setAPIServerDeploymentFields(systemObjs, hash); err != nil { + return err + } + if err := addObjects(add, "06-system.yaml", systemObjs); err != nil { + return err + } + + if provider.Kubernetes() { + credentialProvider, err := e.renderResolve(ctx, e.k8sCredentialProviderPath(k8sCredentialProviderManifest)) + if err != nil { + return err + } + add("07-k8s-credential-provider.yaml", credentialProvider) + } + + egress, err := e.renderAtenetEgressManifest(ctx, provider) + if err != nil { + return err + } + add("08-atenet-egress.yaml", egress) + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating %s: %w", dir, err) + } + for _, f := range files { + path := filepath.Join(dir, f.name) + if err := os.WriteFile(path, f.manifest, 0o644); err != nil { + return fmt.Errorf("while writing %s: %w", path, err) + } + log.Infof("wrote %s", path) + } + return nil +} + +// renderCredentialProvider is the configured credential provider, or the +// bundled Kubernetes Secrets provider when none is set. The bundled provider +// runs in the install namespace and needs nothing outside the cluster, which +// is why kind installs use it too. +func (e *Env) renderCredentialProvider() (config.CredentialProvider, error) { + if e.Cfg.CredentialProviderJSON == "" { + log.Infof("no credential provider set; rendering the bundled %s provider", config.K8sCredentialProviderName) + return config.CredentialProvider{ + Name: config.K8sCredentialProviderName, + Address: config.K8sCredentialProviderAddress, + }, nil + } + return e.Cfg.CredentialProvider() +} + +// takeOtelConfig removes the ate-otel-config ConfigMap from objs and returns +// it, with the --otlp-endpoint override applyOtelEndpointOverride makes. +func (e *Env) takeOtelConfig(objs []*unstructured.Unstructured) (*unstructured.Unstructured, []*unstructured.Unstructured, error) { + for i, obj := range objs { + if obj.GetKind() != "ConfigMap" || obj.GetName() != otelConfigMap { + continue + } + if endpoint := e.Cfg.OtlpEndpoint; endpoint != "" { + if err := unstructured.SetNestedField(obj.Object, endpoint, "data", otelEndpointKey); err != nil { + return nil, nil, fmt.Errorf("while setting %s on %s: %w", otelEndpointKey, kube.Describe(obj), err) + } + } + return obj, append(objs[:i:i], objs[i+1:]...), nil + } + return nil, nil, fmt.Errorf("the system manifest has no configmap/%s", otelConfigMap) +} + +// addObjects encodes objs and adds them as the file name. +func addObjects(add func(string, []byte), name string, objs []*unstructured.Unstructured) error { + manifest, err := kube.EncodeManifest(objs) + if err != nil { + return err + } + add(name, manifest) + return nil +} + +// requireEmptyDir refuses a dir that holds files, so a file left by an earlier +// render with other settings cannot be applied along with this one. +func requireEmptyDir(dir string) error { + if dir == "" { + return errors.New("--output-dir (ATE_RENDER_OUTPUT_DIR) must be set") + } + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("while reading %s: %w", dir, err) + } + if len(entries) > 0 { + return fmt.Errorf("%s is not empty; render into a new or empty directory", dir) + } + return nil +} + +// sandboxConfigManifest is the SandboxConfig admission policy, then the +// default gVisor SandboxConfig, the order DeploySandboxConfig applies them in. +func (e *Env) sandboxConfigManifest() ([]byte, error) { + var out bytes.Buffer + for _, name := range []string{"sandboxconfig-validation.yaml", "sandboxconfig-gvisor.yaml"} { + manifest, err := kube.ReadPath(e.Cfg.Manifest(name)) + if err != nil { + return nil, err + } + out.WriteString("---\n") + out.Write(manifest) + } + return out.Bytes(), nil +} + +// apiServerSettingsObjects builds the settings ConfigMaps and Secrets +// EnsureAPIServerPrerequisites writes to a fresh cluster, from the settings +// alone, and the digest of ate-api-server's environment they give it. With an +// external database the digest leaves out the Secret, which is not rendered. +func (e *Env) apiServerSettingsObjects(snapshotLocation string) ([]*unstructured.Unstructured, string, error) { + apiConfig, err := e.apiConfig() + if err != nil { + return nil, "", err + } + issuer := e.configuredJWTIssuer() + if issuer == "" { + issuer = inClusterIssuer + } + configVars := e.apiServerConfigVars(e.Cfg.PostgresReadWriteRole, e.Cfg.PostgresOwnerRole, + e.Cfg.PostgresPoolMaxConns, snapshotLocation) + + objs := []*unstructured.Unstructured{ + e.configMapObject(ConfigMapAPIConfig, map[string]string{"apiconfig.textproto": apiConfig}), + e.configMapObject(ConfigMapAPIAuthn, map[string]string{"authentication.yaml": buildAuthenticationConfig(issuer)}), + e.configMapObject(ConfigMapAPIEnvVars, configVars), + } + + var secretData map[string][]byte + if e.Cfg.PostgresReadWriteConnectionString == "" { + readWriteDSN, ownerDSN, err := e.postgresReadWriteConnectionStrings() + if err != nil { + return nil, "", err + } + data := buildAPIServerEnvVars(readWriteDSN, ownerDSN, e.Cfg.PostgresSchemaName()) + objs = append(objs, e.secretObject(SecretAPIEnvVars, data)) + secretData = make(map[string][]byte, len(data)) + for k, v := range data { + secretData[k] = []byte(v) + } + } else { + log.Infof("%s is not rendered: it holds the external database's connection strings. "+ + "Create it before applying, with keys ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING, "+ + "ATE_API_POSTGRES_OWNER_CONNECTION_STRING and ATE_API_POSTGRES_SCHEMA", SecretAPIEnvVars) + } + + if path := e.Cfg.PostgresServerCAFile; path != "" { + pem, err := os.ReadFile(path) + if err != nil { + return nil, "", fmt.Errorf("reading ATE_API_POSTGRES_SERVER_CA_FILE: %w", err) + } + objs = append(objs, e.secretObject(SecretPostgresServerCA, map[string]string{"server-ca.pem": string(pem)})) + } + return objs, envHash(configVars, secretData), nil +} + +// configMapObject is the ConfigMap ApplyConfigMap would write. +func (e *Env) configMapObject(name string, data map[string]string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "data": stringMap(data), + }} + obj.SetName(name) + obj.SetNamespace(e.Namespace()) + return obj +} + +// secretObject is the Secret ApplySecret would write. +func (e *Env) secretObject(name string, data map[string]string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "stringData": stringMap(data), + }} + obj.SetName(name) + obj.SetNamespace(e.Namespace()) + return obj +} + +// stringMap converts data to the map type unstructured objects hold. +func stringMap(data map[string]string) map[string]any { + out := make(map[string]any, len(data)) + for k, v := range data { + out[k] = v + } + return out +} diff --git a/cmd/ate-setup/internal/steps/render_test.go b/cmd/ate-setup/internal/steps/render_test.go new file mode 100644 index 0000000000..231ac502fe --- /dev/null +++ b/cmd/ate-setup/internal/steps/render_test.go @@ -0,0 +1,276 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/images" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" +) + +const renderTestDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + +// renderTestEnv is an Env that renders with pre-built images and pins each tag +// to renderTestDigest without a registry. +func renderTestEnv(t *testing.T, edit func(*config.Config)) *Env { + t.Helper() + cfg := &config.Config{ + Root: repoRoot(t), + Router: config.RouterEnvoy, + Images: images.Source{Repo: "example.com/ate", Tag: "v0.0.0"}, + CredentialProviderJSON: `{"name":"k8s.io"}`, + IngressAllowedSPIFFEIDs: []string{"spiffe://example.com/client"}, + SnapshotLocation: "gs://bucket/", + ProjectID: "project", + TransferGSAName: "transfer", + ActorJWTIssuer: "https://idp.ate-system.svc", + PostgresReadWriteRole: config.DefaultPostgresReadWriteRole, + PostgresOwnerRole: config.DefaultPostgresOwnerRole, + } + if edit != nil { + edit(cfg) + } + digest := func(context.Context, string) (string, error) { return renderTestDigest, nil } + return &Env{Cfg: cfg, resolver: images.NewPrebuilt(cfg.Images, digest)} +} + +func TestRenderAteSystem(t *testing.T) { + allFiles := []string{ + "00-namespace.yaml", "01-crds.yaml", "02-podcertificate-controller.yaml", "03-sandboxconfig.yaml", + "04-settings.yaml", "05-postgres.yaml", "06-system.yaml", "07-k8s-credential-provider.yaml", + "08-atenet-egress.yaml", + } + tests := []struct { + name string + edit func(*config.Config) + // files are the files RenderAteSystem must write, in order. + files []string + // secret says whether ate-api-server-secret-envvars is written. + secret bool + // issuer is the issuer ate-api-authentication must name; empty means + // the in-cluster default. + issuer string + // otlpEndpoint is the endpoint ate-otel-config must carry; empty means + // the manifest's own. + otlpEndpoint string + }{ + { + name: "bundled database", + files: allFiles, + secret: true, + }, + { + name: "explicit issuer and OTLP endpoint", + edit: func(c *config.Config) { + c.ExpectedJWTIssuer = "https://issuer.example.com" + c.OtlpEndpoint = "http://collector.example.com:4317" + }, + files: allFiles, + secret: true, + issuer: "https://issuer.example.com", + otlpEndpoint: "http://collector.example.com:4317", + }, + { + name: "kind", + edit: func(c *config.Config) { c.Kind = true }, + files: allFiles, + secret: true, + }, + { + name: "agentgateway", + edit: func(c *config.Config) { c.Router = config.RouterAgentgateway }, + files: allFiles, + secret: true, + }, + { + name: "external database", + edit: func(c *config.Config) { + c.PostgresReadWriteConnectionString = "postgresql://user:password@db.example.com/substrate" + }, + files: []string{ + "00-namespace.yaml", "01-crds.yaml", "02-podcertificate-controller.yaml", "03-sandboxconfig.yaml", + "04-settings.yaml", "06-system.yaml", "07-k8s-credential-provider.yaml", "08-atenet-egress.yaml", + }, + }, + { + name: "credential provider unset renders the bundled one", + edit: func(c *config.Config) { c.CredentialProviderJSON = "" }, + files: allFiles, + secret: true, + }, + { + name: "credential injection off", + edit: func(c *config.Config) { c.CredentialProviderJSON = `{"enabled":false}` }, + files: []string{ + "00-namespace.yaml", "01-crds.yaml", "02-podcertificate-controller.yaml", "03-sandboxconfig.yaml", + "04-settings.yaml", "05-postgres.yaml", "06-system.yaml", "08-atenet-egress.yaml", + }, + secret: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := renderTestEnv(t, tc.edit) + dir := filepath.Join(t.TempDir(), "out") + if err := e.RenderAteSystem(context.Background(), dir); err != nil { + t.Fatalf("RenderAteSystem: %v", err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var files []string + for _, entry := range entries { + files = append(files, entry.Name()) + } + if diff := cmp.Diff(tc.files, files); diff != "" { + t.Errorf("files (-want +got):\n%s", diff) + } + + byName := map[string]*unstructured.Unstructured{} + fileOf := map[string]string{} + for _, name := range files { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "ko://") || strings.Contains(string(data), "${ENVOY_DATAPLANE_IMAGE}") { + t.Errorf("%s has an unresolved image reference", name) + } + for _, line := range strings.Split(string(data), "\n") { + if image, ok := strings.CutPrefix(strings.TrimSpace(line), "image: "); ok && !strings.Contains(image, "@sha256:") { + t.Errorf("%s: image %s is not pinned to a digest", name, image) + } + } + if strings.Contains(string(data), "password") { + t.Errorf("%s holds the external database's connection string", name) + } + objs, err := kube.DecodeManifestBytes(data) + if err != nil { + t.Fatalf("decoding %s: %v", name, err) + } + for _, obj := range objs { + byName[obj.GetKind()+"/"+obj.GetName()] = obj + fileOf[obj.GetKind()+"/"+obj.GetName()] = name + } + } + + // Every workload needs ate-otel-config, so it goes in with the + // settings rather than after the workloads in the system file. + otel := "ConfigMap/" + otelConfigMap + if fileOf[otel] != "04-settings.yaml" { + t.Errorf("%s is in %q, want 04-settings.yaml", otel, fileOf[otel]) + } + if tc.otlpEndpoint != "" { + got, _, _ := unstructured.NestedString(byName[otel].Object, "data", otelEndpointKey) + if got != tc.otlpEndpoint { + t.Errorf("%s %s = %q, want %q", otel, otelEndpointKey, got, tc.otlpEndpoint) + } + } + + issuer := tc.issuer + if issuer == "" { + issuer = inClusterIssuer + } + authn, _, _ := unstructured.NestedString(byName["ConfigMap/"+ConfigMapAPIAuthn].Object, "data", "authentication.yaml") + if want := buildAuthenticationConfig(issuer); authn != want { + t.Errorf("authentication.yaml = %q, want %q", authn, want) + } + + for _, name := range []string{"ConfigMap/" + ConfigMapAPIConfig, "ConfigMap/" + ConfigMapAPIAuthn, "ConfigMap/" + ConfigMapAPIEnvVars} { + if byName[name] == nil { + t.Errorf("no %s rendered", name) + } + } + secret := byName["Secret/"+SecretAPIEnvVars] + if (secret != nil) != tc.secret { + t.Fatalf("%s rendered = %t, want %t", SecretAPIEnvVars, secret != nil, tc.secret) + } + + // The digest on the pod template is the one deploy ate-system + // computes from the rendered settings objects. With an external + // database it covers only the ConfigMap. + configMap, _, _ := unstructured.NestedStringMap(byName["ConfigMap/"+ConfigMapAPIEnvVars].Object, "data") + var secretData map[string][]byte + if secret != nil { + stringData, _, _ := unstructured.NestedStringMap(secret.Object, "stringData") + secretData = map[string][]byte{} + for k, v := range stringData { + secretData[k] = []byte(v) + } + } + got, _, _ := unstructured.NestedString(byName["Deployment/ate-api-server"].Object, + "spec", "template", "metadata", "annotations", envHashAnnotation) + if want := envHash(configMap, secretData); got != want { + t.Errorf("ate-api-server %s = %q, want %q", envHashAnnotation, got, want) + } + }) + } +} + +func TestRenderAteSystemRefuses(t *testing.T) { + tests := []struct { + name string + edit func(*config.Config) + // setup prepares the output directory. + setup func(t *testing.T, dir string) + want string + }{ + { + name: "build from source", + edit: func(c *config.Config) { c.Images = images.Source{} }, + want: "--image-repo", + }, + { + name: "non-empty directory", + setup: func(t *testing.T, dir string) { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "05-postgres.yaml"), nil, 0o644); err != nil { + t.Fatal(err) + } + }, + want: "not empty", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := renderTestEnv(t, tc.edit) + dir := filepath.Join(t.TempDir(), "out") + if tc.setup != nil { + tc.setup(t, dir) + } + err := e.RenderAteSystem(context.Background(), dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("RenderAteSystem error = %v, want one mentioning %q", err, tc.want) + } + entries, _ := os.ReadDir(dir) + if tc.setup == nil && len(entries) > 0 { + t.Errorf("wrote %d files before refusing", len(entries)) + } + }) + } +}