diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index ab39897cf8..b1955d2594 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -25,6 +25,7 @@ import ( "net" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -572,9 +573,10 @@ const credentialProviderKey = "atenet.egress.credentialProvider" // credentialProviderUsage lists the accepted --credential-provider values. const credentialProviderUsage = `{"name":"k8s.io"} for the bundled Kubernetes Secrets provider, ` + `{"enabled":false} to turn egress credential injection off, ` + - `or {"name":"","address":":"} for a provider you deploy` + `{"name":"","address":":"} for a provider you deploy, ` + + `or a JSON array of such providers, each serving its own class` -// credentialProviderSpec is the schema of the --credential-provider JSON. +// credentialProviderSpec is the schema of one --credential-provider object. type credentialProviderSpec struct { // Enabled defaults to true when absent. Enabled *bool `json:"enabled"` @@ -582,22 +584,16 @@ type credentialProviderSpec struct { Address string `json:"address"` } -// CredentialProvider is the credential provider the egress gateway is pointed -// at. The zero value means injection is off. +// CredentialProvider is a credential provider the egress gateway is pointed +// at. type CredentialProvider struct { // Name is the host of the ate-secret:// credential URIs the provider - // serves, e.g. k8s.io. + // serves, e.g. k8s.io: its class. Name string // Address is the host:port the gateway dials the provider at. Address string } -// Enabled reports whether the gateway is given a provider. -func (p CredentialProvider) Enabled() bool { return p.Name != "" } - -// Kubernetes reports whether the provider is the bundled one. -func (p CredentialProvider) Kubernetes() bool { return p.Name == K8sCredentialProviderName } - // ServerName is the SAN the gateway expects on the provider's serving // certificate: the address without its port. func (p CredentialProvider) ServerName() string { @@ -607,53 +603,105 @@ func (p CredentialProvider) ServerName() string { return p.Address } -// CredentialProvider parses --credential-provider, a JSON object that either -// turns injection off with enabled false or names a provider and its address. +// CredentialProviders are the providers the egress gateway selects between by +// the class of a credential URI, at most one per class. Empty means injection +// is off. +type CredentialProviders []CredentialProvider + +// Enabled reports whether the gateway is given a provider. +func (p CredentialProviders) Enabled() bool { return len(p) > 0 } + +// Kubernetes reports whether the bundled provider is among them. +func (p CredentialProviders) Kubernetes() bool { + return slices.ContainsFunc(p, func(c CredentialProvider) bool { return c.Name == K8sCredentialProviderName }) +} + +// CredentialProviders parses --credential-provider: a JSON object that either +// turns injection off with enabled false or names one provider and its +// address, or a non-empty JSON array of providers with distinct names. // Only the bundled provider has a default address. A missing value is reported here rather than in validate // because only the deploys that render the egress gateway need it. -func (c *Config) CredentialProvider() (CredentialProvider, error) { +func (c *Config) CredentialProviders() (CredentialProviders, error) { raw := c.CredentialProviderJSON if raw == "" { // Every channel is named, not just the flag: a provider set in the // configuration document is the documented way to do it, and a reader // told only about the flag goes looking in the wrong place. - return CredentialProvider{}, fmt.Errorf("a credential provider is required; set one of %s: %s", + return nil, fmt.Errorf("a credential provider is required; set one of %s: %s", channelsFor(credentialProviderKey), credentialProviderUsage) } - invalid := func(format string, args ...any) (CredentialProvider, error) { - return CredentialProvider{}, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...)) + invalid := func(format string, args ...any) (CredentialProviders, error) { + return nil, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...)) } - var spec credentialProviderSpec + var specs []credentialProviderSpec + array := strings.HasPrefix(strings.TrimSpace(raw), "[") dec := json.NewDecoder(strings.NewReader(raw)) dec.DisallowUnknownFields() - if err := dec.Decode(&spec); err != nil { + var err error + if array { + err = dec.Decode(&specs) + } else { + var spec credentialProviderSpec + err = dec.Decode(&spec) + specs = []credentialProviderSpec{spec} + } + if err != nil { return invalid("%v; want %s", err, credentialProviderUsage) } if _, err := dec.Token(); err != io.EOF { - return invalid("want a single JSON object") + return invalid("want a single JSON object or array") } - if spec.Enabled != nil && !*spec.Enabled { + if array { + if len(specs) == 0 { + return invalid("the array names no provider; turn injection off with {\"enabled\":false}") + } + } else if spec := specs[0]; spec.Enabled != nil && !*spec.Enabled { if spec.Name != "" || spec.Address != "" { return invalid("enabled false takes no name or address") } - return CredentialProvider{}, nil + return nil, nil + } + providers := make(CredentialProviders, 0, len(specs)) + for i, spec := range specs { + where := "" + if array { + where = fmt.Sprintf("entry %d: ", i) + if spec.Enabled != nil { + return invalid("%senabled is not allowed in an array; turn injection off with {\"enabled\":false}", where) + } + } + provider, err := parseCredentialProvider(spec) + if err != nil { + return invalid("%s%v", where, err) + } + // The gateway selects a provider by the URI host alone, so two + // providers for one class would leave the choice undefined. + if slices.ContainsFunc(providers, func(p CredentialProvider) bool { return p.Name == provider.Name }) { + return invalid("%sprovider %q is named more than once", where, provider.Name) + } + providers = append(providers, provider) } + return providers, nil +} + +// parseCredentialProvider checks one enabled provider object. +func parseCredentialProvider(spec credentialProviderSpec) (CredentialProvider, error) { if spec.Name == "" { - return invalid("name is required; want %s", credentialProviderUsage) + return CredentialProvider{}, fmt.Errorf("name is required; want %s", credentialProviderUsage) } // The name is the host of the credential URIs the provider serves. if errs := validation.IsDNS1123Subdomain(spec.Name); len(errs) > 0 { - return invalid("name %q is not a valid DNS name: %s", spec.Name, strings.Join(errs, "; ")) + return CredentialProvider{}, fmt.Errorf("name %q is not a valid DNS name: %s", spec.Name, strings.Join(errs, "; ")) } if spec.Address == "" { if spec.Name != K8sCredentialProviderName { - return invalid("a provider you deploy needs an address as host:port") + return CredentialProvider{}, fmt.Errorf("a provider you deploy needs an address as host:port") } spec.Address = K8sCredentialProviderAddress } else if host, port, err := net.SplitHostPort(spec.Address); err != nil || host == "" || port == "" { - return invalid("address must be host:port, got %q", spec.Address) + return CredentialProvider{}, fmt.Errorf("address must be host:port, got %q", spec.Address) } return CredentialProvider{Name: spec.Name, Address: spec.Address}, nil } diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 99ae080bbd..fe7755d2d5 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -1020,6 +1020,16 @@ func TestLoadRejectsInvalidValues(t *testing.T) { {"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}}, {"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}}, {"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}}, + {"provider array empty", Options{Router: RouterAgentgateway, CredentialProvider: `[]`}}, + {"provider array with a repeated name", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"k8s.io","address":"secrets.ate-system.svc:50051"}]`}}, + {"provider array with enabled", Options{Router: RouterAgentgateway, CredentialProvider: `[{"enabled":true,"name":"k8s.io"}]`}}, + {"provider array disabling", Options{Router: RouterAgentgateway, CredentialProvider: `[{"enabled":false}]`}}, + {"provider array with a bad name", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"Vault.example.com","address":"vault.ate-system.svc:50051"}]`}}, + {"provider array without an address", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com"}]`}}, + {"provider array with an unknown key", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io","adress":"x:1"}]`}}, + {"provider array with trailing data", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"}] []`}}, + {"provider array of strings", Options{Router: RouterAgentgateway, CredentialProvider: `["k8s.io"]`}}, + {"several providers on envoy", Options{Router: RouterEnvoy, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}]`}}, } { t.Run(tc.name, func(t *testing.T) { if _, err := Load(tc.opts); err == nil { @@ -1030,15 +1040,15 @@ func TestLoadRejectsInvalidValues(t *testing.T) { } // The provider selection is required only where the egress gateway is -// rendered, so Load accepts an absent value and CredentialProvider refuses it. -func TestCredentialProvider(t *testing.T) { +// rendered, so Load accepts an absent value and CredentialProviders refuses it. +func TestCredentialProviders(t *testing.T) { loadEnv(t) for _, tc := range []struct { name string opts Options env map[string]string - want CredentialProvider + want CredentialProviders wantErr bool }{ {name: "absent is an error", opts: Options{}, wantErr: true}, @@ -1047,42 +1057,63 @@ func TestCredentialProvider(t *testing.T) { { name: "kubernetes on agentgateway", opts: Options{Router: RouterAgentgateway, CredentialProvider: `{"name":"k8s.io"}`}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, }, { name: "another provider on agentgateway", opts: Options{Router: RouterAgentgateway, CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}`}, - want: CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}, + want: CredentialProviders{CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}}, }, { name: "enabled true names a provider", opts: Options{CredentialProvider: `{"enabled":true,"name":"k8s.io"}`}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, }, { name: "kubernetes gets the bundled address", opts: Options{CredentialProvider: `{"name":"k8s.io"}`}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, }, { name: "kubernetes keeps a given address", opts: Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc:443"}`}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: "secrets.ate-system.svc:443"}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: "secrets.ate-system.svc:443"}}, }, { name: "another provider", opts: Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}`}, - want: CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}, + want: CredentialProviders{CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}}, }, { name: "whitespace is allowed", opts: Options{CredentialProvider: ` { "name" : "k8s.io" } `}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, + }, + { + name: "an array of one", + opts: Options{CredentialProvider: `[{"name":"k8s.io"}]`}, + want: CredentialProviders{{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, + }, + { + name: "an array of several on agentgateway", + opts: Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}]`}, + want: CredentialProviders{ + {Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + {Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}, + }, + }, + { + name: "an array without the bundled provider", + opts: Options{Router: RouterAgentgateway, CredentialProvider: ` [ {"name":"vault.example.com","address":"vault.ate-system.svc:50051"}, {"name":"tokens.example.com","address":"tokens.ate-system.svc:50051"} ] `}, + want: CredentialProviders{ + {Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}, + {Name: "tokens.example.com", Address: "tokens.ate-system.svc:50051"}, + }, }, { name: "the environment selects too", env: map[string]string{"ATE_CREDENTIAL_PROVIDER": `{"name":"k8s.io"}`}, - want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}, + want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}}, }, { name: "the flag overrides the environment", @@ -1098,21 +1129,25 @@ func TestCredentialProvider(t *testing.T) { if err != nil { t.Fatalf("Load() error = %v", err) } - got, err := cfg.CredentialProvider() + got, err := cfg.CredentialProviders() if tc.wantErr { if err == nil { - t.Fatal("CredentialProvider() succeeded, want an error") + t.Fatal("CredentialProviders() succeeded, want an error") } return } if err != nil { - t.Fatalf("CredentialProvider() error = %v", err) + t.Fatalf("CredentialProviders() error = %v", err) + } + if !slices.Equal(got, tc.want) { + t.Errorf("CredentialProviders() = %+v, want %+v", got, tc.want) } - if got != tc.want { - t.Errorf("CredentialProvider() = %+v, want %+v", got, tc.want) + if got.Enabled() != (len(tc.want) > 0) { + t.Errorf("Enabled() = %v, want %v", got.Enabled(), len(tc.want) > 0) } - if got.Enabled() != (tc.want.Name != "") { - t.Errorf("Enabled() = %v, want %v", got.Enabled(), tc.want.Name != "") + wantKubernetes := slices.ContainsFunc(tc.want, func(p CredentialProvider) bool { return p.Name == K8sCredentialProviderName }) + if got.Kubernetes() != wantKubernetes { + t.Errorf("Kubernetes() = %v, want %v", got.Kubernetes(), wantKubernetes) } }) } diff --git a/cmd/ate-setup/internal/config/errors_test.go b/cmd/ate-setup/internal/config/errors_test.go index aacc43f296..0e59c8349f 100644 --- a/cmd/ate-setup/internal/config/errors_test.go +++ b/cmd/ate-setup/internal/config/errors_test.go @@ -248,9 +248,9 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) { func TestCredentialProviderErrorNamesTheFileChannel(t *testing.T) { useFixtures(t) cfg := &Config{} - _, err := cfg.CredentialProvider() + _, err := cfg.CredentialProviders() if err == nil { - t.Fatal("CredentialProvider() with nothing set returned no error") + t.Fatal("CredentialProviders() with nothing set returned no error") } for _, want := range []string{ "--credential-provider", diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index a85f7d64d5..2d92d1437b 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -458,13 +458,19 @@ func validateResolved(cfg *Config, r *Resolved) error { // configuration rather than part-way through a deploy. Whether one is // required at all is the deploy's question, not this one's. if cfg.CredentialProviderJSON != "" { - if _, err := cfg.CredentialProvider(); err != nil { + provider, _ := r.Value("atenet.egress.credentialProvider") + providers, err := cfg.CredentialProviders() + if err != nil { // The parser's own message says which part is wrong and what to // write instead; only the channel is added. Replacing it with the // generic "must be " would drop the reason. - provider, _ := r.Value("atenet.egress.credentialProvider") return fmt.Errorf("%w (from %s)", err, provider.From.Describe(provider.Setting)) } + // The Envoy dataplane's injector takes its one provider on flags. + if len(providers) > 1 && cfg.Router != RouterAgentgateway { + return &ConflictError{A: provider, B: dataplane, + Why: "more than one credential provider requires dataplane " + RouterAgentgateway} + } } // ate-api-server exits without a bootstrap owner, so catch it before // deploying. diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 389da4d55c..a2d86702ae 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -188,14 +188,14 @@ var Registry = []Setting{ "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, { - // The egress gateway's credential provider, as a JSON object. Upstream + // The egress gateway's credential providers, as JSON. Upstream // takes it as one value on one flag; it is carried the same way here // so the two channels accept the same text. Key: "atenet.egress.credentialProvider", Env: "ATE_CREDENTIAL_PROVIDER", Flag: "credential-provider", Kind: KindString, - Usage: "Credential provider the egress gateway injects credentials from, as JSON; " + + Usage: "Credential providers the egress gateway injects credentials from, as JSON; " + "required by deploy ate-system and deploy atenet. " + credentialProviderUsage + - ". A provider requires --atenet-dataplane=envoy", + ". More than one provider requires --atenet-dataplane=agentgateway", }, { // Both pools use the read-write connection when only it is set; both diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..24d67bc284 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -64,7 +64,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { } // Fail fast on an unusable credential provider selection before touching // the cluster. - provider, err := e.Cfg.CredentialProvider() + providers, err := e.Cfg.CredentialProviders() if err != nil { return err } @@ -163,7 +163,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { } // After the podcertificate controller: the provider serves with a // projected pod certificate. - if provider.Kubernetes() { + if providers.Kubernetes() { err = e.deployK8sCredentialProvider(ctx) } else { err = e.removeK8sCredentialProvider(ctx) @@ -171,7 +171,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err != nil { return err } - if err := e.applyAtenetEgress(ctx, provider); err != nil { + if err := e.applyAtenetEgress(ctx, providers); err != nil { return err } @@ -186,7 +186,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { rollout{kube.KindDeployment, "atenet-egress"}, rollout{kube.KindDaemonSet, "atelet"}, ) - if provider.Kubernetes() { + if providers.Kubernetes() { waits = append(waits, rollout{kube.KindDeployment, k8sCredentialProviderDeployment}) } for _, w := range waits { @@ -534,7 +534,7 @@ func (e *Env) ateletManifestPath() string { func (e *Env) DeployAtenet(ctx context.Context) error { log.Step("deploy_atenet") - provider, err := e.Cfg.CredentialProvider() + providers, err := e.Cfg.CredentialProviders() if err != nil { return err } @@ -564,7 +564,7 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err := e.EnsureEgressMITMCAPoolSecret(ctx); err != nil { return err } - if provider.Kubernetes() { + if providers.Kubernetes() { err = e.deployK8sCredentialProvider(ctx) } else { err = e.removeK8sCredentialProvider(ctx) @@ -572,12 +572,12 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err != nil { return err } - if err := e.applyAtenetEgress(ctx, provider); err != nil { + if err := e.applyAtenetEgress(ctx, providers); err != nil { return err } deployments := []string{"atenet-router", "atenet-egress"} - if provider.Kubernetes() { + if providers.Kubernetes() { deployments = append(deployments, k8sCredentialProviderDeployment) } for _, name := range deployments { diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 8a2d6d3e36..d79632b6af 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -184,7 +184,7 @@ func (e *Env) atenetEgressManifestPath() string { } // renderAtenetEgressManifest produces the atenet egress manifest. -func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.CredentialProvider) ([]byte, error) { +func (e *Env) renderAtenetEgressManifest(ctx context.Context, providers config.CredentialProviders) ([]byte, error) { general := e.Cfg.AdditionalEgressExtprocService != "" if e.Cfg.Router == config.RouterAgentgateway { @@ -198,7 +198,7 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr if err != nil { return nil, err } - raw, err = patchAgentgatewayEgressInject(raw, provider) + raw, err = patchAgentgatewayEgressInject(raw, providers) if err != nil { return nil, err } @@ -223,7 +223,7 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr if err != nil { return nil, err } - raw, err = e.patchAtenetEgressInject(raw, provider) + raw, err = e.patchAtenetEgressInject(raw, providers) if err != nil { return nil, err } @@ -310,22 +310,29 @@ func (e *Env) patchEnvoyDataplaneImage(raw []byte, imageRef string) []byte { // patchAtenetEgressInject replaces the #ATE_EGRESS_INJECT_FLAGS marker in the // egress sidecar's args with the credential-provider flags, or removes it when -// injection is off, which leaves the gateway with no provider. It takes the -// manifest bytes so it can run after the general patch. -func (e *Env) patchAtenetEgressInject(raw []byte, provider config.CredentialProvider) ([]byte, error) { +// injection is off, which leaves the gateway with no provider. The sidecar +// takes one provider. It takes the manifest bytes so it can run after the +// general patch. +func (e *Env) patchAtenetEgressInject(raw []byte, providers config.CredentialProviders) ([]byte, error) { + if len(providers) > 1 { + return nil, fmt.Errorf("dataplane %s takes one credential provider, got %d", config.RouterEnvoy, len(providers)) + } var flagsBlock string - if provider.Enabled() { + if providers.Enabled() { + provider := providers[0] flagsBlock = emitEgressInjectFlags(provider.Name, provider.Address, provider.ServerName()) } return replaceManifestMarker(raw, "#ATE_EGRESS_INJECT_FLAGS", flagsBlock) } -// The HTTP and HTTPS listeners share the egress policy and its providers. -func patchAgentgatewayEgressInject(raw []byte, provider config.CredentialProvider) ([]byte, error) { +// The HTTP and HTTPS listeners share the egress policy and its providers. The +// gateway selects a provider by the authority of the credential URI. +func patchAgentgatewayEgressInject(raw []byte, providers config.CredentialProviders) ([]byte, error) { var block string - if provider.Enabled() { - block = fmt.Sprintf(`credentialProviders: -- uriAuthority: %q + if providers.Enabled() { + entries := []string{"credentialProviders:"} + for _, provider := range providers { + entries = append(entries, fmt.Sprintf(`- uriAuthority: %q target: host: %q policies: @@ -333,7 +340,9 @@ func patchAgentgatewayEgressInject(raw []byte, provider config.CredentialProvide hostname: %q cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns-ca/trust-bundle.pem`, provider.Name, provider.Address, provider.ServerName()) + root: /run/servicedns-ca/trust-bundle.pem`, provider.Name, provider.Address, provider.ServerName())) + } + block = strings.Join(entries, "\n") } return replaceManifestMarker(raw, "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS", block) } @@ -503,8 +512,8 @@ func emitAdditionalEgressExtprocCluster(address, port, serverName string) string port_value: %s`, additionalEgressExtprocCluster, serverName, serverName, additionalEgressExtprocCluster, address, port) } -func (e *Env) applyAtenetEgress(ctx context.Context, provider config.CredentialProvider) error { - manifests, err := e.renderAtenetEgressManifest(ctx, provider) +func (e *Env) applyAtenetEgress(ctx context.Context, providers config.CredentialProviders) error { + manifests, err := e.renderAtenetEgressManifest(ctx, providers) if err != nil { return err } @@ -518,7 +527,7 @@ func (e *Env) applyAtenetEgress(ctx context.Context, provider config.CredentialP return err } - if running && (e.Cfg.AdditionalEgressExtprocService != "" || provider.Enabled()) { + if running && (e.Cfg.AdditionalEgressExtprocService != "" || providers.Enabled()) { if err := e.Kube.RolloutRestartDeployment(ctx, e.Namespace(), "atenet-egress", time.Now()); err != nil { return err } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index a626532e64..555da0480c 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -59,13 +59,13 @@ func TestPatchAtenetEgressInject(t *testing.T) { } for _, tc := range []struct { - name string - provider config.CredentialProvider - want []string + name string + providers config.CredentialProviders + want []string }{ { - name: "kubernetes", - provider: config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}, + name: "kubernetes", + providers: config.CredentialProviders{config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}}, want: []string{ "--credential-provider-name=k8s.io", "--credential-provider-address=k8s-credential-provider.ate-system.svc:50051", @@ -75,8 +75,8 @@ func TestPatchAtenetEgressInject(t *testing.T) { }, }, { - name: "another provider", - provider: config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}, + name: "another provider", + providers: config.CredentialProviders{config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}, want: []string{ "--credential-provider-name=vault.example.com", "--credential-provider-address=vault.ate-system.svc:8200", @@ -86,7 +86,7 @@ func TestPatchAtenetEgressInject(t *testing.T) { {name: "off"}, } { t.Run(tc.name, func(t *testing.T) { - patched, err := env.patchAtenetEgressInject(raw, tc.provider) + patched, err := env.patchAtenetEgressInject(raw, tc.providers) if err != nil { t.Fatalf("patchAtenetEgressInject failed: %v", err) } @@ -100,7 +100,7 @@ func TestPatchAtenetEgressInject(t *testing.T) { t.Errorf("patched manifest is missing spliced flag %q", want) } } - if !tc.provider.Enabled() && strings.Contains(string(patched), "- --credential-provider-") { + if !tc.providers.Enabled() && strings.Contains(string(patched), "- --credential-provider-") { t.Error("patched manifest carries provider flags with injection off") } assertEgressManifestParses(t, patched) @@ -109,9 +109,14 @@ func TestPatchAtenetEgressInject(t *testing.T) { // The marker is what the splice keys on; a manifest without it is a // broken install, not a silent no-op. - if _, err := env.patchAtenetEgressInject([]byte("kind: ConfigMap\n"), config.CredentialProvider{}); err == nil { + if _, err := env.patchAtenetEgressInject([]byte("kind: ConfigMap\n"), nil); err == nil { t.Error("patchAtenetEgressInject accepted a manifest without the marker") } + + // The sidecar's flags name one provider, so a second has nowhere to go. + if _, err := env.patchAtenetEgressInject(raw, config.CredentialProviders{config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}, config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}); err == nil { + t.Error("patchAtenetEgressInject accepted two providers") + } } // assertEgressManifestParses checks that every document is still YAML and @@ -943,12 +948,13 @@ func TestAgentgatewayEgressOverlay(t *testing.T) { func TestRenderAgentgatewayCredentialProvider(t *testing.T) { for _, tc := range []struct { - name string - provider config.CredentialProvider + name string + providers config.CredentialProviders }{ {name: "disabled"}, - {name: "kubernetes", provider: config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}}, - {name: "custom", provider: config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}, + {name: "kubernetes", providers: config.CredentialProviders{config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}}}, + {name: "custom", providers: config.CredentialProviders{config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}}, + {name: "kubernetes and custom", providers: config.CredentialProviders{config.CredentialProvider{Name: config.K8sCredentialProviderName, Address: config.K8sCredentialProviderAddress}, config.CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:8200"}}}, } { t.Run(tc.name, func(t *testing.T) { src := images.Source{Repo: "example.com/substrate", Tag: "v1.2.3"} @@ -958,7 +964,7 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { return "sha256:" + strings.Repeat("2", 64), nil }), } - built, err := e.renderAtenetEgressManifest(t.Context(), tc.provider) + built, err := e.renderAtenetEgressManifest(t.Context(), tc.providers) if err != nil { t.Fatal(err) } @@ -1002,24 +1008,27 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { if err := policy.YNode().Decode(&got); err != nil { t.Fatal(err) } - if !tc.provider.Enabled() { + if !tc.providers.Enabled() { if len(got.Providers) != 0 { t.Errorf("%s unexpectedly has credential providers: %v", protocol, got.Providers) } continue } - want := []map[string]any{{ - "uriAuthority": tc.provider.Name, - "target": map[string]any{ - "host": tc.provider.Address, - "policies": map[string]any{"backendTLS": map[string]any{ - "hostname": tc.provider.ServerName(), - "cert": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", - "key": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", - "root": "/run/servicedns-ca/trust-bundle.pem", - }}, - }, - }} + var want []map[string]any + for _, provider := range tc.providers { + want = append(want, map[string]any{ + "uriAuthority": provider.Name, + "target": map[string]any{ + "host": provider.Address, + "policies": map[string]any{"backendTLS": map[string]any{ + "hostname": provider.ServerName(), + "cert": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", + "key": "/run/podidentity.podcert.ate.dev/credential-bundle.pem", + "root": "/run/servicedns-ca/trust-bundle.pem", + }}, + }, + }) + } if !reflect.DeepEqual(got.Providers, want) { t.Errorf("credential providers = %v, want %v", got.Providers, want) } @@ -1034,7 +1043,7 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) { func TestPatchAgentgatewayCredentialProviderRequiresOneMarker(t *testing.T) { for _, raw := range []string{"", "#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS\n#ATE_AGENTGATEWAY_CREDENTIAL_PROVIDERS\n"} { - if _, err := patchAgentgatewayEgressInject([]byte(raw), config.CredentialProvider{}); err == nil { + if _, err := patchAgentgatewayEgressInject([]byte(raw), nil); err == nil { t.Errorf("accepted a manifest without exactly one marker: %q", raw) } } @@ -1157,7 +1166,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { }{ { name: "egress", - render: func() ([]byte, error) { return e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) }, + render: func() ([]byte, error) { return e.renderAtenetEgressManifest(t.Context(), nil) }, }, { name: "router", @@ -1254,7 +1263,7 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) { return digest, nil }), } - out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + out, err := e.renderAtenetEgressManifest(t.Context(), nil) if err != nil { t.Fatalf("renderAtenetEgressManifest() error = %v", err) } @@ -1276,7 +1285,7 @@ func TestRenderAtenetEgressManifestEnvoyConcurrency(t *testing.T) { return digest, nil }), } - out, err := e.renderAtenetEgressManifest(t.Context(), config.CredentialProvider{}) + out, err := e.renderAtenetEgressManifest(t.Context(), nil) if err != nil { t.Fatalf("renderAtenetEgressManifest() error = %v", err) } diff --git a/docs/egress-credential-injection.md b/docs/egress-credential-injection.md index daa1ea9adf..4a14bd072d 100644 --- a/docs/egress-credential-injection.md +++ b/docs/egress-credential-injection.md @@ -189,12 +189,13 @@ hack/install-ate.sh --deploy-atenet --credential-provider='{"name":"k8s.io"}' ``` `--credential-provider` (or `ATE_CREDENTIAL_PROVIDER` when the flag is -absent) is a JSON object with these keys; any other key is an error: +absent) is a JSON object with these keys, or a JSON array of such objects +without `enabled`, one per provider class; any other key is an error: | Key | Value | |---|---| | `enabled` | Defaults to `true`. `false` turns injection off, and then no other key is allowed | -| `name` | Required unless `enabled` is `false`. A DNS name: `k8s.io` for the bundled Kubernetes Secrets provider, or the name of a provider you deploy, which is the host of the `ate-secret://` URIs it serves, e.g. `vault.example.com` for `ate-secret://vault.example.com/...`. A policy URI naming any other provider fails closed | +| `name` | Required unless `enabled` is `false`. A DNS name: `k8s.io` for the bundled Kubernetes Secrets provider, or the name of a provider you deploy, which is the host of the `ate-secret://` URIs it serves, e.g. `vault.example.com` for `ate-secret://vault.example.com/...`. A policy URI naming a class no provider is given for fails closed. An array names each class once | | `address` | The `host:port` the gateway dials the provider at. Required for a provider you deploy; optional with `k8s.io`, where it defaults to the bundled provider's Service, `k8s-credential-provider.ate-system.svc:50051` | For example: @@ -203,8 +204,12 @@ For example: --credential-provider='{"name":"k8s.io"}' --credential-provider='{"enabled":false}' --credential-provider='{"name":"vault.example.com","address":"vault-provider.ate-system.svc:50051"}' +--credential-provider='[{"name":"k8s.io"},{"name":"vault.example.com","address":"vault-provider.ate-system.svc:50051"}]' ``` +More than one provider requires the agentgateway dataplane; the Envoy +dataplane's injector takes one. + **2. The provider.** With `k8s.io` the install deploys the Kubernetes Secrets provider itself, from `manifests/egress-credential-injection/`, and waits for it alongside the gateway. The install applies: @@ -310,14 +315,22 @@ The provider is a plugin: any gRPC service that implements can back injection. To bring your own — reading HashiCorp Vault, Google Secret Manager, or any other secret store — implement the API under your own provider class (the URI host, e.g. `ate-secret://vault.example.com/...`), then have a -cluster admin point the gateway at it with -`--credential-provider='{"name":"vault.example.com","address":":"}'`. -A gateway currently fronts **one** provider: -a policy URI naming any other class fails closed rather than being sent to the -wrong provider. - -The gateway reads only the URI host, to confirm the URI targets the provider it -serves. Everything after the host is the provider's to interpret. +cluster admin point the gateway at it. + +The agentgateway dataplane fronts the bundled provider and any number of +further ones, each selected by its class. A provider that lives outside this +repository is deployed beside Substrate, as its own Deployment and Service, +and routed with the array form of `--credential-provider`: + +```bash +--credential-provider='[{"name":"k8s.io"},{"name":"vault.example.com","address":":"}]' +``` + +A policy URI naming a class no provider is routed for fails closed rather than +being sent to the wrong provider. The Envoy dataplane fronts one provider. + +The gateway reads only the URI host, to select the provider that serves it. +Everything after the host is the provider's to interpret. `internal/plugins/gcp-secret-manager` is such a provider for Google Cloud Secret Manager.