Allow the egress gateway to use more than one credential provider

cmd/ate-setup/internal/config/config.go

@@ -25,6 +25,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -238,7 +239,7 @@ type Config struct {
 	AdditionalEgressExtprocService string
 
 	// CredentialProviderJSON is the --credential-provider value; see
-	// CredentialProvider for its schema.
+	// CredentialProviders for its schema.
 	CredentialProviderJSON string
 
 	// AnthropicAPIKey is required only by the claude-code-multiplex demo.
@@ -581,7 +582,8 @@ const credentialProviderKey = "atenet.egress.credentialProvider"
 // credentialProviderUsage lists the accepted --credential-provider values.
 const credentialProviderUsage = `{"name":"k8s.io"} for the bundled Kubernetes Secrets provider, ` +
 	`{"enabled":false} to turn egress credential injection off, ` +
-	`or {"name":"<provider>","address":"<host>:<port>"} for a provider you deploy`
+	`or {"name":"<provider>","address":"<host>:<port>"} for a provider you deploy; ` +
+	`a JSON array of provider objects selects several`
 
 // credentialProviderSpec is the schema of the --credential-provider JSON.
 type credentialProviderSpec struct {
@@ -607,6 +609,11 @@ func (p CredentialProvider) Enabled() bool { return p.Name != "" }
 // Kubernetes reports whether the provider is the bundled one.
 func (p CredentialProvider) Kubernetes() bool { return p.Name == K8sCredentialProviderName }
 
+// HasKubernetesProvider reports whether providers include the bundled one.
+func HasKubernetesProvider(providers []CredentialProvider) bool {
+	return slices.ContainsFunc(providers, CredentialProvider.Kubernetes)
+}
+
 // ServerName is the SAN the gateway expects on the provider's serving
 // certificate: the address without its port.
 func (p CredentialProvider) ServerName() string {
@@ -616,19 +623,10 @@ func (p CredentialProvider) ServerName() string {
 	return p.Address
 }
 
-// CredentialProvider parses --credential-provider, a JSON object that either
+// credentialProvider parses a --credential-provider JSON object that either
 // turns injection off with enabled false or names a provider and its address.
-// Only the bundled provider has a default address. A missing value is reported here rather than in validate
-// because only the deploys that render the egress gateway need it.
-func (c *Config) CredentialProvider() (CredentialProvider, error) {
-	raw := c.CredentialProviderJSON
-	if raw == "" {
-		// Every channel is named, not just the flag: a provider set in the
-		// configuration document is the documented way to do it, and a reader
-		// told only about the flag goes looking in the wrong place.
-		return CredentialProvider{}, fmt.Errorf("a credential provider is required; set one of %s: %s",
-			channelsFor(credentialProviderKey), credentialProviderUsage)
-	}
+// Only the bundled provider has a default address.
+func credentialProvider(raw string) (CredentialProvider, error) {
 	invalid := func(format string, args ...any) (CredentialProvider, error) {
 		return CredentialProvider{}, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...))
 	}
@@ -666,3 +664,65 @@ func (c *Config) CredentialProvider() (CredentialProvider, error) {
 	}
 	return CredentialProvider{Name: spec.Name, Address: spec.Address}, nil
 }
+
+// CredentialProviders parses --credential-provider, which is one of:
+//
+//	{"name":"k8s.io"}                                       one provider
+//	[{"name":"k8s.io"},{"name":"<name>","address":"<addr>"}]  several providers
+//	{"enabled":false}                                       none: injection off
+//
+// credentialProvider checks each provider object. A missing value is reported
+// here rather than in validate because only the deploys that render the egress
+// gateway need it.
+func (c *Config) CredentialProviders() ([]CredentialProvider, error) {
+	raw := c.CredentialProviderJSON
+	if raw == "" {
+		return nil, fmt.Errorf("a credential provider is required; set one of %s: %s",
+			channelsFor(credentialProviderKey), credentialProviderUsage)
+	}
+	invalid := func(format string, args ...any) ([]CredentialProvider, error) {
+		return nil, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...))
+	}
+	var value any
+	if err := json.Unmarshal([]byte(raw), &value); err != nil {
+		return invalid("%v; want %s", err, credentialProviderUsage)
+	}
+	var elems []any
+	switch v := value.(type) {
+	case map[string]any:
+		p, err := credentialProvider(raw)
+		if err != nil || !p.Enabled() {
+			return nil, err
+		}
+		return []CredentialProvider{p}, nil
+	case []any:
+		elems = v
+	default:
+		return invalid("want a JSON object or an array of them: %s", credentialProviderUsage)
+	}
+	if len(elems) == 0 {
+		return invalid(`the array names no provider; {"enabled":false} turns injection off`)
+	}
+	var providers []CredentialProvider
+	for _, elem := range elems {
+		if _, ok := elem.(map[string]any); !ok {
+			return invalid("array entries must be JSON objects; want %s", credentialProviderUsage)
+		}
+		obj, err := json.Marshal(elem)
+		if err != nil {
+			return invalid("%v", err)
+		}
+		p, err := credentialProvider(string(obj))
+		if err != nil {
+			return nil, err
+		}
+		if !p.Enabled() {
+			return invalid("enabled false is not allowed in an array")
+		}
+		if slices.ContainsFunc(providers, func(q CredentialProvider) bool { return q.Name == p.Name }) {
+			return invalid("provider name %q repeats", p.Name)
+		}
+		providers = append(providers, p)
+	}
+	return providers, nil
+}

cmd/ate-setup/internal/config/config_test.go

@@ -1059,6 +1059,10 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
 		{"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}},
 		{"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}},
 		{"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}},
+		{"provider array empty", Options{CredentialProvider: `[]`}},
+		{"provider array entry disabled", Options{CredentialProvider: `[{"enabled":false}]`}},
+		{"provider array with a repeated name", Options{CredentialProvider: `[{"name":"k8s.io"},{"name":"k8s.io"}]`}},
+		{"provider array entry without an address", Options{CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com"}]`}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			if _, err := Load(tc.opts); err == nil {
@@ -1069,7 +1073,7 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
 }
 
 // The provider selection is required only where the egress gateway is
-// rendered, so Load accepts an absent value and CredentialProvider refuses it.
+// rendered, so Load accepts an absent value and CredentialProviders refuses it.
 func TestCredentialProvider(t *testing.T) {
 	loadEnv(t)
 
@@ -1137,26 +1141,84 @@ func TestCredentialProvider(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Load() error = %v", err)
 			}
-			got, err := cfg.CredentialProvider()
+			got, err := cfg.CredentialProviders()
 			if tc.wantErr {
 				if err == nil {
-					t.Fatal("CredentialProvider() succeeded, want an error")
+					t.Fatal("CredentialProviders() succeeded, want an error")
 				}
 				return
 			}
 			if err != nil {
-				t.Fatalf("CredentialProvider() error = %v", err)
+				t.Fatalf("CredentialProviders() error = %v", err)
 			}
-			if got != tc.want {
-				t.Errorf("CredentialProvider() = %+v, want %+v", got, tc.want)
+			var want []CredentialProvider
+			if tc.want.Enabled() {
+				want = []CredentialProvider{tc.want}
 			}
-			if got.Enabled() != (tc.want.Name != "") {
-				t.Errorf("Enabled() = %v, want %v", got.Enabled(), tc.want.Name != "")
+			if !slices.Equal(got, want) {
+				t.Errorf("CredentialProviders() = %+v, want %+v", got, want)
 			}
 		})
 	}
 }
 
+func TestCredentialProvidersArray(t *testing.T) {
+	loadEnv(t)
+	k8s := CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}
+	for _, tc := range []struct {
+		name    string
+		raw     string
+		want    []CredentialProvider
+		wantErr string
+	}{
+		{name: "an array of one", raw: `[{"name":"k8s.io"}]`, want: []CredentialProvider{k8s}},
+		{
+			name: "several providers",
+			raw:  `[{"name":"k8s.io"},{"name":"sts.example.com","address":"sts.ate-system.svc:50051"}]`,
+			want: []CredentialProvider{k8s, {Name: "sts.example.com", Address: "sts.ate-system.svc:50051"}},
+		},
+		{name: "a null entry", raw: `[null]`, wantErr: "array entries must be JSON objects"},
+		{name: "a string entry", raw: `["k8s.io"]`, wantErr: "array entries must be JSON objects"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg, err := Load(Options{CredentialProvider: tc.raw})
+			if tc.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
+					t.Fatalf("Load() error = %v, want one containing %q", err, tc.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("Load() error = %v", err)
+			}
+			got, err := cfg.CredentialProviders()
+			if err != nil {
+				t.Fatalf("CredentialProviders() error = %v", err)
+			}
+			if !slices.Equal(got, tc.want) {
+				t.Errorf("CredentialProviders() = %+v, want %+v", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestHasKubernetesProvider(t *testing.T) {
+	k8s := CredentialProvider{Name: K8sCredentialProviderName}
+	vault := CredentialProvider{Name: "vault.example.com"}
+	for _, tc := range []struct {
+		providers []CredentialProvider
+		want      bool
+	}{
+		{nil, false},
+		{[]CredentialProvider{vault}, false},
+		{[]CredentialProvider{vault, k8s}, true},
+	} {
+		if got := HasKubernetesProvider(tc.providers); got != tc.want {
+			t.Errorf("HasKubernetesProvider(%v) = %v, want %v", tc.providers, got, tc.want)
+		}
+	}
+}
+
 func TestCredentialProviderServerName(t *testing.T) {
 	for addr, want := range map[string]string{
 		"k8s-credential-provider.ate-system.svc:50051": "k8s-credential-provider.ate-system.svc",

cmd/ate-setup/internal/config/errors_test.go

@@ -248,9 +248,9 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) {
 func TestCredentialProviderErrorNamesTheFileChannel(t *testing.T) {
 	useFixtures(t)
 	cfg := &Config{}
-	_, err := cfg.CredentialProvider()
+	_, err := cfg.CredentialProviders()
 	if err == nil {
-		t.Fatal("CredentialProvider() with nothing set returned no error")
+		t.Fatal("CredentialProviders() with nothing set returned no error")
 	}
 	for _, want := range []string{
 		"--credential-provider",

cmd/ate-setup/internal/config/load.go

@@ -459,7 +459,7 @@ func validateResolved(cfg *Config, r *Resolved) error {
 	// configuration rather than part-way through a deploy. Whether one is
 	// required at all is the deploy's question, not this one's.
 	if cfg.CredentialProviderJSON != "" {
-		if _, err := cfg.CredentialProvider(); err != nil {
+		if _, err := cfg.CredentialProviders(); err != nil {
 			// The parser's own message says which part is wrong and what to
 			// write instead; only the channel is added. Replacing it with the
 			// generic "must be <usage>" would drop the reason.

cmd/ate-setup/internal/config/setting.go

@@ -195,9 +195,9 @@ var Registry = []Setting{
 			"certificates. The podidentity signer issues spiffe://cluster.local/ns/<namespace>/sa/<service-account>",
 	},
 	{
-		// The egress gateway's credential provider, as a JSON object. Upstream
-		// takes it as one value on one flag; it is carried the same way here
-		// so the two channels accept the same text.
+		// The egress gateway's credential providers: one JSON object, or a JSON array
+		// of them for several. A repeated flag would be ideal, but a Setting holds one
+		// string on every channel, so an array is the least worst option.
 		Key: "atenet.egress.credentialProvider", Env: "ATE_CREDENTIAL_PROVIDER",
 		Flag: "credential-provider", Kind: KindString,
 		Usage: "Credential provider the egress gateway injects credentials from, as JSON; " +

cmd/ate-setup/internal/steps/deploy.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
+	"go.ate.dev/substrate/cmd/ate-setup/internal/config"
 	"go.ate.dev/substrate/cmd/ate-setup/internal/kube"
 	"go.ate.dev/substrate/cmd/ate-setup/internal/log"
 )
@@ -64,7 +65,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
 	}
 	// Fail fast on an unusable credential provider selection before touching
 	// the cluster.
-	provider, err := e.Cfg.CredentialProvider()
+	providers, err := e.Cfg.CredentialProviders()
 	if err != nil {
 		return err
 	}
@@ -159,15 +160,15 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
 	}
 	// After the podcertificate controller: the provider serves with a
 	// projected pod certificate.
-	if provider.Kubernetes() {
+	if config.HasKubernetesProvider(providers) {
 		err = e.deployK8sCredentialProvider(ctx)
 	} else {
 		err = e.removeK8sCredentialProvider(ctx)
 	}
 	if err != nil {
 		return err
 	}
-	if err := e.applyAtenetEgress(ctx, provider); err != nil {
+	if err := e.applyAtenetEgress(ctx, providers); err != nil {
 		return err
 	}
 
@@ -182,7 +183,7 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
 		rollout{kube.KindDeployment, "atenet-egress"},
 		rollout{kube.KindDaemonSet, "atelet"},
 	)
-	if provider.Kubernetes() {
+	if config.HasKubernetesProvider(providers) {
 		waits = append(waits, rollout{kube.KindDeployment, k8sCredentialProviderDeployment})
 	}
 	for _, w := range waits {
@@ -530,7 +531,7 @@ func (e *Env) ateletManifestPath() string {
 func (e *Env) DeployAtenet(ctx context.Context) error {
 	log.Step("deploy_atenet")
 
-	provider, err := e.Cfg.CredentialProvider()
+	providers, err := e.Cfg.CredentialProviders()
 	if err != nil {
 		return err
 	}
@@ -557,20 +558,20 @@ func (e *Env) DeployAtenet(ctx context.Context) error {
 	if err := e.EnsureEgressMITMCAPoolSecret(ctx); err != nil {
 		return err
 	}
-	if provider.Kubernetes() {
+	if config.HasKubernetesProvider(providers) {
 		err = e.deployK8sCredentialProvider(ctx)
 	} else {
 		err = e.removeK8sCredentialProvider(ctx)
 	}
 	if err != nil {
 		return err
 	}
-	if err := e.applyAtenetEgress(ctx, provider); err != nil {
+	if err := e.applyAtenetEgress(ctx, providers); err != nil {
 		return err
 	}
 
 	deployments := []string{"atenet-router", "atenet-egress"}
-	if provider.Kubernetes() {
+	if config.HasKubernetesProvider(providers) {
 		deployments = append(deployments, k8sCredentialProviderDeployment)
 	}
 	for _, name := range deployments {