Skip to content

ate-setup can route several credential providers on the agentgateway egress, so your own provider runs beside the bundled one #2830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Quentin Bisson (QuentinBisson) wants to merge 1 commit into from
Closed

ate-setup can route several credential providers on the agentgateway egress, so your own provider runs beside the bundled one #2830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 74 additions & 26 deletions cmd/ate-setup/internal/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"net"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -572,32 +573,27 @@ const credentialProviderKey = "atenet.egress.credentialProvider"
// credentialProviderUsage lists the accepted --credential-provider values.
const credentialProviderUsage = `{"name":"k8s.io"} for the bundled Kubernetes Secrets provider, ` +
`{"enabled":false} to turn egress credential injection off, ` +
`or {"name":"<provider>","address":"<host>:<port>"} for a provider you deploy`
`{"name":"<provider>","address":"<host>:<port>"} for a provider you deploy, ` +
`or a JSON array of such providers, each serving its own class`

// credentialProviderSpec is the schema of the --credential-provider JSON.
// credentialProviderSpec is the schema of one --credential-provider object.
type credentialProviderSpec struct {
// Enabled defaults to true when absent.
Enabled *bool `json:"enabled"`
Name string `json:"name"`
Address string `json:"address"`
}

// CredentialProvider is the credential provider the egress gateway is pointed
// at. The zero value means injection is off.
// CredentialProvider is a credential provider the egress gateway is pointed
// at.
type CredentialProvider struct {
// Name is the host of the ate-secret:// credential URIs the provider
// serves, e.g. k8s.io.
// serves, e.g. k8s.io: its class.
Name string
// Address is the host:port the gateway dials the provider at.
Address string
}

// Enabled reports whether the gateway is given a provider.
func (p CredentialProvider) Enabled() bool { return p.Name != "" }

// Kubernetes reports whether the provider is the bundled one.
func (p CredentialProvider) Kubernetes() bool { return p.Name == K8sCredentialProviderName }

// ServerName is the SAN the gateway expects on the provider's serving
// certificate: the address without its port.
func (p CredentialProvider) ServerName() string {
Expand All @@ -607,53 +603,105 @@ func (p CredentialProvider) ServerName() string {
return p.Address
}

// CredentialProvider parses --credential-provider, a JSON object that either
// turns injection off with enabled false or names a provider and its address.
// CredentialProviders are the providers the egress gateway selects between by
// the class of a credential URI, at most one per class. Empty means injection
// is off.
type CredentialProviders []CredentialProvider

// Enabled reports whether the gateway is given a provider.
func (p CredentialProviders) Enabled() bool { return len(p) > 0 }

// Kubernetes reports whether the bundled provider is among them.
func (p CredentialProviders) Kubernetes() bool {
return slices.ContainsFunc(p, func(c CredentialProvider) bool { return c.Name == K8sCredentialProviderName })
}

// CredentialProviders parses --credential-provider: a JSON object that either
// turns injection off with enabled false or names one provider and its
// address, or a non-empty JSON array of providers with distinct names.
// Only the bundled provider has a default address. A missing value is reported here rather than in validate
// because only the deploys that render the egress gateway need it.
func (c *Config) CredentialProvider() (CredentialProvider, error) {
func (c *Config) CredentialProviders() (CredentialProviders, error) {
raw := c.CredentialProviderJSON
if raw == "" {
// Every channel is named, not just the flag: a provider set in the
// configuration document is the documented way to do it, and a reader
// told only about the flag goes looking in the wrong place.
return CredentialProvider{}, fmt.Errorf("a credential provider is required; set one of %s: %s",
return nil, fmt.Errorf("a credential provider is required; set one of %s: %s",
channelsFor(credentialProviderKey), credentialProviderUsage)
}
invalid := func(format string, args ...any) (CredentialProvider, error) {
return CredentialProvider{}, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...))
invalid := func(format string, args ...any) (CredentialProviders, error) {
return nil, fmt.Errorf("invalid --credential-provider '%s': %s", raw, fmt.Sprintf(format, args...))
}

var spec credentialProviderSpec
var specs []credentialProviderSpec
array := strings.HasPrefix(strings.TrimSpace(raw), "[")
dec := json.NewDecoder(strings.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&spec); err != nil {
var err error
if array {
err = dec.Decode(&specs)
} else {
var spec credentialProviderSpec
err = dec.Decode(&spec)
specs = []credentialProviderSpec{spec}
}
if err != nil {
return invalid("%v; want %s", err, credentialProviderUsage)
}
if _, err := dec.Token(); err != io.EOF {
return invalid("want a single JSON object")
return invalid("want a single JSON object or array")
}

if spec.Enabled != nil && !*spec.Enabled {
if array {
if len(specs) == 0 {
return invalid("the array names no provider; turn injection off with {\"enabled\":false}")
}
} else if spec := specs[0]; spec.Enabled != nil && !*spec.Enabled {
if spec.Name != "" || spec.Address != "" {
return invalid("enabled false takes no name or address")
}
return CredentialProvider{}, nil
return nil, nil
}
providers := make(CredentialProviders, 0, len(specs))
for i, spec := range specs {
where := ""
if array {
where = fmt.Sprintf("entry %d: ", i)
if spec.Enabled != nil {
return invalid("%senabled is not allowed in an array; turn injection off with {\"enabled\":false}", where)
}
}
provider, err := parseCredentialProvider(spec)
if err != nil {
return invalid("%s%v", where, err)
}
// The gateway selects a provider by the URI host alone, so two
// providers for one class would leave the choice undefined.
if slices.ContainsFunc(providers, func(p CredentialProvider) bool { return p.Name == provider.Name }) {
return invalid("%sprovider %q is named more than once", where, provider.Name)
}
providers = append(providers, provider)
}
return providers, nil
}

// parseCredentialProvider checks one enabled provider object.
func parseCredentialProvider(spec credentialProviderSpec) (CredentialProvider, error) {
if spec.Name == "" {
return invalid("name is required; want %s", credentialProviderUsage)
return CredentialProvider{}, fmt.Errorf("name is required; want %s", credentialProviderUsage)
}
// The name is the host of the credential URIs the provider serves.
if errs := validation.IsDNS1123Subdomain(spec.Name); len(errs) > 0 {
return invalid("name %q is not a valid DNS name: %s", spec.Name, strings.Join(errs, "; "))
return CredentialProvider{}, fmt.Errorf("name %q is not a valid DNS name: %s", spec.Name, strings.Join(errs, "; "))
}
if spec.Address == "" {
if spec.Name != K8sCredentialProviderName {
return invalid("a provider you deploy needs an address as host:port")
return CredentialProvider{}, fmt.Errorf("a provider you deploy needs an address as host:port")
}
spec.Address = K8sCredentialProviderAddress
} else if host, port, err := net.SplitHostPort(spec.Address); err != nil || host == "" || port == "" {
return invalid("address must be host:port, got %q", spec.Address)
return CredentialProvider{}, fmt.Errorf("address must be host:port, got %q", spec.Address)
}
return CredentialProvider{Name: spec.Name, Address: spec.Address}, nil
}
71 changes: 53 additions & 18 deletions cmd/ate-setup/internal/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1020,6 +1020,16 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
{"kubernetes address without a port", Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc"}`}},
{"other provider without an address", Options{CredentialProvider: `{"name":"vault.example.com"}`}},
{"other provider address without a port", Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc"}`}},
{"provider array empty", Options{Router: RouterAgentgateway, CredentialProvider: `[]`}},
{"provider array with a repeated name", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"k8s.io","address":"secrets.ate-system.svc:50051"}]`}},
{"provider array with enabled", Options{Router: RouterAgentgateway, CredentialProvider: `[{"enabled":true,"name":"k8s.io"}]`}},
{"provider array disabling", Options{Router: RouterAgentgateway, CredentialProvider: `[{"enabled":false}]`}},
{"provider array with a bad name", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"Vault.example.com","address":"vault.ate-system.svc:50051"}]`}},
{"provider array without an address", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com"}]`}},
{"provider array with an unknown key", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io","adress":"x:1"}]`}},
{"provider array with trailing data", Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"}] []`}},
{"provider array of strings", Options{Router: RouterAgentgateway, CredentialProvider: `["k8s.io"]`}},
{"several providers on envoy", Options{Router: RouterEnvoy, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}]`}},
} {
t.Run(tc.name, func(t *testing.T) {
if _, err := Load(tc.opts); err == nil {
Expand All @@ -1030,15 +1040,15 @@ func TestLoadRejectsInvalidValues(t *testing.T) {
}

// The provider selection is required only where the egress gateway is
// rendered, so Load accepts an absent value and CredentialProvider refuses it.
func TestCredentialProvider(t *testing.T) {
// rendered, so Load accepts an absent value and CredentialProviders refuses it.
func TestCredentialProviders(t *testing.T) {
loadEnv(t)

for _, tc := range []struct {
name string
opts Options
env map[string]string
want CredentialProvider
want CredentialProviders
wantErr bool
}{
{name: "absent is an error", opts: Options{}, wantErr: true},
Expand All @@ -1047,42 +1057,63 @@ func TestCredentialProvider(t *testing.T) {
{
name: "kubernetes on agentgateway",
opts: Options{Router: RouterAgentgateway, CredentialProvider: `{"name":"k8s.io"}`},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "another provider on agentgateway",
opts: Options{Router: RouterAgentgateway, CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}`},
want: CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"},
want: CredentialProviders{CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}},
},
{
name: "enabled true names a provider",
opts: Options{CredentialProvider: `{"enabled":true,"name":"k8s.io"}`},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "kubernetes gets the bundled address",
opts: Options{CredentialProvider: `{"name":"k8s.io"}`},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "kubernetes keeps a given address",
opts: Options{CredentialProvider: `{"name":"k8s.io","address":"secrets.ate-system.svc:443"}`},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: "secrets.ate-system.svc:443"},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: "secrets.ate-system.svc:443"}},
},
{
name: "another provider",
opts: Options{CredentialProvider: `{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}`},
want: CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"},
want: CredentialProviders{CredentialProvider{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"}},
},
{
name: "whitespace is allowed",
opts: Options{CredentialProvider: ` { "name" : "k8s.io" } `},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "an array of one",
opts: Options{CredentialProvider: `[{"name":"k8s.io"}]`},
want: CredentialProviders{{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "an array of several on agentgateway",
opts: Options{Router: RouterAgentgateway, CredentialProvider: `[{"name":"k8s.io"},{"name":"vault.example.com","address":"vault.ate-system.svc:50051"}]`},
want: CredentialProviders{
{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"},
},
},
{
name: "an array without the bundled provider",
opts: Options{Router: RouterAgentgateway, CredentialProvider: ` [ {"name":"vault.example.com","address":"vault.ate-system.svc:50051"}, {"name":"tokens.example.com","address":"tokens.ate-system.svc:50051"} ] `},
want: CredentialProviders{
{Name: "vault.example.com", Address: "vault.ate-system.svc:50051"},
{Name: "tokens.example.com", Address: "tokens.ate-system.svc:50051"},
},
},
{
name: "the environment selects too",
env: map[string]string{"ATE_CREDENTIAL_PROVIDER": `{"name":"k8s.io"}`},
want: CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress},
want: CredentialProviders{CredentialProvider{Name: K8sCredentialProviderName, Address: K8sCredentialProviderAddress}},
},
{
name: "the flag overrides the environment",
Expand All @@ -1098,21 +1129,25 @@ func TestCredentialProvider(t *testing.T) {
if err != nil {
t.Fatalf("Load() error = %v", err)
}
got, err := cfg.CredentialProvider()
got, err := cfg.CredentialProviders()
if tc.wantErr {
if err == nil {
t.Fatal("CredentialProvider() succeeded, want an error")
t.Fatal("CredentialProviders() succeeded, want an error")
}
return
}
if err != nil {
t.Fatalf("CredentialProvider() error = %v", err)
t.Fatalf("CredentialProviders() error = %v", err)
}
if !slices.Equal(got, tc.want) {
t.Errorf("CredentialProviders() = %+v, want %+v", got, tc.want)
}
if got != tc.want {
t.Errorf("CredentialProvider() = %+v, want %+v", got, tc.want)
if got.Enabled() != (len(tc.want) > 0) {
t.Errorf("Enabled() = %v, want %v", got.Enabled(), len(tc.want) > 0)
}
if got.Enabled() != (tc.want.Name != "") {
t.Errorf("Enabled() = %v, want %v", got.Enabled(), tc.want.Name != "")
wantKubernetes := slices.ContainsFunc(tc.want, func(p CredentialProvider) bool { return p.Name == K8sCredentialProviderName })
if got.Kubernetes() != wantKubernetes {
t.Errorf("Kubernetes() = %v, want %v", got.Kubernetes(), wantKubernetes)
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/ate-setup/internal/config/errors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,9 +248,9 @@ func TestValidationErrorsNameTheSupplyingChannel(t *testing.T) {
func TestCredentialProviderErrorNamesTheFileChannel(t *testing.T) {
useFixtures(t)
cfg := &Config{}
_, err := cfg.CredentialProvider()
_, err := cfg.CredentialProviders()
if err == nil {
t.Fatal("CredentialProvider() with nothing set returned no error")
t.Fatal("CredentialProviders() with nothing set returned no error")
}
for _, want := range []string{
"--credential-provider",
Expand Down
10 changes: 8 additions & 2 deletions cmd/ate-setup/internal/config/load.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,13 +458,19 @@ func validateResolved(cfg *Config, r *Resolved) error {
// configuration rather than part-way through a deploy. Whether one is
// required at all is the deploy's question, not this one's.
if cfg.CredentialProviderJSON != "" {
if _, err := cfg.CredentialProvider(); err != nil {
provider, _ := r.Value("atenet.egress.credentialProvider")
providers, err := cfg.CredentialProviders()
if err != nil {
// The parser's own message says which part is wrong and what to
// write instead; only the channel is added. Replacing it with the
// generic "must be <usage>" would drop the reason.
provider, _ := r.Value("atenet.egress.credentialProvider")
return fmt.Errorf("%w (from %s)", err, provider.From.Describe(provider.Setting))
}
// The Envoy dataplane's injector takes its one provider on flags.
if len(providers) > 1 && cfg.Router != RouterAgentgateway {
return &ConflictError{A: provider, B: dataplane,
Why: "more than one credential provider requires dataplane " + RouterAgentgateway}
}
}
// ate-api-server exits without a bootstrap owner, so catch it before
// deploying.
Expand Down
6 changes: 3 additions & 3 deletions cmd/ate-setup/internal/config/setting.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,14 +188,14 @@ var Registry = []Setting{
"Unset means kubectl-ate's ServiceAccount, system:serviceaccount:<namespace>:ate-client",
},
{
// The egress gateway's credential provider, as a JSON object. Upstream
// The egress gateway's credential providers, as JSON. Upstream
// takes it as one value on one flag; it is carried the same way here
// so the two channels accept the same text.
Key: "atenet.egress.credentialProvider", Env: "ATE_CREDENTIAL_PROVIDER",
Flag: "credential-provider", Kind: KindString,
Usage: "Credential provider the egress gateway injects credentials from, as JSON; " +
Usage: "Credential providers the egress gateway injects credentials from, as JSON; " +
"required by deploy ate-system and deploy atenet. " + credentialProviderUsage +
". A provider requires --atenet-dataplane=envoy",
". More than one provider requires --atenet-dataplane=agentgateway",
},
{
// Both pools use the read-write connection when only it is set; both
Expand Down
Loading
Loading