From 4674d36b4f9d8cfbb638bcde4825c37a70db4e00 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Tue, 27 Oct 2026 23:39:41 +0000 Subject: [PATCH 1/6] authz: let a credential plugin mint actor JWTs Add a credential_minter system role that may mint actor JWTs and nothing else. ate-api-server grants it to --credential-minter-spiffe-id, which defaults to the credential-minter ServiceAccount in the install's namespace. Access policies cannot grant it, and an empty flag grants it to no one. A credential plugin that exchanges actor JWTs for cloud access tokens can then run as its own Deployment instead of inside the atenet-egress pod. Anyone who can run a pod as an arbitrary ServiceAccount in the install's namespace can already run one as atenet-egress, so the default grants no new authority. Signed-off-by: Max Thompson --- cmd/ateapi/internal/authz/authz_util.go | 9 +++++ cmd/ateapi/internal/authz/authz_util_test.go | 30 ++++++++++++++--- cmd/ateapi/internal/authz/model.fga | 10 ++++-- cmd/ateapi/internal/authz/model_test.fga.yaml | 33 ++++++++++++++++++- cmd/ateapi/main.go | 2 ++ docs/authorization.md | 18 ++++++---- internal/installdefaults/installdefaults.go | 12 +++++++ .../installdefaults/installdefaults_test.go | 9 +++++ 8 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index a85729f4d7..3010127682 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -49,6 +49,7 @@ const ( RoleIngressRouter = "ingress_router" RoleAtelet = "atelet" RoleAPIServerSidecar = "api_server_sidecar" + RoleCredentialMinter = "credential_minter" RelationCanCreateAtespace = "can_create_atespace" RelationCanListAtespaces = "can_list_atespaces" @@ -237,6 +238,9 @@ type SystemPrincipals struct { IngressRouterSPIFFEID string AteletSPIFFEID string APIServerSidecarSPIFFEID string + // CredentialMinterSPIFFEID is optional: empty grants credential_minter to + // no one. + CredentialMinterSPIFFEID string } // New provisions the default OpenFGA store and authorization model via @@ -292,15 +296,20 @@ func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) bindings := []struct { role string spiffeID string + optional bool }{ {role: RoleController, spiffeID: system.ControllerSPIFFEID}, {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, + {role: RoleCredentialMinter, spiffeID: system.CredentialMinterSPIFFEID, optional: true}, } out := make(map[string][]string, len(bindings)) for _, b := range bindings { + if b.optional && b.spiffeID == "" { + continue + } if _, err := spiffeid.FromString(b.spiffeID); err != nil { return nil, fmt.Errorf("invalid %s SPIFFE ID %q: %w", b.role, b.spiffeID, err) } diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 77a13c26cf..0b22e7df25 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -30,6 +30,7 @@ func testSystemPrincipals() SystemPrincipals { IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), + CredentialMinterSPIFFEID: installdefaults.CredentialMinterSPIFFEID(installdefaults.SystemNamespace), } } @@ -100,21 +101,41 @@ func TestParseSystemPrincipals(t *testing.T) { IngressRouterSPIFFEID: router, AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", + CredentialMinterSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/credential-minter", }) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) } want := map[string][]string{ - "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/credential-minter": {RoleCredentialMinter}, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("parseSystemPrincipals (-want +got):\n%s", diff) } } +// The credential_minter role is optional: an empty SPIFFE ID grants it to no +// one rather than failing startup. +func TestParseSystemPrincipalsWithoutCredentialMinter(t *testing.T) { + p := testSystemPrincipals() + p.CredentialMinterSPIFFEID = "" + got, err := parseSystemPrincipals(p) + if err != nil { + t.Fatalf("parseSystemPrincipals failed: %v", err) + } + for user, roles := range got { + for _, role := range roles { + if role == RoleCredentialMinter { + t.Errorf("%s holds %s with an empty credential minter SPIFFE ID", user, RoleCredentialMinter) + } + } + } +} + func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { defaults := testSystemPrincipals() for _, tc := range []struct { @@ -127,6 +148,7 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, + {"bare credential minter ServiceAccount name", func(p *SystemPrincipals) { p.CredentialMinterSPIFFEID = "credential-minter" }, "invalid credential_minter SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" diff --git a/cmd/ateapi/internal/authz/model.fga b/cmd/ateapi/internal/authz/model.fga index 1d42a82e3e..fc13094f94 100644 --- a/cmd/ateapi/internal/authz/model.fga +++ b/cmd/ateapi/internal/authz/model.fga @@ -52,6 +52,10 @@ type global define ingress_router: [user] define atelet: [user] define api_server_sidecar: [user] + # An out-of-tree credential plugin that turns actor JWTs into other + # credentials, such as an RFC 8693 token exchange. It may mint actor JWTs + # and nothing else. + define credential_minter: [user] define can_create_atespace: owner define can_list_atespaces: viewer @@ -85,6 +89,7 @@ type atespace define ingress_router: ingress_router from parent_global define atelet: atelet from parent_global define api_server_sidecar: api_server_sidecar from parent_global + define credential_minter: credential_minter from parent_global # --- 2. Policy Governance --- define can_create_access_policy: owner @@ -167,8 +172,9 @@ type actor define can_mint_worker_actor_credential: host_node # Credentials asserting the actor's own identity. Only the egress gateway - # mints them; an editor holding one could impersonate the actor. - define can_mint_actor_jwt: egress_gateway from parent_atespace + # mints them, plus a credential plugin for actor JWTs; an editor holding + # one could impersonate the actor. + define can_mint_actor_jwt: egress_gateway from parent_atespace or credential_minter from parent_atespace define can_mint_actor_certificate: egress_gateway from parent_atespace # Data Plane Ingress: Routers (or editors) can resume diff --git a/cmd/ateapi/internal/authz/model_test.fga.yaml b/cmd/ateapi/internal/authz/model_test.fga.yaml index 02741ca1db..a614f9b977 100644 --- a/cmd/ateapi/internal/authz/model_test.fga.yaml +++ b/cmd/ateapi/internal/authz/model_test.fga.yaml @@ -93,6 +93,9 @@ tuples: - user: user:sys-api-server-sidecar relation: api_server_sidecar object: global:root + - user: user:sys-credential-minter + relation: credential_minter + object: global:root tests: - name: 1. Global Superadmin & Viewer Propagation @@ -561,7 +564,7 @@ tests: can_create_tag: false can_list_tags: false - - name: 9. Actor Credentials (Egress Gateway Only) + - name: 9. Actor Credentials (Egress Gateway, and Credential Minter for JWTs) check: - user: user:carol # Global owner object: actor:runner-1 @@ -578,3 +581,31 @@ tests: assertions: can_mint_actor_jwt: false can_mint_actor_certificate: false + - user: user:sys-credential-minter # mints actor JWTs and nothing else + object: actor:runner-1 + assertions: + can_mint_actor_jwt: true + can_mint_actor_certificate: false + can_get: false + can_update: false + can_delete: false + can_get_egress_policy: false + can_update_egress_policy: false + can_resume: false + can_suspend: false + - user: user:sys-credential-minter + object: atespace:space-1 + assertions: + viewer: false + can_get: false + can_list_actors: false + can_create_actor: false + can_create_tag: false + - user: user:sys-credential-minter + object: global:root + assertions: + can_list_actors: false + can_list_workers: false + can_create_worker: false + can_create_atespace: false + can_get_access_policy: false diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 8e9d74d6f8..6a8b8d0f36 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -97,6 +97,7 @@ var ( egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") + credentialMinterSPIFFEID = pflag.String("credential-minter-spiffe-id", installdefaults.CredentialMinterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of an out-of-tree credential plugin. Holds the credential_minter role, which mints actor JWTs and nothing else. Empty grants the role to no one.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") @@ -261,6 +262,7 @@ func main() { IngressRouterSPIFFEID: *ingressRouterSPIFFEID, AteletSPIFFEID: ateletSPIFFEID, APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, + CredentialMinterSPIFFEID: *credentialMinterSPIFFEID, }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize OpenFGA authz", err) diff --git a/docs/authorization.md b/docs/authorization.md index ab5cb19cd5..a9ea7f39c6 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -128,9 +128,10 @@ one **atespace**. Substrate's components hold roles that access policies cannot grant, and only through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, -`--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id` and -`--api-server-sidecar-spiffe-id` flags bind them, defaulting to the install's; -`atelet`'s comes from `--atelet-service-account`. +`--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id`, +`--api-server-sidecar-spiffe-id` and `--credential-minter-spiffe-id` flags bind +them, defaulting to the install's; `atelet`'s comes from +`--atelet-service-account`. | Role | Held by | Allows | |---|---|---| @@ -139,9 +140,14 @@ through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | - -Only `egress_gateway` can mint actor credentials: not even a global owner can, -since a credential lets its holder act as the actor. +| `credential_minter` | an out-of-tree credential plugin running as the `credential-minter` ServiceAccount in the install's namespace. Substrate deploys none. | Mint actor JWTs, for example to exchange them for a cloud access token | + +Only `egress_gateway` can mint actor certificates, and only `egress_gateway` +and `credential_minter` can mint actor JWTs: not even a global owner can, since +a credential lets its holder act as the actor. Anyone who can run a pod as an +arbitrary ServiceAccount in the install's namespace can already run one as +`atenet-egress`, so `credential_minter` grants no one new authority. Set +`--credential-minter-spiffe-id=""` to grant it to no one. ## Checks per RPC diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 2402e7986a..8907f55594 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -52,6 +52,12 @@ const ( ControllerServiceAccount = "ate-controller" APIServerServiceAccount = "ate-api-server" + // CredentialMinterServiceAccount is the ServiceAccount an out-of-tree + // credential plugin runs as to mint actor JWTs. Substrate creates no + // ServiceAccount or Pod with this name; an operator who deploys such a + // plugin in the system namespace opts in by using it. + CredentialMinterServiceAccount = "credential-minter" + // TransferServiceAccountName is the default name of the Google service // account snapshot transfers run as on GCS, which tools/setup-gcp creates. TransferServiceAccountName = "ate-snapshot-transfer" @@ -129,6 +135,12 @@ func ControllerSPIFFEID(namespace string) string { return SPIFFEID(namespace, ControllerServiceAccount) } +// CredentialMinterSPIFFEID returns the SPIFFE ID a credential plugin running +// as CredentialMinterServiceAccount in namespace presents. +func CredentialMinterSPIFFEID(namespace string) string { + return SPIFFEID(namespace, CredentialMinterServiceAccount) +} + // APIServerSPIFFEID returns the SPIFFE ID the ate-api-server pod presents when // it runs in namespace. Its objectstore-plugin sidecar shares it. func APIServerSPIFFEID(namespace string) string { diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go index 2329431e20..612d97840d 100644 --- a/internal/installdefaults/installdefaults_test.go +++ b/internal/installdefaults/installdefaults_test.go @@ -88,6 +88,15 @@ func TestAPIServerSPIFFEID(t *testing.T) { } } +func TestCredentialMinterSPIFFEID(t *testing.T) { + // Matches the --credential-minter-spiffe-id default ate-api-server ships + // with. Out-of-tree credential plugins run as this ServiceAccount. + const want = "spiffe://cluster.local/ns/ate-system/sa/credential-minter" + if got := CredentialMinterSPIFFEID(SystemNamespace); got != want { + t.Errorf("CredentialMinterSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) + } +} + func TestAPIServiceHost(t *testing.T) { for _, tt := range []struct{ namespace, want string }{ {"ate-system", "api.ate-system.svc"}, From a4f1aaa0ae27261ed8153d99e2f14a60c39e07e3 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Tue, 27 Oct 2026 17:10:39 -0700 Subject: [PATCH 2/6] ateapi: accept a list of credential minter SPIFFE IDs Replace --credential-minter-spiffe-id with --credential-minter-spiffe-ids so more than one credential plugin can hold the credential_minter role. The default is still the credential-minter ServiceAccount in the install's namespace, and an empty list grants the role to no one. Signed-off-by: Max Thompson --- cmd/ateapi/internal/authz/authz_util.go | 19 ++++++------ cmd/ateapi/internal/authz/authz_util_test.go | 31 ++++++++++++------- cmd/ateapi/main.go | 22 ++++++------- docs/authorization.md | 8 ++--- .../installdefaults/installdefaults_test.go | 2 +- 5 files changed, 44 insertions(+), 38 deletions(-) diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index 3010127682..b358fc33a4 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -238,9 +238,9 @@ type SystemPrincipals struct { IngressRouterSPIFFEID string AteletSPIFFEID string APIServerSidecarSPIFFEID string - // CredentialMinterSPIFFEID is optional: empty grants credential_minter to - // no one. - CredentialMinterSPIFFEID string + // CredentialMinterSPIFFEIDs may be empty, which grants credential_minter + // to no one. + CredentialMinterSPIFFEIDs []string } // New provisions the default OpenFGA store and authorization model via @@ -293,23 +293,22 @@ func parseBootstrapOwners(ids []string) (map[string]struct{}, error) { // parseSystemPrincipals validates system's SPIFFE IDs and maps each OpenFGA // user string to the system component roles it holds. func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) { - bindings := []struct { + type binding struct { role string spiffeID string - optional bool - }{ + } + bindings := []binding{ {role: RoleController, spiffeID: system.ControllerSPIFFEID}, {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, - {role: RoleCredentialMinter, spiffeID: system.CredentialMinterSPIFFEID, optional: true}, + } + for _, id := range system.CredentialMinterSPIFFEIDs { + bindings = append(bindings, binding{role: RoleCredentialMinter, spiffeID: id}) } out := make(map[string][]string, len(bindings)) for _, b := range bindings { - if b.optional && b.spiffeID == "" { - continue - } if _, err := spiffeid.FromString(b.spiffeID); err != nil { return nil, fmt.Errorf("invalid %s SPIFFE ID %q: %w", b.role, b.spiffeID, err) } diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index 0b22e7df25..eb2d861677 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -25,12 +25,12 @@ import ( func testSystemPrincipals() SystemPrincipals { return SystemPrincipals{ - ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), - IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), - AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), - APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), - CredentialMinterSPIFFEID: installdefaults.CredentialMinterSPIFFEID(installdefaults.SystemNamespace), + ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), + CredentialMinterSPIFFEIDs: []string{installdefaults.CredentialMinterSPIFFEID(installdefaults.SystemNamespace)}, } } @@ -101,7 +101,10 @@ func TestParseSystemPrincipals(t *testing.T) { IngressRouterSPIFFEID: router, AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", - CredentialMinterSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/credential-minter", + CredentialMinterSPIFFEIDs: []string{ + "spiffe://cluster.local/ns/ate-system/sa/credential-minter", + "spiffe://cluster.local/ns/plugins/sa/token-exchange", + }, }) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) @@ -112,17 +115,18 @@ func TestParseSystemPrincipals(t *testing.T) { "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, "user:spiffe%3A//cluster.local/ns/ate-system/sa/credential-minter": {RoleCredentialMinter}, + "user:spiffe%3A//cluster.local/ns/plugins/sa/token-exchange": {RoleCredentialMinter}, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("parseSystemPrincipals (-want +got):\n%s", diff) } } -// The credential_minter role is optional: an empty SPIFFE ID grants it to no -// one rather than failing startup. +// The credential_minter role is optional: an empty list grants it to no one +// rather than failing startup. func TestParseSystemPrincipalsWithoutCredentialMinter(t *testing.T) { p := testSystemPrincipals() - p.CredentialMinterSPIFFEID = "" + p.CredentialMinterSPIFFEIDs = nil got, err := parseSystemPrincipals(p) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) @@ -130,7 +134,7 @@ func TestParseSystemPrincipalsWithoutCredentialMinter(t *testing.T) { for user, roles := range got { for _, role := range roles { if role == RoleCredentialMinter { - t.Errorf("%s holds %s with an empty credential minter SPIFFE ID", user, RoleCredentialMinter) + t.Errorf("%s holds %s with no credential minter SPIFFE IDs", user, RoleCredentialMinter) } } } @@ -148,7 +152,10 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, - {"bare credential minter ServiceAccount name", func(p *SystemPrincipals) { p.CredentialMinterSPIFFEID = "credential-minter" }, "invalid credential_minter SPIFFE ID"}, + {"bare credential minter ServiceAccount name", func(p *SystemPrincipals) { p.CredentialMinterSPIFFEIDs = []string{"credential-minter"} }, "invalid credential_minter SPIFFE ID"}, + {"empty credential minter SPIFFE ID in list", func(p *SystemPrincipals) { + p.CredentialMinterSPIFFEIDs = []string{defaults.CredentialMinterSPIFFEIDs[0], ""} + }, "invalid credential_minter SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 6a8b8d0f36..09ea95fded 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -93,11 +93,11 @@ var ( authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. Required: at least one.") // These SPIFFE IDs hold the matching system component role when they call // over mTLS. - controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") - ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") - apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") - credentialMinterSPIFFEID = pflag.String("credential-minter-spiffe-id", installdefaults.CredentialMinterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of an out-of-tree credential plugin. Holds the credential_minter role, which mints actor JWTs and nothing else. Empty grants the role to no one.") + controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") + egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") + ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") + apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") + credentialMinterSPIFFEIDs = pflag.StringSlice("credential-minter-spiffe-ids", []string{installdefaults.CredentialMinterSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of out-of-tree credential plugins. Comma-separated or repeated. Each holds the credential_minter role, which mints actor JWTs and nothing else. Empty grants the role to no one.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") @@ -257,12 +257,12 @@ func main() { } defer fgaServer.Close() authorizer, policyManager, err := authz.New(shutdownCtx, pool, fgaServer, *authzBootstrapOwners, authz.SystemPrincipals{ - ControllerSPIFFEID: *controllerSPIFFEID, - EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, - IngressRouterSPIFFEID: *ingressRouterSPIFFEID, - AteletSPIFFEID: ateletSPIFFEID, - APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, - CredentialMinterSPIFFEID: *credentialMinterSPIFFEID, + ControllerSPIFFEID: *controllerSPIFFEID, + EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, + IngressRouterSPIFFEID: *ingressRouterSPIFFEID, + AteletSPIFFEID: ateletSPIFFEID, + APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, + CredentialMinterSPIFFEIDs: *credentialMinterSPIFFEIDs, }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize OpenFGA authz", err) diff --git a/docs/authorization.md b/docs/authorization.md index a9ea7f39c6..640b05d7c4 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -129,7 +129,7 @@ one **atespace**. Substrate's components hold roles that access policies cannot grant, and only through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, `--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id`, -`--api-server-sidecar-spiffe-id` and `--credential-minter-spiffe-id` flags bind +`--api-server-sidecar-spiffe-id` and `--credential-minter-spiffe-ids` flags bind them, defaulting to the install's; `atelet`'s comes from `--atelet-service-account`. @@ -140,14 +140,14 @@ them, defaulting to the install's; `atelet`'s comes from | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | -| `credential_minter` | an out-of-tree credential plugin running as the `credential-minter` ServiceAccount in the install's namespace. Substrate deploys none. | Mint actor JWTs, for example to exchange them for a cloud access token | +| `credential_minter` | out-of-tree credential plugins, by default one running as the `credential-minter` ServiceAccount in the install's namespace. Substrate deploys none. | Mint actor JWTs, for example to exchange them for a cloud access token | Only `egress_gateway` can mint actor certificates, and only `egress_gateway` and `credential_minter` can mint actor JWTs: not even a global owner can, since a credential lets its holder act as the actor. Anyone who can run a pod as an arbitrary ServiceAccount in the install's namespace can already run one as -`atenet-egress`, so `credential_minter` grants no one new authority. Set -`--credential-minter-spiffe-id=""` to grant it to no one. +`atenet-egress`, so the default `credential_minter` binding grants no one new +authority. Set `--credential-minter-spiffe-ids=""` to grant it to no one. ## Checks per RPC diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go index 612d97840d..06e2046519 100644 --- a/internal/installdefaults/installdefaults_test.go +++ b/internal/installdefaults/installdefaults_test.go @@ -89,7 +89,7 @@ func TestAPIServerSPIFFEID(t *testing.T) { } func TestCredentialMinterSPIFFEID(t *testing.T) { - // Matches the --credential-minter-spiffe-id default ate-api-server ships + // Matches the --credential-minter-spiffe-ids default ate-api-server ships // with. Out-of-tree credential plugins run as this ServiceAccount. const want = "spiffe://cluster.local/ns/ate-system/sa/credential-minter" if got := CredentialMinterSPIFFEID(SystemNamespace); got != want { From 3c0f22d9d1f118e2512ce687b05e1ca5757a6631 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Tue, 27 Oct 2026 17:23:43 -0700 Subject: [PATCH 3/6] ateapi: accept more than one egress gateway SPIFFE ID --egress-gateway-spiffe-id now takes a list, comma-separated or repeated, and grants egress_gateway to every ID in it. A credential plugin that exchanges actor JWTs for cloud access tokens can then run as its own Deployment instead of inside the atenet-egress pod. The default is still atenet-egress alone, and an empty list fails startup. Signed-off-by: Max Thompson --- cmd/ateapi/internal/authz/authz_util.go | 14 ++--- cmd/ateapi/internal/authz/authz_util_test.go | 57 ++++++------------- cmd/ateapi/internal/authz/interceptor_test.go | 2 +- cmd/ateapi/internal/authz/model.fga | 10 +--- cmd/ateapi/internal/authz/model_test.fga.yaml | 33 +---------- .../internal/store/atepg/common_test.go | 2 +- .../internal/store/storetest/storetest.go | 2 +- cmd/ateapi/main.go | 20 +++---- docs/authorization.md | 20 +++---- internal/installdefaults/installdefaults.go | 12 ---- .../installdefaults/installdefaults_test.go | 9 --- 11 files changed, 44 insertions(+), 137 deletions(-) diff --git a/cmd/ateapi/internal/authz/authz_util.go b/cmd/ateapi/internal/authz/authz_util.go index b358fc33a4..1e1734d6cd 100644 --- a/cmd/ateapi/internal/authz/authz_util.go +++ b/cmd/ateapi/internal/authz/authz_util.go @@ -49,7 +49,6 @@ const ( RoleIngressRouter = "ingress_router" RoleAtelet = "atelet" RoleAPIServerSidecar = "api_server_sidecar" - RoleCredentialMinter = "credential_minter" RelationCanCreateAtespace = "can_create_atespace" RelationCanListAtespaces = "can_list_atespaces" @@ -234,13 +233,10 @@ func EnsureStoreAndModel(ctx context.Context, pool *pgxpool.Pool, fgaServer *ser // every check, independent of any stored AccessPolicy. type SystemPrincipals struct { ControllerSPIFFEID string - EgressGatewaySPIFFEID string + EgressGatewaySPIFFEIDs []string IngressRouterSPIFFEID string AteletSPIFFEID string APIServerSidecarSPIFFEID string - // CredentialMinterSPIFFEIDs may be empty, which grants credential_minter - // to no one. - CredentialMinterSPIFFEIDs []string } // New provisions the default OpenFGA store and authorization model via @@ -293,19 +289,21 @@ func parseBootstrapOwners(ids []string) (map[string]struct{}, error) { // parseSystemPrincipals validates system's SPIFFE IDs and maps each OpenFGA // user string to the system component roles it holds. func parseSystemPrincipals(system SystemPrincipals) (map[string][]string, error) { + if len(system.EgressGatewaySPIFFEIDs) == 0 { + return nil, fmt.Errorf("no %s SPIFFE ID", RoleEgressGateway) + } type binding struct { role string spiffeID string } bindings := []binding{ {role: RoleController, spiffeID: system.ControllerSPIFFEID}, - {role: RoleEgressGateway, spiffeID: system.EgressGatewaySPIFFEID}, {role: RoleIngressRouter, spiffeID: system.IngressRouterSPIFFEID}, {role: RoleAtelet, spiffeID: system.AteletSPIFFEID}, {role: RoleAPIServerSidecar, spiffeID: system.APIServerSidecarSPIFFEID}, } - for _, id := range system.CredentialMinterSPIFFEIDs { - bindings = append(bindings, binding{role: RoleCredentialMinter, spiffeID: id}) + for _, id := range system.EgressGatewaySPIFFEIDs { + bindings = append(bindings, binding{role: RoleEgressGateway, spiffeID: id}) } out := make(map[string][]string, len(bindings)) for _, b := range bindings { diff --git a/cmd/ateapi/internal/authz/authz_util_test.go b/cmd/ateapi/internal/authz/authz_util_test.go index eb2d861677..da08a0887f 100644 --- a/cmd/ateapi/internal/authz/authz_util_test.go +++ b/cmd/ateapi/internal/authz/authz_util_test.go @@ -25,12 +25,11 @@ import ( func testSystemPrincipals() SystemPrincipals { return SystemPrincipals{ - ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), - IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), - AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), - APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), - CredentialMinterSPIFFEIDs: []string{installdefaults.CredentialMinterSPIFFEID(installdefaults.SystemNamespace)}, + ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, + IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), + AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), + APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), } } @@ -97,49 +96,26 @@ func TestParseSystemPrincipals(t *testing.T) { const router = "spiffe://cluster.local/ns/ate-system/sa/atenet-router" got, err := parseSystemPrincipals(SystemPrincipals{ ControllerSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-controller", - EgressGatewaySPIFFEID: router, + EgressGatewaySPIFFEIDs: []string{router, "spiffe://cluster.local/ns/plugins/sa/token-exchange"}, IngressRouterSPIFFEID: router, AteletSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/atelet", APIServerSidecarSPIFFEID: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server", - CredentialMinterSPIFFEIDs: []string{ - "spiffe://cluster.local/ns/ate-system/sa/credential-minter", - "spiffe://cluster.local/ns/plugins/sa/token-exchange", - }, }) if err != nil { t.Fatalf("parseSystemPrincipals failed: %v", err) } want := map[string][]string{ - "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleEgressGateway, RoleIngressRouter}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, - "user:spiffe%3A//cluster.local/ns/ate-system/sa/credential-minter": {RoleCredentialMinter}, - "user:spiffe%3A//cluster.local/ns/plugins/sa/token-exchange": {RoleCredentialMinter}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-controller": {RoleController}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atenet-router": {RoleIngressRouter, RoleEgressGateway}, + "user:spiffe%3A//cluster.local/ns/plugins/sa/token-exchange": {RoleEgressGateway}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/atelet": {RoleAtelet}, + "user:spiffe%3A//cluster.local/ns/ate-system/sa/ate-api-server": {RoleAPIServerSidecar}, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("parseSystemPrincipals (-want +got):\n%s", diff) } } -// The credential_minter role is optional: an empty list grants it to no one -// rather than failing startup. -func TestParseSystemPrincipalsWithoutCredentialMinter(t *testing.T) { - p := testSystemPrincipals() - p.CredentialMinterSPIFFEIDs = nil - got, err := parseSystemPrincipals(p) - if err != nil { - t.Fatalf("parseSystemPrincipals failed: %v", err) - } - for user, roles := range got { - for _, role := range roles { - if role == RoleCredentialMinter { - t.Errorf("%s holds %s with no credential minter SPIFFE IDs", user, RoleCredentialMinter) - } - } - } -} - func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { defaults := testSystemPrincipals() for _, tc := range []struct { @@ -148,17 +124,16 @@ func TestNewRejectsInvalidSystemPrincipals(t *testing.T) { wantErr string }{ {"empty controller SPIFFE ID", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "" }, "invalid controller SPIFFE ID"}, - {"empty egress gateway SPIFFE ID", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEID = "" }, "invalid egress_gateway SPIFFE ID"}, + {"no egress gateway SPIFFE IDs", func(p *SystemPrincipals) { p.EgressGatewaySPIFFEIDs = nil }, "no egress_gateway SPIFFE ID"}, + {"empty egress gateway SPIFFE ID in list", func(p *SystemPrincipals) { + p.EgressGatewaySPIFFEIDs = []string{defaults.EgressGatewaySPIFFEIDs[0], ""} + }, "invalid egress_gateway SPIFFE ID"}, {"empty ingress router SPIFFE ID", func(p *SystemPrincipals) { p.IngressRouterSPIFFEID = "" }, "invalid ingress_router SPIFFE ID"}, {"empty atelet SPIFFE ID", func(p *SystemPrincipals) { p.AteletSPIFFEID = "" }, "invalid atelet SPIFFE ID"}, {"empty api server sidecar SPIFFE ID", func(p *SystemPrincipals) { p.APIServerSidecarSPIFFEID = "" }, "invalid api_server_sidecar SPIFFE ID"}, - {"bare credential minter ServiceAccount name", func(p *SystemPrincipals) { p.CredentialMinterSPIFFEIDs = []string{"credential-minter"} }, "invalid credential_minter SPIFFE ID"}, - {"empty credential minter SPIFFE ID in list", func(p *SystemPrincipals) { - p.CredentialMinterSPIFFEIDs = []string{defaults.CredentialMinterSPIFFEIDs[0], ""} - }, "invalid credential_minter SPIFFE ID"}, {"bare ServiceAccount name", func(p *SystemPrincipals) { p.ControllerSPIFFEID = "ate-controller" }, "invalid controller SPIFFE ID"}, {"non-spiffe URI", func(p *SystemPrincipals) { - p.EgressGatewaySPIFFEID = "https://cluster.local/ns/ate-system/sa/atenet-egress" + p.EgressGatewaySPIFFEIDs = []string{"https://cluster.local/ns/ate-system/sa/atenet-egress"} }, "invalid egress_gateway SPIFFE ID"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/ateapi/internal/authz/interceptor_test.go b/cmd/ateapi/internal/authz/interceptor_test.go index 110ef6bf1d..b20bd6a7a6 100644 --- a/cmd/ateapi/internal/authz/interceptor_test.go +++ b/cmd/ateapi/internal/authz/interceptor_test.go @@ -626,7 +626,7 @@ func TestUnaryServerInterceptor_SystemPrincipals(t *testing.T) { ) authorizer, _, err := New(ctx, pool, fgaServer, nil, SystemPrincipals{ ControllerSPIFFEID: controllerID, - EgressGatewaySPIFFEID: egressID, + EgressGatewaySPIFFEIDs: []string{egressID}, IngressRouterSPIFFEID: routerID, AteletSPIFFEID: ateletID, APIServerSidecarSPIFFEID: pluginID, diff --git a/cmd/ateapi/internal/authz/model.fga b/cmd/ateapi/internal/authz/model.fga index fc13094f94..1d42a82e3e 100644 --- a/cmd/ateapi/internal/authz/model.fga +++ b/cmd/ateapi/internal/authz/model.fga @@ -52,10 +52,6 @@ type global define ingress_router: [user] define atelet: [user] define api_server_sidecar: [user] - # An out-of-tree credential plugin that turns actor JWTs into other - # credentials, such as an RFC 8693 token exchange. It may mint actor JWTs - # and nothing else. - define credential_minter: [user] define can_create_atespace: owner define can_list_atespaces: viewer @@ -89,7 +85,6 @@ type atespace define ingress_router: ingress_router from parent_global define atelet: atelet from parent_global define api_server_sidecar: api_server_sidecar from parent_global - define credential_minter: credential_minter from parent_global # --- 2. Policy Governance --- define can_create_access_policy: owner @@ -172,9 +167,8 @@ type actor define can_mint_worker_actor_credential: host_node # Credentials asserting the actor's own identity. Only the egress gateway - # mints them, plus a credential plugin for actor JWTs; an editor holding - # one could impersonate the actor. - define can_mint_actor_jwt: egress_gateway from parent_atespace or credential_minter from parent_atespace + # mints them; an editor holding one could impersonate the actor. + define can_mint_actor_jwt: egress_gateway from parent_atespace define can_mint_actor_certificate: egress_gateway from parent_atespace # Data Plane Ingress: Routers (or editors) can resume diff --git a/cmd/ateapi/internal/authz/model_test.fga.yaml b/cmd/ateapi/internal/authz/model_test.fga.yaml index a614f9b977..02741ca1db 100644 --- a/cmd/ateapi/internal/authz/model_test.fga.yaml +++ b/cmd/ateapi/internal/authz/model_test.fga.yaml @@ -93,9 +93,6 @@ tuples: - user: user:sys-api-server-sidecar relation: api_server_sidecar object: global:root - - user: user:sys-credential-minter - relation: credential_minter - object: global:root tests: - name: 1. Global Superadmin & Viewer Propagation @@ -564,7 +561,7 @@ tests: can_create_tag: false can_list_tags: false - - name: 9. Actor Credentials (Egress Gateway, and Credential Minter for JWTs) + - name: 9. Actor Credentials (Egress Gateway Only) check: - user: user:carol # Global owner object: actor:runner-1 @@ -581,31 +578,3 @@ tests: assertions: can_mint_actor_jwt: false can_mint_actor_certificate: false - - user: user:sys-credential-minter # mints actor JWTs and nothing else - object: actor:runner-1 - assertions: - can_mint_actor_jwt: true - can_mint_actor_certificate: false - can_get: false - can_update: false - can_delete: false - can_get_egress_policy: false - can_update_egress_policy: false - can_resume: false - can_suspend: false - - user: user:sys-credential-minter - object: atespace:space-1 - assertions: - viewer: false - can_get: false - can_list_actors: false - can_create_actor: false - can_create_tag: false - - user: user:sys-credential-minter - object: global:root - assertions: - can_list_actors: false - can_list_workers: false - can_create_worker: false - can_create_atespace: false - can_get_access_policy: false diff --git a/cmd/ateapi/internal/store/atepg/common_test.go b/cmd/ateapi/internal/store/atepg/common_test.go index 30a21388ba..c25a3ecbfa 100644 --- a/cmd/ateapi/internal/store/atepg/common_test.go +++ b/cmd/ateapi/internal/store/atepg/common_test.go @@ -57,7 +57,7 @@ func setTestPolicyManager(t *testing.T, p *Persistence) { t.Cleanup(fgaServer.Close) _, policyManager, err := authz.New(t.Context(), p.pool, fgaServer, nil, authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index 7bc219b3d2..2d0f4a5dff 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -101,7 +101,7 @@ func SetupPostgresPersistence(t *testing.T) *atepg.Persistence { func DefaultSystemPrincipals() authz.SystemPrincipals { return authz.SystemPrincipals{ ControllerSPIFFEID: installdefaults.ControllerSPIFFEID(installdefaults.SystemNamespace), - EgressGatewaySPIFFEID: installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace), + EgressGatewaySPIFFEIDs: []string{installdefaults.EgressSPIFFEID(installdefaults.SystemNamespace)}, IngressRouterSPIFFEID: installdefaults.RouterSPIFFEID(installdefaults.SystemNamespace), AteletSPIFFEID: installdefaults.AteletSPIFFEID(installdefaults.SystemNamespace), APIServerSidecarSPIFFEID: installdefaults.APIServerSPIFFEID(installdefaults.SystemNamespace), diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 09ea95fded..ba6cffbc2c 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -93,11 +93,10 @@ var ( authzBootstrapOwners = pflag.StringSlice("authz-bootstrap-owners", nil, "Principal IDs that are always global owners while listed, independent of the stored global AccessPolicy. Removing an ID revokes its access on restart. Required: at least one.") // These SPIFFE IDs hold the matching system component role when they call // over mTLS. - controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEID = pflag.String("egress-gateway-spiffe-id", installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-egress. Holds the egress_gateway authz role, which reads and lists actors and reads their egress policies.") - ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") - apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") - credentialMinterSPIFFEIDs = pflag.StringSlice("credential-minter-spiffe-ids", []string{installdefaults.CredentialMinterSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of out-of-tree credential plugins. Comma-separated or repeated. Each holds the credential_minter role, which mints actor JWTs and nothing else. Empty grants the role to no one.") + controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") + egressGatewaySPIFFEIDs = pflag.StringSlice("egress-gateway-spiffe-id", []string{installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of atenet-egress and any other client that holds the egress_gateway authz role, which reads and lists actors and reads their egress policies. Comma-separated or repeated.") + ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") + apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") actorIDJWTPoolFile = pflag.String("actor-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing actor JWTs") actorJWTIssuer = pflag.String("actor-jwt-issuer", "", "Issuer URL placed in the iss claim of actor JWTs. Relying parties fetch /.well-known/openid-configuration to verify them. Must be https with no query or fragment. Required.") @@ -257,12 +256,11 @@ func main() { } defer fgaServer.Close() authorizer, policyManager, err := authz.New(shutdownCtx, pool, fgaServer, *authzBootstrapOwners, authz.SystemPrincipals{ - ControllerSPIFFEID: *controllerSPIFFEID, - EgressGatewaySPIFFEID: *egressGatewaySPIFFEID, - IngressRouterSPIFFEID: *ingressRouterSPIFFEID, - AteletSPIFFEID: ateletSPIFFEID, - APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, - CredentialMinterSPIFFEIDs: *credentialMinterSPIFFEIDs, + ControllerSPIFFEID: *controllerSPIFFEID, + EgressGatewaySPIFFEIDs: *egressGatewaySPIFFEIDs, + IngressRouterSPIFFEID: *ingressRouterSPIFFEID, + AteletSPIFFEID: ateletSPIFFEID, + APIServerSidecarSPIFFEID: *apiServerSidecarSPIFFEID, }) if err != nil { serverboot.Fatal(ctx, "Failed to initialize OpenFGA authz", err) diff --git a/docs/authorization.md b/docs/authorization.md index 640b05d7c4..669ee2c401 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -128,26 +128,20 @@ one **atespace**. Substrate's components hold roles that access policies cannot grant, and only through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, -`--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id`, -`--api-server-sidecar-spiffe-id` and `--credential-minter-spiffe-ids` flags bind -them, defaulting to the install's; `atelet`'s comes from -`--atelet-service-account`. +`--egress-gateway-spiffe-id`, `--ingress-router-spiffe-id` and +`--api-server-sidecar-spiffe-id` flags bind them, defaulting to the install's; +`atelet`'s comes from `--atelet-service-account`. | Role | Held by | Allows | |---|---|---| | `controller` | `ate-controller` | Create, get, list, update, delete and drain workers | -| `egress_gateway` | `atenet-egress` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | +| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs listed in `--egress-gateway-spiffe-id` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | -| `credential_minter` | out-of-tree credential plugins, by default one running as the `credential-minter` ServiceAccount in the install's namespace. Substrate deploys none. | Mint actor JWTs, for example to exchange them for a cloud access token | - -Only `egress_gateway` can mint actor certificates, and only `egress_gateway` -and `credential_minter` can mint actor JWTs: not even a global owner can, since -a credential lets its holder act as the actor. Anyone who can run a pod as an -arbitrary ServiceAccount in the install's namespace can already run one as -`atenet-egress`, so the default `credential_minter` binding grants no one new -authority. Set `--credential-minter-spiffe-ids=""` to grant it to no one. + +Only `egress_gateway` can mint actor credentials: not even a global owner can, +since a credential lets its holder act as the actor. ## Checks per RPC diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go index 8907f55594..2402e7986a 100644 --- a/internal/installdefaults/installdefaults.go +++ b/internal/installdefaults/installdefaults.go @@ -52,12 +52,6 @@ const ( ControllerServiceAccount = "ate-controller" APIServerServiceAccount = "ate-api-server" - // CredentialMinterServiceAccount is the ServiceAccount an out-of-tree - // credential plugin runs as to mint actor JWTs. Substrate creates no - // ServiceAccount or Pod with this name; an operator who deploys such a - // plugin in the system namespace opts in by using it. - CredentialMinterServiceAccount = "credential-minter" - // TransferServiceAccountName is the default name of the Google service // account snapshot transfers run as on GCS, which tools/setup-gcp creates. TransferServiceAccountName = "ate-snapshot-transfer" @@ -135,12 +129,6 @@ func ControllerSPIFFEID(namespace string) string { return SPIFFEID(namespace, ControllerServiceAccount) } -// CredentialMinterSPIFFEID returns the SPIFFE ID a credential plugin running -// as CredentialMinterServiceAccount in namespace presents. -func CredentialMinterSPIFFEID(namespace string) string { - return SPIFFEID(namespace, CredentialMinterServiceAccount) -} - // APIServerSPIFFEID returns the SPIFFE ID the ate-api-server pod presents when // it runs in namespace. Its objectstore-plugin sidecar shares it. func APIServerSPIFFEID(namespace string) string { diff --git a/internal/installdefaults/installdefaults_test.go b/internal/installdefaults/installdefaults_test.go index 06e2046519..2329431e20 100644 --- a/internal/installdefaults/installdefaults_test.go +++ b/internal/installdefaults/installdefaults_test.go @@ -88,15 +88,6 @@ func TestAPIServerSPIFFEID(t *testing.T) { } } -func TestCredentialMinterSPIFFEID(t *testing.T) { - // Matches the --credential-minter-spiffe-ids default ate-api-server ships - // with. Out-of-tree credential plugins run as this ServiceAccount. - const want = "spiffe://cluster.local/ns/ate-system/sa/credential-minter" - if got := CredentialMinterSPIFFEID(SystemNamespace); got != want { - t.Errorf("CredentialMinterSPIFFEID(%q) = %q, want %q", SystemNamespace, got, want) - } -} - func TestAPIServiceHost(t *testing.T) { for _, tt := range []struct{ namespace, want string }{ {"ate-system", "api.ate-system.svc"}, From 2eb56a4bf1f8474e8560060885b54c37e4f1082a Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Tue, 27 Oct 2026 18:08:57 -0700 Subject: [PATCH 4/6] ateapi: take one egress gateway SPIFFE ID per flag Make --egress-gateway-spiffe-id a StringArray so each value is one SPIFFE ID. StringSlice also split values on commas. Signed-off-by: Max Thompson --- cmd/ateapi/main.go | 2 +- docs/authorization.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index ba6cffbc2c..9ade6ee253 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -94,7 +94,7 @@ var ( // These SPIFFE IDs hold the matching system component role when they call // over mTLS. controllerSPIFFEID = pflag.String("controller-spiffe-id", installdefaults.ControllerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of ate-controller. Holds the controller authz role, which manages workers.") - egressGatewaySPIFFEIDs = pflag.StringSlice("egress-gateway-spiffe-id", []string{installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of atenet-egress and any other client that holds the egress_gateway authz role, which reads and lists actors and reads their egress policies. Comma-separated or repeated.") + egressGatewaySPIFFEIDs = pflag.StringArray("egress-gateway-spiffe-id", []string{installdefaults.EgressSPIFFEID(installdefaults.NamespaceFromPodEnv())}, "mTLS SPIFFE IDs of atenet-egress and any other client that holds the egress_gateway authz role, which reads and lists actors and reads their egress policies. Repeat the flag for each ID.") ingressRouterSPIFFEID = pflag.String("ingress-router-spiffe-id", installdefaults.RouterSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of atenet-router. Holds the ingress_router authz role, which resumes and lists actors.") apiServerSidecarSPIFFEID = pflag.String("api-server-sidecar-spiffe-id", installdefaults.APIServerSPIFFEID(installdefaults.NamespaceFromPodEnv()), "mTLS SPIFFE ID of the ate-api-server pod, which its sidecars share. Holds the api_server_sidecar role, which reads actors so the objectstore control plugin can scope their storage credentials.") diff --git a/docs/authorization.md b/docs/authorization.md index 669ee2c401..631268eb99 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -135,7 +135,7 @@ through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, | Role | Held by | Allows | |---|---|---| | `controller` | `ate-controller` | Create, get, list, update, delete and drain workers | -| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs listed in `--egress-gateway-spiffe-id` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | +| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs passed with `--egress-gateway-spiffe-id` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | From 675d8638311214f1aae48d52bcd4903c03f03bbf Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Wed, 28 Oct 2026 11:08:15 -0700 Subject: [PATCH 5/6] ate-setup: share the ate-api-server flag rewrite Move the arg rewrite out of setAPIServerAuthzArgs into setAPIServerFlag so other settings can render ate-api-server flags. Signed-off-by: Max Thompson --- cmd/ate-setup/internal/steps/apienv.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 481f72de1e..8537570d90 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -365,9 +365,19 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er const apiServerAuthzFlag = "--authz-bootstrap-owners" // setAPIServerAuthzArgs sets ate-api-server's bootstrap owners flag on its -// container in objs, replacing any the manifest already carries. A changed -// flag changes the pod template, so it rolls the Deployment. +// container in objs, replacing any the manifest already carries. func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners string) error { + var values []string + if strings.Trim(bootstrapOwners, " ,") != "" { + values = []string{bootstrapOwners} + } + return setAPIServerFlag(objs, apiServerAuthzFlag, values) +} + +// setAPIServerFlag replaces every flag arg on ate-api-server's container in +// objs with one flag=value arg per value, in order. A changed flag changes the +// pod template, so it rolls the Deployment. +func setAPIServerFlag(objs []*unstructured.Unstructured, flag string, values []string) error { var dep *unstructured.Unstructured for _, obj := range objs { if obj.GetKind() == "Deployment" && obj.GetName() == "ate-api-server" { @@ -376,7 +386,7 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st } } if dep == nil { - return fmt.Errorf("the manifest has no deployment/ate-api-server to set authorization flags on") + return fmt.Errorf("the manifest has no deployment/ate-api-server to set %s on", flag) } containers, found, err := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") if err != nil || !found { @@ -392,10 +402,10 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st return fmt.Errorf("%s: %w", kube.Describe(dep), err) } args = slices.DeleteFunc(args, func(arg string) bool { - return arg == apiServerAuthzFlag || strings.HasPrefix(arg, apiServerAuthzFlag+"=") + return arg == flag || strings.HasPrefix(arg, flag+"=") }) - if strings.Trim(bootstrapOwners, " ,") != "" { - args = append(args, apiServerAuthzFlag+"="+bootstrapOwners) + for _, v := range values { + args = append(args, flag+"="+v) } if err := unstructured.SetNestedStringSlice(container, args, "args"); err != nil { return fmt.Errorf("%s: %w", kube.Describe(dep), err) From 8e335d49ce23b7104eda31b02a3bee74773e4076 Mon Sep 17 00:00:00 2001 From: Max Thompson Date: Wed, 28 Oct 2026 11:10:42 -0700 Subject: [PATCH 6/6] ate-setup: add egress gateway extra SPIFFE IDs setting ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS lists SPIFFE IDs that hold ate-api-server's egress_gateway role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress first because --egress-gateway-spiffe-id replaces its default when set. Signed-off-by: Max Thompson --- cmd/ate-setup/commands.md | 1 + cmd/ate-setup/internal/config/config.go | 8 +++ cmd/ate-setup/internal/config/config_test.go | 39 +++++++++++++ .../internal/config/coverage_test.go | 2 +- cmd/ate-setup/internal/config/load.go | 13 ++++- cmd/ate-setup/internal/config/setting.go | 7 +++ cmd/ate-setup/internal/steps/apienv.go | 24 ++++++++ cmd/ate-setup/internal/steps/apienv_test.go | 57 +++++++++++++++++++ docs/authorization.md | 2 +- docs/operator-install.md | 1 + 10 files changed, 149 insertions(+), 5 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index b10ce64d40..b551215364 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -32,6 +32,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--cordon-system-infra-components` | `--cordon-system-infra-components` | Keep the system infra components, such as ate-api-server, atenet-router, atenet-egress and postgres, off the worker nodes. Assumes a small shared pool labeled and tainted `ate.dev/workloadType=ate-system-infra:NoSchedule`, across which each workload's replicas are spread, and a one-node pool labeled and tainted `ate.dev/workloadType=ate-postgres:NoSchedule` for postgres alone. `ATE_INSTALL_CORDON_SYSTEM_INFRA_COMPONENTS=true` when the flag is absent | | `--fake-data-plane` | `--fake-data-plane` | Install the fake data plane for control-plane benchmarks: fake-atelet takes atelet's place in its DaemonSet, taking its objectstore-plugin's storage settings, ate-controller is scaled to zero, and fake-workersync honors every WorkerPool with fake Workers on the fake-atelet nodes. Reaches `deploy ate-system`, `deploy atelet`, and `deploy ate-controller`; only `deploy ate-system` switches a cluster between the real and fake data planes, and the other two refuse a data plane other than the installed one. `deploy ate-system --fake-data-plane` refuses a cluster whose WorkerPools have worker pods. fake-atelet runs with its defaults; a benchmark tunes it through the optional `fake-atelet-tuning` ConfigMap, whose `FAKE_ATELET_*` variables set its flags, and which no install creates or resets. Actors do not exist; never use it on a cluster that serves real actors. `ATE_INSTALL_FAKE_DATA_PLANE=true` when the flag is absent | | `--ingress-allowed-spiffe-ids IDS` | `--ingress-allowed-spiffe-ids IDS`, or `ATE_INGRESS_ALLOWED_SPIFFE_IDS=IDS` | Comma-separated SPIFFE IDs (exact match) of the clients the ingress router lets reach actors; any listed client may reach any actor. Unset, no client may reach actors through the router. Both dataplanes require a Pod Identity CA client certificate on HTTPS and TLS CONNECT. The SPIFFE URI SAN allowlist may be checked during or after TLS, including after CONNECT acceptance, but unlisted clients must be denied before actor routing. The podidentity signer issues `spiffe://cluster.local/ns//sa/` | +| `--egress-gateway-extra-spiffe-ids IDS` | — | New. Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent | | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter | | `--credential-provider JSON` | `--credential-provider JSON`, or `ATE_CREDENTIAL_PROVIDER` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"","address":":"}` uses a provider you deploy yourself. See [`docs/egress-credential-injection.md`](../../docs/egress-credential-injection.md) | | `--ateapi-log-level LEVEL` | `--ateapi-log-level LEVEL`, or `ATE_API_LOG_LEVEL=LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 7a80966d89..d52c925f6f 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -121,6 +121,10 @@ type Config struct { // owner on global:root (ATE_API_AUTHZ_BOOTSTRAP_OWNERS). APIAuthzBootstrapOwners string + // EgressGatewayExtraSPIFFEIDs hold ate-api-server's egress_gateway role + // next to atenet-egress (ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS). + EgressGatewayExtraSPIFFEIDs []string + // ActorJWTAlgorithm is the signing algorithm of the key in a new actor JWT // pool (ACTOR_JWT_ALGORITHM): ES256 or RS256. ActorJWTAlgorithm string @@ -518,6 +522,10 @@ func (c *Config) ScriptEnv() []string { if len(c.IngressAllowedSPIFFEIDs) > 0 { merged["ATE_INGRESS_ALLOWED_SPIFFE_IDS"] = strings.Join(c.IngressAllowedSPIFFEIDs, ",") } + delete(merged, "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS") + if len(c.EgressGatewayExtraSPIFFEIDs) > 0 { + merged["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] = strings.Join(c.EgressGatewayExtraSPIFFEIDs, ",") + } if c.AdditionalEgressExtprocService != "" { merged["ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE"] = c.AdditionalEgressExtprocService } diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index b50366e1d9..3bf7be5662 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -61,6 +61,7 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", "ATE_CREDENTIAL_PROVIDER", + "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", "ATE_INGRESS_ALLOWED_SPIFFE_IDS", @@ -316,6 +317,44 @@ func TestLoadIngressAllowedSPIFFEIDs(t *testing.T) { } } +func TestLoadEgressGatewayExtraSPIFFEIDs(t *testing.T) { + const a, b = "spiffe://cluster.local/ns/a/sa/a", "spiffe://cluster.local/ns/b/sa/b" + for _, tc := range []struct { + name string + env string + want []string + wantErr string + }{ + {name: "unset", want: nil}, + {name: "several, with stray separators", env: " " + a + ",," + b + ", ", want: []string{a, b}}, + {name: "not a SPIFFE ID", env: a + ",https://example.com/x", wantErr: "ateapi.authz.egressGatewayExtraSPIFFEIDs must be a comma-separated list of SPIFFE IDs"}, + } { + t.Run(tc.name, func(t *testing.T) { + loadEnv(t) + if tc.env != "" { + t.Setenv("ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", tc.env) + } + cfg, err := Load(Options{}) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Load() error = %v, want one containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !slices.Equal(cfg.EgressGatewayExtraSPIFFEIDs, tc.want) { + t.Errorf("EgressGatewayExtraSPIFFEIDs = %q, want %q", cfg.EgressGatewayExtraSPIFFEIDs, tc.want) + } + got, exported := scriptEnvMap(t, cfg)["ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS"] + if want := strings.Join(tc.want, ","); exported != (want != "") || got != want { + t.Errorf("ScriptEnv() ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS = %q (exported %v), want %q", got, exported, want) + } + }) + } +} + // The credential exchange impersonates the transfer account only on GCS, so // only a GKE deploy needs one, and its email needs both the name and the // project. diff --git a/cmd/ate-setup/internal/config/coverage_test.go b/cmd/ate-setup/internal/config/coverage_test.go index 8f23e55ea5..5585769f7a 100644 --- a/cmd/ate-setup/internal/config/coverage_test.go +++ b/cmd/ate-setup/internal/config/coverage_test.go @@ -90,7 +90,7 @@ func valueFor(key string) string { return RouterEnvoy case "clusterSize": return ClusterSizeSize0 - case "ingressAllowedSPIFFEIDs": + case "ingressAllowedSPIFFEIDs", "ateapi.authz.egressGatewayExtraSPIFFEIDs": return "spiffe://cluster.local/ns/a/sa/b" case "ateapi.postgres.cloudsql.ipType": return CloudSQLIPTypePSC diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index fa16bd5176..143efd52cc 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -230,7 +230,8 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro ClusterLocation: r.String("gcp.clusterLocation"), ExpectedJWTIssuer: r.String("ateapi.expectedJWTIssuer"), - APIAuthzBootstrapOwners: bootstrapOwners, + APIAuthzBootstrapOwners: bootstrapOwners, + EgressGatewayExtraSPIFFEIDs: splitList(r.String("ateapi.authz.egressGatewayExtraSPIFFEIDs")), BucketName: r.String("storage.bucketName"), SnapshotLocation: r.String("storage.snapshotLocation"), @@ -413,14 +414,20 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: size, Want: ClusterSizeSize0 + " or " + ClusterSizeSize10} } - // The router rejects a malformed ID at startup; catch it before the - // rollout does. + // The router and ate-api-server reject a malformed ID at startup; catch it + // before the rollout does. for _, id := range cfg.IngressAllowedSPIFFEIDs { if _, err := spiffeid.FromString(id); err != nil { ids, _ := r.Value("ingressAllowedSPIFFEIDs") return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} } } + for _, id := range cfg.EgressGatewayExtraSPIFFEIDs { + if _, err := spiffeid.FromString(id); err != nil { + ids, _ := r.Value("ateapi.authz.egressGatewayExtraSPIFFEIDs") + return &InvalidError{Value: ids, Want: fmt.Sprintf("a comma-separated list of SPIFFE IDs (%q: %v)", id, err)} + } + } switch cfg.APILogLevel { case "", "debug", "info", "warn", "error": diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 1fdea0effb..d13a1aaf1c 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -187,6 +187,13 @@ var Registry = []Setting{ Usage: "Comma-separated principals that always hold owner on global:root. " + "Unset means kubectl-ate's ServiceAccount, system:serviceaccount::ate-client", }, + { + Key: "ateapi.authz.egressGatewayExtraSPIFFEIDs", Env: "ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS", + Flag: "egress-gateway-extra-spiffe-ids", Kind: KindString, + Usage: "Comma-separated SPIFFE IDs that hold ate-api-server's egress_gateway role next to atenet-egress, " + + "such as a credential provider that mints actor JWTs. The role also reads actors and mints actor " + + "certificates. The podidentity signer issues spiffe://cluster.local/ns//sa/", + }, { // The egress gateway's credential provider, as a JSON object. Upstream // takes it as one value on one flag; it is carried the same way here diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 8537570d90..3cbf16a3cc 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -355,6 +355,9 @@ func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) er if err := setAPIServerAuthzArgs(objs, e.Cfg.APIAuthzBootstrapOwners); err != nil { return err } + if err := setAPIServerEgressGatewayArgs(objs, e.Namespace(), e.Cfg.EgressGatewayExtraSPIFFEIDs); err != nil { + return err + } if msg := sharedBootstrapOwnerWarning(e.Cfg.APIAuthzBootstrapOwners, e.Namespace()); msg != "" { log.Warnf("%s", msg) } @@ -374,6 +377,27 @@ func setAPIServerAuthzArgs(objs []*unstructured.Unstructured, bootstrapOwners st return setAPIServerFlag(objs, apiServerAuthzFlag, values) } +// apiServerEgressGatewayFlag is the ate-api-server flag +// setAPIServerEgressGatewayArgs owns. +const apiServerEgressGatewayFlag = "--egress-gateway-spiffe-id" + +// setAPIServerEgressGatewayArgs passes atenet-egress's SPIFFE ID and then each +// of extras, without duplicates, to ate-api-server's egress gateway flag. With +// no extras it drops the flag, leaving ate-api-server's default. +func setAPIServerEgressGatewayArgs(objs []*unstructured.Unstructured, namespace string, extras []string) error { + var ids []string + if len(extras) > 0 { + // Passing the flag replaces ate-api-server's default. + ids = []string{installdefaults.EgressSPIFFEID(namespace)} + for _, id := range extras { + if !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + return setAPIServerFlag(objs, apiServerEgressGatewayFlag, ids) +} + // setAPIServerFlag replaces every flag arg on ate-api-server's container in // objs with one flag=value arg per value, in order. A changed flag changes the // pod template, so it rolls the Deployment. diff --git a/cmd/ate-setup/internal/steps/apienv_test.go b/cmd/ate-setup/internal/steps/apienv_test.go index 3c97dea3dc..db6694a706 100644 --- a/cmd/ate-setup/internal/steps/apienv_test.go +++ b/cmd/ate-setup/internal/steps/apienv_test.go @@ -695,6 +695,63 @@ spec: } } +func TestSetAPIServerEgressGatewayArgs(t *testing.T) { + const manifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: ate-api-server + namespace: ate-system +spec: + template: + spec: + containers: + - name: ate-api-server + args: + - --config-file=/run/apiconfig.ate.dev/apiconfig.textproto + - --egress-gateway-spiffe-id=stale +` + const ( + egress = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" + a = "spiffe://cluster.local/ns/ate-system/sa/sts-credential-provider" + b = "spiffe://cluster.local/ns/plugins/sa/other" + ) + for _, tc := range []struct { + name string + extras []string + want []string + }{ + { + name: "gateway first, then extras without duplicates", + extras: []string{a, egress, b, a}, + want: []string{ + "--config-file=/run/apiconfig.ate.dev/apiconfig.textproto", + "--egress-gateway-spiffe-id=" + egress, + "--egress-gateway-spiffe-id=" + a, + "--egress-gateway-spiffe-id=" + b, + }, + }, + { + name: "no extras drops the flag", + want: []string{"--config-file=/run/apiconfig.ate.dev/apiconfig.textproto"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + objs, err := kube.DecodeManifestBytes([]byte(manifest)) + if err != nil { + t.Fatalf("DecodeManifestBytes() error = %v", err) + } + if err := setAPIServerEgressGatewayArgs(objs, "ate-system", tc.extras); err != nil { + t.Fatalf("setAPIServerEgressGatewayArgs() error = %v", err) + } + containers, _, _ := unstructured.NestedSlice(objs[0].Object, "spec", "template", "spec", "containers") + args, _, _ := unstructured.NestedStringSlice(containers[0].(map[string]any), "args") + if diff := cmp.Diff(tc.want, args); diff != "" { + t.Errorf("ate-api-server args (-want +got):\n%s", diff) + } + }) + } +} + func TestCreateAPIServerEnvVarsOmitsAuthz(t *testing.T) { e := &Env{Cfg: &config.Config{ TransferGSAName: testTransferGSAName, ProjectID: testProjectID, diff --git a/docs/authorization.md b/docs/authorization.md index 631268eb99..7879449a21 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -135,7 +135,7 @@ through their pod certificates. `ate-api-server`'s `--controller-spiffe-id`, | Role | Held by | Allows | |---|---|---| | `controller` | `ate-controller` | Create, get, list, update, delete and drain workers | -| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs passed with `--egress-gateway-spiffe-id` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | +| `egress_gateway` | `atenet-egress`, and any other SPIFFE IDs passed with `--egress-gateway-spiffe-id`, which ate-setup sets from `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` | Get actors and their egress policies; list actors across all atespaces; mint actor JWTs and certificates | | `ingress_router` | `atenet-router` | Resume actors; list actors across all atespaces | | `atelet` | `atelet` | Get actors' egress policies, for the worker that enforces them | | `api_server_sidecar` | the `ate-api-server` pod identity, which its sidecars share, today the objectstore control plugin | Get actors, to derive the snapshot prefixes a per-actor storage credential may cover | diff --git a/docs/operator-install.md b/docs/operator-install.md index 15b649a8a3..a9c1cd4f7e 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -277,6 +277,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `actorJWT.algorithm` | `ACTOR_JWT_ALGORITHM` | `--actor-jwt-algorithm` | `ES256` | (global) | | `actorJWT.issuer` | `ACTOR_JWT_ISSUER` | `--actor-jwt-issuer` | — | (global) | | `ateapi.authz.bootstrapOwners` | `ATE_API_AUTHZ_BOOTSTRAP_OWNERS` | `--ateapi-authz-bootstrap-owners` | `system:serviceaccount::ate-client` | (global) | +| `ateapi.authz.egressGatewayExtraSPIFFEIDs` | `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` | `--egress-gateway-extra-spiffe-ids` | — | (global) | | `ateapi.expectedJWTIssuer` | `EXPECTED_JWT_ISSUER` | `--ateapi-expected-jwt-issuer` | — | (global) | | `ateapi.logLevel` | `ATE_API_LOG_LEVEL` | `--ateapi-log-level` | — | (global) | | `ateapi.postgres.cloudsql.gsa` | `ATE_API_POSTGRES_CLOUDSQL_GSA` | `--ateapi-postgres-cloudsql-gsa` | — | (global) |