fix(auth): honor custom OAuth M2M scopes without forcing all-apis

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Release History
 
+## Unreleased
+- **Behavior change:** `m2m.NewAuthenticatorWithScopes` now requests exactly the given scopes instead of always appending `all-apis`, so service principals with scoped OAuth secrets (e.g. `sql` only) can authenticate (databricks/databricks-sql-go#476). Empty scopes and `m2m.NewAuthenticator` still request `all-apis`; callers that relied on it being appended should pass no scopes, or add `all-apis` to their list if they need both. The SEA/kernel backend now forwards custom M2M scopes instead of rejecting them.
+
 ## v1.16.0 (2026-09-24)
 - Upgrade the kernel bindings to v1.1.0; the kernel dependency is now stable.
 - Expand the kernel backend with client query timeouts, configurable idle HTTP connections, and per-statement query tags.

CONNECTION_PARAMETERS.md

@@ -63,8 +63,6 @@ Notes for the SEA/kernel backend:
 - The kernel snapshots one federated-provider token during setup;
   `WithFederatedTokenProviderAndClientID` also forwards the SP-wide client ID. Expired
   tokens require a new connection.
-- Custom OAuth **M2M scopes** are rejected on the kernel path (the kernel applies its
-  own default scopes). Default scopes work on both.
 - `WithAuthenticator` is kernel-compatible only when its concrete authenticator is
   one of the supported PAT, M2M, or U2M implementations. An arbitrary custom
   `auth.Authenticator` is supported on Thrift and rejected on the kernel path.

README.md

@@ -294,6 +294,9 @@ groups. Telemetry parameters are covered under [Telemetry](#telemetry).
 ```
 
 `authType=OauthM2M` is optional — supplying `clientID` + `clientSecret` selects M2M.
+M2M requests the `all-apis` scope by default. For a secret limited to other scopes
+(e.g. `sql`), use
+`WithAuthenticator(m2m.NewAuthenticatorWithScopes(id, secret, host, []string{"sql"}))`.
 
 **OAuth U2M** (interactive browser login):
 
@@ -305,8 +308,6 @@ Notes for the SEA/kernel backend:
 
 - The kernel snapshots one `WithFederatedTokenProvider*` token during setup;
   `AndClientID` also forwards the SP-wide client ID. Expired tokens require a new connection.
-- Custom OAuth **M2M scopes** are rejected on the kernel path (the kernel applies its
-  own default scopes). Default scopes work on both.
 - **U2M** is interactive: on a cache miss, connecting launches the browser and a
   connect-context **deadline is not honored** during the login window. U2M scopes are at
   parity with Thrift. Use PAT or M2M for headless/deadline-bound connects.

auth/oauth/m2m/m2m.go

@@ -17,6 +17,8 @@ func NewAuthenticator(clientID, clientSecret, hostName string) auth.Authenticato
 	return NewAuthenticatorWithScopes(clientID, clientSecret, hostName, []string{})
 }
 
+// NewAuthenticatorWithScopes requests exactly the given scopes, or "all-apis" when
+// scopes is empty.
 func NewAuthenticatorWithScopes(clientID, clientSecret, hostName string, scopes []string) auth.Authenticator {
 	scopes = GetScopes(hostName, scopes)
 	return &authClient{
@@ -45,8 +47,7 @@ func (c *authClient) M2MCredentials() (clientID, clientSecret string) {
 	return c.clientID, c.clientSecret
 }
 
-// M2MScopes exposes the configured scopes so the kernel backend can reject a
-// custom set its scopes-less M2M setter can't carry (M2MScopesSupported).
+// M2MScopes exposes the configured scopes so the kernel backend can forward them.
 func (c *authClient) M2MScopes() []string {
 	return c.scopes
 }
@@ -108,9 +109,11 @@ func GetConfig(ctx context.Context, issuerURL, clientID, clientSecret string, sc
 	return config, nil
 }
 
+// GetScopes returns scopes unchanged, defaulting to "all-apis" only when empty so a
+// least-privilege secret (e.g. scoped to "sql") can authenticate.
 func GetScopes(hostName string, scopes []string) []string {
-	if !oauth.HasScope(scopes, "all-apis") {
-		scopes = append(scopes, "all-apis")
+	if len(scopes) == 0 {
+		return []string{"all-apis"}
 	}
 
 	return scopes

auth/oauth/m2m/m2m_test.go

@@ -24,14 +24,14 @@ func TestM2MScopes(t *testing.T) {
 		assert.Equal(t, []string{"all-apis"}, auth.scopes)
 	})
 
-	t.Run("should add all-apis to passed scopes", func(t *testing.T) {
-		auth := NewAuthenticatorWithScopes("id", "secret", "staging.cloud.company.com", []string{"my-scope"}).(*authClient)
+	t.Run("should not add all-apis to passed scopes", func(t *testing.T) {
+		auth := NewAuthenticatorWithScopes("id", "secret", "staging.cloud.company.com", []string{"sql"}).(*authClient)
 		assert.Equal(t, "id", auth.clientID)
 		assert.Equal(t, "secret", auth.clientSecret)
-		assert.Equal(t, []string{"my-scope", "all-apis"}, auth.scopes)
+		assert.Equal(t, []string{"sql"}, auth.scopes)
 	})
 
-	t.Run("should not add all-apis if already in passed scopes", func(t *testing.T) {
+	t.Run("should keep all-apis if already in passed scopes", func(t *testing.T) {
 		auth := NewAuthenticatorWithScopes("id", "secret", "staging.cloud.company.com", []string{"all-apis", "my-scope"}).(*authClient)
 		assert.Equal(t, "id", auth.clientID)
 		assert.Equal(t, "secret", auth.clientSecret)

doc.go

@@ -226,9 +226,8 @@ applied post-connect via USE CATALOG / USE SCHEMA); metric-view metadata
 RetryWaitMin / RetryWaitMax / RetryMax, including the disable form, forwarded to the
 kernel's HTTP retry config); and the TLS, proxy, and session-conf (query tags,
 statement timeout, time zone) options. Federated token providers use one token snapshot
-during setup. Nothing is silently ignored: WithTimeout, token-provider / external /
-static authenticators, and custom M2M OAuth scopes
-(the kernel applies its own) are rejected at connect; staging (PUT/GET/REMOVE on a
+during setup. Nothing is silently ignored: WithTimeout and token-provider / external /
+static authenticators are rejected at connect; staging (PUT/GET/REMOVE on a
 Unity Catalog volume) is rejected at execute. WithMaxRows is accepted but inert (the
 kernel manages fetching below the C ABI).
 

internal/backend/kernel/auth.go

@@ -19,8 +19,9 @@ const (
 // for Mode are populated. The connector fills it from the driver config (see
 // validateKernelConfig); OpenSession maps it to exactly one
 // kernel_session_config_set_auth_* call.
-// Scopes and RedirectPort map to the optional args of set_auth_u2m and are wired
-// through to it by setAuth. For U2M, resolveKernelAuth populates ClientID/Scopes
+// For M2M, setAuth forwards Scopes via set_oauth_scopes. For U2M, Scopes and
+// RedirectPort map to the optional args of set_auth_u2m and are wired through to
+// it by setAuth; resolveKernelAuth populates ClientID/Scopes
 // with the fixed in-house databricks-sql-connector client and offline_access + sql
 // on every cloud (NOT the cloud-inferred Thrift values), because the kernel runs one
 // in-house workspace-federated flow with no Azure branching. RedirectPort stays zero
@@ -33,7 +34,7 @@ type Auth struct {
 	Token        string   // PAT
 	ClientID     string   // M2M + U2M (U2M: fixed in-house client, cloud-agnostic); federated PAT uses the optional SP-wide client id
 	ClientSecret string   // M2M
-	Scopes       []string // U2M — fixed offline_access + sql (cloud-agnostic); nil → kernel default
+	Scopes       []string // M2M — the authenticator's scopes; U2M — fixed offline_access + sql (cloud-agnostic); nil → kernel default
 	RedirectPort uint16   // U2M — no user option today; 0 → kernel default port (8030)
 }
 
@@ -50,27 +51,11 @@ type Auth struct {
 type M2MCredentialsProvider interface {
 	// M2MCredentials returns the client id and client secret.
 	M2MCredentials() (clientID, clientSecret string)
-	// M2MScopes returns the configured OAuth scopes. The kernel's C-ABI M2M setter
-	// takes no scopes (it applies its own default set), so resolveKernelAuth rejects
-	// a custom set via M2MScopesSupported rather than silently dropping it.
+	// M2MScopes returns the configured OAuth scopes, forwarded to the kernel via
+	// set_oauth_scopes.
 	M2MScopes() []string
 }
 
-// M2MScopesSupported reports whether an M2M authenticator's scopes can be honored
-// on the kernel path. The kernel's set_auth_m2m has no scopes argument and applies
-// "all-apis" itself, so only an empty set or exactly {"all-apis"} is forwardable;
-// any other set would silently downgrade to the kernel default, so it is rejected.
-func M2MScopesSupported(scopes []string) bool {
-	switch len(scopes) {
-	case 0:
-		return true
-	case 1:
-		return scopes[0] == "all-apis"
-	default:
-		return false
-	}
-}
-
 // U2MCredentialsProvider is implemented by the OAuth U2M authenticator. The kernel
 // backend asserts this interface only to DETECT that the interactive U2M flow is
 // selected — it does NOT forward the returned (cloud-inferred) client id. The kernel

internal/backend/kernel/backend.go

@@ -559,10 +559,11 @@ func describeRetry(r *RetryConfig) string {
 }
 
 // setAuth applies the resolved auth form to the session config via exactly one
-// kernel_session_config_set_auth_* call. PAT and M2M are plain value setters; U2M
-// records the client id / redirect port / scopes and the kernel owns the browser
-// (PKCE) flow, started when the session opens. Empty string args are passed as NULL
-// so the kernel applies its own defaults (e.g. U2M's public client / default port).
+// kernel_session_config_set_auth_* call (M2M scopes go through set_oauth_scopes).
+// PAT and M2M are plain value setters; U2M records the client id / redirect port /
+// scopes and the kernel owns the browser (PKCE) flow, started when the session
+// opens. Empty string args are passed as NULL so the kernel applies its own
+// defaults (e.g. U2M's public client / default port).
 func (k *KernelBackend) setAuth(cfg *C.KernelSessionConfig) error {
 	switch k.cfg.Auth.Mode {
 	case AuthM2M:
@@ -575,6 +576,16 @@ func (k *KernelBackend) setAuth(cfg *C.KernelSessionConfig) error {
 		}); err != nil {
 			return fmt.Errorf("kernel: set_auth_m2m: %w", toConnError(err))
 		}
+		// set_auth_m2m has no scopes arg; without this the kernel requests "all-apis".
+		if len(k.cfg.Auth.Scopes) > 0 {
+			scopes := newCStr(joinScopes(k.cfg.Auth.Scopes))
+			defer scopes.free()
+			if err := call(func() C.KernelStatusCode {
+				return C.kernel_session_config_set_oauth_scopes(cfg, scopes.c)
+			}); err != nil {
+				return fmt.Errorf("kernel: set_oauth_scopes: %w", toConnError(err))
+			}
+		}
 	case AuthU2M:
 		// client id / scopes are optional: NULL when empty lets the kernel use its
 		// public client / default scopes. resolveKernelAuth fills these with the
@@ -619,9 +630,9 @@ func (k *KernelBackend) setAuth(cfg *C.KernelSessionConfig) error {
 	return nil
 }
 
-// joinScopes renders U2M scopes as the comma-separated form the kernel U2M setter
-// expects. Empty (no scopes) yields "" so setAuth passes NULL and the kernel
-// applies its default scope set.
+// joinScopes renders OAuth scopes as the comma-separated form the kernel scope
+// setters expect. Empty (no scopes) yields "" so setAuth passes NULL (U2M) or skips
+// set_oauth_scopes (M2M) and the kernel applies its default scope set.
 func joinScopes(scopes []string) string {
 	return strings.Join(scopes, ",")
 }

internal/backend/kernel/kernel_test.go

@@ -118,6 +118,7 @@ func TestSetAuthByMode(t *testing.T) {
 		{"PAT", Auth{Mode: AuthPAT, Token: "dapi-x"}},
 		{"federated PAT", Auth{Mode: AuthPAT, Token: "subject-token", ClientID: "federation-client"}},
 		{"M2M", Auth{Mode: AuthM2M, ClientID: "cid", ClientSecret: "sec"}},
+		{"M2M with scopes", Auth{Mode: AuthM2M, ClientID: "cid", ClientSecret: "sec", Scopes: []string{"sql"}}},
 		// "U2M full" populates Scopes/RedirectPort, which no production path sets today
 		// (resolveKernelAuth sources only the client id — see kernel.Auth docs). It is
 		// kept deliberately to pin the marshalling of those optional set_auth_u2m args

kernel_auth_real_test.go

@@ -38,6 +38,22 @@ func TestResolveKernelAuthRealAuthenticators(t *testing.T) {
 		if a.Mode != kernel.AuthM2M || a.ClientID != "real-cid" || a.ClientSecret != "real-secret" {
 			t.Errorf("auth = %+v, want mode=M2M clientID=real-cid clientSecret=real-secret", a)
 		}
+		if want := []string{"all-apis"}; !reflect.DeepEqual(a.Scopes, want) {
+			t.Errorf("M2M scopes = %v, want %v", a.Scopes, want)
+		}
+	})
+
+	t.Run("real M2M authenticator forwards custom scopes without all-apis", func(t *testing.T) {
+		c := baseKernelConfig()
+		c.AccessToken = ""
+		c.Authenticator = m2m.NewAuthenticatorWithScopes("real-cid", "real-secret", "staging.cloud.databricks.com", []string{"sql"})
+		a, err := validateKernelConfig(c)
+		if err != nil {
+			t.Fatalf("real M2M authenticator with scopes should validate: %v", err)
+		}
+		if want := []string{"sql"}; !reflect.DeepEqual(a.Scopes, want) {
+			t.Errorf("M2M scopes = %v, want %v", a.Scopes, want)
+		}
 	})
 
 	t.Run("real U2M authenticator resolves to a U2M descriptor", func(t *testing.T) {

kernel_config.go

@@ -401,16 +401,8 @@ func resolveKernelAuthContext(ctx context.Context, cfg *config.Config) (kernel.A
 		}
 		return kernel.Auth{Mode: kernel.AuthPAT, Token: token.AccessToken, ClientID: a.clientID}, nil
 	case kernel.M2MCredentialsProvider:
-		// The kernel's set_auth_m2m takes no scopes and applies "all-apis" itself, so
-		// a custom scope set can't be forwarded — reject it instead of silently
-		// downgrading (a least-privilege caller would get broader-than-asked access).
-		if !kernel.M2MScopesSupported(a.M2MScopes()) {
-			return kernel.Auth{}, fmt.Errorf("databricks: custom M2M OAuth scopes are %w "+
-				"(the kernel applies its default scopes); drop the custom scopes "+
-				"(use m2m.NewAuthenticator) or use the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel)
-		}
 		clientID, clientSecret := a.M2MCredentials()
-		return kernel.Auth{Mode: kernel.AuthM2M, ClientID: clientID, ClientSecret: clientSecret}, nil
+		return kernel.Auth{Mode: kernel.AuthM2M, ClientID: clientID, ClientSecret: clientSecret, Scopes: a.M2MScopes()}, nil
 	case kernel.U2MCredentialsProvider:
 		// The kernel runs a single, cloud-agnostic in-house U2M flow: it does OIDC
 		// discovery against {host}/oidc and uses that authorize endpoint verbatim —

kernel_config_test.go

@@ -154,8 +154,7 @@ func TestValidateKernelConfig(t *testing.T) {
 		c := baseKernelConfig()
 		c.AccessToken = ""
 		// An M2M authenticator is the single source of truth; resolveKernelAuth reads
-		// the creds off it via the auth.M2MCredentialsProvider interface. Default
-		// scopes ({"all-apis"}, matching the kernel default) forward fine.
+		// the creds off it via the auth.M2MCredentialsProvider interface.
 		c.Authenticator = fakeM2MAuth{id: "cid", secret: "sec", scopes: []string{"all-apis"}}
 		a, err := validateKernelConfig(c)
 		if err != nil {
@@ -166,16 +165,16 @@ func TestValidateKernelConfig(t *testing.T) {
 		}
 	})
 
-	t.Run("OAuth M2M with custom scopes rejected", func(t *testing.T) {
+	t.Run("OAuth M2M forwards custom scopes", func(t *testing.T) {
 		c := baseKernelConfig()
 		c.AccessToken = ""
-		// The kernel's set_auth_m2m can't carry scopes, so a custom set must be
-		// rejected (not silently downgraded to the kernel default) and wrap
-		// ErrNotSupportedByKernel like every other unsupported option.
-		c.Authenticator = fakeM2MAuth{id: "cid", secret: "sec", scopes: []string{"all-apis", "custom-scope"}}
-		_, err := validateKernelConfig(c)
-		if !errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) {
-			t.Errorf("custom-scope M2M rejection should wrap ErrNotSupportedByKernel, got %v", err)
+		c.Authenticator = fakeM2MAuth{id: "cid", secret: "sec", scopes: []string{"sql", "custom-scope"}}
+		a, err := validateKernelConfig(c)
+		if err != nil {
+			t.Fatalf("M2M with custom scopes should validate, got %v", err)
+		}
+		if want := []string{"sql", "custom-scope"}; !reflect.DeepEqual(a.Scopes, want) {
+			t.Errorf("M2M scopes = %v, want %v", a.Scopes, want)
 		}
 	})