fix(kernel): Thrift-parity fixes + GetArrowBatches + decimal-as-float opt-in + U2M scopes/telemetry + SPOG warehouse-id routing

auth/oauth/m2m/m2m.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/databricks/databricks-sql-go/auth"
 	"github.com/databricks/databricks-sql-go/auth/oauth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 )
@@ -72,12 +72,14 @@ func (c *authClient) Authenticate(r *http.Request) error {
 
 	c.tokenSource = GetTokenSource(config)
 	token, err := c.tokenSource.Token()
-	log.Debug().Msgf("databricks OAuth token fetched successfully")
 	if err != nil {
-		log.Err(err).Msg("failed to get token")
+		logger.Err(err).Msg("failed to get token")
 
 		return err
 	}
+	// Log via the driver's configurable logger (defaults to Warn) rather than the
+	// global zerolog logger, so this line honors SetLogLevel like every other.
+	logger.Debug().Msgf("databricks OAuth token fetched successfully")
 	token.SetAuthHeader(r)
 
 	return nil

auth/oauth/oauth.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-oidc/v3/oidc"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 )
 
@@ -99,21 +99,21 @@ func resolveOIDCIssuer(ctx context.Context, client *http.Client, hostName string
 	cfgURL := fmt.Sprintf("https://%s/.well-known/databricks-config", hostName)
 	meta, ok := fetchHostMetadata(ctx, client, cfgURL)
 	if !ok || meta.OIDCEndpoint == "" {
-		log.Debug().Msgf("oauth: no usable databricks-config for %q; using bare-host OIDC issuer", hostName)
+		logger.Debug().Msgf("oauth: no usable databricks-config for %q; using bare-host OIDC issuer", hostName)
 		return fallback
 	}
 
 	// An account-rooted endpoint needs a non-empty account_id; otherwise the
 	// placeholder would resolve to a malformed ".../accounts/" issuer. Fall back
 	// rather than emit it (the function's documented contract).
 	if strings.Contains(meta.OIDCEndpoint, accountIDPlaceholder) && meta.AccountID == "" {
-		log.Warn().Msgf("oauth: databricks-config for %q has an %s placeholder but empty account_id; using bare-host OIDC issuer", hostName, accountIDPlaceholder)
+		logger.Warn().Msgf("oauth: databricks-config for %q has an %s placeholder but empty account_id; using bare-host OIDC issuer", hostName, accountIDPlaceholder)
 		return fallback
 	}
 
 	issuer := substituteAccountID(meta)
 	if !isValidDatabricksIssuer(issuer) {
-		log.Warn().Msgf("oauth: databricks-config for %q advertised an unusable oidc_endpoint %q; using bare-host OIDC issuer", hostName, issuer)
+		logger.Warn().Msgf("oauth: databricks-config for %q advertised an unusable oidc_endpoint %q; using bare-host OIDC issuer", hostName, issuer)
 		return fallback
 	}
 	return issuer

auth/oauth/oauth_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -62,6 +63,28 @@ func TestGetAzureDnsZone(t *testing.T) {
 	}
 }
 
+// TestGetScopes pins the literal cloud-specific scope sets (not GetScopes's own
+// output) so a regression in the AWS/GCP-vs-Azure branching — the source of the
+// kernel U2M access_denied incident — is caught rather than asserted tautologically.
+func TestGetScopes(t *testing.T) {
+	cases := []struct {
+		name string
+		host string
+		want []string
+	}{
+		{"AWS", "dbc-1234.cloud.databricks.com", []string{"offline_access", "sql"}},
+		{"GCP", "x.gcp.databricks.com", []string{"offline_access", "sql"}},
+		{"Azure", "adb-123.azuredatabricks.net", []string{"offline_access", "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/user_impersonation"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := GetScopes(tc.host, nil); !reflect.DeepEqual(got, tc.want) {
+				t.Fatalf("GetScopes(%q, nil) = %v, want %v", tc.host, got, tc.want)
+			}
+		})
+	}
+}
+
 // TestResolveOIDCIssuer drives resolveOIDCIssuer end-to-end against an httptest
 // server: the server stands in for the connection host, so a 200 with a given
 // databricks-config body exercises the real fetch + substitution + validation +

auth/oauth/u2m/authenticator.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/databricks/databricks-sql-go/auth"
 	"github.com/databricks/databricks-sql-go/auth/oauth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 )
 
@@ -84,15 +84,9 @@ type u2mAuthenticator struct {
 // mode. It structurally satisfies the U2MCredentialsProvider interface the kernel
 // backend asserts (defined in internal/backend/kernel, off the public API).
 //
-// Only the client id crosses to the kernel — NOT the scopes. The Thrift path
-// requests offline_access + sql (AWS/GCP) or offline_access + <tenant>/
-// user_impersonation (Azure) via oauth.GetScopes, while the kernel's U2M flow
-// applies its own default set (all-apis + offline_access). Neither backend exposes
-// a user-facing U2M-scopes option, so this is a fixed-vs-fixed difference, not a
-// dropped caller choice; against the built-in databricks-sql-connector public
-// client (which all-apis targets) both authorize successfully. If a U2M-scopes
-// option is ever added, forward it via kernel.Auth.Scopes (already wired through
-// set_auth_u2m) so the two paths request the same set.
+// Only the client id is read here; resolveKernelAuth pairs it with the same
+// oauth.GetScopes set the Thrift path requests, so both backends authorize against
+// the built-in databricks-sql-connector client identically (not the kernel default).
 func (c *u2mAuthenticator) U2MClientID() string { return c.clientID }
 
 // Auth will start the OAuth Authorization Flow to authenticate the cli client
@@ -160,7 +154,7 @@ func (tsp *tokenSourceProvider) GetTokenSource() (oauth2.TokenSource, error) {
 	loginURL := tsp.config.AuthCodeURL(state, challenge, challengeMethod)
 	tsp.state = state
 
-	log.Info().Msgf("listening on %s://%s/", tsp.redirectURL.Scheme, tsp.redirectURL.Host)
+	logger.Info().Msgf("listening on %s://%s/", tsp.redirectURL.Scheme, tsp.redirectURL.Host)
 	listener, err := net.Listen("tcp", tsp.redirectURL.Host)
 	if err != nil {
 		return nil, err
@@ -225,28 +219,28 @@ func (tsp *tokenSourceProvider) ServeHTTP(w http.ResponseWriter, r *http.Request
 
 	// Do some checking of the response here to show more relevant content
 	if resp.err != "" {
-		log.Error().Msg(resp.err)
+		logger.Error().Msg(resp.err)
 		w.WriteHeader(http.StatusBadRequest)
 		_, err := w.Write([]byte(errorHTML("Identity Provider returned an error: " + resp.err))) //nolint:gosec // XSS not a concern for local OAuth callback
 		if err != nil {
-			log.Error().Err(err).Msg("unable to write error response")
+			logger.Error().Err(err).Msg("unable to write error response")
 		}
 		return
 	}
 	if resp.state != tsp.state && r.URL.String() != "/favicon.ico" {
 		msg := "Authentication state received did not match original request. Please try to login again."
-		log.Error().Msg(msg)
+		logger.Error().Msg(msg)
 		w.WriteHeader(http.StatusBadRequest)
 		_, err := w.Write([]byte(errorHTML(msg)))
 		if err != nil {
-			log.Error().Err(err).Msg("unable to write error response")
+			logger.Error().Err(err).Msg("unable to write error response")
 		}
 		return
 	}
 
 	_, err := w.Write([]byte(infoHTML("CLI Login Success", "You may close this window anytime now and go back to terminal")))
 	if err != nil {
-		log.Error().Err(err).Msg("unable to write success response")
+		logger.Error().Err(err).Msg("unable to write success response")
 	}
 }
 

auth/tokenprovider/authenticator.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 
 	"github.com/databricks/databricks-sql-go/auth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 )
 
 // TokenProviderAuthenticator implements auth.Authenticator using a TokenProvider.
@@ -47,7 +47,7 @@ func (a *TokenProviderAuthenticator) Authenticate(r *http.Request) error {
 	}
 
 	token.SetAuthHeader(r)
-	log.Debug().Msgf("token provider authenticator: authenticated using provider %s", a.provider.Name())
+	logger.Debug().Msgf("token provider authenticator: authenticated using provider %s", a.provider.Name())
 
 	return nil
 }

auth/tokenprovider/cached.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 )
 
 // CachedTokenProvider wraps another provider and caches tokens
@@ -43,7 +43,7 @@ func (p *CachedTokenProvider) GetToken(ctx context.Context) (*Token, error) {
 
 	// If cache is valid and not being refreshed, return a copy
 	if cached != nil && !needsRefresh {
-		log.Debug().Msgf("cached token provider: using cached token for provider %s", p.provider.Name())
+		logger.Debug().Msgf("cached token provider: using cached token for provider %s", p.provider.Name())
 		// Return a copy to avoid concurrent modification issues
 		return copyToken(cached), nil
 	}
@@ -75,7 +75,7 @@ func (p *CachedTokenProvider) GetToken(ctx context.Context) (*Token, error) {
 	p.mutex.Unlock()
 
 	// Fetch new token WITHOUT holding the lock
-	log.Debug().Msgf("cached token provider: fetching new token from provider %s", p.provider.Name())
+	logger.Debug().Msgf("cached token provider: fetching new token from provider %s", p.provider.Name())
 	token, err := p.provider.GetToken(ctx)
 
 	// Update cache with result

auth/tokenprovider/exchange.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/databricks/databricks-sql-go/logger"
 	"github.com/golang-jwt/jwt/v5"
-	"github.com/rs/zerolog/log"
 )
 
 // FederationProvider wraps another token provider and automatically handles token exchange
@@ -61,16 +61,16 @@ func (p *FederationProvider) GetToken(ctx context.Context) (*Token, error) {
 
 	// Check if token is a JWT and needs exchange
 	if p.needsTokenExchange(baseToken.AccessToken) {
-		log.Debug().Msgf("federation provider: attempting token exchange for %s", p.baseProvider.Name())
+		logger.Debug().Msgf("federation provider: attempting token exchange for %s", p.baseProvider.Name())
 
 		// Try token exchange
 		exchangedToken, err := p.tryTokenExchange(ctx, baseToken.AccessToken)
 		if err != nil {
-			log.Warn().Err(err).Msg("federation provider: token exchange failed, using original token")
+			logger.Warn().Err(err).Msg("federation provider: token exchange failed, using original token")
 			return baseToken, nil // Fall back to original token
 		}
 
-		log.Debug().Msg("federation provider: token exchange successful")
+		logger.Debug().Msg("federation provider: token exchange successful")
 		return exchangedToken, nil
 	}
 
@@ -87,7 +87,7 @@ func (p *FederationProvider) needsTokenExchange(tokenString string) bool {
 	// 3. Token validation will be done by Databricks during exchange
 	token, _, err := new(jwt.Parser).ParseUnverified(tokenString, jwt.MapClaims{})
 	if err != nil {
-		log.Debug().Err(err).Msg("federation provider: not a JWT token, skipping exchange")
+		logger.Debug().Err(err).Msg("federation provider: not a JWT token, skipping exchange")
 		return false
 	}
 
@@ -114,7 +114,7 @@ func (p *FederationProvider) tryTokenExchange(ctx context.Context, subjectToken
 		exchangeURL = "https://" + exchangeURL
 	} else if strings.HasPrefix(exchangeURL, "http://") {
 		// Warn if using insecure HTTP for token exchange
-		log.Warn().Msgf("federation provider: using insecure HTTP for token exchange: %s", exchangeURL)
+		logger.Warn().Msgf("federation provider: using insecure HTTP for token exchange: %s", exchangeURL)
 	}
 	if !strings.HasSuffix(exchangeURL, "/") {
 		exchangeURL += "/"
@@ -179,7 +179,7 @@ func (p *FederationProvider) tryTokenExchange(ctx context.Context, subjectToken
 		return nil, fmt.Errorf("token exchange returned empty access token")
 	}
 	if tokenResp.TokenType == "" {
-		log.Debug().Msg("token exchange: token_type not specified, defaulting to Bearer")
+		logger.Debug().Msg("token exchange: token_type not specified, defaulting to Bearer")
 		tokenResp.TokenType = "Bearer"
 	}
 	if tokenResp.ExpiresIn < 0 {
@@ -211,15 +211,15 @@ func (p *FederationProvider) isSameHost(url1, url2 string) bool {
 	u2, err2 := url.Parse(parsedURL2)
 
 	if err1 != nil || err2 != nil {
-		log.Debug().Msgf("federation provider: failed to parse URLs for comparison: url1=%s err1=%v, url2=%s err2=%v",
+		logger.Debug().Msgf("federation provider: failed to parse URLs for comparison: url1=%s err1=%v, url2=%s err2=%v",
 			url1, err1, parsedURL2, err2)
 		return false
 	}
 
 	// Use Hostname() instead of Host to ignore port differences
 	// This handles cases like "host.com:443" == "host.com" for HTTPS
 	isSame := u1.Hostname() == u2.Hostname()
-	log.Debug().Msgf("federation provider: host comparison: %s vs %s = %v", u1.Hostname(), u2.Hostname(), isSame)
+	logger.Debug().Msgf("federation provider: host comparison: %s vs %s = %v", u1.Hostname(), u2.Hostname(), isSame)
 	return isSame
 }
 

connector.go

@@ -31,6 +31,13 @@ type connector struct {
 	client *http.Client
 }
 
+// interactiveU2MAuthenticator is satisfied only by the browser-based U2M
+// authenticator (U2MClientID is unique to it; PAT/M2M lack it). Matches the
+// structural check the kernel backend uses to detect U2M.
+type interactiveU2MAuthenticator interface {
+	U2MClientID() string
+}
+
 // Connect returns a connection to the Databricks database from a connection pool.
 func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	defer debuglog.Track(ctx, "connector.Connect", "host=%s", c.cfg.Host)()
@@ -86,16 +93,31 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 		telemetryClient = withSpogHeaders(c.client, spogHeaders)
 	}
 
+	// Skip telemetry on the kernel U2M path: the kernel owns the interactive browser
+	// flow, so the telemetry/feature-flag call through the interactive authenticator
+	// would launch a second, redundant browser (and can block connect on its
+	// callback). Telemetry is best-effort, so it's dropped here rather than made to
+	// prompt. Unauthenticated telemetry (Python/Node parity) is tracked in PECOBLR-3839.
+	skipTelemetry := false
+	if c.cfg.UseKernel {
+		if _, isU2M := c.cfg.Authenticator.(interactiveU2MAuthenticator); isU2M {
+			skipTelemetry = true
+			log.Debug().Msg("telemetry skipped: kernel U2M owns the interactive auth flow")
+		}
+	}
+
 	// Initialize telemetry: client config overlay decides; if unset, feature flags decide
-	conn.telemetry = telemetry.InitializeForConnection(ctx, telemetry.TelemetryInitOptions{
-		Host:            c.cfg.Host,
-		DriverVersion:   c.cfg.DriverVersion,
-		UserAgent:       client.BuildUserAgent(c.cfg),
-		HTTPClient:      telemetryClient,
-		EnableTelemetry: c.cfg.EnableTelemetry,
-		BatchSize:       c.cfg.TelemetryBatchSize,
-		FlushInterval:   c.cfg.TelemetryFlushInterval,
-	})
+	if !skipTelemetry {
+		conn.telemetry = telemetry.InitializeForConnection(ctx, telemetry.TelemetryInitOptions{
+			Host:            c.cfg.Host,
+			DriverVersion:   c.cfg.DriverVersion,
+			UserAgent:       client.BuildUserAgent(c.cfg),
+			HTTPClient:      telemetryClient,
+			EnableTelemetry: c.cfg.EnableTelemetry,
+			BatchSize:       c.cfg.TelemetryBatchSize,
+			FlushInterval:   c.cfg.TelemetryFlushInterval,
+		})
+	}
 	if conn.telemetry != nil {
 		log.Debug().Msg("telemetry initialized for connection")
 		conn.telemetry.RecordOperation(ctx, conn.id, "", telemetry.OperationTypeCreateSession, sessionLatencyMs, nil)
@@ -580,6 +602,21 @@ func kernelExperimental(c *config.Config) *config.KernelExperimentalConfig {
 	return c.KernelExperimental
 }
 
+// WithKernelDecimalAsFloat makes the kernel path scan top-level DECIMAL columns to
+// a lossy float64 instead of the exact fixed-point string. The kernel still
+// receives native Arrow Decimal128; this only changes how the Go scanner
+// materializes each cell, skipping per-cell string formatting for a cheap scalar.
+// Precision beyond ~15-17 digits is lost, so it is opt-in and off by default;
+// it mirrors the Thrift driver's pre-UseArrowNativeDecimal behavior.
+//
+// EXPERIMENTAL, kernel-only: the default (Thrift) backend rejects this at connect
+// (use WithArrowNativeDecimal there instead).
+func WithKernelDecimalAsFloat(asFloat bool) ConnOption {
+	return func(c *config.Config) {
+		kernelExperimental(c).DecimalAsFloat = asFloat
+	}
+}
+
 // WithKernelTrustedCerts adds a PEM CA-certificate bundle to the kernel's TLS
 // trust store on top of the system roots — for a corporate re-signing proxy or an
 // on-prem CA. Required (rather than relying on SSL_CERT_FILE) because the kernel's

connector_kernel_u2m_test.go

@@ -0,0 +1,32 @@
+package dbsql
+
+import (
+	"testing"
+
+	"github.com/databricks/databricks-sql-go/auth/oauth/m2m"
+	"github.com/databricks/databricks-sql-go/auth/pat"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// u2mShaped stands in for the browser-based U2M authenticator, which exposes
+// U2MClientID() (its real constructor does live OIDC discovery, unusable in a unit
+// test). The kernel-U2M telemetry guard keys off exactly this method.
+type u2mShaped struct{}
+
+func (u2mShaped) U2MClientID() string { return "databricks-sql-connector" }
+
+// TestInteractiveU2MAuthenticatorDetection pins the structural check the kernel-U2M
+// telemetry guard relies on: only a U2M-shaped authenticator satisfies it, so
+// PAT/M2M keep authenticated telemetry while U2M does not trigger a 2nd browser.
+func TestInteractiveU2MAuthenticatorDetection(t *testing.T) {
+	_, isU2M := interface{}(u2mShaped{}).(interactiveU2MAuthenticator)
+	assert.True(t, isU2M, "a U2M-shaped authenticator must satisfy interactiveU2MAuthenticator")
+
+	_, patIsU2M := interface{}(&pat.PATAuth{AccessToken: "dapi-x"}).(interactiveU2MAuthenticator)
+	assert.False(t, patIsU2M, "PAT must NOT satisfy interactiveU2MAuthenticator")
+
+	m2mAuth := m2m.NewAuthenticator("cid", "secret", "dbc-1234.cloud.databricks.com")
+	_, m2mIsU2M := m2mAuth.(interactiveU2MAuthenticator)
+	assert.False(t, m2mIsU2M, "M2M must NOT satisfy interactiveU2MAuthenticator")
+}