From a93fe144ea52fd3e8ef1e90931fc1ce405ebc51e Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Wed, 11 Nov 2026 12:00:51 -0500 Subject: [PATCH 1/5] RHINENG-31827: build Kessel credentials via OIDC discovery The OAuth2 client was built at package init with KESSEL_AUTH_OIDC_ISSUER passed as the token endpoint. The shared env sets that to the SSO realm URL, so token fetches could never succeed; it went unnoticed because KESSEL_INSECURE=true drops call credentials. Build one credentials object lazily, resolve the token endpoint through OIDC discovery, and fail with a clear error when issuer, client id or secret is missing. Failures are not cached so a transient discovery error is retried. The object is exposed via getBearerToken so HTTP callers behind the east-west gateway can share it with the gRPC client. --- manager/middlewares/kessel.go | 57 +++++++++++++++++---- manager/middlewares/kessel_test.go | 80 ++++++++++++++++++++++++------ 2 files changed, 112 insertions(+), 25 deletions(-) diff --git a/manager/middlewares/kessel.go b/manager/middlewares/kessel.go index 8d1e4936c..8035f0a91 100644 --- a/manager/middlewares/kessel.go +++ b/manager/middlewares/kessel.go @@ -6,6 +6,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/gin-gonic/gin" @@ -18,19 +19,55 @@ import ( kesselRbacV2 "github.com/project-kessel/kessel-sdk-go/kessel/rbac/v2" ) -var credentials = auth.NewOAuth2ClientCredentials( - utils.CoreCfg.KesselAuthClientID, - utils.CoreCfg.KesselAuthClientSecret, - utils.CoreCfg.KesselAuthOIDCIssuer, +var ( + credentialsMu sync.Mutex + credentials *auth.OAuth2ClientCredentials ) -func setupClient() (kesselv2.KesselInventoryServiceClient, *grpc.ClientConn, error) { - clientBuilder := kesselv2.NewClientBuilder(utils.CoreCfg.KesselURL) - if utils.CoreCfg.KesselAuthEnabled { - clientBuilder = clientBuilder.OAuth2ClientAuthenticated(&credentials, nil) +// getCredentials lazily builds the single OAuth2 client shared by Kessel gRPC and authenticated RBAC calls. +// Failures are not cached so a transient OIDC discovery error is retried on the next call. +func getCredentials(ctx context.Context) (*auth.OAuth2ClientCredentials, error) { + credentialsMu.Lock() + defer credentialsMu.Unlock() + if credentials != nil { + return credentials, nil + } + cfg := &utils.CoreCfg + if cfg.KesselAuthOIDCIssuer == "" || cfg.KesselAuthClientID == "" || cfg.KesselAuthClientSecret == "" { + return nil, errors.New("KESSEL_AUTH_OIDC_ISSUER, KESSEL_AUTH_CLIENT_ID and KESSEL_AUTH_CLIENT_SECRET must be set") + } + discovery, err := auth.FetchOIDCDiscovery(ctx, cfg.KesselAuthOIDCIssuer, auth.FetchOIDCDiscoveryOptions{}) + if err != nil { + return nil, errors.Wrap(err, "OIDC discovery failed") + } + creds := auth.NewOAuth2ClientCredentials(cfg.KesselAuthClientID, cfg.KesselAuthClientSecret, discovery.TokenEndpoint) + credentials = &creds + return credentials, nil +} + +// getBearerToken returns an SSO service-account access token; the SDK caches and refreshes it. +func getBearerToken(ctx context.Context) (string, error) { + creds, err := getCredentials(ctx) + if err != nil { + return "", err } - if utils.CoreCfg.KesselInsecure { // insecure TLS + token, err := creds.GetToken(ctx, auth.GetTokenOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to get access token") + } + return token.AccessToken, nil +} + +func setupClient(ctx context.Context) (kesselv2.KesselInventoryServiceClient, *grpc.ClientConn, error) { + clientBuilder := kesselv2.NewClientBuilder(utils.CoreCfg.KesselURL) + if utils.CoreCfg.KesselInsecure { // insecure TLS, the SDK drops call credentials here clientBuilder = clientBuilder.Insecure() + } else if utils.CoreCfg.KesselAuthEnabled { + creds, err := getCredentials(ctx) + if err != nil { + return nil, nil, err + } + clientBuilder = clientBuilder.OAuth2ClientAuthenticated(creds, nil) } return clientBuilder.Build() } @@ -91,7 +128,7 @@ func useStreamedListObjects( } func hasPermissionKessel(c *gin.Context) { - client, conn, err := setupClient() + client, conn, err := setupClient(c) if err != nil { utils.LogError("err", err, "failed to setup Kessel service client") c.AbortWithStatusJSON(http.StatusInternalServerError, utils.ErrorResponse{ diff --git a/manager/middlewares/kessel_test.go b/manager/middlewares/kessel_test.go index 2648cfedb..3bd1889fd 100644 --- a/manager/middlewares/kessel_test.go +++ b/manager/middlewares/kessel_test.go @@ -78,40 +78,37 @@ func TestSetupClient(t *testing.T) { originalKesselAuthEnabled := utils.CoreCfg.KesselAuthEnabled originalKesselAuthClientID := utils.CoreCfg.KesselAuthClientID originalKesselAuthClientSecret := utils.CoreCfg.KesselAuthClientSecret + originalKesselAuthOIDCIssuer := utils.CoreCfg.KesselAuthOIDCIssuer + originalCredentials := credentials + ctx := t.Context() // insecure TLS and no auth utils.CoreCfg.KesselInsecure = true utils.CoreCfg.KesselAuthEnabled = false - client, conn, err := setupClient() - defer conn.Close() + client, conn, err := setupClient(ctx) if assert.NoError(t, err) { + defer conn.Close() assert.NotNil(t, client) } // secure TLS and no auth utils.CoreCfg.KesselInsecure = false - client, conn, err = setupClient() - defer conn.Close() + client, conn, err = setupClient(ctx) if assert.NoError(t, err) { + defer conn.Close() assert.NotNil(t, client) } - // insecure TLS and auth + // insecure TLS and auth: credentials are never built + credentials = nil utils.CoreCfg.KesselInsecure = true utils.CoreCfg.KesselAuthEnabled = true utils.CoreCfg.KesselAuthClientID = "test-client-id" utils.CoreCfg.KesselAuthClientSecret = "test-client-secret" - client, conn, err = setupClient() - defer conn.Close() - if assert.NoError(t, err) { - assert.NotNil(t, client) - } - - // secure TLS and auth - utils.CoreCfg.KesselInsecure = false - client, conn, err = setupClient() - defer conn.Close() + utils.CoreCfg.KesselAuthOIDCIssuer = "" + client, conn, err = setupClient(ctx) if assert.NoError(t, err) { + defer conn.Close() assert.NotNil(t, client) } @@ -120,6 +117,59 @@ func TestSetupClient(t *testing.T) { utils.CoreCfg.KesselAuthEnabled = originalKesselAuthEnabled utils.CoreCfg.KesselAuthClientID = originalKesselAuthClientID utils.CoreCfg.KesselAuthClientSecret = originalKesselAuthClientSecret + utils.CoreCfg.KesselAuthOIDCIssuer = originalKesselAuthOIDCIssuer + credentials = originalCredentials +} + +// newOIDCServer serves OIDC discovery and a client-credentials token endpoint. +func newOIDCServer(t *testing.T, token string) *httptest.Server { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/.well-known/openid-configuration": + _, _ = w.Write([]byte(`{"issuer":"` + srv.URL + `","token_endpoint":"` + srv.URL + `/token"}`)) + case "/token": + _, _ = w.Write([]byte(`{"access_token":"` + token + `","token_type":"Bearer","expires_in":3600}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +// resetKesselAuth restores CoreCfg and the shared credentials after the test. +func resetKesselAuth(t *testing.T) { + orig := utils.CoreCfg + credentialsMu.Lock() + origCreds := credentials + credentials = nil + credentialsMu.Unlock() + t.Cleanup(func() { + utils.CoreCfg = orig + credentials = origCreds + }) +} + +func TestSetupClientSecureAuth(t *testing.T) { + resetKesselAuth(t) + utils.CoreCfg.KesselInsecure = false + utils.CoreCfg.KesselAuthEnabled = true + utils.CoreCfg.KesselAuthClientID = "client-id" + utils.CoreCfg.KesselAuthClientSecret = "client-secret" + utils.CoreCfg.KesselAuthOIDCIssuer = "" + + // no issuer: fails instead of building an unauthenticated client + _, _, err := setupClient(t.Context()) + assert.Error(t, err) + + utils.CoreCfg.KesselAuthOIDCIssuer = newOIDCServer(t, "sso-token").URL + client, conn, err := setupClient(t.Context()) + if assert.NoError(t, err) { + defer conn.Close() + assert.NotNil(t, client) + } } func TestBuildPermission(t *testing.T) { From d24b0df9b632d993b21681d8bc46582a1a6137b9 Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Wed, 11 Nov 2026 12:01:10 -0500 Subject: [PATCH 2/5] RHINENG-31827: resolve RBAC from Clowder v2 and add gateway bearer auth Clowder writes dependencyEndpoints.v2 on every cluster, so read RBAC only from v2 rbac/service and drop the v1 fallback. The v2 entry also says whether the endpoint sits behind the east-west gateway (authenticated) and which CA to trust. When authenticated, send an SSO service-account bearer token alongside x-rh-identity; RBAC still evaluates the identity, the gateway only checks the bearer. If no token can be obtained the manager returns 502 without calling RBAC, so it never makes an unauthenticated call to an authenticated endpoint. A provided ca_certificate is added to the system trust store. The RBAC address is now validated when the middleware is built, so a missing dependency fails at startup instead of on the first request. RBAC_ADDRESS still overrides the address for local runs. --- base/utils/config.go | 27 ++++--- base/utils/config_clowder_v2_test.go | 99 +++++++++++------------ conf/cdappconfig.json | 17 ++-- manager/middlewares/rbac.go | 62 ++++++++++---- manager/middlewares/rbac_auth_test.go | 111 ++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 88 deletions(-) create mode 100644 manager/middlewares/rbac_auth_test.go diff --git a/base/utils/config.go b/base/utils/config.go index 79c933d32..d2dae14a6 100644 --- a/base/utils/config.go +++ b/base/utils/config.go @@ -73,6 +73,8 @@ type coreConfig struct { // services VmaasAddress string RbacAddress string + RbacAuthenticated bool + RbacCACert string CandlepinAddress string CandlepinCert string CandlepinKey string @@ -276,16 +278,22 @@ func initKafkaFromClowder() { type Endpoint clowder.DependencyEndpoint type PrivateEndpoint clowder.PrivateDependencyEndpoint -// resolveRbacV2Address returns the RBAC address from V2 dependency endpoints, -// or empty string if V2 is not available. -func resolveRbacV2Address() string { - if endpoint, ok := clowder.GetV2DependencyEndpoint("rbac", "service"); ok && endpoint.Uri != "" { - return endpoint.Uri +// initRbacFromClowder reads RBAC from the Clowder V2 dependency endpoint only. +// A missing endpoint leaves RbacAddress empty; the manager fails at startup unless RBAC_ADDRESS is set. +func initRbacFromClowder() { + endpoint, ok := clowder.GetV2DependencyEndpoint("rbac", "service") + if !ok || endpoint.Uri == "" { + return + } + CoreCfg.RbacAddress = endpoint.Uri + CoreCfg.RbacAuthenticated = endpoint.Authenticated + if endpoint.CaCertificate != nil { + CoreCfg.RbacCACert = *endpoint.CaCertificate } - return "" } func initServicesFromClowder() { + initRbacFromClowder() webappName := "webapp-service" if PodConfig.GetBool("use_vmaas_go", true) { webappName = "webapp-go" @@ -296,13 +304,6 @@ func initServicesFromClowder() { if strings.Contains(endpoint.Name, webappName) { CoreCfg.VmaasAddress = (*Endpoint)(&endpoint).buildURL() } - case "rbac": - // Prefer V2 dependency endpoint; fall back to V1 flat list. - if addr := resolveRbacV2Address(); addr != "" { - CoreCfg.RbacAddress = addr - } else { - CoreCfg.RbacAddress = (*Endpoint)(&endpoint).buildURL() - } case "rbac-service": CoreCfg.RbacURL = (*Endpoint)(&endpoint).buildURL() case "content-sources-backend": diff --git a/base/utils/config_clowder_v2_test.go b/base/utils/config_clowder_v2_test.go index 0e1a25e71..5b648231d 100644 --- a/base/utils/config_clowder_v2_test.go +++ b/base/utils/config_clowder_v2_test.go @@ -8,64 +8,55 @@ import ( "github.com/stretchr/testify/assert" ) -func TestResolveRbacV2Address(t *testing.T) { - origV2 := clowder.DependencyEndpointsV2 - defer func() { clowder.DependencyEndpointsV2 = origV2 }() - - t.Run("V2 available with URI", func(t *testing.T) { - clowder.DependencyEndpointsV2 = map[string]map[string]clowder.DependencyEndpointV2{ - "rbac": { - "service": {Uri: "https://rbac-service.svc:8443", Authenticated: false}, - }, - } - assert.Equal(t, "https://rbac-service.svc:8443", resolveRbacV2Address()) - }) - - t.Run("V2 nil map", func(t *testing.T) { - clowder.DependencyEndpointsV2 = nil - assert.Equal(t, "", resolveRbacV2Address()) - }) +const testRbacURI = "https://rbac-service.svc:8443" +const testRbacApp = "rbac" - t.Run("V2 empty URI", func(t *testing.T) { - clowder.DependencyEndpointsV2 = map[string]map[string]clowder.DependencyEndpointV2{ - "rbac": { - "service": {Uri: "", Authenticated: false}, - }, - } - assert.Equal(t, "", resolveRbacV2Address()) - }) +func TestInitRbacFromClowder(t *testing.T) { + origV2 := clowder.DependencyEndpointsV2 + origCfg := CoreCfg + defer func() { + clowder.DependencyEndpointsV2 = origV2 + CoreCfg = origCfg + }() - t.Run("V2 wrong app key", func(t *testing.T) { - clowder.DependencyEndpointsV2 = map[string]map[string]clowder.DependencyEndpointV2{ - "other-app": { - "service": {Uri: "https://other.svc:8443", Authenticated: false}, - }, - } - assert.Equal(t, "", resolveRbacV2Address()) - }) + run := func(v2 map[string]map[string]clowder.DependencyEndpointV2) { + CoreCfg.RbacAddress, CoreCfg.RbacAuthenticated, CoreCfg.RbacCACert = "", false, "" + clowder.DependencyEndpointsV2 = v2 + initRbacFromClowder() + } - t.Run("V2 wrong deployment key", func(t *testing.T) { - clowder.DependencyEndpointsV2 = map[string]map[string]clowder.DependencyEndpointV2{ - "rbac": { - "wrong-deploy": {Uri: "https://rbac.svc:8443", Authenticated: false}, - }, - } - assert.Equal(t, "", resolveRbacV2Address()) + t.Run("V2 unauthenticated", func(t *testing.T) { + run(map[string]map[string]clowder.DependencyEndpointV2{ + testRbacApp: {"service": {Uri: testRbacURI}}, + }) + assert.Equal(t, testRbacURI, CoreCfg.RbacAddress) + assert.False(t, CoreCfg.RbacAuthenticated) + assert.Equal(t, "", CoreCfg.RbacCACert) }) - t.Run("V2 with CA certificate", func(t *testing.T) { + t.Run("V2 authenticated with CA certificate", func(t *testing.T) { caPath := "/tmp/ca.crt" - clowder.DependencyEndpointsV2 = map[string]map[string]clowder.DependencyEndpointV2{ - "rbac": { - "service": { - Uri: "https://rbac-service.svc:8443", - Authenticated: false, - CaCertificate: &caPath, - }, - }, - } - assert.Equal(t, "https://rbac-service.svc:8443", resolveRbacV2Address()) + run(map[string]map[string]clowder.DependencyEndpointV2{ + testRbacApp: {"service": {Uri: "https://ewgw.example:443", Authenticated: true, CaCertificate: &caPath}}, + }) + assert.Equal(t, "https://ewgw.example:443", CoreCfg.RbacAddress) + assert.True(t, CoreCfg.RbacAuthenticated) + assert.Equal(t, caPath, CoreCfg.RbacCACert) }) + + missing := map[string]map[string]map[string]clowder.DependencyEndpointV2{ + "nil map": nil, + "empty URI": {testRbacApp: {"service": {Uri: ""}}}, + "wrong app": {"other-app": {"service": {Uri: "https://other.svc:8443"}}}, + "wrong endpoint": {testRbacApp: {"wrong-deploy": {Uri: "https://rbac.svc:8443"}}}, + } + for name, v2 := range missing { + t.Run("V2 missing: "+name, func(t *testing.T) { + run(v2) + assert.Equal(t, "", CoreCfg.RbacAddress) + assert.False(t, CoreCfg.RbacAuthenticated) + }) + } } func TestRbacAddressEnvOverride(t *testing.T) { @@ -73,7 +64,7 @@ func TestRbacAddressEnvOverride(t *testing.T) { defer func() { CoreCfg.RbacAddress = origAddr }() // Simulate V2-resolved address - CoreCfg.RbacAddress = "https://rbac-service.svc:8443" + CoreCfg.RbacAddress = testRbacURI // Env var override takes precedence (initServicesFromEnv behavior) t.Setenv("RBAC_ADDRESS", "http://localhost:8080") @@ -86,7 +77,7 @@ func TestRbacAddressEnvPreservesV2WhenUnset(t *testing.T) { defer func() { CoreCfg.RbacAddress = origAddr }() // Simulate V2-resolved address - CoreCfg.RbacAddress = "https://rbac-service.svc:8443" + CoreCfg.RbacAddress = testRbacURI // No env override: Getenv returns current value as default. // t.Setenv registers cleanup to restore original value after test; @@ -94,5 +85,5 @@ func TestRbacAddressEnvPreservesV2WhenUnset(t *testing.T) { t.Setenv("RBAC_ADDRESS", "") os.Unsetenv("RBAC_ADDRESS") initServicesFromEnv() - assert.Equal(t, "https://rbac-service.svc:8443", CoreCfg.RbacAddress) + assert.Equal(t, testRbacURI, CoreCfg.RbacAddress) } diff --git a/conf/cdappconfig.json b/conf/cdappconfig.json index 6fddbee82..8e9252351 100644 --- a/conf/cdappconfig.json +++ b/conf/cdappconfig.json @@ -67,15 +67,18 @@ "hostname": "platform", "name": "webapp-go", "port": 9001 - }, - { - "apiPath": "api/patch/v3", - "app": "rbac", - "hostname": "platform", - "name": "", - "port": 9001 } ], + "dependencyEndpoints": { + "v2": { + "rbac": { + "service": { + "uri": "http://platform:9001", + "authenticated": false + } + } + } + }, "privateEndpoints": [ { "app": "patchman", diff --git a/manager/middlewares/rbac.go b/manager/middlewares/rbac.go index 4ef373805..0c3d63f6b 100644 --- a/manager/middlewares/rbac.go +++ b/manager/middlewares/rbac.go @@ -7,8 +7,11 @@ import ( "app/base/telemetry" "app/base/utils" "app/manager/config" + "crypto/tls" + "crypto/x509" "fmt" "net/http" + "os" "strconv" "strings" "sync" @@ -40,23 +43,34 @@ var granularPerms = map[string]string{ "SystemDeleteHandler": "patch:system:write", } -// Make RBAC client on demand, with specified identity -func makeClient(identity string) *api.Client { - httpOnce.Do(func() { - httpClient = telemetry.InstrumentHTTPClient(httpClient) - }) +// Make RBAC client on demand, with specified identity and optional gateway bearer token +func makeClient(identity, bearer string) *api.Client { debugRequest := log.IsLevelEnabled(log.TraceLevel) - client := api.Client{ + headers := map[string]string{xRHIdentity: identity} + if bearer != "" { + headers["Authorization"] = "Bearer " + bearer + } + return &api.Client{ HTTPClient: httpClient, Debug: debugRequest, - DefaultHeaders: map[string]string{xRHIdentity: identity}, + DefaultHeaders: headers, } - if rbacURL == "" { - rbacURL = utils.FailIfEmpty(utils.CoreCfg.RbacAddress, "RBAC_ADDRESS") + base.RBACApiPrefix + - "/access/?application=patch,inventory,content-sources" +} + +func rbacTLSConfig() *tls.Config { + pem, err := os.ReadFile(utils.CoreCfg.RbacCACert) + if err != nil { + panic(fmt.Sprintf("failed to read RBAC CA certificate %s: %v", utils.CoreCfg.RbacCACert, err)) + } + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() } - return &client + if !pool.AppendCertsFromPEM(pem) { + panic(fmt.Sprintf("no certificates found in RBAC CA certificate %s", utils.CoreCfg.RbacCACert)) + } + return &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} } // for short lists like that is slice.Contains() faster than map lookup _, ok := map[key] @@ -110,8 +124,8 @@ func checkPermissions(access *rbac.AccessPagination, handlerName, method string) return false } -func isAccessGranted(c *gin.Context) bool { - client := makeClient(c.GetHeader("x-rh-identity")) +func isAccessGranted(c *gin.Context, bearer string) bool { + client := makeClient(c.GetHeader("x-rh-identity"), bearer) access := rbac.AccessPagination{} res, err := client.Request(&base.Context, http.MethodGet, rbacURL, nil, &access) if res != nil && res.Body != nil { @@ -193,8 +207,28 @@ func RBAC() gin.HandlerFunc { return func(_ *gin.Context) {} } + rbacURL = utils.FailIfEmpty(utils.CoreCfg.RbacAddress, "RBAC_ADDRESS (or Clowder v2 dependency rbac/service)") + + base.RBACApiPrefix + "/access/?application=patch,inventory,content-sources" + httpOnce.Do(func() { + if utils.CoreCfg.RbacCACert != "" { + httpClient = &http.Client{Transport: &http.Transport{TLSClientConfig: rbacTLSConfig()}} + } + httpClient = telemetry.InstrumentHTTPClient(httpClient) + }) + return func(c *gin.Context) { - if isAccessGranted(c) { + var bearer string + if utils.CoreCfg.RbacAuthenticated { + var err error + // fail closed: never call an authenticated RBAC endpoint without the bearer + if bearer, err = getBearerToken(c); err != nil { + utils.LogError("err", err, "Failed to get token for authenticated RBAC endpoint") + serviceErrorCnt.WithLabelValues("rbac", strconv.Itoa(http.StatusBadGateway)).Inc() + c.AbortWithStatusJSON(http.StatusBadGateway, utils.ErrorResponse{Error: "Communication with RBAC failed"}) + return + } + } + if isAccessGranted(c, bearer) { return } c.AbortWithStatusJSON(http.StatusUnauthorized, diff --git a/manager/middlewares/rbac_auth_test.go b/manager/middlewares/rbac_auth_test.go new file mode 100644 index 000000000..7c627c31a --- /dev/null +++ b/manager/middlewares/rbac_auth_test.go @@ -0,0 +1,111 @@ +package middlewares + +import ( + "app/base/utils" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withRbacServer points the RBAC middleware at a fake RBAC that grants full access and records request headers. +func withRbacServer(t *testing.T, authenticated bool, issuer string) (*http.Header, *int) { + headers := &http.Header{} + calls := new(int) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *calls++ + *headers = r.Header.Clone() + _, _ = w.Write([]byte(`{"data":[{"permission":"patch:*:*"},{"permission":"inventory:*:*"}]}`)) + })) + t.Cleanup(srv.Close) + + resetKesselAuth(t) + utils.CoreCfg.RbacAddress = srv.URL + utils.CoreCfg.RbacAuthenticated = authenticated + utils.CoreCfg.KesselAuthOIDCIssuer = issuer + utils.CoreCfg.KesselAuthClientID = "client-id" + utils.CoreCfg.KesselAuthClientSecret = "client-secret" + return headers, calls +} + +func serveRBAC() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/", nil) + req.Header.Set(xRHIdentity, testXRHIdentity) + router := gin.New() + router.Use(RBAC()) + router.GET("/", okHandler) + router.ServeHTTP(w, req) + return w +} + +func TestRBACAuthenticatedAddsBearer(t *testing.T) { + oidc := newOIDCServer(t, "sso-token") + headers, calls := withRbacServer(t, true, oidc.URL) + + w := serveRBAC() + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, 1, *calls) + assert.Equal(t, "Bearer sso-token", headers.Get("Authorization")) + assert.Equal(t, testXRHIdentity, headers.Get(xRHIdentity)) +} + +func TestRBACUnauthenticatedNoToken(t *testing.T) { + // empty issuer would fail any token fetch, so a 200 proves none was attempted + headers, calls := withRbacServer(t, false, "") + + w := serveRBAC() + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, 1, *calls) + assert.Empty(t, headers.Get("Authorization")) + assert.Equal(t, testXRHIdentity, headers.Get(xRHIdentity)) +} + +func TestRBACAuthenticatedTokenFailureFailsClosed(t *testing.T) { + _, calls := withRbacServer(t, true, "") + + w := serveRBAC() + assert.Equal(t, http.StatusBadGateway, w.Code) + assert.Equal(t, 0, *calls) +} + +func TestRBACAndKesselShareCredentials(t *testing.T) { + oidc := newOIDCServer(t, "sso-token") + withRbacServer(t, true, oidc.URL) + utils.CoreCfg.KesselInsecure = false + utils.CoreCfg.KesselAuthEnabled = true + + _, err := getBearerToken(t.Context()) + require.NoError(t, err) + first := credentials + _, conn, err := setupClient(t.Context()) + require.NoError(t, err) + defer conn.Close() + assert.Same(t, first, credentials) +} + +func TestRbacTLSConfigTrustsCA(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + caPath := filepath.Join(t.TempDir(), "ca.crt") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + + orig := utils.CoreCfg.RbacCACert + defer func() { utils.CoreCfg.RbacCACert = orig }() + utils.CoreCfg.RbacCACert = caPath + + client := &http.Client{Transport: &http.Transport{TLSClientConfig: rbacTLSConfig()}} + res, err := client.Get(srv.URL) + require.NoError(t, err) + res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) +} From 8bcbdf652aa7a0b5bd30ffee5426cd30cd643a4f Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Wed, 11 Nov 2026 12:11:40 -0500 Subject: [PATCH 3/5] RHINENG-31827: redact Authorization header in HTTP debug dumps At trace level CallAPI dumps outgoing requests including headers, so authenticated RBAC calls would write the gateway bearer token to logs. The header is masked in place only for the dump; the request still carries the real token. --- base/utils/http.go | 8 ++++++++ base/utils/http_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/base/utils/http.go b/base/utils/http.go index c4eafff11..10bb60824 100644 --- a/base/utils/http.go +++ b/base/utils/http.go @@ -65,7 +65,15 @@ func closeHTTPResponse(resp *http.Response) { func CallAPI(client *http.Client, request *http.Request, debugEnabled bool) (*http.Response, error) { if debugEnabled { + // redact in place: DumpRequestOut restores the body of the request it is given, so a clone would lose it + auth := request.Header.Get("Authorization") + if auth != "" { + request.Header.Set("Authorization", "REDACTED") + } dump, err := httputil.DumpRequestOut(request, true) + if auth != "" { + request.Header.Set("Authorization", auth) + } if err != nil { return nil, err } diff --git a/base/utils/http_test.go b/base/utils/http_test.go index a659ef0f5..27b34bd22 100644 --- a/base/utils/http_test.go +++ b/base/utils/http_test.go @@ -1,13 +1,16 @@ package utils import ( + "bytes" "errors" "io" "net/http" + "net/http/httptest" "strings" "sync/atomic" "testing" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -139,3 +142,29 @@ func TestCloseHTTPResponseLimitsDrain(t *testing.T) { assert.True(t, body.closed.Load()) assert.Equal(t, int64(maxHTTPResponseDrain), body.readBytes.Load()) } + +func TestCallAPIDebugRedactsAuthorization(t *testing.T) { + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + })) + defer srv.Close() + + var logged bytes.Buffer + origOut, origLevel := log.StandardLogger().Out, log.GetLevel() + log.SetOutput(&logged) + log.SetLevel(log.DebugLevel) + defer func() { + log.SetOutput(origOut) + log.SetLevel(origLevel) + }() + + req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) + req.Header.Set("Authorization", "Bearer secret-token") + resp, err := CallAPI(srv.Client(), req, true) + assert.NoError(t, err) + resp.Body.Close() + + assert.NotContains(t, logged.String(), "secret-token") + assert.Equal(t, "Bearer secret-token", gotAuth) // real request still carries the token +} From 2da70220e3dfd3a313ac8771d01759aff0e4d14b Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Wed, 11 Nov 2026 12:11:42 -0500 Subject: [PATCH 4/5] RHINENG-31827: time out OIDC discovery and token calls The request context passed in is a gin.Context, which is never cancelled, and discovery runs while holding credentialsMu. A stalled SSO endpoint would therefore hang every RBAC-checked request, not just one. Bound each discovery/token HTTP call to 10s. --- manager/middlewares/kessel.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/manager/middlewares/kessel.go b/manager/middlewares/kessel.go index 8035f0a91..d7ba9f8f1 100644 --- a/manager/middlewares/kessel.go +++ b/manager/middlewares/kessel.go @@ -22,6 +22,8 @@ import ( var ( credentialsMu sync.Mutex credentials *auth.OAuth2ClientCredentials + // bounds OIDC discovery and token calls; gin.Context is never cancelled, and discovery runs under credentialsMu + oidcHTTPClient = &http.Client{Timeout: 10 * time.Second} ) // getCredentials lazily builds the single OAuth2 client shared by Kessel gRPC and authenticated RBAC calls. @@ -34,9 +36,11 @@ func getCredentials(ctx context.Context) (*auth.OAuth2ClientCredentials, error) } cfg := &utils.CoreCfg if cfg.KesselAuthOIDCIssuer == "" || cfg.KesselAuthClientID == "" || cfg.KesselAuthClientSecret == "" { - return nil, errors.New("KESSEL_AUTH_OIDC_ISSUER, KESSEL_AUTH_CLIENT_ID and KESSEL_AUTH_CLIENT_SECRET must be set") + return nil, errors.New( + "KESSEL_AUTH_OIDC_ISSUER, KESSEL_AUTH_CLIENT_ID and KESSEL_AUTH_CLIENT_SECRET must be set") } - discovery, err := auth.FetchOIDCDiscovery(ctx, cfg.KesselAuthOIDCIssuer, auth.FetchOIDCDiscoveryOptions{}) + discovery, err := auth.FetchOIDCDiscovery( + ctx, cfg.KesselAuthOIDCIssuer, auth.FetchOIDCDiscoveryOptions{HttpClient: oidcHTTPClient}) if err != nil { return nil, errors.Wrap(err, "OIDC discovery failed") } @@ -51,7 +55,7 @@ func getBearerToken(ctx context.Context) (string, error) { if err != nil { return "", err } - token, err := creds.GetToken(ctx, auth.GetTokenOptions{}) + token, err := creds.GetToken(ctx, auth.GetTokenOptions{HttpClient: oidcHTTPClient}) if err != nil { return "", errors.Wrap(err, "failed to get access token") } From d0601c803c52481d0bc5cf82e20d7f5d394a8115 Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Wed, 11 Nov 2026 12:11:42 -0500 Subject: [PATCH 5/5] RHINENG-31827: drop v2 RBAC auth when RBAC_ADDRESS overrides it authenticated and ca_certificate describe the Clowder v2 endpoint. If RBAC_ADDRESS points elsewhere, keeping them would send the service account bearer token to a host Clowder never vouched for. An override equal to the v2 address keeps the v2 settings. --- base/utils/config.go | 5 ++++- base/utils/config_clowder_v2_test.go | 33 ++++++++++++++++++---------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/base/utils/config.go b/base/utils/config.go index d2dae14a6..1afb4d863 100644 --- a/base/utils/config.go +++ b/base/utils/config.go @@ -190,7 +190,10 @@ func initTopicsFromEnv() { func initServicesFromEnv() { CoreCfg.VmaasAddress = Getenv("VMAAS_ADDRESS", CoreCfg.VmaasAddress) - CoreCfg.RbacAddress = Getenv("RBAC_ADDRESS", CoreCfg.RbacAddress) + if addr := Getenv("RBAC_ADDRESS", CoreCfg.RbacAddress); addr != CoreCfg.RbacAddress { + // v2 auth metadata describes the Clowder endpoint only; never send its bearer or CA to an override + CoreCfg.RbacAddress, CoreCfg.RbacAuthenticated, CoreCfg.RbacCACert = addr, false, "" + } CoreCfg.CandlepinAddress = Getenv("CANDLEPIN_ADDRESS", CoreCfg.CandlepinAddress) CoreCfg.CandlepinCert = Getenv("CANDLEPIN_CERT", CoreCfg.CandlepinCert) CoreCfg.CandlepinKey = Getenv("CANDLEPIN_KEY", CoreCfg.CandlepinKey) diff --git a/base/utils/config_clowder_v2_test.go b/base/utils/config_clowder_v2_test.go index 5b648231d..192edfc2d 100644 --- a/base/utils/config_clowder_v2_test.go +++ b/base/utils/config_clowder_v2_test.go @@ -59,25 +59,35 @@ func TestInitRbacFromClowder(t *testing.T) { } } -func TestRbacAddressEnvOverride(t *testing.T) { - origAddr := CoreCfg.RbacAddress - defer func() { CoreCfg.RbacAddress = origAddr }() +// setAuthenticatedV2Rbac simulates an authenticated V2-resolved RBAC endpoint. +func setAuthenticatedV2Rbac(t *testing.T) { + origCfg := CoreCfg + t.Cleanup(func() { CoreCfg = origCfg }) + CoreCfg.RbacAddress, CoreCfg.RbacAuthenticated, CoreCfg.RbacCACert = testRbacURI, true, "/tmp/ca.crt" +} - // Simulate V2-resolved address - CoreCfg.RbacAddress = testRbacURI +func TestRbacAddressEnvOverrideDropsV2Auth(t *testing.T) { + setAuthenticatedV2Rbac(t) - // Env var override takes precedence (initServicesFromEnv behavior) + // Env var override takes precedence; the V2 bearer/CA must not follow it t.Setenv("RBAC_ADDRESS", "http://localhost:8080") initServicesFromEnv() assert.Equal(t, "http://localhost:8080", CoreCfg.RbacAddress) + assert.False(t, CoreCfg.RbacAuthenticated) + assert.Equal(t, "", CoreCfg.RbacCACert) } -func TestRbacAddressEnvPreservesV2WhenUnset(t *testing.T) { - origAddr := CoreCfg.RbacAddress - defer func() { CoreCfg.RbacAddress = origAddr }() +func TestRbacAddressEnvSameAsV2KeepsAuth(t *testing.T) { + setAuthenticatedV2Rbac(t) - // Simulate V2-resolved address - CoreCfg.RbacAddress = testRbacURI + t.Setenv("RBAC_ADDRESS", testRbacURI) + initServicesFromEnv() + assert.True(t, CoreCfg.RbacAuthenticated) + assert.Equal(t, "/tmp/ca.crt", CoreCfg.RbacCACert) +} + +func TestRbacAddressEnvPreservesV2WhenUnset(t *testing.T) { + setAuthenticatedV2Rbac(t) // No env override: Getenv returns current value as default. // t.Setenv registers cleanup to restore original value after test; @@ -86,4 +96,5 @@ func TestRbacAddressEnvPreservesV2WhenUnset(t *testing.T) { os.Unsetenv("RBAC_ADDRESS") initServicesFromEnv() assert.Equal(t, testRbacURI, CoreCfg.RbacAddress) + assert.True(t, CoreCfg.RbacAuthenticated) }