feat: handle revoked OAuth grants for MCP servers gracefully (#27264)

Closes
[CODAGT-792](https://linear.app/codercom/issue/CODAGT-792/handle-revoked-oauth-grants-for-mcp-servers-gracefully).

When a user revokes an upstream OAuth grant for an MCP server used by
Coder Agents, Coder kept treating the cached token as valid:
`invalid_grant` refresh failures were logged and swallowed, the dead
bearer token kept being attached, the list endpoints re-attempted the
refresh on every call, and the UI kept showing the server as
authenticated.

## Changes

Backend, mirroring the `external_auth_links` prior art:

- New migration adds
`mcp_server_user_tokens.oauth_refresh_failure_reason`.
`UpsertMCPServerUserToken` clears it, so completing the OAuth flow again
recovers the row.
- New `MarkMCPServerUserTokenRefreshFailure` query records the failure
and clears all token material, guarded by an `updated_at` optimistic
lock so a stale failure never clobbers a concurrently refreshed token
(on a lock miss the winner's row is used).
- `mcpclient.IsPermanentRefreshError` classifies `*oauth2.RetrieveError`
codes: only `invalid_grant` and `bad_refresh_token` are permanent.
Client/config errors (`invalid_client`, `unauthorized_client`, ...) stay
transient for the user row since reconnecting cannot fix them.
- chatd token refresh and the MCP list/get endpoints persist permanent
failures, return cleared tokens for the in-flight request, and skip
provider calls for already-failed rows.
- `buildAuthHeaders` no longer attaches an Authorization header for
failed tokens, so chat degrades by omitting that server's tools instead
of sending a dead bearer.

API and UI:

- No new API surface. A permanently failed token simply reports
`auth_connected: false`, so the existing "Auth" button and "Not
authenticated" tooltip appear and the user re-runs the same OAuth flow
to recover. An earlier revision added an `auth_status` enum (`connected`
/ `not_connected` / `reconnect_required`) with a dedicated "Reconnect"
button; it was collapsed to keep the API minimal since both states lead
to the identical re-auth action.

Out of scope (follow-up): typed 401-on-connect detection and forced
refresh. mcp-go exposes no stable typed 401 signal in the static-header
path, so a revocation while the access token still looks valid locally
stays undetected until expiry triggers a refresh.

## Testing

- Unit and integration tests: classifier, chatd refresh paths
(permanent/transient/race/persist-failure), API endpoints (revoked,
transient, no-retry caching, re-auth recovery, stale-lock), dbauthz,
dbcrypt, migrations.
- Dogfood UAT against a dev instance with a mock IdP returning
`invalid_grant`: revoked grant detected on refresh and persisted once
(no repeated IdP calls), chat with the revoked server selected completes
with the server's tools omitted, and re-auth restores the connected
state.

> This PR was authored by Mux, working on Mike's behalf.

coderd/database/dbauthz/dbauthz.go

@@ -6869,6 +6869,13 @@ func (q *querier) MarkChatsContextDirtyByAgent(ctx context.Context, arg database
 	return q.db.MarkChatsContextDirtyByAgent(ctx, arg)
 }
 
+func (q *querier) MarkMCPServerUserTokenRefreshFailure(ctx context.Context, arg database.MarkMCPServerUserTokenRefreshFailureParams) (database.MCPServerUserToken, error) {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil {
+		return database.MCPServerUserToken{}, err
+	}
+	return q.db.MarkMCPServerUserTokenRefreshFailure(ctx, arg)
+}
+
 func (q *querier) OIDCClaimFieldValues(ctx context.Context, args database.OIDCClaimFieldValuesParams) ([]string, error) {
 	resource := rbac.ResourceIdpsyncSettings
 	if args.OrganizationID != uuid.Nil {

coderd/database/dbauthz/dbauthz_test.go

@@ -1925,6 +1925,16 @@ func (s *MethodTestSuite) TestChats() {
 		dbm.EXPECT().UpsertMCPServerUserToken(gomock.Any(), arg).Return(token, nil).AnyTimes()
 		check.Args(arg).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate).Returns(token)
 	}))
+	s.Run("MarkMCPServerUserTokenRefreshFailure", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		token := testutil.Fake(s.T(), faker, database.MCPServerUserToken{})
+		arg := database.MarkMCPServerUserTokenRefreshFailureParams{
+			ID:                        token.ID,
+			UpdatedAt:                 token.UpdatedAt,
+			OauthRefreshFailureReason: "invalid_grant",
+		}
+		dbm.EXPECT().MarkMCPServerUserTokenRefreshFailure(gomock.Any(), arg).Return(token, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate).Returns(token)
+	}))
 }
 
 func (s *MethodTestSuite) TestFile() {

coderd/database/dbmetrics/querymetrics.go

@@ -4841,6 +4841,14 @@ func (m queryMetricsStore) MarkChatsContextDirtyByAgent(ctx context.Context, arg
 	return r0, r1
 }
 
+func (m queryMetricsStore) MarkMCPServerUserTokenRefreshFailure(ctx context.Context, arg database.MarkMCPServerUserTokenRefreshFailureParams) (database.MCPServerUserToken, error) {
+	start := time.Now()
+	r0, r1 := m.s.MarkMCPServerUserTokenRefreshFailure(ctx, arg)
+	m.queryLatencies.WithLabelValues("MarkMCPServerUserTokenRefreshFailure").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "MarkMCPServerUserTokenRefreshFailure").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) OIDCClaimFieldValues(ctx context.Context, arg database.OIDCClaimFieldValuesParams) ([]string, error) {
 	start := time.Now()
 	r0, r1 := m.s.OIDCClaimFieldValues(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -9114,6 +9114,21 @@ func (mr *MockStoreMockRecorder) MarkChatsContextDirtyByAgent(ctx, arg any) *gom
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkChatsContextDirtyByAgent", reflect.TypeOf((*MockStore)(nil).MarkChatsContextDirtyByAgent), ctx, arg)
 }
 
+// MarkMCPServerUserTokenRefreshFailure mocks base method.
+func (m *MockStore) MarkMCPServerUserTokenRefreshFailure(ctx context.Context, arg database.MarkMCPServerUserTokenRefreshFailureParams) (database.MCPServerUserToken, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MarkMCPServerUserTokenRefreshFailure", ctx, arg)
+	ret0, _ := ret[0].(database.MCPServerUserToken)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// MarkMCPServerUserTokenRefreshFailure indicates an expected call of MarkMCPServerUserTokenRefreshFailure.
+func (mr *MockStoreMockRecorder) MarkMCPServerUserTokenRefreshFailure(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkMCPServerUserTokenRefreshFailure", reflect.TypeOf((*MockStore)(nil).MarkMCPServerUserTokenRefreshFailure), ctx, arg)
+}
+
 // OIDCClaimFieldValues mocks base method.
 func (m *MockStore) OIDCClaimFieldValues(ctx context.Context, arg database.OIDCClaimFieldValuesParams) ([]string, error) {
 	m.ctrl.T.Helper()

coderd/database/dump.sql

@@ -2490,7 +2490,8 @@ CREATE TABLE mcp_server_user_tokens (
     token_type text DEFAULT 'Bearer'::text NOT NULL,
     expiry timestamp with time zone,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
-    updated_at timestamp with time zone DEFAULT now() NOT NULL
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
+    oauth_refresh_failure_reason text DEFAULT ''::text NOT NULL
 );
 
 CREATE TABLE notification_messages (

coderd/database/migrations/000544_mcp_token_refresh_failure.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mcp_server_user_tokens
+	DROP COLUMN oauth_refresh_failure_reason
+;

coderd/database/migrations/000544_mcp_token_refresh_failure.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mcp_server_user_tokens
+	ADD COLUMN oauth_refresh_failure_reason TEXT NOT NULL DEFAULT ''
+;

coderd/database/models.go

@@ -5438,17 +5438,18 @@ type MCPServerConfig struct {
 }
 
 type MCPServerUserToken struct {
-	ID                uuid.UUID      `db:"id" json:"id"`
-	MCPServerConfigID uuid.UUID      `db:"mcp_server_config_id" json:"mcp_server_config_id"`
-	UserID            uuid.UUID      `db:"user_id" json:"user_id"`
-	AccessToken       string         `db:"access_token" json:"access_token"`
-	AccessTokenKeyID  sql.NullString `db:"access_token_key_id" json:"access_token_key_id"`
-	RefreshToken      string         `db:"refresh_token" json:"refresh_token"`
-	RefreshTokenKeyID sql.NullString `db:"refresh_token_key_id" json:"refresh_token_key_id"`
-	TokenType         string         `db:"token_type" json:"token_type"`
-	Expiry            sql.NullTime   `db:"expiry" json:"expiry"`
-	CreatedAt         time.Time      `db:"created_at" json:"created_at"`
-	UpdatedAt         time.Time      `db:"updated_at" json:"updated_at"`
+	ID                        uuid.UUID      `db:"id" json:"id"`
+	MCPServerConfigID         uuid.UUID      `db:"mcp_server_config_id" json:"mcp_server_config_id"`
+	UserID                    uuid.UUID      `db:"user_id" json:"user_id"`
+	AccessToken               string         `db:"access_token" json:"access_token"`
+	AccessTokenKeyID          sql.NullString `db:"access_token_key_id" json:"access_token_key_id"`
+	RefreshToken              string         `db:"refresh_token" json:"refresh_token"`
+	RefreshTokenKeyID         sql.NullString `db:"refresh_token_key_id" json:"refresh_token_key_id"`
+	TokenType                 string         `db:"token_type" json:"token_type"`
+	Expiry                    sql.NullTime   `db:"expiry" json:"expiry"`
+	CreatedAt                 time.Time      `db:"created_at" json:"created_at"`
+	UpdatedAt                 time.Time      `db:"updated_at" json:"updated_at"`
+	OauthRefreshFailureReason string         `db:"oauth_refresh_failure_reason" json:"oauth_refresh_failure_reason"`
 }
 
 type NotificationMessage struct {

coderd/database/querier.go

@@ -1205,6 +1205,12 @@ type sqlcQuerier interface {
 	// re-pins it. Returns the chats that transitioned so the caller can
 	// emit watch events after the transaction commits.
 	MarkChatsContextDirtyByAgent(ctx context.Context, arg MarkChatsContextDirtyByAgentParams) ([]MarkChatsContextDirtyByAgentRow, error)
+	// Records a permanent refresh failure (e.g. revoked grant) and clears
+	// the dead token material so it is never attached to a request again.
+	// The updated_at predicate provides optimistic concurrency: if another
+	// request refreshed or replaced the token since it was read, this
+	// update matches zero rows and returns sql.ErrNoRows.
+	MarkMCPServerUserTokenRefreshFailure(ctx context.Context, arg MarkMCPServerUserTokenRefreshFailureParams) (MCPServerUserToken, error)
 	OIDCClaimFieldValues(ctx context.Context, arg OIDCClaimFieldValuesParams) ([]string, error)
 	// OIDCClaimFields returns a list of distinct keys in the the merged_claims fields.
 	// This query is used to generate the list of available sync fields for idp sync settings.

coderd/database/queries.sql.go

@@ -16864,7 +16864,7 @@ func (q *sqlQuerier) GetMCPServerConfigsByIDs(ctx context.Context, ids []uuid.UU
 
 const getMCPServerUserToken = `-- name: GetMCPServerUserToken :one
 SELECT
-    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at
+    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at, oauth_refresh_failure_reason
 FROM
     mcp_server_user_tokens
 WHERE
@@ -16892,13 +16892,14 @@ func (q *sqlQuerier) GetMCPServerUserToken(ctx context.Context, arg GetMCPServer
 		&i.Expiry,
 		&i.CreatedAt,
 		&i.UpdatedAt,
+		&i.OauthRefreshFailureReason,
 	)
 	return i, err
 }
 
 const getMCPServerUserTokensByUserID = `-- name: GetMCPServerUserTokensByUserID :many
 SELECT
-    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at
+    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at, oauth_refresh_failure_reason
 FROM
     mcp_server_user_tokens
 WHERE
@@ -16926,6 +16927,7 @@ func (q *sqlQuerier) GetMCPServerUserTokensByUserID(ctx context.Context, userID
 			&i.Expiry,
 			&i.CreatedAt,
 			&i.UpdatedAt,
+			&i.OauthRefreshFailureReason,
 		); err != nil {
 			return nil, err
 		}
@@ -17098,6 +17100,54 @@ func (q *sqlQuerier) InsertMCPServerConfig(ctx context.Context, arg InsertMCPSer
 	return i, err
 }
 
+const markMCPServerUserTokenRefreshFailure = `-- name: MarkMCPServerUserTokenRefreshFailure :one
+UPDATE mcp_server_user_tokens
+SET
+    access_token = '',
+    access_token_key_id = NULL,
+    refresh_token = '',
+    refresh_token_key_id = NULL,
+    expiry = NULL,
+    oauth_refresh_failure_reason = $1::text,
+    updated_at = NOW()
+WHERE
+    id = $2::uuid
+    AND updated_at = $3::timestamptz
+RETURNING
+    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at, oauth_refresh_failure_reason
+`
+
+type MarkMCPServerUserTokenRefreshFailureParams struct {
+	OauthRefreshFailureReason string    `db:"oauth_refresh_failure_reason" json:"oauth_refresh_failure_reason"`
+	ID                        uuid.UUID `db:"id" json:"id"`
+	UpdatedAt                 time.Time `db:"updated_at" json:"updated_at"`
+}
+
+// Records a permanent refresh failure (e.g. revoked grant) and clears
+// the dead token material so it is never attached to a request again.
+// The updated_at predicate provides optimistic concurrency: if another
+// request refreshed or replaced the token since it was read, this
+// update matches zero rows and returns sql.ErrNoRows.
+func (q *sqlQuerier) MarkMCPServerUserTokenRefreshFailure(ctx context.Context, arg MarkMCPServerUserTokenRefreshFailureParams) (MCPServerUserToken, error) {
+	row := q.db.QueryRowContext(ctx, markMCPServerUserTokenRefreshFailure, arg.OauthRefreshFailureReason, arg.ID, arg.UpdatedAt)
+	var i MCPServerUserToken
+	err := row.Scan(
+		&i.ID,
+		&i.MCPServerConfigID,
+		&i.UserID,
+		&i.AccessToken,
+		&i.AccessTokenKeyID,
+		&i.RefreshToken,
+		&i.RefreshTokenKeyID,
+		&i.TokenType,
+		&i.Expiry,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.OauthRefreshFailureReason,
+	)
+	return i, err
+}
+
 const updateMCPServerConfig = `-- name: UpdateMCPServerConfig :one
 UPDATE
     mcp_server_configs
@@ -17258,9 +17308,12 @@ ON CONFLICT (mcp_server_config_id, user_id) DO UPDATE SET
     refresh_token_key_id = $6::text,
     token_type = $7::text,
     expiry = $8::timestamptz,
+    -- New token material means the user re-authenticated, so any
+    -- cached permanent refresh failure no longer applies.
+    oauth_refresh_failure_reason = '',
     updated_at = NOW()
 RETURNING
-    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at
+    id, mcp_server_config_id, user_id, access_token, access_token_key_id, refresh_token, refresh_token_key_id, token_type, expiry, created_at, updated_at, oauth_refresh_failure_reason
 `
 
 type UpsertMCPServerUserTokenParams struct {
@@ -17298,6 +17351,7 @@ func (q *sqlQuerier) UpsertMCPServerUserToken(ctx context.Context, arg UpsertMCP
 		&i.Expiry,
 		&i.CreatedAt,
 		&i.UpdatedAt,
+		&i.OauthRefreshFailureReason,
 	)
 	return i, err
 }

coderd/database/queries/mcpserverconfigs.sql

@@ -200,10 +200,34 @@ ON CONFLICT (mcp_server_config_id, user_id) DO UPDATE SET
     refresh_token_key_id = sqlc.narg('refresh_token_key_id')::text,
     token_type = @token_type::text,
     expiry = sqlc.narg('expiry')::timestamptz,
+    -- New token material means the user re-authenticated, so any
+    -- cached permanent refresh failure no longer applies.
+    oauth_refresh_failure_reason = '',
     updated_at = NOW()
 RETURNING
     *;
 
+-- name: MarkMCPServerUserTokenRefreshFailure :one
+-- Records a permanent refresh failure (e.g. revoked grant) and clears
+-- the dead token material so it is never attached to a request again.
+-- The updated_at predicate provides optimistic concurrency: if another
+-- request refreshed or replaced the token since it was read, this
+-- update matches zero rows and returns sql.ErrNoRows.
+UPDATE mcp_server_user_tokens
+SET
+    access_token = '',
+    access_token_key_id = NULL,
+    refresh_token = '',
+    refresh_token_key_id = NULL,
+    expiry = NULL,
+    oauth_refresh_failure_reason = @oauth_refresh_failure_reason::text,
+    updated_at = NOW()
+WHERE
+    id = @id::uuid
+    AND updated_at = @updated_at::timestamptz
+RETURNING
+    *;
+
 -- name: DeleteMCPServerUserToken :exec
 DELETE FROM
     mcp_server_user_tokens