feat: log rate-limited external auth token validation (#26754)

When `ValidateToken` keeps a token because the external auth validation
endpoint was rate-limited (a `403` with rate-limit headers or a `429`),
it returns `valid=true` without provider confirmation. Previously this
happened silently, so operators couldn't tell a provider-confirmed token
from one kept optimistically during a rate limit.

This adds a `Logger` to `externalauth.Config` and emits a `Warn` (with
`provider_id`, `provider_type`, `status_code`, and `reason`) on those
rate-limit branches. It also adds a
`coderd_oauth2_external_requests_rate_limited_total{name, source,
status_code}` counter, incremented in the instrumented round tripper
whenever a provider returns a rate-limited response. The rate-limit
detection is the shared `xhttp.IsRateLimited` (in `coderd/util/xhttp`),
used by both the tripper and `ValidateToken` so the metric and the
validation decision share one definition; no extra wiring is needed
since `ValidateToken` already routes through the instrumented client
with `source="ValidateToken"`.

One deliberate behavioral change rides along: rate-limit detection now
also recognizes the unprefixed `RateLimit-Remaining` header (GitLab, and
the IETF draft rate-limit headers), so a `403` with
`RateLimit-Remaining: 0` is treated as optimistically valid where it was
previously treated as revoked. All other valid/invalid decisions are
unchanged. `TestValidateToken` asserts the warning's fields on the
rate-limited cases and no warning for revocations, `401`, and confirmed
responses; `promoauth` and `xhttp` tests cover the detector and the new
counter.

<details>
<summary>Manual testing</summary>

The signals fire on the external-auth status check (`GET
/api/v2/external-auth/{id}`), which calls `ValidateToken`. To force a
rate-limited response, point a provider's `validate_url` at a mock that
returns the rate-limit shape:

1. Run a mock returning `429` on one path and `403` +
`X-RateLimit-Remaining: 0` on another.
2. Start `coder server` with `--prometheus-enable` and external auth
providers whose `validate_url` point at those mock paths (e.g.
`CODER_EXTERNAL_AUTH_0_VALIDATE_URL=http://127.0.0.1:5599/429`).
3. Create a stored link, either complete the OAuth flow, or insert a row
into `external_auth_links` with a future `oauth_expiry` (token contents
are irrelevant; the mock rejects regardless).
4. `curl` the status endpoint with a session token, then check:
- coderd logs for the `Warn` (`reason=status_code` for `429`,
`reason=rate_limit_headers` for `403`),
- the metrics endpoint for
`coderd_oauth2_external_requests_rate_limited_total{...,status_code="429"|"403"}`.

Notes: `scripts/testidp -429` only rate-limits `/oauth2/userinfo`, not
the `/external-auth-validate/...` path, so it does not exercise this;
use a mock `validate_url`. The default Prometheus port `2112` may
already be taken on dogfood workspaces, set `CODER_PROMETHEUS_ADDRESS`
to a free port.

</details>

🤖 Generated with the help of Coder Agents on behalf of @jscottmiller.

coderd/externalauth/externalauth.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -22,11 +23,13 @@ import (
 	"golang.org/x/sync/singleflight"
 	"golang.org/x/xerrors"
 
+	"cdr.dev/slog/v3"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/externalauth/gitprovider"
 	"github.com/coder/coder/v2/coderd/promoauth"
 	"github.com/coder/coder/v2/coderd/util/slice"
+	"github.com/coder/coder/v2/coderd/util/xhttp"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/retry"
 )
@@ -63,6 +66,10 @@ type SingleflightGroup interface {
 // Config is used for authentication for Git operations.
 type Config struct {
 	promoauth.InstrumentedOAuth2Config
+	// Logs rate-limited validation warnings. Zero value discards output.
+	Logger slog.Logger
+	// rateLimitLogThrottle throttles rate-limited validation warnings.
+	rateLimitLogThrottle logThrottle
 	// ID is a unique identifier for the authenticator.
 	ID string
 	// Type is the type of provider.
@@ -520,7 +527,8 @@ func (c *Config) ValidateToken(ctx context.Context, link *oauth2.Token) (bool, *
 		// validation endpoint is rejecting for a transient reason.
 		// Treat it as optimistically valid rather than discarding
 		// the token.
-		if isRateLimited(res) {
+		if xhttp.IsRateLimited(res) {
+			c.logRateLimitedValidation(ctx, http.StatusForbidden, "rate_limit_headers")
 			return true, nil, nil
 		}
 		// No rate-limit headers: genuine token revocation or
@@ -532,6 +540,7 @@ func (c *Config) ValidateToken(ctx context.Context, link *oauth2.Token) (bool, *
 		// Treat 429 the same as a rate-limited 403: optimistically
 		// valid. The token was likely just issued by the IDP; the
 		// validation endpoint is transiently overloaded.
+		c.logRateLimitedValidation(ctx, http.StatusTooManyRequests, "status_code")
 		return true, nil, nil
 
 	case http.StatusOK:
@@ -560,6 +569,57 @@ func (c *Config) ValidateToken(ctx context.Context, link *oauth2.Token) (bool, *
 	return true, user, nil
 }
 
+// rateLimitLogInterval is the minimum time between rate-limited validation
+// warnings emitted per Config.
+const rateLimitLogInterval = time.Minute
+
+// logRateLimitedValidation warns that a token was kept valid without
+// provider confirmation due to a rate-limited response. At most one
+// warning is emitted per Config per rateLimitLogInterval; the line
+// carries the number of occurrences suppressed since the previous one.
+func (c *Config) logRateLimitedValidation(ctx context.Context, statusCode int, reason string) {
+	suppressed, ok := c.rateLimitLogThrottle.shouldLog(time.Now(), rateLimitLogInterval)
+	if !ok {
+		return
+	}
+	c.Logger.Warn(ctx, "external auth validation endpoint rate-limited; keeping token without provider confirmation",
+		slog.F("status_code", statusCode),
+		slog.F("reason", reason),
+		slog.F("suppressed", suppressed),
+	)
+}
+
+// logThrottle allows one event per interval and counts the events
+// suppressed in between. Safe for concurrent use; the zero value is
+// ready for use.
+type logThrottle struct {
+	mu         sync.Mutex
+	lastLog    time.Time
+	suppressed int64
+}
+
+// shouldLog reports whether an event occurring at now may be logged,
+// allowing at most one event per interval. When it returns true, it also
+// returns the number of events suppressed since the last allowed one;
+// if two or more intervals have elapsed, the stale count is discarded
+// and zero is returned.
+func (t *logThrottle) shouldLog(now time.Time, interval time.Duration) (int64, bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	sinceLast := now.Sub(t.lastLog)
+	if sinceLast < interval {
+		t.suppressed++
+		return 0, false
+	}
+	n := t.suppressed
+	if sinceLast >= 2*interval {
+		n = 0
+	}
+	t.suppressed = 0
+	t.lastLog = now
+	return n, true
+}
+
 type AppInstallation struct {
 	ID int
 	// Login is the username of the installation.
@@ -852,7 +912,7 @@ func (c *DeviceAuth) formatDeviceCodeURL() (string, error) {
 
 // ConvertConfig converts the SDK configuration entry format
 // to the parsed and ready-to-consume in coderd provider type.
-func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([]*Config, error) {
+func ConvertConfig(logger slog.Logger, instrument *promoauth.Factory, entries []codersdk.ExternalAuthConfig, accessURL *url.URL) ([]*Config, error) {
 	ids := map[string]struct{}{}
 	configs := []*Config{}
 	for _, entry := range entries {
@@ -936,6 +996,7 @@ func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAut
 
 		cfg := &Config{
 			InstrumentedOAuth2Config:      instrumented,
+			Logger:                        logger.Named("externalauth").With(slog.F("provider_id", entry.ID), slog.F("provider_type", entry.Type)),
 			ID:                            entry.ID,
 			ClientID:                      entry.ClientID,
 			ClientSecret:                  entry.ClientSecret,
@@ -1483,32 +1544,6 @@ func IsGithubDotComURL(str string) bool {
 	return ghURL.Host == "github.com"
 }
 
-// isRateLimited checks whether an HTTP response indicates a rate
-// limit rather than a genuine authorization failure. It returns
-// true if either X-RateLimit-Remaining is "0" (primary) or
-// Retry-After is present (secondary). OR logic is intentional:
-// GitHub secondary limits can include Retry-After without
-// X-RateLimit-Remaining: 0 (the remaining count tracks the
-// primary quota, not secondary).
-//
-// Does not catch every secondary rate limit. GitHub can return
-// 403 with positive X-RateLimit-Remaining and no Retry-After.
-// Reliable detection of those requires response body inspection.
-// Missing them is not a regression since all 403s were previously
-// treated as invalid.
-func isRateLimited(resp *http.Response) bool {
-	if resp == nil {
-		return false
-	}
-	if resp.Header.Get("Retry-After") != "" {
-		return true
-	}
-	if resp.Header.Get("X-RateLimit-Remaining") == "0" {
-		return true
-	}
-	return false
-}
-
 // isFailedRefresh returns true if the error returned by the refresh attempt
 // is due to a failed refresh. The failure being the refresh token itself.
 // If this returns true, no amount of retries will fix the issue.