mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: log rate-limited external auth token validation (#26754)
When `ValidateToken` keeps a token because the external auth validation
endpoint was rate-limited (a `403` with rate-limit headers or a `429`),
it returns `valid=true` without provider confirmation. Previously this
happened silently, so operators couldn't tell a provider-confirmed token
from one kept optimistically during a rate limit.
This adds a `Logger` to `externalauth.Config` and emits a `Warn` (with
`provider_id`, `provider_type`, `status_code`, and `reason`) on those
rate-limit branches. It also adds a
`coderd_oauth2_external_requests_rate_limited_total{name, source,
status_code}` counter, incremented in the instrumented round tripper
whenever a provider returns a rate-limited response. The rate-limit
detection is the shared `xhttp.IsRateLimited` (in `coderd/util/xhttp`),
used by both the tripper and `ValidateToken` so the metric and the
validation decision share one definition; no extra wiring is needed
since `ValidateToken` already routes through the instrumented client
with `source="ValidateToken"`.
One deliberate behavioral change rides along: rate-limit detection now
also recognizes the unprefixed `RateLimit-Remaining` header (GitLab, and
the IETF draft rate-limit headers), so a `403` with
`RateLimit-Remaining: 0` is treated as optimistically valid where it was
previously treated as revoked. All other valid/invalid decisions are
unchanged. `TestValidateToken` asserts the warning's fields on the
rate-limited cases and no warning for revocations, `401`, and confirmed
responses; `promoauth` and `xhttp` tests cover the detector and the new
counter.
<details>
<summary>Manual testing</summary>
The signals fire on the external-auth status check (`GET
/api/v2/external-auth/{id}`), which calls `ValidateToken`. To force a
rate-limited response, point a provider's `validate_url` at a mock that
returns the rate-limit shape:
1. Run a mock returning `429` on one path and `403` +
`X-RateLimit-Remaining: 0` on another.
2. Start `coder server` with `--prometheus-enable` and external auth
providers whose `validate_url` point at those mock paths (e.g.
`CODER_EXTERNAL_AUTH_0_VALIDATE_URL=http://127.0.0.1:5599/429`).
3. Create a stored link, either complete the OAuth flow, or insert a row
into `external_auth_links` with a future `oauth_expiry` (token contents
are irrelevant; the mock rejects regardless).
4. `curl` the status endpoint with a session token, then check:
- coderd logs for the `Warn` (`reason=status_code` for `429`,
`reason=rate_limit_headers` for `403`),
- the metrics endpoint for
`coderd_oauth2_external_requests_rate_limited_total{...,status_code="429"|"403"}`.
Notes: `scripts/testidp -429` only rate-limits `/oauth2/userinfo`, not
the `/external-auth-validate/...` path, so it does not exercise this;
use a mock `validate_url`. The default Prometheus port `2112` may
already be taken on dogfood workspaces, set `CODER_PROMETHEUS_ADDRESS`
to a free port.
</details>
🤖 Generated with the help of Coder Agents on behalf of @jscottmiller.
This commit is contained in:
@@ -1,7 +1,12 @@
|
||||
package externalauth
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -9,10 +14,97 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"cdr.dev/slog/v3/sloggers/slogjson"
|
||||
"github.com/coder/coder/v2/coderd/promoauth"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
)
|
||||
|
||||
func TestLogThrottle(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const interval = time.Minute
|
||||
var th logThrottle
|
||||
start := time.Now()
|
||||
|
||||
suppressed, ok := th.shouldLog(start, interval)
|
||||
require.True(t, ok, "the first event should log")
|
||||
require.EqualValues(t, 0, suppressed)
|
||||
|
||||
for i := range 3 {
|
||||
_, ok := th.shouldLog(start.Add(time.Duration(i+1)*time.Second), interval)
|
||||
require.False(t, ok, "events within the interval should be suppressed")
|
||||
}
|
||||
_, ok = th.shouldLog(start.Add(interval-time.Millisecond), interval)
|
||||
require.False(t, ok, "an event just inside the interval should be suppressed")
|
||||
|
||||
suppressed, ok = th.shouldLog(start.Add(interval), interval)
|
||||
require.True(t, ok, "the first event after the interval should log")
|
||||
require.EqualValues(t, 4, suppressed, "suppressed should count events since the last log")
|
||||
|
||||
suppressed, ok = th.shouldLog(start.Add(2*interval), interval)
|
||||
require.True(t, ok)
|
||||
require.EqualValues(t, 0, suppressed, "suppressed should reset after each log")
|
||||
|
||||
// Suppress one event, then let more than two intervals elapse.
|
||||
_, ok = th.shouldLog(start.Add(2*interval+time.Second), interval)
|
||||
require.False(t, ok)
|
||||
suppressed, ok = th.shouldLog(start.Add(5*interval), interval)
|
||||
require.True(t, ok)
|
||||
require.EqualValues(t, 0, suppressed, "counts from a burst that ended more than an interval ago are discarded")
|
||||
}
|
||||
|
||||
func TestLogThrottleConcurrent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
interval = time.Minute
|
||||
events = 32
|
||||
)
|
||||
var th logThrottle
|
||||
now := time.Now()
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
logged atomic.Int64
|
||||
)
|
||||
for range events {
|
||||
wg.Go(func() {
|
||||
if _, ok := th.shouldLog(now, interval); ok {
|
||||
logged.Add(1)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
require.EqualValues(t, 1, logged.Load(), "exactly one concurrent event should log")
|
||||
|
||||
suppressed, ok := th.shouldLog(now.Add(interval), interval)
|
||||
require.True(t, ok)
|
||||
require.EqualValues(t, events-1, suppressed, "every other concurrent event should be counted")
|
||||
}
|
||||
|
||||
// TestLogRateLimitedValidationSuppressed verifies the suppressed count
|
||||
// reaches the emitted log line.
|
||||
func TestLogRateLimitedValidationSuppressed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
logs := &bytes.Buffer{}
|
||||
c := &Config{Logger: slog.Make(slogjson.Sink(logs)).Leveled(slog.LevelDebug)}
|
||||
c.rateLimitLogThrottle.lastLog = time.Now().Add(-rateLimitLogInterval - time.Second)
|
||||
c.rateLimitLogThrottle.suppressed = 5
|
||||
|
||||
c.logRateLimitedValidation(context.Background(), http.StatusTooManyRequests, "status_code")
|
||||
|
||||
var entry struct {
|
||||
Fields struct {
|
||||
Suppressed *int64 `json:"suppressed"`
|
||||
} `json:"fields"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
|
||||
require.NotNil(t, entry.Fields.Suppressed, "the log line should carry the suppressed field")
|
||||
require.EqualValues(t, 5, *entry.Fields.Suppressed)
|
||||
}
|
||||
|
||||
func TestGitlabDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user