mirror of
https://github.com/coder/coder.git
synced 2026-10-16 14:48:56 +08:00
feat: let the AI Gateway emit the structured interception records (#30030)
coderd emitted the structured interception records as they arrived over
DRPC, so
a record the gateway never sends cannot be exported.
- Add `--ai-gateway-structured-logging-source` (`coderd`, `gateway`,
`both`) and
teach the gateway's `LogRecorder` to emit the same format where the
records
originate. Gateway lines omit `thread_parent_id` and `thread_root_id`,
which
are resolved by a database lookup only coderd can perform.
- Reaching the serving pool required repairing the pool options: they
were
derived at construction sites and discarded, so the server always built
its
pool with the defaults. They are now derived from the deployment
configuration
in one place and passed to the pool that serves requests. The server
reports
those options, and its ownership of the `key_pool_state` collector, so a
second
registration cannot silently reappear and panic startup.
- The standalone gateway inherits both structured logging options, since
it is
now one of the processes that can emit.
This is the keystone of the stack: the flag, the mechanism that carries
it and
the behavior it selects land together, so
`TestAIGatewayStart_InheritedOptions`
and the gateway help output stay truthful at every commit.
<details>
<summary>Why the stack is ordered this way</summary>
The final design is a chain of recorder decorators:
+---------------------+
| AsyncRecorder | all records except the interception start
+---------------------+
| LogRecorder | emit the structured record
+---------------------+
| ValidatingRecorder | refuse malformed records
+---------------------+
| caller middleware | record policy, e.g. drop content
+---------------------+
| TraceRecorder |
+---------------------+
| DRPCRecorder | acquire client, send to coderd
+---------------------+
Ordering constraints:
- The record type constants (1) are the vocabulary both emitters and the
validator use, so they come first.
- Validation (5) rejects unset timestamps, so records must already be
stamped
where they are created (3). Stamping at creation leaves
`WrappedRecorder`
doing nothing but client acquisition, which is why it is folded into
`DRPCRecorder` (4) and not before.
- coderd's server-time fallback (2) is what keeps a gateway of an older
version
from persisting zero times once the gateway owns stamping (3).
- PRs 1 to 5 change no user-visible surface. Each deployment option then
lands
with the code that reads it: the log source with gateway-side emission
(6),
content recording with the filter that drops the records (7). Pool
options are
plumbed to the serving pool in 6, where the first option that needs them
arrives, rather than as an unused hook beforehand.
</details>
---
🤖 Raised by [Coder Agents](https://coder.com) on behalf of @SasSwart.
This commit is contained in:
+2
-2
@@ -81,9 +81,9 @@ func NewMetrics(reg prometheus.Registerer) *metrics.Metrics {
|
||||
|
||||
// NewRecorder creates a [Recorder] which logs each record and refuses
|
||||
// malformed ones before handing it to base.
|
||||
func NewRecorder(logger slog.Logger, tracer trace.Tracer, base Recorder) Recorder {
|
||||
func NewRecorder(logger slog.Logger, tracer trace.Tracer, apiKeyID string, structured bool, base Recorder) Recorder {
|
||||
return recorder.ChainMiddleware(
|
||||
recorder.WithLogging(logger),
|
||||
recorder.WithLogging(logger, apiKeyID, structured),
|
||||
recorder.WithValidation(logger),
|
||||
recorder.WithTracing(tracer),
|
||||
)(base)
|
||||
|
||||
@@ -56,6 +56,10 @@ type bridgeConfig struct {
|
||||
userID string
|
||||
metadata recorder.Metadata
|
||||
logger slog.Logger
|
||||
apiKeyID string
|
||||
// structuredLogging makes the bridge emit AI Gateway interception
|
||||
// records in the format described by [recorder.InterceptionLogMarker].
|
||||
structuredLogging bool
|
||||
}
|
||||
|
||||
// bridgeTestServer wraps an httptest.Server running a RequestBridge.
|
||||
@@ -172,7 +176,7 @@ func newBridgeTestServer(
|
||||
}
|
||||
|
||||
mockRec := &testutil.MockRecorder{}
|
||||
rec := aibridge.NewRecorder(cfg.logger, cfg.tracer, mockRec)
|
||||
rec := aibridge.NewRecorder(cfg.logger, cfg.tracer, cfg.apiKeyID, cfg.structuredLogging, mockRec)
|
||||
|
||||
bridge, err := aibridge.NewRequestBridge(
|
||||
ctx, providers, rec, cfg.mcpProxy,
|
||||
|
||||
@@ -2,6 +2,8 @@ package recorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"maps"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"github.com/coder/coder/v2/coderd/util/ptr"
|
||||
@@ -37,26 +39,83 @@ const (
|
||||
// compatibility. A single failure therefore still produces two lines for
|
||||
// most records. This is suboptimal, but a requirement for backwards compatibility.
|
||||
type LogRecorder struct {
|
||||
logger slog.Logger
|
||||
wrapped Recorder
|
||||
logger slog.Logger
|
||||
apiKeyID string
|
||||
structured bool
|
||||
wrapped Recorder
|
||||
}
|
||||
|
||||
// NewLogRecorder creates a [LogRecorder] which logs each record and then
|
||||
// delegates it to wrapped. wrapped may be nil, in which case records are only
|
||||
// logged.
|
||||
func NewLogRecorder(logger slog.Logger, wrapped Recorder) *LogRecorder {
|
||||
return &LogRecorder{logger: logger, wrapped: wrapped}
|
||||
func NewLogRecorder(logger slog.Logger, apiKeyID string, structured bool, wrapped Recorder) *LogRecorder {
|
||||
return &LogRecorder{logger: logger, apiKeyID: apiKeyID, structured: structured, wrapped: wrapped}
|
||||
}
|
||||
|
||||
// WithLogging returns a [Decorator] which wraps a [Recorder] in a
|
||||
// WithLogging returns a [Middleware] which wraps a [Recorder] in a
|
||||
// [LogRecorder], for composition by [ChainMiddleware].
|
||||
func WithLogging(logger slog.Logger) Middleware {
|
||||
func WithLogging(logger slog.Logger, apiKeyID string, structured bool) Middleware {
|
||||
return func(next Recorder) Recorder {
|
||||
return NewLogRecorder(logger, next)
|
||||
return NewLogRecorder(logger, apiKeyID, structured, next)
|
||||
}
|
||||
}
|
||||
|
||||
// logStructured emits one record in the format described by
|
||||
// [InterceptionLogMarker], when the deployment has asked for it.
|
||||
//
|
||||
// It runs before the record is delegated, so that a record dropped lower in the
|
||||
// chain is still reported.
|
||||
func (r *LogRecorder) logStructured(ctx context.Context, recordType string, fields ...slog.Field) {
|
||||
if !r.structured {
|
||||
return
|
||||
}
|
||||
r.logger.Info(ctx, InterceptionLogMarker, append([]slog.Field{slog.F("record_type", recordType)}, fields...)...)
|
||||
}
|
||||
|
||||
// metadataWithUserAgent copies metadata rather than writing through: the
|
||||
// record's own metadata travels on to coderd, which merges the user agent
|
||||
// itself and warns when the key is already set.
|
||||
func metadataWithUserAgent(metadata Metadata, userAgent string) Metadata {
|
||||
if userAgent == "" {
|
||||
return metadata
|
||||
}
|
||||
merged := make(Metadata, len(metadata)+1)
|
||||
maps.Copy(merged, metadata)
|
||||
merged[MetadataUserAgentKey] = userAgent
|
||||
return merged
|
||||
}
|
||||
|
||||
// marshalToolArgs renders tool call arguments the way [DRPCRecorder] sends
|
||||
// them, so that both emitters report an identical input field.
|
||||
func (r *LogRecorder) marshalToolArgs(ctx context.Context, args ToolArgs) string {
|
||||
serialized, err := json.Marshal(args)
|
||||
if err != nil {
|
||||
// An empty input field is indistinguishable from a tool called with
|
||||
// no arguments, so the difference is reported here.
|
||||
r.logger.Warn(ctx, "failed to marshal tool call arguments, reporting empty input", slog.Error(err))
|
||||
return ""
|
||||
}
|
||||
return string(serialized)
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordInterception(ctx context.Context, req *InterceptionRecord) error {
|
||||
// thread_parent_id and thread_root_id are deliberately absent: they are
|
||||
// resolved from recorded tool usages by a database lookup which only
|
||||
// coderd can perform. An absent key fails a consumer loudly; a nil UUID
|
||||
// would assert, wrongly, that the interception has no parent.
|
||||
r.logStructured(ctx, RecordTypeInterceptionStart,
|
||||
slog.F("interception_id", req.ID),
|
||||
slog.F("initiator_id", req.InitiatorID),
|
||||
slog.F("api_key_id", r.apiKeyID),
|
||||
slog.F("provider", req.Provider),
|
||||
slog.F("model", req.Model),
|
||||
slog.F("client", req.Client),
|
||||
slog.F("client_session_id", ptr.NilToEmpty(req.ClientSessionID)),
|
||||
slog.F("started_at", req.StartedAt),
|
||||
slog.F("metadata", metadataWithUserAgent(req.Metadata, req.UserAgent)),
|
||||
slog.F("correlating_tool_call_id", ptr.NilToEmpty(req.CorrelatingToolCallID)),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordInterception(ctx, req)
|
||||
@@ -85,6 +144,11 @@ func (r *LogRecorder) RecordInterception(ctx context.Context, req *InterceptionR
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordInterceptionEnded(ctx context.Context, req *InterceptionRecordEnded) error {
|
||||
r.logStructured(ctx, RecordTypeInterceptionEnd,
|
||||
slog.F("interception_id", req.ID),
|
||||
slog.F("ended_at", req.EndedAt),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordInterceptionEnded(ctx, req)
|
||||
@@ -106,6 +170,14 @@ func (r *LogRecorder) RecordInterceptionEnded(ctx context.Context, req *Intercep
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordPromptUsage(ctx context.Context, req *PromptUsageRecord) error {
|
||||
r.logStructured(ctx, RecordTypePromptUsage,
|
||||
slog.F("interception_id", req.InterceptionID),
|
||||
slog.F("msg_id", req.MsgID),
|
||||
slog.F("prompt", req.Prompt),
|
||||
slog.F("created_at", req.CreatedAt),
|
||||
slog.F("metadata", req.Metadata),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordPromptUsage(ctx, req)
|
||||
@@ -127,6 +199,17 @@ func (r *LogRecorder) RecordPromptUsage(ctx context.Context, req *PromptUsageRec
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordTokenUsage(ctx context.Context, req *TokenUsageRecord) error {
|
||||
r.logStructured(ctx, RecordTypeTokenUsage,
|
||||
slog.F("interception_id", req.InterceptionID),
|
||||
slog.F("msg_id", req.MsgID),
|
||||
slog.F("input_tokens", req.Input),
|
||||
slog.F("output_tokens", req.Output),
|
||||
slog.F("cache_read_input_tokens", req.CacheReadInputTokens),
|
||||
slog.F("cache_write_input_tokens", req.CacheWriteInputTokens),
|
||||
slog.F("created_at", req.CreatedAt),
|
||||
slog.F("metadata", req.Metadata),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordTokenUsage(ctx, req)
|
||||
@@ -152,6 +235,24 @@ func (r *LogRecorder) RecordTokenUsage(ctx context.Context, req *TokenUsageRecor
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordToolUsage(ctx context.Context, req *ToolUsageRecord) error {
|
||||
var structuredInvocationErr string
|
||||
if req.InvocationError != nil {
|
||||
structuredInvocationErr = req.InvocationError.Error()
|
||||
}
|
||||
r.logStructured(ctx, RecordTypeToolUsage,
|
||||
slog.F("interception_id", req.InterceptionID),
|
||||
slog.F("msg_id", req.MsgID),
|
||||
slog.F("tool_call_id", req.ToolCallID),
|
||||
slog.F("item_id", req.ItemID),
|
||||
slog.F("tool", req.Tool),
|
||||
slog.F("input", r.marshalToolArgs(ctx, req.Args)),
|
||||
slog.F("server_url", ptr.NilToEmpty(req.ServerURL)),
|
||||
slog.F("injected", req.Injected),
|
||||
slog.F("invocation_error", structuredInvocationErr),
|
||||
slog.F("created_at", req.CreatedAt),
|
||||
slog.F("metadata", req.Metadata),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordToolUsage(ctx, req)
|
||||
@@ -182,6 +283,13 @@ func (r *LogRecorder) RecordToolUsage(ctx context.Context, req *ToolUsageRecord)
|
||||
}
|
||||
|
||||
func (r *LogRecorder) RecordModelThought(ctx context.Context, req *ModelThoughtRecord) error {
|
||||
r.logStructured(ctx, RecordTypeModelThought,
|
||||
slog.F("interception_id", req.InterceptionID),
|
||||
slog.F("content", req.Content),
|
||||
slog.F("created_at", req.CreatedAt),
|
||||
slog.F("metadata", req.Metadata),
|
||||
)
|
||||
|
||||
var err error
|
||||
if r.wrapped != nil {
|
||||
err = r.wrapped.RecordModelThought(ctx, req)
|
||||
|
||||
+42
-7
@@ -18,17 +18,43 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/aibridged"
|
||||
"github.com/coder/coder/v2/coderd/aibridged/proto"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/pubsub"
|
||||
"github.com/coder/coder/v2/coderd/tracing"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/quartz"
|
||||
)
|
||||
|
||||
// newAIBridgeDaemon constructs the in-memory aibridge daemon and wires
|
||||
// AIBridgeDaemonOptions configures [NewAIBridgeDaemon].
|
||||
type AIBridgeDaemonOptions struct {
|
||||
// API is the coderd instance the daemon serves and dials over the
|
||||
// in-memory RPC.
|
||||
API *coderd.API
|
||||
// Config supplies both the provider build settings and the pool's record
|
||||
// policy.
|
||||
Config codersdk.AIBridgeConfig
|
||||
// Registerer receives the daemon's collectors, so it must not already
|
||||
// hold them. Nil skips registration.
|
||||
Registerer prometheus.Registerer
|
||||
// BridgeMetrics is the metrics instance the bridge and key pools report
|
||||
// to. Nil disables those metrics.
|
||||
BridgeMetrics *aibridge.Metrics
|
||||
// ProviderMetrics receives provider reload events. Nil builds one from
|
||||
// Registerer; supply one to assert on reload metrics without owning the
|
||||
// registerer.
|
||||
ProviderMetrics *aibridged.Metrics
|
||||
// Pubsub triggers provider hot reloads. Nil uses API.Pubsub.
|
||||
Pubsub pubsub.Pubsub
|
||||
}
|
||||
|
||||
// NewAIBridgeDaemon constructs the in-memory aibridge daemon and wires
|
||||
// up a subscription that hot-reloads providers over the in-memory
|
||||
// RPC on every ai_providers change event. The returned unsubscribe
|
||||
// function tears down the subscription; callers must invoke it
|
||||
// alongside Server.Close on shutdown.
|
||||
//
|
||||
// Tests reach this through [github.com/coder/coder/v2/coderd/aibridgedtest],
|
||||
// so a deployment option that does not reach the serving pool fails there too.
|
||||
//
|
||||
// Reloads fetch the provider set from coderd over the in-memory DRPC
|
||||
// (GetAIProviders) rather than reading the database directly, so embedded and
|
||||
// standalone gateways construct providers identically. Pubsub remains the
|
||||
@@ -40,31 +66,40 @@ import (
|
||||
// context, so only the daemon lifecycle bounds the wait. That is acceptable
|
||||
// here: the embedded daemon's connection is an in-memory pipe that comes up
|
||||
// immediately.
|
||||
func newAIBridgeDaemon(coderAPI *coderd.API, cfg codersdk.AIBridgeConfig, reg prometheus.Registerer, metrics *aibridge.Metrics) (*aibridged.Server, func(), error) {
|
||||
ctx := context.Background()
|
||||
func NewAIBridgeDaemon(ctx context.Context, opts AIBridgeDaemonOptions) (*aibridged.Server, func(), error) {
|
||||
coderAPI, cfg, reg, metrics := opts.API, opts.Config, opts.Registerer, opts.BridgeMetrics
|
||||
coderAPI.Logger.Debug(ctx, "starting in-memory aibridge daemon")
|
||||
|
||||
logger := coderAPI.Logger.Named("ai-gateway")
|
||||
|
||||
providerMetrics := aibridged.NewMetrics(reg)
|
||||
providerMetrics := opts.ProviderMetrics
|
||||
if providerMetrics == nil {
|
||||
providerMetrics = aibridged.NewMetrics(reg)
|
||||
}
|
||||
ps := opts.Pubsub
|
||||
if ps == nil {
|
||||
ps = coderAPI.Pubsub
|
||||
}
|
||||
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
|
||||
|
||||
// Create daemon. Construct it before subscribing so the reloader can use
|
||||
// srv.Client to fetch providers over the in-memory RPC.
|
||||
srv, err := aibridged.New(ctx, func(dialCtx context.Context) (aibridged.DRPCClient, error) {
|
||||
return coderAPI.CreateInMemoryAIBridgeServer(dialCtx)
|
||||
}, logger, tracer, coderAPI.Experiments, metrics)
|
||||
}, logger, tracer, coderAPI.Experiments, metrics, aibridged.WithPoolOptions(aibridged.PoolOptionsFromConfig(cfg)))
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("start in-memory aibridge daemon: %w", err)
|
||||
}
|
||||
|
||||
reg.MustRegister(srv.KeyPoolStateCollector())
|
||||
if reg != nil {
|
||||
reg.MustRegister(srv.KeyPoolStateCollector())
|
||||
}
|
||||
|
||||
// Subscribe to ai_providers change events so the backend tracks the database
|
||||
// without a restart, and perform the initial reload. The reload data path
|
||||
// is the in-memory RPC.
|
||||
reloader := NewProviderRPCReloader(srv.ReplaceProviders, srv.Client, cfg, logger.Named("provider-loader"), metrics, providerMetrics)
|
||||
unsubscribe, err := aibridged.SubscribeProviderReload(ctx, coderAPI.Pubsub, reloader, logger.Named("provider-reload"))
|
||||
unsubscribe, err := aibridged.SubscribeProviderReload(ctx, ps, reloader, logger.Named("provider-reload"))
|
||||
if err != nil {
|
||||
// Without the subscription the backend cannot track provider changes,
|
||||
// so fail startup rather than serve a permanently stale snapshot.
|
||||
|
||||
@@ -11,10 +11,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/aibridge"
|
||||
"github.com/coder/coder/v2/aibridge/keypool"
|
||||
"github.com/coder/coder/v2/coderd/aibridged"
|
||||
"github.com/coder/coder/v2/coderd/coderdtest"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
"github.com/coder/serpent"
|
||||
)
|
||||
|
||||
// TestNewAIBridgeDaemonBackend asserts the embedded daemon reaches its mode
|
||||
@@ -38,7 +40,11 @@ func TestNewAIBridgeDaemonBackend(t *testing.T) {
|
||||
}
|
||||
client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{DeploymentValues: dv})
|
||||
firstUser := coderdtest.CreateFirstUser(t, client)
|
||||
srv, unsubscribe, err := newAIBridgeDaemon(api, dv.AI.BridgeConfig, prometheus.NewRegistry(), nil)
|
||||
srv, unsubscribe, err := NewAIBridgeDaemon(t.Context(), AIBridgeDaemonOptions{
|
||||
API: api,
|
||||
Config: dv.AI.BridgeConfig,
|
||||
Registerer: prometheus.NewRegistry(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, srv.Close()) })
|
||||
t.Cleanup(unsubscribe)
|
||||
@@ -56,3 +62,73 @@ func TestNewAIBridgeDaemonBackend(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAIBridgeDaemonOwnsKeyPoolStateCollector guards the daemon's ownership
|
||||
// of the key pool collector. Anything else registering one alongside it fails
|
||||
// MustRegister, which takes coderd's whole startup down.
|
||||
func TestNewAIBridgeDaemonOwnsKeyPoolStateCollector(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dv := coderdtest.DeploymentValues(t)
|
||||
_, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{DeploymentValues: dv})
|
||||
|
||||
reg := prometheus.NewRegistry()
|
||||
srv, unsubscribe, err := NewAIBridgeDaemon(t.Context(), AIBridgeDaemonOptions{
|
||||
API: api,
|
||||
Config: dv.AI.BridgeConfig,
|
||||
Registerer: reg,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, srv.Close()) })
|
||||
t.Cleanup(unsubscribe)
|
||||
|
||||
// The collector reports nothing until providers load, so ownership is
|
||||
// asserted by registration rather than by gathering.
|
||||
err = reg.Register(keypool.NewStateCollector(func() []*keypool.Pool { return nil }))
|
||||
require.Error(t, err, "the daemon should already have registered the key pool state collector")
|
||||
require.Contains(t, err.Error(), "duplicate metrics collector registration")
|
||||
}
|
||||
|
||||
// TestNewAIBridgeDaemonAppliesRecordPolicy asserts the deployment's record
|
||||
// policy reaches the daemon. The policy is only observable on the pool the
|
||||
// daemon builds for itself, so a daemon constructed from configuration that
|
||||
// never reaches that pool fails here.
|
||||
func TestNewAIBridgeDaemonAppliesRecordPolicy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mutate func(*codersdk.AIBridgeConfig)
|
||||
}{
|
||||
{
|
||||
name: "Defaults",
|
||||
mutate: func(*codersdk.AIBridgeConfig) {},
|
||||
},
|
||||
{
|
||||
name: "GatewayEmitsStructuredLogs",
|
||||
mutate: func(cfg *codersdk.AIBridgeConfig) {
|
||||
cfg.StructuredLogging = serpent.Bool(true)
|
||||
cfg.StructuredLoggingSource = string(codersdk.AIStructuredLoggingSourceGateway)
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dv := coderdtest.DeploymentValues(t)
|
||||
tc.mutate(&dv.AI.BridgeConfig)
|
||||
_, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{DeploymentValues: dv})
|
||||
|
||||
srv, unsubscribe, err := NewAIBridgeDaemon(t.Context(), AIBridgeDaemonOptions{
|
||||
API: api,
|
||||
Config: dv.AI.BridgeConfig,
|
||||
Registerer: prometheus.NewRegistry(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, srv.Close()) })
|
||||
t.Cleanup(unsubscribe)
|
||||
|
||||
require.Equal(t, aibridged.PoolOptionsFromConfig(dv.AI.BridgeConfig), srv.PoolOptions())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+6
-1
@@ -1200,7 +1200,12 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
|
||||
defer closeBlockedUsersFunc()
|
||||
}
|
||||
var unsubscribeProviderReload func()
|
||||
aibridgeDaemon, unsubscribeProviderReload, err = newAIBridgeDaemon(coderAPI, vals.AI.BridgeConfig, aibridgeReg, aibridgeMetrics)
|
||||
aibridgeDaemon, unsubscribeProviderReload, err = NewAIBridgeDaemon(ctx, AIBridgeDaemonOptions{
|
||||
API: coderAPI,
|
||||
Config: vals.AI.BridgeConfig,
|
||||
Registerer: aibridgeReg,
|
||||
BridgeMetrics: aibridgeMetrics,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create aibridged: %w", err)
|
||||
}
|
||||
|
||||
+10
@@ -167,6 +167,16 @@ AI GATEWAY OPTIONS:
|
||||
Emit structured logs for AI Gateway interception records. Use this for
|
||||
exporting these records to external SIEM or observability systems.
|
||||
|
||||
--ai-gateway-structured-logging-source coderd|gateway|both, $CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE (default: coderd)
|
||||
Which process emits AI Gateway interception records when structured
|
||||
logging is enabled: coderd, the gateway, or both. The gateway emits
|
||||
records that are never persisted, such as records a deployment has
|
||||
chosen not to store, but cannot report thread_parent_id or
|
||||
thread_root_id. Use both to verify a move from one to the other;
|
||||
records reaching coderd are then reported twice. A standalone gateway
|
||||
must be configured to emit its own records, and its logs shipped
|
||||
rather than coderd's.
|
||||
|
||||
AI GATEWAY PROXY OPTIONS:
|
||||
--ai-gateway-proxy-dump-dir string, $CODER_AI_GATEWAY_PROXY_DUMP_DIR
|
||||
Directory for dumping MITM request/response pairs to disk for
|
||||
|
||||
+9
@@ -998,6 +998,15 @@ ai_gateway:
|
||||
# these records to external SIEM or observability systems.
|
||||
# (default: false, type: bool)
|
||||
structured_logging: false
|
||||
# Which process emits AI Gateway interception records when structured logging is
|
||||
# enabled: coderd, the gateway, or both. The gateway emits records that are never
|
||||
# persisted, such as records a deployment has chosen not to store, but cannot
|
||||
# report thread_parent_id or thread_root_id. Use both to verify a move from one to
|
||||
# the other; records reaching coderd are then reported twice. A standalone gateway
|
||||
# must be configured to emit its own records, and its logs shipped rather than
|
||||
# coderd's.
|
||||
# (default: coderd, type: enum[coderd\|gateway\|both])
|
||||
structured_logging_source: coderd
|
||||
# Once enabled, extra headers will be added to upstream requests to identify the
|
||||
# user (actor) making requests to AI Gateway. This is only needed if you are using
|
||||
# a proxy between AI Gateway and an upstream AI provider. This will send
|
||||
|
||||
@@ -66,9 +66,22 @@ type Server struct {
|
||||
// cancelFn closes the lifecycleCtx with the reason it closed.
|
||||
cancelFn context.CancelCauseFunc
|
||||
|
||||
// poolOptions configures the interception pool created at startup.
|
||||
poolOptions PoolOptions
|
||||
|
||||
shutdownOnce sync.Once
|
||||
}
|
||||
|
||||
// ServerOption configures a [Server] at construction.
|
||||
type ServerOption func(*Server)
|
||||
|
||||
// WithPoolOptions builds the server's interception pool with options instead of
|
||||
// [DefaultPoolOptions]. The deployment's record policy reaches the pool that
|
||||
// serves requests this way; see [PoolOptionsFromConfig].
|
||||
func WithPoolOptions(options PoolOptions) ServerOption {
|
||||
return func(s *Server) { s.poolOptions = options }
|
||||
}
|
||||
|
||||
// backend holds either an interception pool or a proxy router.
|
||||
// Its fields are fixed after publication. Both pool and proxyRouter are nil when
|
||||
// proxy mode is selected but providers are not yet loaded.
|
||||
@@ -80,7 +93,7 @@ type backend struct {
|
||||
|
||||
// New starts a gateway server. Creates a request pool only when interception
|
||||
// mode is selected. Mode is fixed at startup and requires a restart to change.
|
||||
func New(ctx context.Context, rpcDialer Dialer, logger slog.Logger, tracer trace.Tracer, experiments codersdk.Experiments, metrics *aibridge.Metrics) (*Server, error) {
|
||||
func New(ctx context.Context, rpcDialer Dialer, logger slog.Logger, tracer trace.Tracer, experiments codersdk.Experiments, metrics *aibridge.Metrics, opts ...ServerOption) (*Server, error) {
|
||||
if rpcDialer == nil {
|
||||
return nil, xerrors.Errorf("nil rpcDialer given")
|
||||
}
|
||||
@@ -97,6 +110,10 @@ func New(ctx context.Context, rpcDialer Dialer, logger slog.Logger, tracer trace
|
||||
reverseProxyExp: experiments.Enabled(codersdk.ExperimentAIGatewayReverseProxy),
|
||||
metrics: metrics,
|
||||
inflight: aibridge.NewInflightGate(logger),
|
||||
poolOptions: DefaultPoolOptions,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(daemon)
|
||||
}
|
||||
|
||||
if !daemon.reverseProxyExp {
|
||||
@@ -237,7 +254,7 @@ func (s *Server) initializeBackend(ctx context.Context, client DRPCClient) error
|
||||
|
||||
// initializeInterception creates and publishes the server-owned request pool.
|
||||
func (s *Server) initializeInterception() error {
|
||||
pool, err := NewCachedBridgePool(DefaultPoolOptions, nil, s.logger.Named("pool"), s.metrics, s.tracer)
|
||||
pool, err := NewCachedBridgePool(s.poolOptions, nil, s.logger.Named("pool"), s.metrics, s.tracer)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create request pool: %w", err)
|
||||
}
|
||||
@@ -334,6 +351,21 @@ func (s *Server) ReplaceProviders(ctx context.Context, providers []aibridge.Prov
|
||||
return nil
|
||||
}
|
||||
|
||||
// PoolOptions reports the options the published interception pool was built
|
||||
// with, so a caller can assert that its configuration reached the pool serving
|
||||
// requests. It reports the zero value in proxy mode, which has no pool.
|
||||
func (s *Server) PoolOptions() PoolOptions {
|
||||
current := s.backend.Load()
|
||||
if current == nil {
|
||||
return PoolOptions{}
|
||||
}
|
||||
pool, ok := current.pool.(*CachedBridgePool)
|
||||
if !ok {
|
||||
return PoolOptions{}
|
||||
}
|
||||
return pool.Options()
|
||||
}
|
||||
|
||||
// KeyPoolStateCollector reports key states from the current provider snapshot.
|
||||
func (s *Server) KeyPoolStateCollector() prometheus.Collector {
|
||||
return keypool.NewStateCollector(s.keyPools)
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/coder/coder/v2/aibridge/recorder"
|
||||
"github.com/coder/coder/v2/aibridge/tracing"
|
||||
"github.com/coder/coder/v2/coderd/aibridged/proto"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/quartz"
|
||||
)
|
||||
|
||||
@@ -54,10 +55,24 @@ type PoolOptions struct {
|
||||
MaxItems int64
|
||||
TTL time.Duration
|
||||
Clock quartz.Clock
|
||||
|
||||
// StructuredLogging makes each bridge emit AI Gateway interception
|
||||
// records in the format described by [recorder.InterceptionLogMarker].
|
||||
StructuredLogging bool
|
||||
}
|
||||
|
||||
var DefaultPoolOptions = PoolOptions{MaxItems: 5000, TTL: time.Minute * 15}
|
||||
|
||||
// PoolOptionsFromConfig returns DefaultPoolOptions with the record policy the
|
||||
// deployment configured. Every construction site uses it, so the in-process
|
||||
// daemon, the standalone gateway and the test harness cannot drift.
|
||||
func PoolOptionsFromConfig(cfg codersdk.AIBridgeConfig) PoolOptions {
|
||||
options := DefaultPoolOptions
|
||||
options.StructuredLogging = cfg.EmitsStructuredLogs(codersdk.AIStructuredLoggingSourceGateway)
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
var _ Pooler = &CachedBridgePool{}
|
||||
|
||||
type CachedBridgePool struct {
|
||||
@@ -84,6 +99,11 @@ type CachedBridgePool struct {
|
||||
cacheWG sync.WaitGroup
|
||||
}
|
||||
|
||||
// Options reports the options the pool was built with.
|
||||
func (p *CachedBridgePool) Options() PoolOptions {
|
||||
return p.options
|
||||
}
|
||||
|
||||
func NewCachedBridgePool(options PoolOptions, providers []aibridge.Provider, logger slog.Logger, metrics *aibridge.Metrics, tracer trace.Tracer) (*CachedBridgePool, error) {
|
||||
cache, err := ristretto.NewCache(&ristretto.Config[string, *aibridge.RequestBridge]{
|
||||
NumCounters: options.MaxItems * 10, // Docs suggest setting this 10x number of keys.
|
||||
@@ -228,6 +248,8 @@ func (p *CachedBridgePool) Acquire(ctx context.Context, req Request, clientFn Cl
|
||||
rec := aibridge.NewRecorder(
|
||||
p.logger.Named("recorder"),
|
||||
p.tracer,
|
||||
req.APIKeyID,
|
||||
p.options.StructuredLogging,
|
||||
recorder.NewDRPCRecorder(req.APIKeyID, func(clientCtx context.Context) (proto.DRPCRecorderClient, error) {
|
||||
// The recorder outlives this Acquire call, so the client is acquired
|
||||
// against the context of the record call being served.
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package aibridged_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/aibridged"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/serpent"
|
||||
)
|
||||
|
||||
func TestPoolOptionsFromConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cfg codersdk.AIBridgeConfig
|
||||
wantStructuredLogging bool
|
||||
}{
|
||||
{
|
||||
name: "Defaults",
|
||||
},
|
||||
{
|
||||
name: "StructuredLoggingWithoutSource",
|
||||
cfg: codersdk.AIBridgeConfig{StructuredLogging: serpent.Bool(true)},
|
||||
},
|
||||
{
|
||||
name: "SourceCoderd",
|
||||
cfg: codersdk.AIBridgeConfig{
|
||||
StructuredLogging: serpent.Bool(true),
|
||||
StructuredLoggingSource: string(codersdk.AIStructuredLoggingSourceCoderd),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SourceGateway",
|
||||
cfg: codersdk.AIBridgeConfig{
|
||||
StructuredLogging: serpent.Bool(true),
|
||||
StructuredLoggingSource: string(codersdk.AIStructuredLoggingSourceGateway),
|
||||
},
|
||||
wantStructuredLogging: true,
|
||||
},
|
||||
{
|
||||
name: "SourceBoth",
|
||||
cfg: codersdk.AIBridgeConfig{
|
||||
StructuredLogging: serpent.Bool(true),
|
||||
StructuredLoggingSource: string(codersdk.AIStructuredLoggingSourceBoth),
|
||||
},
|
||||
wantStructuredLogging: true,
|
||||
},
|
||||
{
|
||||
name: "SourceGatewayWithoutStructuredLogging",
|
||||
cfg: codersdk.AIBridgeConfig{
|
||||
StructuredLoggingSource: string(codersdk.AIStructuredLoggingSourceGateway),
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
options := aibridged.PoolOptionsFromConfig(tc.cfg)
|
||||
|
||||
require.Equal(t, tc.wantStructuredLogging, options.StructuredLogging, "StructuredLogging")
|
||||
|
||||
// The record policy must not disturb the cache sizing the
|
||||
// deployment relies on.
|
||||
require.Equal(t, aibridged.DefaultPoolOptions.MaxItems, options.MaxItems, "MaxItems")
|
||||
require.Equal(t, aibridged.DefaultPoolOptions.TTL, options.TTL, "TTL")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func NewServer(lifecycleCtx context.Context, opts Options) (*Server, error) {
|
||||
pubsub: opts.Pubsub,
|
||||
logger: opts.Logger,
|
||||
externalAuthConfigs: eac,
|
||||
structuredLogging: opts.GatewayCfg.StructuredLogging.Value(),
|
||||
structuredLogging: opts.GatewayCfg.EmitsStructuredLogs(codersdk.AIStructuredLoggingSourceCoderd),
|
||||
aiSeatTracker: opts.AISeatTracker,
|
||||
experiments: opts.Experiments,
|
||||
budgetPolicy: codersdk.NewAIBudgetPolicyFromString(opts.GatewayCfg.BudgetPolicy),
|
||||
|
||||
@@ -9,9 +9,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"github.com/coder/coder/v2/cli"
|
||||
"github.com/coder/coder/v2/coderd"
|
||||
"github.com/coder/coder/v2/coderd/aibridged"
|
||||
@@ -19,13 +17,11 @@ import (
|
||||
)
|
||||
|
||||
// StartTestAIBridgeDaemon wires an in-process aibridged daemon onto the
|
||||
// supplied API, mirroring what cli/server.go does in production. Tests that
|
||||
// create AI provider rows with BaseURL pointing at fake upstream HTTP servers
|
||||
// (e.g. chattest.NewOpenAI) will have their requests proxied through the real
|
||||
// aibridged stack as they would in production.
|
||||
//
|
||||
// The daemon fetches providers from coderd over the in-memory DRPC, then
|
||||
// refreshes on ai_providers change events, exactly like cli.newAIBridgeDaemon.
|
||||
// supplied API through [cli.NewAIBridgeDaemon], the same constructor
|
||||
// cli/server.go uses. Tests that create AI provider rows with BaseURL pointing
|
||||
// at fake upstream HTTP servers (e.g. chattest.NewOpenAI) will have their
|
||||
// requests proxied through the real aibridged stack as they would in
|
||||
// production, under the API's own deployment configuration.
|
||||
//
|
||||
// metrics is the registry the daemon reports provider reload events to.
|
||||
// The caller owns the metrics instance and can assert on it after the daemon
|
||||
@@ -53,30 +49,18 @@ func StartTestAIBridgeDaemonWithPubsub(
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
logger := api.Logger.Named("aibridged").Leveled(slog.LevelDebug)
|
||||
cfg := api.DeploymentValues.AI.BridgeConfig
|
||||
tracer := otel.Tracer("aibridge-test")
|
||||
|
||||
if metrics == nil {
|
||||
metrics = aibridged.NewMetrics(prometheus.NewRegistry())
|
||||
}
|
||||
|
||||
srv, err := aibridged.New(ctx, func(dialCtx context.Context) (aibridged.DRPCClient, error) {
|
||||
return api.CreateInMemoryAIBridgeServer(dialCtx)
|
||||
}, logger, tracer, api.Experiments, nil)
|
||||
srv, unsubscribe, err := cli.NewAIBridgeDaemon(ctx, cli.AIBridgeDaemonOptions{
|
||||
API: api,
|
||||
Config: api.DeploymentValues.AI.BridgeConfig,
|
||||
Registerer: prometheus.NewRegistry(),
|
||||
ProviderMetrics: metrics,
|
||||
Pubsub: ps,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create aibridged server: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = srv.Close() })
|
||||
|
||||
// The reloader fetches providers from coderd over srv's DRPC client; the
|
||||
// subscription drives an initial load and refreshes on change events.
|
||||
reloader := cli.NewProviderRPCReloader(srv.ReplaceProviders, srv.Client, cfg, logger.Named("reloader"), nil, metrics)
|
||||
unsubscribe, err := aibridged.SubscribeProviderReload(ctx, ps, reloader, logger.Named("subscriber"))
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe provider reload: %v", err)
|
||||
}
|
||||
t.Cleanup(unsubscribe)
|
||||
t.Cleanup(func() { _ = srv.Close() })
|
||||
|
||||
api.RegisterInMemoryAIBridgedHTTPHandler(srv)
|
||||
}
|
||||
|
||||
Generated
+4
@@ -17882,6 +17882,10 @@ const docTemplate = `{
|
||||
},
|
||||
"structured_logging": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"structured_logging_source": {
|
||||
"description": "StructuredLoggingSource selects which process emits the records that\nStructuredLogging enables. See AIStructuredLoggingSource.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
Generated
+4
@@ -15942,6 +15942,10 @@
|
||||
},
|
||||
"structured_logging": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"structured_logging_source": {
|
||||
"description": "StructuredLoggingSource selects which process emits the records that\nStructuredLogging enables. See AIStructuredLoggingSource.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
+63
-2
@@ -615,6 +615,42 @@ var PostgresAuthDrivers = []string{
|
||||
// based on max open connections.
|
||||
const PostgresConnMaxIdleAuto = "auto"
|
||||
|
||||
// AIStructuredLoggingSource selects which process emits AI Gateway
|
||||
// interception records when structured logging is enabled. Both processes
|
||||
// produce the same format; the gateway cannot resolve thread_parent_id and
|
||||
// thread_root_id, which are looked up in the database by coderd.
|
||||
type AIStructuredLoggingSource string
|
||||
|
||||
const (
|
||||
// AIStructuredLoggingSourceCoderd emits from coderd, as records arrive
|
||||
// over DRPC. Records the gateway does not send are not reported.
|
||||
AIStructuredLoggingSourceCoderd AIStructuredLoggingSource = "coderd"
|
||||
// AIStructuredLoggingSourceGateway emits from the AI Gateway, where the
|
||||
// records originate, so that records which are never persisted are still
|
||||
// reported. A standalone gateway must be configured to emit them, and its
|
||||
// logs shipped, rather than coderd's.
|
||||
AIStructuredLoggingSourceGateway AIStructuredLoggingSource = "gateway"
|
||||
// AIStructuredLoggingSourceBoth emits from both, for verifying a move
|
||||
// from one to the other. Records that reach coderd are reported twice.
|
||||
AIStructuredLoggingSourceBoth AIStructuredLoggingSource = "both"
|
||||
)
|
||||
|
||||
var AIStructuredLoggingSources = []string{
|
||||
string(AIStructuredLoggingSourceCoderd),
|
||||
string(AIStructuredLoggingSourceGateway),
|
||||
string(AIStructuredLoggingSourceBoth),
|
||||
}
|
||||
|
||||
// NewAIStructuredLoggingSourceFromString converts s to an
|
||||
// AIStructuredLoggingSource, falling back to AIStructuredLoggingSourceCoderd
|
||||
// when s is empty or not a recognized source.
|
||||
func NewAIStructuredLoggingSourceFromString(s string) AIStructuredLoggingSource {
|
||||
if slices.Contains(AIStructuredLoggingSources, s) {
|
||||
return AIStructuredLoggingSource(s)
|
||||
}
|
||||
return AIStructuredLoggingSourceCoderd
|
||||
}
|
||||
|
||||
// AIBudgetPolicy determines how the effective group is selected when a user
|
||||
// belongs to multiple groups with AI budgets configured.
|
||||
type AIBudgetPolicy string
|
||||
@@ -2002,6 +2038,16 @@ communicating directly.`,
|
||||
Group: &deploymentGroupAIGateway,
|
||||
YAML: "structured_logging",
|
||||
}
|
||||
aiGatewayStructuredLoggingSource := serpent.Option{
|
||||
Name: "AI Gateway Structured Logging Source",
|
||||
Description: "Which process emits AI Gateway interception records when structured logging is enabled: coderd, the gateway, or both. The gateway emits records that are never persisted, such as records a deployment has chosen not to store, but cannot report thread_parent_id or thread_root_id. Use both to verify a move from one to the other; records reaching coderd are then reported twice. A standalone gateway must be configured to emit its own records, and its logs shipped rather than coderd's.",
|
||||
Flag: "ai-gateway-structured-logging-source",
|
||||
Env: "CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE",
|
||||
Value: serpent.EnumOf(&c.AI.BridgeConfig.StructuredLoggingSource, AIStructuredLoggingSources...),
|
||||
Default: string(AIStructuredLoggingSourceCoderd),
|
||||
Group: &deploymentGroupAIGateway,
|
||||
YAML: "structured_logging_source",
|
||||
}
|
||||
aiGatewayAPIDumpDir := serpent.Option{
|
||||
Name: "AI Gateway API Dump Directory",
|
||||
Description: "Base directory for dumping AI Gateway request/response pairs to disk for debugging. When set, each provider writes under a subdirectory named after the provider. Sensitive headers are redacted. Leave empty to disable.",
|
||||
@@ -4556,6 +4602,7 @@ Write out the current server config as YAML to stdout.`,
|
||||
UseInstead: serpent.OptionSet{aiGatewayStructuredLogging},
|
||||
},
|
||||
aiGatewayStructuredLogging,
|
||||
aiGatewayStructuredLoggingSource,
|
||||
{
|
||||
Name: "AI Bridge Send Actor Headers",
|
||||
Description: "Deprecated: use --ai-gateway-send-actor-headers or CODER_AI_GATEWAY_SEND_ACTOR_HEADERS instead. Once enabled, extra headers will be added to upstream requests to identify the user (actor) making requests to AI Bridge. " +
|
||||
@@ -4920,8 +4967,11 @@ type AIBridgeConfig struct {
|
||||
MaxConcurrency serpent.Int64 `json:"max_concurrency" typescript:",notnull"`
|
||||
RateLimit serpent.Int64 `json:"rate_limit" typescript:",notnull"`
|
||||
StructuredLogging serpent.Bool `json:"structured_logging" typescript:",notnull"`
|
||||
SendActorHeaders serpent.Bool `json:"send_actor_headers" typescript:",notnull"`
|
||||
AllowBYOK serpent.Bool `json:"allow_byok" typescript:",notnull"`
|
||||
// StructuredLoggingSource selects which process emits the records that
|
||||
// StructuredLogging enables. See AIStructuredLoggingSource.
|
||||
StructuredLoggingSource string `json:"structured_logging_source,omitempty" typescript:",notnull"`
|
||||
SendActorHeaders serpent.Bool `json:"send_actor_headers" typescript:",notnull"`
|
||||
AllowBYOK serpent.Bool `json:"allow_byok" typescript:",notnull"`
|
||||
// Budget settings for AI Governance cost controls.
|
||||
BudgetPolicy string `json:"budget_policy,omitempty" typescript:",notnull"`
|
||||
BudgetPeriod string `json:"budget_period,omitempty" typescript:",notnull"`
|
||||
@@ -4938,6 +4988,17 @@ type AIBridgeConfig struct {
|
||||
APIDumpDir serpent.String `json:"api_dump_dir" typescript:",notnull"`
|
||||
}
|
||||
|
||||
// EmitsStructuredLogs reports whether source should emit AI Gateway
|
||||
// interception records. Both processes consult this, so that exactly the
|
||||
// configured one emits and a record is not reported twice by accident.
|
||||
func (c AIBridgeConfig) EmitsStructuredLogs(source AIStructuredLoggingSource) bool {
|
||||
if !c.StructuredLogging.Value() {
|
||||
return false
|
||||
}
|
||||
configured := NewAIStructuredLoggingSourceFromString(c.StructuredLoggingSource)
|
||||
return configured == source || configured == AIStructuredLoggingSourceBoth
|
||||
}
|
||||
|
||||
type AIBridgeProxyConfig struct {
|
||||
Enabled serpent.Bool `json:"enabled" typescript:",notnull"`
|
||||
ListenAddr serpent.String `json:"listen_addr" typescript:",notnull"`
|
||||
|
||||
@@ -325,6 +325,15 @@ Emit structured logs for AI Gateway interception records. Use this for exporting
|
||||
- YAML key: `ai_gateway.structured_logging`
|
||||
- Default value: `false`
|
||||
|
||||
### Structured logging source
|
||||
|
||||
Which process emits AI Gateway interception records when structured logging is enabled: coderd, the gateway, or both. The gateway emits records that are never persisted, such as records a deployment has chosen not to store, but cannot report thread_parent_id or thread_root_id. Use both to verify a move from one to the other; records reaching coderd are then reported twice. A standalone gateway must be configured to emit its own records, and its logs shipped rather than coderd's.
|
||||
|
||||
- Environment variable: `CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE`
|
||||
- CLI flag: [`--ai-gateway-structured-logging-source`](../../reference/cli/server/index.md#--ai-gateway-structured-logging-source)
|
||||
- YAML key: `ai_gateway.structured_logging_source`
|
||||
- Default value: `coderd`
|
||||
|
||||
## AI Gateway Proxy
|
||||
|
||||
### API dump directory
|
||||
|
||||
@@ -188,9 +188,10 @@ Refer to the [`coder ai-gateway start` logging options](../../reference/cli/ai-g
|
||||
### Structured interception logs
|
||||
|
||||
AI Gateway can emit a structured log for every interception record to an external SIEM or observability platform.
|
||||
The `CODER_AI_GATEWAY_STRUCTURED_LOGGING` setting belongs to `coderd`, and the standalone gateway does not consume it.
|
||||
Standalone replicas send interception records to `coderd`, which writes the structured logs to the `coderd` log output.
|
||||
Refer to [structured logging](./setup.md#structured-logging) for configuration and record types.
|
||||
`CODER_AI_GATEWAY_STRUCTURED_LOGGING` enables the records, and `CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE` selects which process emits them.
|
||||
Under the default source, `coderd`, standalone replicas send interception records to `coderd`, which writes the structured logs to the `coderd` log output.
|
||||
Set the source to `gateway` or `both` on every process to have the gateway emit its own records, and ship the gateway's logs as well as `coderd`'s.
|
||||
Refer to [structured logging](./setup.md#structured-logging) for configuration and record types, and [choose which process emits the records](./setup.md#choose-which-process-emits-the-records) for the trade-offs between the sources.
|
||||
|
||||
## Export data
|
||||
|
||||
|
||||
@@ -146,8 +146,33 @@ stderr) or [`--log-json`](../../reference/cli/server/index.md#--log-json). For m
|
||||
ingestion, set `--log-json` to a file path or `/dev/stderr` so that records are
|
||||
emitted as JSON.
|
||||
|
||||
This setting belongs to `coderd`.
|
||||
A [standalone gateway](./standalone.md) does not consume it.
|
||||
### Choose which process emits the records
|
||||
|
||||
By default `coderd` emits these records as they arrive from the gateway.
|
||||
Set `--ai-gateway-structured-logging-source` to change that:
|
||||
|
||||
| Value | Emitted by | Use it when |
|
||||
|-----------|------------------------|---------------------------------------------------------------------------------------------------------|
|
||||
| `coderd` | `coderd` (default) | You want today's behavior. |
|
||||
| `gateway` | The AI Gateway process | You need records that the gateway never persists, such as records your deployment chooses not to store. |
|
||||
| `both` | Both processes | You are moving from one to the other and want to compare the two streams. |
|
||||
|
||||
```sh
|
||||
coder server --ai-gateway-structured-logging=true \
|
||||
--ai-gateway-structured-logging-source=gateway
|
||||
```
|
||||
|
||||
Gateway-emitted records carry the same message, the same `record_type` values and the same fields, with one exception: they omit `thread_parent_id` and `thread_root_id`.
|
||||
Those are resolved from recorded tool usage by a database lookup that only `coderd` can perform.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> With `both`, every record that reaches `coderd` is logged twice.
|
||||
> Deduplicate on `interception_id`, `record_type` and `msg_id` if your pipeline counts records.
|
||||
|
||||
When the gateway emits the records, they are written to the gateway's log output rather than `coderd`'s, and under a different logger name.
|
||||
Match on the `"interception log"` message rather than the logger name so that your pipeline works with either source.
|
||||
|
||||
On a [standalone gateway](./standalone.md), set `CODER_AI_GATEWAY_STRUCTURED_LOGGING` and `CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE` on **both** processes, and ship the gateway's logs.
|
||||
|
||||
Filter for AI Gateway records in your logging pipeline by matching on the
|
||||
`"interception log"` message. Each log line includes a `record_type` field that
|
||||
|
||||
Generated
+2
-1
@@ -208,7 +208,8 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
|
||||
"rate_limit": 0,
|
||||
"retention": 0,
|
||||
"send_actor_headers": true,
|
||||
"structured_logging": true
|
||||
"structured_logging": true,
|
||||
"structured_logging_source": "string"
|
||||
},
|
||||
"chat": {
|
||||
"acquire_batch_size": 0,
|
||||
|
||||
Generated
+9
-4
@@ -405,7 +405,8 @@ title: Schemas
|
||||
"rate_limit": 0,
|
||||
"retention": 0,
|
||||
"send_actor_headers": true,
|
||||
"structured_logging": true
|
||||
"structured_logging": true,
|
||||
"structured_logging_source": "string"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -429,6 +430,7 @@ title: Schemas
|
||||
| `retention` | integer | false | | |
|
||||
| `send_actor_headers` | boolean | false | | |
|
||||
| `structured_logging` | boolean | false | | |
|
||||
| `structured_logging_source` | string | false | | Structured logging source selects which process emits the records that StructuredLogging enables. See AIStructuredLoggingSource. |
|
||||
|
||||
## codersdk.AIBridgeListSessionsResponse
|
||||
|
||||
@@ -1046,7 +1048,8 @@ title: Schemas
|
||||
"rate_limit": 0,
|
||||
"retention": 0,
|
||||
"send_actor_headers": true,
|
||||
"structured_logging": true
|
||||
"structured_logging": true,
|
||||
"structured_logging_source": "string"
|
||||
},
|
||||
"chat": {
|
||||
"acquire_batch_size": 0,
|
||||
@@ -7573,7 +7576,8 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"rate_limit": 0,
|
||||
"retention": 0,
|
||||
"send_actor_headers": true,
|
||||
"structured_logging": true
|
||||
"structured_logging": true,
|
||||
"structured_logging_source": "string"
|
||||
},
|
||||
"chat": {
|
||||
"acquire_batch_size": 0,
|
||||
@@ -8206,7 +8210,8 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"rate_limit": 0,
|
||||
"retention": 0,
|
||||
"send_actor_headers": true,
|
||||
"structured_logging": true
|
||||
"structured_logging": true,
|
||||
"structured_logging_source": "string"
|
||||
},
|
||||
"chat": {
|
||||
"acquire_batch_size": 0,
|
||||
|
||||
@@ -202,6 +202,28 @@ Maximum number of concurrent AI Gateway requests per replica. Set to 0 to disabl
|
||||
|
||||
Maximum number of AI Gateway requests per second per replica. Set to 0 to disable (unlimited).
|
||||
|
||||
### --ai-gateway-structured-logging
|
||||
|
||||
| | |
|
||||
|-------------|---------------------------------------------------|
|
||||
| Type | <code>bool</code> |
|
||||
| Environment | <code>$CODER_AI_GATEWAY_STRUCTURED_LOGGING</code> |
|
||||
| YAML | <code>ai_gateway.structured_logging</code> |
|
||||
| Default | <code>false</code> |
|
||||
|
||||
Emit structured logs for AI Gateway interception records. Use this for exporting these records to external SIEM or observability systems.
|
||||
|
||||
### --ai-gateway-structured-logging-source
|
||||
|
||||
| | |
|
||||
|-------------|----------------------------------------------------------|
|
||||
| Type | <code>coderd\|gateway\|both</code> |
|
||||
| Environment | <code>$CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE</code> |
|
||||
| YAML | <code>ai_gateway.structured_logging_source</code> |
|
||||
| Default | <code>coderd</code> |
|
||||
|
||||
Which process emits AI Gateway interception records when structured logging is enabled: coderd, the gateway, or both. The gateway emits records that are never persisted, such as records a deployment has chosen not to store, but cannot report thread_parent_id or thread_root_id. Use both to verify a move from one to the other; records reaching coderd are then reported twice. A standalone gateway must be configured to emit its own records, and its logs shipped rather than coderd's.
|
||||
|
||||
### --ai-gateway-send-actor-headers
|
||||
|
||||
| | |
|
||||
|
||||
@@ -1932,6 +1932,17 @@ Maximum number of AI Gateway requests per second per replica. Set to 0 to disabl
|
||||
|
||||
Emit structured logs for AI Gateway interception records. Use this for exporting these records to external SIEM or observability systems.
|
||||
|
||||
### --ai-gateway-structured-logging-source
|
||||
|
||||
| | |
|
||||
|-------------|----------------------------------------------------------|
|
||||
| Type | <code>coderd\|gateway\|both</code> |
|
||||
| Environment | <code>$CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE</code> |
|
||||
| YAML | <code>ai_gateway.structured_logging_source</code> |
|
||||
| Default | <code>coderd</code> |
|
||||
|
||||
Which process emits AI Gateway interception records when structured logging is enabled: coderd, the gateway, or both. The gateway emits records that are never persisted, such as records a deployment has chosen not to store, but cannot report thread_parent_id or thread_root_id. Use both to verify a move from one to the other; records reaching coderd are then reported twice. A standalone gateway must be configured to emit its own records, and its logs shipped rather than coderd's.
|
||||
|
||||
### --ai-gateway-send-actor-headers
|
||||
|
||||
| | |
|
||||
|
||||
@@ -85,6 +85,10 @@ var aiGatewayInheritedEnvs = map[string]struct{}{
|
||||
"CODER_AI_GATEWAY_MAX_CONCURRENCY": {},
|
||||
"CODER_AI_GATEWAY_RATE_LIMIT": {},
|
||||
"CODER_AI_GATEWAY_SEND_ACTOR_HEADERS": {},
|
||||
// Structured logging is inherited because the gateway is now one of the
|
||||
// processes that can emit the records; which one does is the source.
|
||||
"CODER_AI_GATEWAY_STRUCTURED_LOGGING": {},
|
||||
"CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE": {},
|
||||
|
||||
// Prometheus
|
||||
"CODER_PROMETHEUS_ADDRESS": {},
|
||||
@@ -310,7 +314,8 @@ func runStandaloneGateway(ctx context.Context, params standaloneGatewayParams) e
|
||||
func newStandaloneGateway(params standaloneGatewayParams) (*standaloneGateway, error) {
|
||||
// The aibridged daemon must outlive the serving context so in-flight HTTP
|
||||
// requests retain their DRPC connection during graceful HTTP shutdown.
|
||||
daemon, err := aibridged.New(context.Background(), params.dialer, params.logger.Named("aibridged"), params.tracer, params.experiments, params.metrics)
|
||||
daemon, err := aibridged.New(context.Background(), params.dialer, params.logger.Named("aibridged"), params.tracer, params.experiments, params.metrics,
|
||||
aibridged.WithPoolOptions(aibridged.PoolOptionsFromConfig(params.bridgeConfig)))
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("start AI Gateway daemon: %w", err)
|
||||
}
|
||||
|
||||
@@ -611,7 +611,6 @@ func TestAIGatewayStart_InheritedOptions(t *testing.T) {
|
||||
"CODER_AI_GATEWAY_ENABLED": {},
|
||||
"CODER_AI_GATEWAY_INJECT_CODER_MCP_TOOLS": {},
|
||||
"CODER_AI_GATEWAY_RETENTION": {},
|
||||
"CODER_AI_GATEWAY_STRUCTURED_LOGGING": {},
|
||||
|
||||
// Prometheus (coderd-only: agent/database collectors)
|
||||
"CODER_PROMETHEUS_AGGREGATE_AGENT_STATS_BY": {},
|
||||
|
||||
@@ -68,6 +68,20 @@ AI GATEWAY OPTIONS:
|
||||
making the request) and X-Ai-Bridge-Actor-Metadata-Username (their
|
||||
username).
|
||||
|
||||
--ai-gateway-structured-logging bool, $CODER_AI_GATEWAY_STRUCTURED_LOGGING (default: false)
|
||||
Emit structured logs for AI Gateway interception records. Use this for
|
||||
exporting these records to external SIEM or observability systems.
|
||||
|
||||
--ai-gateway-structured-logging-source coderd|gateway|both, $CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE (default: coderd)
|
||||
Which process emits AI Gateway interception records when structured
|
||||
logging is enabled: coderd, the gateway, or both. The gateway emits
|
||||
records that are never persisted, such as records a deployment has
|
||||
chosen not to store, but cannot report thread_parent_id or
|
||||
thread_root_id. Use both to verify a move from one to the other;
|
||||
records reaching coderd are then reported twice. A standalone gateway
|
||||
must be configured to emit its own records, and its logs shipped
|
||||
rather than coderd's.
|
||||
|
||||
CONFIG OPTIONS:
|
||||
Use a YAML configuration file when your server launch become unwieldy.
|
||||
|
||||
|
||||
@@ -168,6 +168,16 @@ AI GATEWAY OPTIONS:
|
||||
Emit structured logs for AI Gateway interception records. Use this for
|
||||
exporting these records to external SIEM or observability systems.
|
||||
|
||||
--ai-gateway-structured-logging-source coderd|gateway|both, $CODER_AI_GATEWAY_STRUCTURED_LOGGING_SOURCE (default: coderd)
|
||||
Which process emits AI Gateway interception records when structured
|
||||
logging is enabled: coderd, the gateway, or both. The gateway emits
|
||||
records that are never persisted, such as records a deployment has
|
||||
chosen not to store, but cannot report thread_parent_id or
|
||||
thread_root_id. Use both to verify a move from one to the other;
|
||||
records reaching coderd are then reported twice. A standalone gateway
|
||||
must be configured to emit its own records, and its logs shipped
|
||||
rather than coderd's.
|
||||
|
||||
AI GATEWAY PROXY OPTIONS:
|
||||
--ai-gateway-proxy-dump-dir string, $CODER_AI_GATEWAY_PROXY_DUMP_DIR
|
||||
Directory for dumping MITM request/response pairs to disk for
|
||||
|
||||
Generated
+14
@@ -47,6 +47,11 @@ export interface AIBridgeConfig {
|
||||
readonly max_concurrency: number;
|
||||
readonly rate_limit: number;
|
||||
readonly structured_logging: boolean;
|
||||
/**
|
||||
* StructuredLoggingSource selects which process emits the records that
|
||||
* StructuredLogging enables. See AIStructuredLoggingSource.
|
||||
*/
|
||||
readonly structured_logging_source?: string;
|
||||
readonly send_actor_headers: boolean;
|
||||
readonly allow_byok: boolean;
|
||||
/**
|
||||
@@ -651,6 +656,15 @@ export interface AISpendPeriodWindow {
|
||||
readonly period_end: string;
|
||||
}
|
||||
|
||||
// From codersdk/deployment.go
|
||||
export type AIStructuredLoggingSource = "both" | "coderd" | "gateway";
|
||||
|
||||
export const AIStructuredLoggingSources: AIStructuredLoggingSource[] = [
|
||||
"both",
|
||||
"coderd",
|
||||
"gateway",
|
||||
];
|
||||
|
||||
// From codersdk/allowlist.go
|
||||
/**
|
||||
* APIAllowListTarget represents a single allow-list entry using the canonical
|
||||
|
||||
Reference in New Issue
Block a user