mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
perf(coderd/x/chatd): add process-wide config cache for hot DB queries (#23272)
## Summary Adds a process-wide cache for three hot database queries in `chatd` that were hitting Postgres on **every chat turn** despite returning rarely-changing configuration data: | Query | Before (50k turns) | After | Reduction | |---|---|---|---| | `GetEnabledChatProviders` | ~98.6k calls | ~500-1000 | ~99% | | `GetChatModelConfigByID` | ~49.2k calls | ~500-1000 | ~98% | | `GetUserChatCustomPrompt` | ~46.7k calls | ~1000-2000 | ~97% | These were identified via `coder exp scaletest chat` (5000 concurrent chats × 10 turns) as the dominant source of Postgres load during chat processing. ## Design Follows the established **webpush subscription cache pattern** (`coderd/webpush/webpush.go`): - `sync.RWMutex` + `tailscale.com/util/singleflight` (generic) + generation-based stale prevention + TTL - 10s TTL for provider/model config, 5s TTL for user prompts - Negative caching for `sql.ErrNoRows` on user prompts (the common case — most users don't set custom prompts) - Deep-clones `ChatModelConfig.Options` (`json.RawMessage` = `[]byte`) on both store and read paths ### Invalidation Single pubsub channel (`chat:config_change`) with kind discriminator for cross-replica cache invalidation. Seven publish points in `coderd/chats.go` cover all admin mutation endpoints (create/update/delete for providers and model configs, put for user prompts). _This PR was generated with mux and was reviewed by a human_
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
package pubsub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
// ChatConfigEventChannel is the pubsub channel for chat config
|
||||
// changes (providers, model configs, user prompts). All replicas
|
||||
// subscribe to this channel to invalidate their local caches.
|
||||
const ChatConfigEventChannel = "chat:config_change"
|
||||
|
||||
// HandleChatConfigEvent wraps a typed callback for ChatConfigEvent
|
||||
// messages, following the same pattern as HandleChatEvent.
|
||||
func HandleChatConfigEvent(cb func(ctx context.Context, payload ChatConfigEvent, err error)) func(ctx context.Context, message []byte, err error) {
|
||||
return func(ctx context.Context, message []byte, err error) {
|
||||
if err != nil {
|
||||
cb(ctx, ChatConfigEvent{}, xerrors.Errorf("chat config event pubsub: %w", err))
|
||||
return
|
||||
}
|
||||
var payload ChatConfigEvent
|
||||
if err := json.Unmarshal(message, &payload); err != nil {
|
||||
cb(ctx, ChatConfigEvent{}, xerrors.Errorf("unmarshal chat config event: %w", err))
|
||||
return
|
||||
}
|
||||
|
||||
cb(ctx, payload, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ChatConfigEvent is published when chat configuration changes
|
||||
// (provider CRUD, model config CRUD, or user prompt updates).
|
||||
// Subscribers use this to invalidate their local caches.
|
||||
type ChatConfigEvent struct {
|
||||
Kind ChatConfigEventKind `json:"kind"`
|
||||
// EntityID carries context for the invalidation:
|
||||
// - For providers: uuid.Nil (all providers are invalidated).
|
||||
// - For model configs: the specific config ID.
|
||||
// - For user prompts: the user ID.
|
||||
EntityID uuid.UUID `json:"entity_id"`
|
||||
}
|
||||
|
||||
type ChatConfigEventKind string
|
||||
|
||||
const (
|
||||
ChatConfigEventProviders ChatConfigEventKind = "providers"
|
||||
ChatConfigEventModelConfig ChatConfigEventKind = "model_config"
|
||||
ChatConfigEventUserPrompt ChatConfigEventKind = "user_prompt"
|
||||
)
|
||||
Reference in New Issue
Block a user