mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
Ethan
15f2fa55c6
## Summary Adds a process-wide cache for three hot database queries in `chatd` that were hitting Postgres on **every chat turn** despite returning rarely-changing configuration data: | Query | Before (50k turns) | After | Reduction | |---|---|---|---| | `GetEnabledChatProviders` | ~98.6k calls | ~500-1000 | ~99% | | `GetChatModelConfigByID` | ~49.2k calls | ~500-1000 | ~98% | | `GetUserChatCustomPrompt` | ~46.7k calls | ~1000-2000 | ~97% | These were identified via `coder exp scaletest chat` (5000 concurrent chats × 10 turns) as the dominant source of Postgres load during chat processing. ## Design Follows the established **webpush subscription cache pattern** (`coderd/webpush/webpush.go`): - `sync.RWMutex` + `tailscale.com/util/singleflight` (generic) + generation-based stale prevention + TTL - 10s TTL for provider/model config, 5s TTL for user prompts - Negative caching for `sql.ErrNoRows` on user prompts (the common case — most users don't set custom prompts) - Deep-clones `ChatModelConfig.Options` (`json.RawMessage` = `[]byte`) on both store and read paths ### Invalidation Single pubsub channel (`chat:config_change`) with kind discriminator for cross-replica cache invalidation. Seven publish points in `coderd/chats.go` cover all admin mutation endpoints (create/update/delete for providers and model configs, put for user prompts). _This PR was generated with mux and was reviewed by a human_
53 lines
1.7 KiB
Go
53 lines
1.7 KiB
Go
package pubsub
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
|
|
"github.com/google/uuid"
|
|
"golang.org/x/xerrors"
|
|
)
|
|
|
|
// ChatConfigEventChannel is the pubsub channel for chat config
|
|
// changes (providers, model configs, user prompts). All replicas
|
|
// subscribe to this channel to invalidate their local caches.
|
|
const ChatConfigEventChannel = "chat:config_change"
|
|
|
|
// HandleChatConfigEvent wraps a typed callback for ChatConfigEvent
|
|
// messages, following the same pattern as HandleChatEvent.
|
|
func HandleChatConfigEvent(cb func(ctx context.Context, payload ChatConfigEvent, err error)) func(ctx context.Context, message []byte, err error) {
|
|
return func(ctx context.Context, message []byte, err error) {
|
|
if err != nil {
|
|
cb(ctx, ChatConfigEvent{}, xerrors.Errorf("chat config event pubsub: %w", err))
|
|
return
|
|
}
|
|
var payload ChatConfigEvent
|
|
if err := json.Unmarshal(message, &payload); err != nil {
|
|
cb(ctx, ChatConfigEvent{}, xerrors.Errorf("unmarshal chat config event: %w", err))
|
|
return
|
|
}
|
|
|
|
cb(ctx, payload, err)
|
|
}
|
|
}
|
|
|
|
// ChatConfigEvent is published when chat configuration changes
|
|
// (provider CRUD, model config CRUD, or user prompt updates).
|
|
// Subscribers use this to invalidate their local caches.
|
|
type ChatConfigEvent struct {
|
|
Kind ChatConfigEventKind `json:"kind"`
|
|
// EntityID carries context for the invalidation:
|
|
// - For providers: uuid.Nil (all providers are invalidated).
|
|
// - For model configs: the specific config ID.
|
|
// - For user prompts: the user ID.
|
|
EntityID uuid.UUID `json:"entity_id"`
|
|
}
|
|
|
|
type ChatConfigEventKind string
|
|
|
|
const (
|
|
ChatConfigEventProviders ChatConfigEventKind = "providers"
|
|
ChatConfigEventModelConfig ChatConfigEventKind = "model_config"
|
|
ChatConfigEventUserPrompt ChatConfigEventKind = "user_prompt"
|
|
)
|