From ec89abd6e5ac1df40dd38d6cb923cbd6947c8c36 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 4 Mar 2026 11:03:03 -0500 Subject: [PATCH] feat(chatd): use lightweight model candidates for title generation (#22605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Title generation uses the same model the user selected for chat. This breaks when: 1. **Thinking/extended thinking models** — `ToolChoice: None` conflicts with extended thinking on Anthropic. The bare call has no thinking config, so provider-level defaults can conflict. 2. **Expensive models** — User picks `o3` or `claude-opus-4`, and a trivial 8-word title generation burns through tokens/cost unnecessarily. 3. **Provider quirks** — Different providers have different constraints around thinking mode + tool choice combinations. ## Solution Modeled after how `coder/mux` handles this with `NAME_GEN_PREFERRED_MODELS` + ordered candidate fallback: ### Phase 1: Candidate model list with fallback - New `TitleModelFunc` type returns an ordered list of candidate models - Tries `claude-haiku-4-5` → `gpt-4o-mini` → user's model - Gracefully skips unavailable candidates (missing API key, provider not configured) - Falls back to the user's chat model as last resort ### Phase 2: Provider-safe call options - Removed `ToolChoice: None` which conflicts with extended thinking on some providers - Added `MaxOutputTokens: 256` to cap token usage - Improved title prompt with verb-noun format guidance (`Fix sidebar layout`, `Add user authentication`) and explicit no-markdown/no-code-fences instructions ### Files changed - `coderd/chatd/title.go` — Candidate loop, improved prompt, safe call options - `coderd/chatd/chatd.go` — Build `TitleModelFunc` closure with lightweight candidates --- coderd/chatd/chatd.go | 14 +++--- coderd/chatd/title.go | 106 ++++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 32 deletions(-) diff --git a/coderd/chatd/chatd.go b/coderd/chatd/chatd.go index d239217f91..23682f3a13 100644 --- a/coderd/chatd/chatd.go +++ b/coderd/chatd/chatd.go @@ -1904,7 +1904,7 @@ func (p *Server) runChat( chat database.Chat, logger slog.Logger, ) error { - model, modelConfig, err := p.resolveChatModel(ctx, chat) + model, modelConfig, providerKeys, err := p.resolveChatModel(ctx, chat) if err != nil { return err } @@ -1926,7 +1926,7 @@ func (p *Server) runChat( p.inflight.Add(1) go func() { defer p.inflight.Done() - p.maybeGenerateChatTitle(context.WithoutCancel(ctx), chat, messages, model, logger) + p.maybeGenerateChatTitle(context.WithoutCancel(ctx), chat, messages, model, providerKeys, logger) }() prompt, err := chatprompt.ConvertMessages(messages) @@ -2406,17 +2406,17 @@ func (p *Server) persistChatContextSummary( func (p *Server) resolveChatModel( ctx context.Context, chat database.Chat, -) (fantasy.LanguageModel, database.ChatModelConfig, error) { +) (fantasy.LanguageModel, database.ChatModelConfig, chatprovider.ProviderAPIKeys, error) { dbConfig, err := p.resolveModelConfig(ctx, chat) if err != nil { - return nil, database.ChatModelConfig{}, xerrors.Errorf( + return nil, database.ChatModelConfig{}, chatprovider.ProviderAPIKeys{}, xerrors.Errorf( "resolve model config: %w", err, ) } providers, err := p.db.GetEnabledChatProviders(ctx) if err != nil { - return nil, database.ChatModelConfig{}, xerrors.Errorf( + return nil, database.ChatModelConfig{}, chatprovider.ProviderAPIKeys{}, xerrors.Errorf( "get enabled chat providers: %w", err, ) } @@ -2438,11 +2438,11 @@ func (p *Server) resolveChatModel( dbConfig.Provider, dbConfig.Model, keys, ) if err != nil { - return nil, database.ChatModelConfig{}, xerrors.Errorf( + return nil, database.ChatModelConfig{}, chatprovider.ProviderAPIKeys{}, xerrors.Errorf( "create model: %w", err, ) } - return model, dbConfig, nil + return model, dbConfig, keys, nil } // resolveModelConfig looks up the chat's model config by its diff --git a/coderd/chatd/title.go b/coderd/chatd/title.go index 238acc3eec..0ab706a634 100644 --- a/coderd/chatd/title.go +++ b/coderd/chatd/title.go @@ -6,28 +6,59 @@ import ( "time" "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + fantasyazure "charm.land/fantasy/providers/azure" + fantasybedrock "charm.land/fantasy/providers/bedrock" + fantasygoogle "charm.land/fantasy/providers/google" + fantasyopenai "charm.land/fantasy/providers/openai" + fantasyopenrouter "charm.land/fantasy/providers/openrouter" + fantasyvercel "charm.land/fantasy/providers/vercel" "golang.org/x/xerrors" "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/chatd/chatprompt" + "github.com/coder/coder/v2/coderd/chatd/chatprovider" "github.com/coder/coder/v2/coderd/chatd/chatretry" "github.com/coder/coder/v2/coderd/database" coderdpubsub "github.com/coder/coder/v2/coderd/pubsub" ) -const titleGenerationPrompt = "Generate a concise title (max 8 words, under 128 characters) for " + - "the user's first message. Return plain text only — no quotes, no emoji, " + - "no markdown, no special characters." +const titleGenerationPrompt = "Generate a concise title (2-8 words) for the user's message. " + + "Use verb-noun format describing the primary intent (e.g. \"Fix sidebar layout\", " + + "\"Add user authentication\", \"Refactor database queries\"). " + + "Return plain text only — no quotes, no emoji, no markdown, no code fences, " + + "no special characters, no trailing punctuation. Sentence case." + +// preferredTitleModels are lightweight models used for title +// generation, one per provider type. Each entry uses the +// cheapest/fastest small model for that provider as identified +// by the charmbracelet/catwalk model catalog. Providers that +// aren't configured (no API key) are silently skipped. +var preferredTitleModels = []struct { + provider string + model string +}{ + {fantasyanthropic.Name, "claude-haiku-4-5"}, + {fantasyopenai.Name, "gpt-4o-mini"}, + {fantasygoogle.Name, "gemini-2.5-flash"}, + {fantasyazure.Name, "gpt-4o-mini"}, + {fantasybedrock.Name, "anthropic.claude-haiku-4-5-20251001-v1:0"}, + {fantasyopenrouter.Name, "anthropic/claude-3.5-haiku"}, + {fantasyvercel.Name, "anthropic/claude-haiku-4.5"}, +} // maybeGenerateChatTitle generates an AI title for the chat when // appropriate (first user message, no assistant reply yet, and the // current title is either empty or still the fallback truncation). -// It is a best-effort operation that logs and swallows errors. +// It tries cheap, fast models first and falls back to the user's +// chat model. It is a best-effort operation that logs and swallows +// errors. func (p *Server) maybeGenerateChatTitle( ctx context.Context, chat database.Chat, messages []database.ChatMessage, - model fantasy.LanguageModel, + fallbackModel fantasy.LanguageModel, + keys chatprovider.ProviderAPIKeys, logger slog.Logger, ) { input, ok := titleInput(chat, messages) @@ -38,31 +69,55 @@ func (p *Server) maybeGenerateChatTitle( titleCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - title, err := generateTitle(titleCtx, model, input) - if err != nil { - logger.Debug(ctx, "failed to generate chat title", - slog.F("chat_id", chat.ID), - slog.Error(err), + // Build candidate list: preferred lightweight models first, + // then the user's chat model as last resort. + candidates := make([]fantasy.LanguageModel, 0, len(preferredTitleModels)+1) + for _, c := range preferredTitleModels { + m, err := chatprovider.ModelFromConfig( + c.provider, c.model, keys, ) - return + if err == nil { + candidates = append(candidates, m) + } } - if title == "" || title == chat.Title { + candidates = append(candidates, fallbackModel) + var lastErr error + for _, model := range candidates { + title, err := generateTitle(titleCtx, model, input) + if err != nil { + lastErr = err + logger.Debug(ctx, "title model candidate failed", + slog.F("chat_id", chat.ID), + slog.Error(err), + ) + continue + } + if title == "" || title == chat.Title { + return + } + + _, err = p.db.UpdateChatByID(ctx, database.UpdateChatByIDParams{ + ID: chat.ID, + Title: title, + }) + if err != nil { + logger.Warn(ctx, "failed to update generated chat title", + slog.F("chat_id", chat.ID), + slog.Error(err), + ) + return + } + chat.Title = title + p.publishChatPubsubEvent(chat, coderdpubsub.ChatEventKindTitleChange) return } - _, err = p.db.UpdateChatByID(ctx, database.UpdateChatByIDParams{ - ID: chat.ID, - Title: title, - }) - if err != nil { - logger.Warn(ctx, "failed to update generated chat title", + if lastErr != nil { + logger.Debug(ctx, "all title model candidates failed", slog.F("chat_id", chat.ID), - slog.Error(err), + slog.Error(lastErr), ) - return } - chat.Title = title - p.publishChatPubsubEvent(chat, coderdpubsub.ChatEventKindTitleChange) } // generateTitle calls the model with a title-generation system prompt @@ -87,14 +142,15 @@ func generateTitle( }, }, } - toolChoice := fantasy.ToolChoiceNone + + var maxOutputTokens int64 = 256 var response *fantasy.Response err := chatretry.Retry(ctx, func(retryCtx context.Context) error { var genErr error response, genErr = model.Generate(retryCtx, fantasy.Call{ - Prompt: prompt, - ToolChoice: &toolChoice, + Prompt: prompt, + MaxOutputTokens: &maxOutputTokens, }) return genErr }, nil)