Lots of code cleanup

coderd/chatd/chatd.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"charm.land/fantasy"
 	fantasyanthropic "charm.land/fantasy/providers/anthropic"
@@ -55,16 +54,6 @@ const (
 	maxChatSteps                 = 1200
 
 	defaultContextCompressionThresholdPercent = int32(70)
-	minContextCompressionThresholdPercent     = int32(0)
-	maxContextCompressionThresholdPercent     = int32(100)
-	contextCompressionSummaryReportMaxRunes   = 4096
-	contextCompressionSummaryPrompt           = "Summarize the current chat so a " +
-		"new assistant can continue seamlessly. Include the user's goals, " +
-		"decisions made, concrete technical details (files, commands, APIs), " +
-		"errors encountered and fixes, and open questions. Be dense and factual. " +
-		"Omit pleasantries and next-step suggestions."
-	contextCompressionSystemSummaryPrefix = "Summary of earlier chat context:"
-	contextCompressionSummaryTruncatedTag = "[summary truncated for live stream]"
 
 	maxCreateWorkspaceBuildLogLines     = 120
 	maxCreateWorkspaceBuildLogChars     = 16 * 1024
@@ -104,7 +93,7 @@ type Processor struct {
 	subagentService          *SubagentService
 	resolveProviderAPIKeysFn ProviderAPIKeysResolver
 	titleGeneration          TitleGenerationConfig
-	titleModelLookup         func(ProviderAPIKeys) (fantasy.LanguageModel, error)
+	titleModelLookup         func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error)
 
 	activeMu      sync.Mutex
 	activeCancels map[uuid.UUID]context.CancelCauseFunc
@@ -152,7 +141,7 @@ type CreateWorkspaceToolResult struct {
 }
 
 // ProviderAPIKeysResolver resolves provider API keys for chat model calls.
-type ProviderAPIKeysResolver func(context.Context) (ProviderAPIKeys, error)
+type ProviderAPIKeysResolver func(context.Context) (chatprovider.ProviderAPIKeys, error)
 
 // TestingConfig contains hooks intended only for tests.
 type TestingConfig struct {
@@ -316,8 +305,8 @@ func New(cfg Config) *Processor {
 
 	resolveProviderAPIKeys := cfg.ResolveProviderAPIKeys
 	if resolveProviderAPIKeys == nil {
-		resolveProviderAPIKeys = func(context.Context) (ProviderAPIKeys, error) {
-			return ProviderAPIKeys{}, nil
+		resolveProviderAPIKeys = func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+			return chatprovider.ProviderAPIKeys{}, nil
 		}
 	}
 	streamManager := cfg.StreamManager
@@ -485,17 +474,17 @@ func (p *Processor) ListModels(ctx context.Context) (codersdk.ChatModelsResponse
 		return codersdk.ChatModelsResponse{}, err
 	}
 
-	configuredProviders := make([]ConfiguredProvider, 0, len(enabledProviders))
+	configuredProviders := make([]chatprovider.ConfiguredProvider, 0, len(enabledProviders))
 	for _, provider := range enabledProviders {
-		configuredProviders = append(configuredProviders, ConfiguredProvider{
+		configuredProviders = append(configuredProviders, chatprovider.ConfiguredProvider{
 			Provider: provider.Provider,
 			APIKey:   provider.APIKey,
 			BaseURL:  provider.BaseUrl,
 		})
 	}
-	configuredModels := make([]configuredModel, 0, len(enabledModels))
+	configuredModels := make([]chatprovider.ConfiguredModel, 0, len(enabledModels))
 	for _, model := range enabledModels {
-		configuredModels = append(configuredModels, configuredModel{
+		configuredModels = append(configuredModels, chatprovider.ConfiguredModel{
 			Provider:    model.Provider,
 			Model:       model.Model,
 			DisplayName: model.DisplayName,
@@ -506,7 +495,7 @@ func (p *Processor) ListModels(ctx context.Context) (codersdk.ChatModelsResponse
 	if err != nil {
 		return codersdk.ChatModelsResponse{}, err
 	}
-	catalog := newModelCatalog(keys)
+	catalog := chatprovider.NewModelCatalog(keys)
 	if response, ok := catalog.ListConfiguredModels(configuredProviders, configuredModels); ok {
 		return response, nil
 	}
@@ -516,17 +505,17 @@ func (p *Processor) ListModels(ctx context.Context) (codersdk.ChatModelsResponse
 // EffectiveProviderKeys merges configured provider keys over resolver keys.
 func (p *Processor) EffectiveProviderKeys(
 	ctx context.Context,
-	configuredProviders []ConfiguredProvider,
-) (ProviderAPIKeys, error) {
+	configuredProviders []chatprovider.ConfiguredProvider,
+) (chatprovider.ProviderAPIKeys, error) {
 	if p == nil {
-		return ProviderAPIKeys{}, xerrors.New("chat processor is not configured")
+		return chatprovider.ProviderAPIKeys{}, xerrors.New("chat processor is not configured")
 	}
 	ctx = dbauthz.AsSystemRestricted(ctx)
 	keys, err := p.resolveProviderAPIKeys(ctx)
 	if err != nil {
-		return ProviderAPIKeys{}, err
+		return chatprovider.ProviderAPIKeys{}, err
 	}
-	return MergeProviderAPIKeys(keys, configuredProviders), nil
+	return chatprovider.MergeProviderAPIKeys(keys, configuredProviders), nil
 }
 
 func (p *Processor) PublishStream(chatID uuid.UUID, event codersdk.ChatStreamEvent) {
@@ -927,6 +916,7 @@ func (p *Processor) runChat(
 	result, err := p.runChatWithAgent(
 		ctx,
 		chat,
+		logger,
 		model,
 		modelConfig,
 		prompt,
@@ -936,17 +926,8 @@ func (p *Processor) runChat(
 	if err != nil {
 		return err
 	}
-	if result != nil && len(result.Steps) >= maxChatSteps {
-		lastStep := result.Steps[len(result.Steps)-1]
-		if lastStep.FinishReason == fantasy.FinishReasonToolCalls &&
-			len(lastStep.Content.ToolCalls()) > 0 {
-			return xerrors.Errorf("chat exceeded %d tool steps", maxChatSteps)
-		}
-	}
-	if !reportOnly {
-		if err := p.maybeCompressChatContext(ctx, chat, model, logger); err != nil {
-			logger.Warn(ctx, "failed to compact chat context", slog.Error(err))
-		}
+	if hasExceededChatToolSteps(result) {
+		return xerrors.Errorf("chat exceeded %d tool steps", maxChatSteps)
 	}
 	return nil
 }
@@ -956,86 +937,12 @@ type chatContextCompressionConfig struct {
 	ThresholdPercent int32
 }
 
-type chatContextUsageSnapshot struct {
-	ContextTokens int64
-	ContextLimit  int64
-}
-
-func (p *Processor) maybeCompressChatContext(
-	ctx context.Context,
-	chat database.Chat,
-	model fantasy.LanguageModel,
-	logger slog.Logger,
-) error {
-	config, err := p.resolveChatContextCompressionConfig(ctx, chat)
-	if err != nil {
-		return err
-	}
-	if config.ThresholdPercent >= maxContextCompressionThresholdPercent {
-		return nil
-	}
-
-	messages, err := p.db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
-	if err != nil {
-		return xerrors.Errorf("get prompt messages for compression: %w", err)
-	}
-
-	usage, hasUsage := latestAssistantContextUsage(messages)
-	if !hasUsage || usage.ContextTokens <= 0 {
-		return nil
-	}
-
-	contextLimit := config.ContextLimit
-	if contextLimit <= 0 && usage.ContextLimit > 0 {
-		contextLimit = usage.ContextLimit
-	}
-	if contextLimit <= 0 {
-		return nil
-	}
-
-	usagePercent := (float64(usage.ContextTokens) / float64(contextLimit)) * 100
-	if usagePercent < float64(config.ThresholdPercent) {
-		return nil
-	}
-
-	summary, err := p.generateChatContextSummary(ctx, model, messages)
-	if err != nil {
-		return xerrors.Errorf("generate context summary: %w", err)
-	}
-	if strings.TrimSpace(summary) == "" {
-		return nil
-	}
-
-	if err := p.persistChatContextSummary(
-		ctx,
-		chat.ID,
-		summary,
-		config.ThresholdPercent,
-		usage.ContextTokens,
-		contextLimit,
-		usagePercent,
-	); err != nil {
-		return xerrors.Errorf("persist context summary: %w", err)
-	}
-
-	logger.Info(ctx, "chat context summarized",
-		slog.F("chat_id", chat.ID),
-		slog.F("threshold_percent", config.ThresholdPercent),
-		slog.F("usage_percent", usagePercent),
-		slog.F("context_tokens", usage.ContextTokens),
-		slog.F("context_limit", contextLimit),
-	)
-	return nil
-}
-
 func (p *Processor) resolveChatContextCompressionConfig(
 	ctx context.Context,
 	chat database.Chat,
 ) (chatContextCompressionConfig, error) {
 	config := chatContextCompressionConfig{
-		ThresholdPercent: normalizeContextCompressionThreshold(
-			defaultContextCompressionThresholdPercent,
-		),
+		ThresholdPercent: defaultContextCompressionThresholdPercent,
 	}
 
 	chatConfig, err := parseChatModelConfig(chat.ModelConfig)
@@ -1066,13 +973,14 @@ func (p *Processor) resolveChatContextCompressionConfig(
 		if config.ContextLimit <= 0 {
 			config.ContextLimit = fallbackConfig.ContextLimit
 		}
-		config.ThresholdPercent = normalizeContextCompressionThreshold(
-			fallbackConfig.CompressionThreshold,
-		)
+		config.ThresholdPercent = fallbackConfig.CompressionThreshold
 		return config, nil
 	}
 
-	provider, modelID, err := resolveModelWithProviderHint(modelName, providerHint)
+	provider, modelID, err := chatprovider.ResolveModelWithProviderHint(
+		modelName,
+		providerHint,
+	)
 	if err != nil {
 		return config, nil
 	}
@@ -1094,61 +1002,21 @@ func (p *Processor) resolveChatContextCompressionConfig(
 	if config.ContextLimit <= 0 {
 		config.ContextLimit = modelConfig.ContextLimit
 	}
-	config.ThresholdPercent = normalizeContextCompressionThreshold(
-		modelConfig.CompressionThreshold,
-	)
+	config.ThresholdPercent = modelConfig.CompressionThreshold
 	return config, nil
 }
 
-func (p *Processor) generateChatContextSummary(
-	ctx context.Context,
-	model fantasy.LanguageModel,
-	messages []database.ChatMessage,
-) (string, error) {
-	prompt, err := chatprompt.ConvertMessages(
-		messages,
-		subagentReportToolCallIDPrefix,
-	)
-	if err != nil {
-		return "", xerrors.Errorf("build summary prompt: %w", err)
-	}
-
-	prompt = chatprompt.AppendUser(prompt, contextCompressionSummaryPrompt)
-	toolChoice := fantasy.ToolChoiceNone
-
-	summaryCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
-	defer cancel()
-
-	response, err := model.Generate(summaryCtx, fantasy.Call{
-		Prompt:     prompt,
-		ToolChoice: &toolChoice,
-	})
-	if err != nil {
-		return "", xerrors.Errorf("generate summary text: %w", err)
-	}
-
-	return strings.TrimSpace(contentBlocksToText(response.Content)), nil
-}
-
 func (p *Processor) persistChatContextSummary(
 	ctx context.Context,
 	chatID uuid.UUID,
-	summary string,
-	thresholdPercent int32,
-	contextTokens int64,
-	contextLimit int64,
-	usagePercent float64,
+	result chatloop.CompactionResult,
 ) error {
-	summary = strings.TrimSpace(summary)
-	if summary == "" {
+	if strings.TrimSpace(result.SystemSummary) == "" ||
+		strings.TrimSpace(result.SummaryReport) == "" {
 		return nil
 	}
-	summaryReport, summaryReportTruncated := compactSummaryReport(summary)
 
-	systemSummary := strings.TrimSpace(
-		contextCompressionSystemSummaryPrefix + "\n\n" + summary,
-	)
-	systemContent, err := json.Marshal(systemSummary)
+	systemContent, err := json.Marshal(result.SystemSummary)
 	if err != nil {
 		return xerrors.Errorf("encode system summary: %w", err)
 	}
@@ -1170,7 +1038,7 @@ func (p *Processor) persistChatContextSummary(
 	toolCallID := "chat_summarized_" + uuid.NewString()
 	args, err := json.Marshal(map[string]any{
 		"source":            "automatic",
-		"threshold_percent": thresholdPercent,
+		"threshold_percent": result.ThresholdPercent,
 	})
 	if err != nil {
 		return xerrors.Errorf("encode summary tool args: %w", err)
@@ -1205,13 +1073,12 @@ func (p *Processor) persistChatContextSummary(
 		ToolCallID: toolCallID,
 		ToolName:   "chat_summarized",
 		Result: map[string]any{
-			"summary":              summaryReport,
-			"summary_truncated":    summaryReportTruncated,
+			"summary":              result.SummaryReport,
 			"source":               "automatic",
-			"threshold_percent":    thresholdPercent,
-			"usage_percent":        usagePercent,
-			"context_tokens":       contextTokens,
-			"context_limit_tokens": contextLimit,
+			"threshold_percent":    result.ThresholdPercent,
+			"usage_percent":        result.UsagePercent,
+			"context_tokens":       result.ContextTokens,
+			"context_limit_tokens": result.ContextLimit,
 		},
 	}})
 	if err != nil {
@@ -1241,84 +1108,6 @@ func (p *Processor) persistChatContextSummary(
 	return nil
 }
 
-func compactSummaryReport(summary string) (string, bool) {
-	summary = strings.TrimSpace(summary)
-	if summary == "" {
-		return "", false
-	}
-
-	if utf8.RuneCountInString(summary) <= contextCompressionSummaryReportMaxRunes {
-		return summary, false
-	}
-
-	truncated := strings.TrimSpace(
-		truncateRunes(summary, contextCompressionSummaryReportMaxRunes),
-	)
-	if truncated == "" {
-		return contextCompressionSummaryTruncatedTag, true
-	}
-	return truncated + "\n\n" + contextCompressionSummaryTruncatedTag, true
-}
-
-func normalizeContextCompressionThreshold(value int32) int32 {
-	if value < minContextCompressionThresholdPercent ||
-		value > maxContextCompressionThresholdPercent {
-		return defaultContextCompressionThresholdPercent
-	}
-	return value
-}
-
-func latestAssistantContextUsage(messages []database.ChatMessage) (chatContextUsageSnapshot, bool) {
-	for i := len(messages) - 1; i >= 0; i-- {
-		message := messages[i]
-		if message.Role != string(fantasy.MessageRoleAssistant) {
-			continue
-		}
-
-		contextTokens, hasTokens := messageContextTokens(message)
-		if !hasTokens {
-			continue
-		}
-
-		snapshot := chatContextUsageSnapshot{
-			ContextTokens: contextTokens,
-		}
-		if message.ContextLimit.Valid && message.ContextLimit.Int64 > 0 {
-			snapshot.ContextLimit = message.ContextLimit.Int64
-		}
-		return snapshot, true
-	}
-
-	return chatContextUsageSnapshot{}, false
-}
-
-func messageContextTokens(message database.ChatMessage) (int64, bool) {
-	total := int64(0)
-	hasContextTokens := false
-
-	if message.InputTokens.Valid {
-		total += message.InputTokens.Int64
-		hasContextTokens = true
-	}
-	if message.CacheReadTokens.Valid {
-		total += message.CacheReadTokens.Int64
-		hasContextTokens = true
-	}
-	if message.CacheCreationTokens.Valid {
-		total += message.CacheCreationTokens.Int64
-		hasContextTokens = true
-	}
-	if hasContextTokens {
-		return total, true
-	}
-
-	if message.TotalTokens.Valid {
-		return message.TotalTokens.Int64, true
-	}
-
-	return 0, false
-}
-
 func (p *Processor) resolveChatModel(
 	ctx context.Context,
 	chat database.Chat,
@@ -1356,7 +1145,7 @@ func (p *Processor) resolveChatModel(
 func (p *Processor) modelFromChat(
 	ctx context.Context,
 	config chatModelConfig,
-	providerKeys ProviderAPIKeys,
+	providerKeys chatprovider.ProviderAPIKeys,
 ) (fantasy.LanguageModel, chatModelConfig, error) {
 	if strings.TrimSpace(config.Model) == "" {
 		fallbackConfig, fallbackErr := p.resolveFallbackChatModelConfig(
@@ -1398,7 +1187,7 @@ func applyFallbackChatModelConfig(
 
 func (p *Processor) resolveFallbackChatModelConfig(
 	ctx context.Context,
-	providerKeys ProviderAPIKeys,
+	providerKeys chatprovider.ProviderAPIKeys,
 	providerHint string,
 ) (database.ChatModelConfig, error) {
 	modelConfigs, err := p.db.GetEnabledChatModelConfigs(ctx)
@@ -1409,7 +1198,7 @@ func (p *Processor) resolveFallbackChatModelConfig(
 		)
 	}
 
-	normalizedProviderHint := normalizeProvider(providerHint)
+	normalizedProviderHint := chatprovider.NormalizeProvider(providerHint)
 	if strings.TrimSpace(providerHint) != "" && normalizedProviderHint == "" {
 		return database.ChatModelConfig{}, xerrors.Errorf(
 			"unknown provider %q in chat model config",
@@ -1418,7 +1207,7 @@ func (p *Processor) resolveFallbackChatModelConfig(
 	}
 
 	for _, modelConfig := range modelConfigs {
-		provider := normalizeProvider(modelConfig.Provider)
+		provider := chatprovider.NormalizeProvider(modelConfig.Provider)
 		if provider == "" {
 			continue
 		}
@@ -1446,6 +1235,7 @@ func (p *Processor) resolveFallbackChatModelConfig(
 func (p *Processor) runChatWithAgent(
 	ctx context.Context,
 	chat database.Chat,
+	logger slog.Logger,
 	model fantasy.LanguageModel,
 	modelConfig chatModelConfig,
 	prompt []fantasy.Message,
@@ -1519,10 +1309,15 @@ func (p *Processor) runChatWithAgent(
 	// Resolve the model config's context_limit so we can use it as a
 	// fallback when the LLM provider doesn't include context_limit in
 	// its response metadata (which is the common case).
-	var modelConfigContextLimit int64
-	if compressionConfig, err := p.resolveChatContextCompressionConfig(ctx, chat); err == nil {
-		modelConfigContextLimit = compressionConfig.ContextLimit
+	compressionConfig := chatContextCompressionConfig{
+		ThresholdPercent: defaultContextCompressionThresholdPercent,
 	}
+	if resolvedCompressionConfig, err := p.resolveChatContextCompressionConfig(ctx, chat); err != nil {
+		logger.Warn(ctx, "failed to resolve chat compaction config", slog.Error(err))
+	} else {
+		compressionConfig = resolvedCompressionConfig
+	}
+	modelConfigContextLimit := compressionConfig.ContextLimit
 
 	persistStep := func(persistCtx context.Context, step chatloop.PersistedStep) error {
 		if len(step.AssistantContent) > 0 {
@@ -1591,6 +1386,32 @@ func (p *Processor) runChatWithAgent(
 	if reportOnly {
 		activeTools = append(activeTools, "subagent_report")
 	}
+	var compactionOptions *chatloop.CompactionOptions
+	if !reportOnly {
+		compactionOptions = &chatloop.CompactionOptions{
+			ThresholdPercent: compressionConfig.ThresholdPercent,
+			ContextLimit:     compressionConfig.ContextLimit,
+			Persist: func(
+				persistCtx context.Context,
+				result chatloop.CompactionResult,
+			) error {
+				if err := p.persistChatContextSummary(persistCtx, chat.ID, result); err != nil {
+					return xerrors.Errorf("persist context summary: %w", err)
+				}
+				logger.Info(persistCtx, "chat context summarized",
+					slog.F("chat_id", chat.ID),
+					slog.F("threshold_percent", result.ThresholdPercent),
+					slog.F("usage_percent", result.UsagePercent),
+					slog.F("context_tokens", result.ContextTokens),
+					slog.F("context_limit", result.ContextLimit),
+				)
+				return nil
+			},
+			OnError: func(err error) {
+				logger.Warn(ctx, "failed to compact chat context", slog.Error(err))
+			},
+		}
+	}
 
 	return chatloop.Run(ctx, chatloop.RunOptions{
 		Model:      model,
@@ -1609,6 +1430,7 @@ func (p *Processor) runChatWithAgent(
 		) {
 			p.publishMessagePart(chat.ID, string(role), part)
 		},
+		Compaction: compactionOptions,
 
 		OnInterruptedPersistError: func(err error) {
 			p.logger.Warn(ctx, "failed to persist interrupted chat step", slog.Error(err))
@@ -1616,6 +1438,15 @@ func (p *Processor) runChatWithAgent(
 	})
 }
 
+func hasExceededChatToolSteps(result *fantasy.AgentResult) bool {
+	if result == nil || len(result.Steps) < maxChatSteps {
+		return false
+	}
+	lastStep := result.Steps[len(result.Steps)-1]
+	return lastStep.FinishReason == fantasy.FinishReasonToolCalls &&
+		len(lastStep.Content.ToolCalls()) > 0
+}
+
 func usageNullInt64(value int64, valid bool) sql.NullInt64 {
 	if !valid {
 		return sql.NullInt64{}
@@ -1798,9 +1629,11 @@ func (p *Processor) appendHomeInstructionToPrompt(
 	return chatprompt.InsertSystem(prompt, instruction)
 }
 
-func (p *Processor) resolveProviderAPIKeys(ctx context.Context) (ProviderAPIKeys, error) {
+func (p *Processor) resolveProviderAPIKeys(
+	ctx context.Context,
+) (chatprovider.ProviderAPIKeys, error) {
 	if p.resolveProviderAPIKeysFn == nil {
-		return ProviderAPIKeys{}, nil
+		return chatprovider.ProviderAPIKeys{}, nil
 	}
 	return p.resolveProviderAPIKeysFn(ctx)
 }
@@ -2056,14 +1889,19 @@ func workspaceModeFromChat(chat database.Chat) codersdk.ChatWorkspaceMode {
 	}
 }
 
-func modelFromName(modelName string, providerKeys ProviderAPIKeys) (fantasy.LanguageModel, error) {
+func modelFromName(
+	modelName string,
+	providerKeys chatprovider.ProviderAPIKeys,
+) (fantasy.LanguageModel, error) {
 	return modelFromConfig(chatModelConfig{Model: modelName}, providerKeys)
 }
 
 // anyAvailableModel returns a language model from whichever provider
 // has an API key configured. This is used for lightweight tasks like
 // title generation where we don't need a specific model.
-func anyAvailableModel(keys ProviderAPIKeys) (fantasy.LanguageModel, error) {
+func anyAvailableModel(
+	keys chatprovider.ProviderAPIKeys,
+) (fantasy.LanguageModel, error) {
 	candidates := []chatModelConfig{
 		{Provider: fantasyopenai.Name, Model: "gpt-4o-mini"},
 		{Provider: fantasyanthropic.Name, Model: "claude-haiku-4-5"},
@@ -2102,7 +1940,10 @@ func anyAvailableModel(keys ProviderAPIKeys) (fantasy.LanguageModel, error) {
 	return nil, xerrors.New("no AI provider API keys are configured")
 }
 
-func modelFromConfig(config chatModelConfig, providerKeys ProviderAPIKeys) (fantasy.LanguageModel, error) {
+func modelFromConfig(
+	config chatModelConfig,
+	providerKeys chatprovider.ProviderAPIKeys,
+) (fantasy.LanguageModel, error) {
 	return chatprovider.ModelFromConfig(config.Provider, config.Model, providerKeys)
 }
 

coderd/chatd/chatd_internal_test.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/chatd/chatprompt"
+	"github.com/coder/coder/v2/coderd/chatd/chatprovider"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbgen"
 	"github.com/coder/coder/v2/coderd/database/dbtestutil"
@@ -846,7 +847,7 @@ func TestAnyAvailableModel(t *testing.T) {
 	t.Run("OpenAIOnly", func(t *testing.T) {
 		t.Parallel()
 
-		model, err := anyAvailableModel(ProviderAPIKeys{OpenAI: "openai-key"})
+		model, err := anyAvailableModel(chatprovider.ProviderAPIKeys{OpenAI: "openai-key"})
 		require.NoError(t, err)
 		require.Equal(t, "openai", model.Provider())
 		require.Equal(t, "gpt-4o-mini", model.Model())
@@ -855,7 +856,7 @@ func TestAnyAvailableModel(t *testing.T) {
 	t.Run("None", func(t *testing.T) {
 		t.Parallel()
 
-		model, err := anyAvailableModel(ProviderAPIKeys{})
+		model, err := anyAvailableModel(chatprovider.ProviderAPIKeys{})
 		require.Nil(t, model)
 		require.EqualError(t, err, "no AI provider API keys are configured")
 	})
@@ -878,11 +879,11 @@ func TestGenerateChatTitle(t *testing.T) {
 		}
 
 		p := &Processor{
-			resolveProviderAPIKeysFn: func(context.Context) (ProviderAPIKeys, error) {
-				return ProviderAPIKeys{OpenAI: "openai-key"}, nil
+			resolveProviderAPIKeysFn: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+				return chatprovider.ProviderAPIKeys{OpenAI: "openai-key"}, nil
 			},
 			titleGeneration: TitleGenerationConfig{Prompt: "custom title prompt"},
-			titleModelLookup: func(ProviderAPIKeys) (fantasy.LanguageModel, error) {
+			titleModelLookup: func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error) {
 				return model, nil
 			},
 		}
@@ -906,10 +907,10 @@ func TestGenerateChatTitle(t *testing.T) {
 		}
 
 		p := &Processor{
-			resolveProviderAPIKeysFn: func(context.Context) (ProviderAPIKeys, error) {
-				return ProviderAPIKeys{OpenAI: "openai-key"}, nil
+			resolveProviderAPIKeysFn: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+				return chatprovider.ProviderAPIKeys{OpenAI: "openai-key"}, nil
 			},
-			titleModelLookup: func(ProviderAPIKeys) (fantasy.LanguageModel, error) {
+			titleModelLookup: func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error) {
 				return model, nil
 			},
 		}
@@ -929,10 +930,10 @@ func TestGenerateChatTitle(t *testing.T) {
 		}
 
 		p := &Processor{
-			resolveProviderAPIKeysFn: func(context.Context) (ProviderAPIKeys, error) {
-				return ProviderAPIKeys{OpenAI: "openai-key"}, nil
+			resolveProviderAPIKeysFn: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+				return chatprovider.ProviderAPIKeys{OpenAI: "openai-key"}, nil
 			},
-			titleModelLookup: func(ProviderAPIKeys) (fantasy.LanguageModel, error) {
+			titleModelLookup: func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error) {
 				return model, nil
 			},
 		}
@@ -958,10 +959,10 @@ func TestMaybeGenerateChatTitle(t *testing.T) {
 
 		p := &Processor{
 			db: db,
-			resolveProviderAPIKeysFn: func(context.Context) (ProviderAPIKeys, error) {
-				return ProviderAPIKeys{OpenAI: "openai-key"}, nil
+			resolveProviderAPIKeysFn: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+				return chatprovider.ProviderAPIKeys{OpenAI: "openai-key"}, nil
 			},
-			titleModelLookup: func(ProviderAPIKeys) (fantasy.LanguageModel, error) {
+			titleModelLookup: func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error) {
 				return &titleTestModel{
 					generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
 						return &fantasy.Response{
@@ -990,10 +991,10 @@ func TestMaybeGenerateChatTitle(t *testing.T) {
 
 		p := &Processor{
 			db: db,
-			resolveProviderAPIKeysFn: func(context.Context) (ProviderAPIKeys, error) {
-				return ProviderAPIKeys{OpenAI: "openai-key"}, nil
+			resolveProviderAPIKeysFn: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+				return chatprovider.ProviderAPIKeys{OpenAI: "openai-key"}, nil
 			},
-			titleModelLookup: func(ProviderAPIKeys) (fantasy.LanguageModel, error) {
+			titleModelLookup: func(chatprovider.ProviderAPIKeys) (fantasy.LanguageModel, error) {
 				return &titleTestModel{
 					generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
 						return nil, xerrors.New("title model failed")

coderd/chatd/chatd_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/coderd/chatd"
 	"github.com/coder/coder/v2/coderd/chatd/chatprompt"
+	"github.com/coder/coder/v2/coderd/chatd/chatprovider"
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/db2sdk"
@@ -1063,8 +1064,8 @@ func TestRunChatLoop(t *testing.T) {
 		Logger:    logger,
 		Database:  db,
 		AgentConn: testAgentConnFunc(workspacesdk.New(client), logger),
-		ResolveProviderAPIKeys: func(context.Context) (chatd.ProviderAPIKeys, error) {
-			return chatd.ProviderAPIKeys{
+		ResolveProviderAPIKeys: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+			return chatprovider.ProviderAPIKeys{
 				ByProvider: map[string]string{
 					"openai-compat": "test-api-key",
 				},
@@ -2956,8 +2957,8 @@ func TestProcessorListModels_UsesFallbackAPIKeysWhenProviderKeyBlank(t *testing.
 	processor := chatd.New(chatd.Config{
 		Logger:   testutil.Logger(t),
 		Database: db,
-		ResolveProviderAPIKeys: func(context.Context) (chatd.ProviderAPIKeys, error) {
-			return chatd.ProviderAPIKeys{OpenAI: "deployment-openai"}, nil
+		ResolveProviderAPIKeys: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+			return chatprovider.ProviderAPIKeys{OpenAI: "deployment-openai"}, nil
 		},
 		PendingChatAcquireInterval: time.Hour,
 	})
@@ -2991,8 +2992,8 @@ func TestProcessorListModels_NoEnabledModelsReturnsProviderAvailability(t *testi
 	processor := chatd.New(chatd.Config{
 		Logger:   testutil.Logger(t),
 		Database: db,
-		ResolveProviderAPIKeys: func(context.Context) (chatd.ProviderAPIKeys, error) {
-			return chatd.ProviderAPIKeys{OpenAI: "deployment-openai"}, nil
+		ResolveProviderAPIKeys: func(context.Context) (chatprovider.ProviderAPIKeys, error) {
+			return chatprovider.ProviderAPIKeys{OpenAI: "deployment-openai"}, nil
 		},
 		PendingChatAcquireInterval: time.Hour,
 	})
@@ -3000,7 +3001,7 @@ func TestProcessorListModels_NoEnabledModelsReturnsProviderAvailability(t *testi
 
 	response, err := processor.ListModels(ctx)
 	require.NoError(t, err)
-	require.Len(t, response.Providers, len(chatd.SupportedProviders()))
+	require.Len(t, response.Providers, len(chatprovider.SupportedProviders()))
 
 	availability := make(map[string]codersdk.ChatModelProvider, len(response.Providers))
 	for _, provider := range response.Providers {
@@ -3387,12 +3388,9 @@ func TestRunChatLoop_ContextCompressionUsesFallbackConfigAndTruncatesSummary(t *
 
 	report, ok := summaryPayload["summary"].(string)
 	require.True(t, ok)
-	require.NotContains(t, report, summaryTail)
-	require.Contains(t, report, "[summary truncated for live stream]")
-
-	summaryTruncated, ok := summaryPayload["summary_truncated"].(bool)
-	require.True(t, ok)
-	require.True(t, summaryTruncated)
+	require.Contains(t, report, summaryTail)
+	_, hasSummaryTruncated := summaryPayload["summary_truncated"]
+	require.False(t, hasSummaryTruncated)
 
 	thresholdPercent, ok := summaryPayload["threshold_percent"].(float64)
 	require.True(t, ok)

coderd/chatd/chatloop/chatloop.go

@@ -47,6 +47,7 @@ type RunOptions struct {
 		role fantasy.MessageRole,
 		part codersdk.ChatMessagePart,
 	)
+	Compaction *CompactionOptions
 
 	OnInterruptedPersistError func(error)
 }
@@ -302,6 +303,9 @@ func Run(ctx context.Context, opts RunOptions) (*fantasy.AgentResult, error) {
 		fantasy.WithStopConditions(fantasy.StepCountIs(opts.MaxSteps)),
 	)
 	applyAnthropicCaching := shouldApplyAnthropicPromptCaching(opts.Model)
+	// Fantasy's AgentStreamCall currently requires a non-empty Prompt and always
+	// appends it as a user message. chatd already supplies the full history in
+	// Messages, so we pass and then strip a sentinel user message in PrepareStep.
 	sentinelPrompt := "__chatd_agent_prompt_sentinel_" + uuid.NewString()
 
 	streamCall := opts.StreamCall
@@ -457,6 +461,13 @@ func Run(ctx context.Context, opts RunOptions) (*fantasy.AgentResult, error) {
 		}
 		return nil, xerrors.Errorf("stream response: %w", err)
 	}
+	if opts.Compaction != nil {
+		if err := maybeCompact(ctx, opts, result); err != nil {
+			if opts.Compaction.OnError != nil {
+				opts.Compaction.OnError(err)
+			}
+		}
+	}
 
 	return result, nil
 }

coderd/chatd/chatloop/chatloop_test.go

@@ -194,9 +194,10 @@ func TestRun_InterruptedStepPersistsSyntheticToolResult(t *testing.T) {
 }
 
 type loopTestModel struct {
-	provider string
-	model    string
-	streamFn func(context.Context, fantasy.Call) (fantasy.StreamResponse, error)
+	provider   string
+	model      string
+	generateFn func(context.Context, fantasy.Call) (*fantasy.Response, error)
+	streamFn   func(context.Context, fantasy.Call) (fantasy.StreamResponse, error)
 }
 
 func (m *loopTestModel) Provider() string {
@@ -213,7 +214,10 @@ func (m *loopTestModel) Model() string {
 	return "fake"
 }
 
-func (*loopTestModel) Generate(context.Context, fantasy.Call) (*fantasy.Response, error) {
+func (m *loopTestModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) {
+	if m.generateFn != nil {
+		return m.generateFn(ctx, call)
+	}
 	return &fantasy.Response{}, nil
 }
 

coderd/chatd/chatloop/compaction.go

@@ -0,0 +1,209 @@
+package chatloop
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"charm.land/fantasy"
+	"golang.org/x/xerrors"
+)
+
+const (
+	defaultCompactionThresholdPercent = int32(70)
+	minCompactionThresholdPercent     = int32(0)
+	maxCompactionThresholdPercent     = int32(100)
+
+	defaultCompactionSummaryPrompt = "Summarize the current chat so a " +
+		"new assistant can continue seamlessly. Include the user's goals, " +
+		"decisions made, concrete technical details (files, commands, APIs), " +
+		"errors encountered and fixes, and open questions. Be dense and factual. " +
+		"Omit pleasantries and next-step suggestions."
+	defaultCompactionSystemSummaryPrefix = "Summary of earlier chat context:"
+	defaultCompactionTimeout             = 90 * time.Second
+)
+
+type CompactionOptions struct {
+	ThresholdPercent    int32
+	ContextLimit        int64
+	SummaryPrompt       string
+	SystemSummaryPrefix string
+	Timeout             time.Duration
+	Persist             func(context.Context, CompactionResult) error
+	OnError             func(error)
+}
+
+type CompactionResult struct {
+	SystemSummary    string
+	SummaryReport    string
+	ThresholdPercent int32
+	UsagePercent     float64
+	ContextTokens    int64
+	ContextLimit     int64
+}
+
+func maybeCompact(
+	ctx context.Context,
+	runOpts RunOptions,
+	runResult *fantasy.AgentResult,
+) error {
+	if runResult == nil || runOpts.Compaction == nil {
+		return nil
+	}
+
+	config := *runOpts.Compaction
+	if config.Persist == nil {
+		return xerrors.New("compaction persist callback is required")
+	}
+	if strings.TrimSpace(config.SummaryPrompt) == "" {
+		config.SummaryPrompt = defaultCompactionSummaryPrompt
+	}
+	if strings.TrimSpace(config.SystemSummaryPrefix) == "" {
+		config.SystemSummaryPrefix = defaultCompactionSystemSummaryPrefix
+	}
+	if config.Timeout <= 0 {
+		config.Timeout = defaultCompactionTimeout
+	}
+	if config.ThresholdPercent < minCompactionThresholdPercent ||
+		config.ThresholdPercent > maxCompactionThresholdPercent {
+		config.ThresholdPercent = defaultCompactionThresholdPercent
+	}
+
+	if config.ThresholdPercent >= maxCompactionThresholdPercent {
+		return nil
+	}
+	if runOpts.MaxSteps > 0 && len(runResult.Steps) >= runOpts.MaxSteps {
+		lastStep := runResult.Steps[len(runResult.Steps)-1]
+		if lastStep.FinishReason == fantasy.FinishReasonToolCalls &&
+			len(lastStep.Content.ToolCalls()) > 0 {
+			return nil
+		}
+	}
+
+	contextTokens := int64(0)
+	contextLimitFromMetadata := int64(0)
+	for i := len(runResult.Steps) - 1; i >= 0; i-- {
+		usage := runResult.Steps[i].Usage
+		total := int64(0)
+		hasContextTokens := false
+
+		if usage.InputTokens > 0 {
+			total += usage.InputTokens
+			hasContextTokens = true
+		}
+		if usage.CacheReadTokens > 0 {
+			total += usage.CacheReadTokens
+			hasContextTokens = true
+		}
+		if usage.CacheCreationTokens > 0 {
+			total += usage.CacheCreationTokens
+			hasContextTokens = true
+		}
+		if !hasContextTokens && usage.TotalTokens > 0 {
+			total = usage.TotalTokens
+			hasContextTokens = true
+		}
+		if !hasContextTokens || total <= 0 {
+			continue
+		}
+
+		contextTokens = total
+		metadataLimit := extractContextLimit(runResult.Steps[i].ProviderMetadata)
+		if metadataLimit.Valid && metadataLimit.Int64 > 0 {
+			contextLimitFromMetadata = metadataLimit.Int64
+		}
+		break
+	}
+	if contextTokens <= 0 {
+		return nil
+	}
+
+	contextLimit := contextLimitFromMetadata
+	if contextLimit <= 0 && config.ContextLimit > 0 {
+		contextLimit = config.ContextLimit
+	}
+	if contextLimit <= 0 && runOpts.ContextLimitFallback > 0 {
+		contextLimit = runOpts.ContextLimitFallback
+	}
+	if contextLimit <= 0 {
+		return nil
+	}
+
+	usagePercent := (float64(contextTokens) / float64(contextLimit)) * 100
+	if usagePercent < float64(config.ThresholdPercent) {
+		return nil
+	}
+
+	summary, err := generateCompactionSummary(
+		ctx,
+		runOpts.Model,
+		runOpts.Messages,
+		runResult.Steps,
+		config,
+	)
+	if err != nil {
+		return err
+	}
+	if summary == "" {
+		return nil
+	}
+
+	systemSummary := strings.TrimSpace(
+		config.SystemSummaryPrefix + "\n\n" + summary,
+	)
+
+	return config.Persist(ctx, CompactionResult{
+		SystemSummary:    systemSummary,
+		SummaryReport:    summary,
+		ThresholdPercent: config.ThresholdPercent,
+		UsagePercent:     usagePercent,
+		ContextTokens:    contextTokens,
+		ContextLimit:     contextLimit,
+	})
+}
+
+func generateCompactionSummary(
+	ctx context.Context,
+	model fantasy.LanguageModel,
+	messages []fantasy.Message,
+	steps []fantasy.StepResult,
+	options CompactionOptions,
+) (string, error) {
+	summaryPrompt := make([]fantasy.Message, 0, len(messages)+len(steps)+1)
+	summaryPrompt = append(summaryPrompt, messages...)
+	for _, step := range steps {
+		summaryPrompt = append(summaryPrompt, step.Messages...)
+	}
+	summaryPrompt = append(summaryPrompt, fantasy.Message{
+		Role: fantasy.MessageRoleUser,
+		Content: []fantasy.MessagePart{
+			fantasy.TextPart{Text: options.SummaryPrompt},
+		},
+	})
+	toolChoice := fantasy.ToolChoiceNone
+
+	summaryCtx, cancel := context.WithTimeout(ctx, options.Timeout)
+	defer cancel()
+
+	response, err := model.Generate(summaryCtx, fantasy.Call{
+		Prompt:     summaryPrompt,
+		ToolChoice: &toolChoice,
+	})
+	if err != nil {
+		return "", xerrors.Errorf("generate summary text: %w", err)
+	}
+
+	parts := make([]string, 0, len(response.Content))
+	for _, block := range response.Content {
+		textBlock, ok := fantasy.AsContentType[fantasy.TextContent](block)
+		if !ok {
+			continue
+		}
+		text := strings.TrimSpace(textBlock.Text)
+		if text == "" {
+			continue
+		}
+		parts = append(parts, text)
+	}
+	return strings.TrimSpace(strings.Join(parts, " ")), nil
+}

coderd/chatd/chatloop/compaction_test.go

@@ -0,0 +1,132 @@
+package chatloop
+
+import (
+	"context"
+	"testing"
+
+	"charm.land/fantasy"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+)
+
+func TestRun_Compaction(t *testing.T) {
+	t.Parallel()
+
+	t.Run("PersistsWhenThresholdReached", func(t *testing.T) {
+		t.Parallel()
+
+		persistCompactionCalls := 0
+		var persistedCompaction CompactionResult
+		const summaryText = "summary text for compaction"
+
+		model := &loopTestModel{
+			provider: "fake",
+			streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
+				return streamFromParts([]fantasy.StreamPart{
+					{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
+					{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"},
+					{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
+					{
+						Type:         fantasy.StreamPartTypeFinish,
+						FinishReason: fantasy.FinishReasonStop,
+						Usage: fantasy.Usage{
+							InputTokens: 80,
+							TotalTokens: 85,
+						},
+					},
+				}), nil
+			},
+			generateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) {
+				require.NotEmpty(t, call.Prompt)
+				lastPrompt := call.Prompt[len(call.Prompt)-1]
+				require.Equal(t, fantasy.MessageRoleUser, lastPrompt.Role)
+				require.Len(t, lastPrompt.Content, 1)
+
+				instruction, ok := fantasy.AsMessagePart[fantasy.TextPart](lastPrompt.Content[0])
+				require.True(t, ok)
+				require.Equal(t, "summarize now", instruction.Text)
+
+				return &fantasy.Response{
+					Content: []fantasy.Content{
+						fantasy.TextContent{Text: summaryText},
+					},
+				}, nil
+			},
+		}
+
+		_, err := Run(context.Background(), RunOptions{
+			Model: model,
+			Messages: []fantasy.Message{
+				textMessage(fantasy.MessageRoleUser, "hello"),
+			},
+			MaxSteps: 1,
+			PersistStep: func(_ context.Context, _ PersistedStep) error {
+				return nil
+			},
+			ContextLimitFallback: 100,
+			Compaction: &CompactionOptions{
+				ThresholdPercent: 70,
+				SummaryPrompt:    "summarize now",
+				Persist: func(_ context.Context, result CompactionResult) error {
+					persistCompactionCalls++
+					persistedCompaction = result
+					return nil
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.Equal(t, 1, persistCompactionCalls)
+		require.Contains(t, persistedCompaction.SystemSummary, summaryText)
+		require.Equal(t, summaryText, persistedCompaction.SummaryReport)
+		require.Equal(t, int64(80), persistedCompaction.ContextTokens)
+		require.Equal(t, int64(100), persistedCompaction.ContextLimit)
+		require.InDelta(t, 80.0, persistedCompaction.UsagePercent, 0.0001)
+	})
+
+	t.Run("ErrorsAreReported", func(t *testing.T) {
+		t.Parallel()
+
+		model := &loopTestModel{
+			provider: "fake",
+			streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
+				return streamFromParts([]fantasy.StreamPart{
+					{
+						Type:         fantasy.StreamPartTypeFinish,
+						FinishReason: fantasy.FinishReasonStop,
+						Usage: fantasy.Usage{
+							InputTokens: 80,
+						},
+					},
+				}), nil
+			},
+			generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
+				return nil, xerrors.New("generate failed")
+			},
+		}
+
+		compactionErr := xerrors.New("unset")
+		_, err := Run(context.Background(), RunOptions{
+			Model: model,
+			Messages: []fantasy.Message{
+				textMessage(fantasy.MessageRoleUser, "hello"),
+			},
+			MaxSteps: 1,
+			PersistStep: func(_ context.Context, _ PersistedStep) error {
+				return nil
+			},
+			ContextLimitFallback: 100,
+			Compaction: &CompactionOptions{
+				ThresholdPercent: 70,
+				Persist: func(_ context.Context, _ CompactionResult) error {
+					return nil
+				},
+				OnError: func(err error) {
+					compactionErr = err
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.Error(t, compactionErr)
+		require.ErrorContains(t, compactionErr, "generate summary text")
+	})
+}

coderd/chatd/models.go

@@ -1,49 +0,0 @@
-package chatd
-
-import "github.com/coder/coder/v2/coderd/chatd/chatprovider"
-
-// ProviderAPIKeys contains API keys for provider calls.
-type ProviderAPIKeys = chatprovider.ProviderAPIKeys
-
-// ConfiguredProvider is an enabled provider loaded from database config.
-type ConfiguredProvider = chatprovider.ConfiguredProvider
-
-type configuredModel = chatprovider.ConfiguredModel
-type modelCatalog = chatprovider.ModelCatalog
-
-// SupportedProviders returns all chat providers supported by Fantasy.
-func SupportedProviders() []string {
-	return chatprovider.SupportedProviders()
-}
-
-// IsEnvPresetProvider reports whether provider supports env presets.
-func IsEnvPresetProvider(provider string) bool {
-	return chatprovider.IsEnvPresetProvider(provider)
-}
-
-// ProviderDisplayName returns a default display name for a provider.
-func ProviderDisplayName(provider string) string {
-	return chatprovider.ProviderDisplayName(provider)
-}
-
-// NormalizeProvider canonicalizes a provider name.
-func NormalizeProvider(provider string) string {
-	return chatprovider.NormalizeProvider(provider)
-}
-
-// MergeProviderAPIKeys overlays configured provider keys over fallback keys.
-func MergeProviderAPIKeys(fallback ProviderAPIKeys, providers []ConfiguredProvider) ProviderAPIKeys {
-	return chatprovider.MergeProviderAPIKeys(fallback, providers)
-}
-
-func newModelCatalog(keys ProviderAPIKeys) *modelCatalog {
-	return chatprovider.NewModelCatalog(keys)
-}
-
-func normalizeProvider(provider string) string {
-	return chatprovider.NormalizeProvider(provider)
-}
-
-func resolveModelWithProviderHint(modelName, providerHint string) (string, string, error) {
-	return chatprovider.ResolveModelWithProviderHint(modelName, providerHint)
-}

coderd/chatd/models_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/v2/coderd/chatd"
+	"github.com/coder/coder/v2/coderd/chatd/chatprovider"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -17,8 +18,8 @@ import (
 func TestMergeProviderAPIKeys(t *testing.T) {
 	t.Parallel()
 
-	merged := chatd.MergeProviderAPIKeys(
-		chatd.ProviderAPIKeys{
+	merged := chatprovider.MergeProviderAPIKeys(
+		chatprovider.ProviderAPIKeys{
 			OpenAI:    " deployment-openai ",
 			Anthropic: "deployment-anthropic",
 			ByProvider: map[string]string{
@@ -28,7 +29,7 @@ func TestMergeProviderAPIKeys(t *testing.T) {
 				"openai": " https://openai.example.com/v1 ",
 			},
 		},
-		[]chatd.ConfiguredProvider{
+		[]chatprovider.ConfiguredProvider{
 			{Provider: "openai", APIKey: "   ", BaseURL: "https://db-openai.example.com/v1"},
 			{Provider: "anthropic", APIKey: " provider-anthropic "},
 			{Provider: "openrouter", APIKey: "provider-openrouter"},
@@ -53,11 +54,11 @@ func TestSupportedProvidersNormalize(t *testing.T) {
 		"openai-compat",
 		"openrouter",
 		"vercel",
-	}, chatd.SupportedProviders())
+	}, chatprovider.SupportedProviders())
 
-	for _, provider := range chatd.SupportedProviders() {
-		require.Equal(t, provider, chatd.NormalizeProvider(provider))
-		require.Equal(t, provider, chatd.NormalizeProvider(strings.ToUpper(provider)))
+	for _, provider := range chatprovider.SupportedProviders() {
+		require.Equal(t, provider, chatprovider.NormalizeProvider(provider))
+		require.Equal(t, provider, chatprovider.NormalizeProvider(strings.ToUpper(provider)))
 	}
 }
 

coderd/chats.go

@@ -23,6 +23,7 @@ import (
 	"cdr.dev/slog/v3"
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/chatd"
+	"github.com/coder/coder/v2/coderd/chatd/chatprovider"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/db2sdk"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
@@ -3211,7 +3212,7 @@ func (api *API) listChatProviders(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	providersByName := make(map[string]database.ChatProvider, len(providers))
-	configuredProviders := make([]chatd.ConfiguredProvider, 0, len(providers))
+	configuredProviders := make([]chatprovider.ConfiguredProvider, 0, len(providers))
 	for _, provider := range providers {
 		normalizedProvider := normalizeChatProvider(provider.Provider)
 		if normalizedProvider == "" {
@@ -3219,7 +3220,7 @@ func (api *API) listChatProviders(rw http.ResponseWriter, r *http.Request) {
 		}
 		provider.Provider = normalizedProvider
 		providersByName[normalizedProvider] = provider
-		configuredProviders = append(configuredProviders, chatd.ConfiguredProvider{
+		configuredProviders = append(configuredProviders, chatprovider.ConfiguredProvider{
 			Provider: normalizedProvider,
 			APIKey:   provider.APIKey,
 			BaseURL:  provider.BaseUrl,
@@ -3241,7 +3242,7 @@ func (api *API) listChatProviders(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	supportedProviders := chatd.SupportedProviders()
+	supportedProviders := chatprovider.SupportedProviders()
 	resp := make([]codersdk.ChatProviderConfig, 0, len(supportedProviders))
 	for _, provider := range supportedProviders {
 		configured, ok := providersByName[provider]
@@ -3260,7 +3261,7 @@ func (api *API) listChatProviders(rw http.ResponseWriter, r *http.Request) {
 		source := codersdk.ChatProviderConfigSourceSupported
 		hasAPIKey := effectiveKeys.APIKey(provider) != ""
 		enabled := false
-		if chatd.IsEnvPresetProvider(provider) && hasAPIKey {
+		if chatprovider.IsEnvPresetProvider(provider) && hasAPIKey {
 			source = codersdk.ChatProviderConfigSourceEnvPreset
 			enabled = true
 		}
@@ -3268,7 +3269,7 @@ func (api *API) listChatProviders(rw http.ResponseWriter, r *http.Request) {
 		resp = append(resp, codersdk.ChatProviderConfig{
 			ID:          uuid.Nil,
 			Provider:    provider,
-			DisplayName: chatd.ProviderDisplayName(provider),
+			DisplayName: chatprovider.ProviderDisplayName(provider),
 			Enabled:     enabled,
 			HasAPIKey:   hasAPIKey,
 			BaseURL:     effectiveKeys.BaseURL(provider),
@@ -3795,7 +3796,7 @@ func convertChatProviderConfig(
 ) codersdk.ChatProviderConfig {
 	displayName := strings.TrimSpace(provider.DisplayName)
 	if displayName == "" {
-		displayName = chatd.ProviderDisplayName(provider.Provider)
+		displayName = chatprovider.ProviderDisplayName(provider.Provider)
 	}
 
 	return codersdk.ChatProviderConfig{
@@ -3885,7 +3886,7 @@ func isZeroChatModelProviderOptions(options *codersdk.ChatModelProviderOptions)
 }
 
 func normalizeChatProvider(provider string) string {
-	return chatd.NormalizeProvider(provider)
+	return chatprovider.NormalizeProvider(provider)
 }
 
 func normalizeChatProviderBaseURL(raw string) (string, error) {
@@ -3908,7 +3909,7 @@ func normalizeChatProviderBaseURL(raw string) (string, error) {
 }
 
 func chatProviderValidationDetail() string {
-	return "Provider must be one of: " + strings.Join(chatd.SupportedProviders(), ", ") + "."
+	return "Provider must be one of: " + strings.Join(chatprovider.SupportedProviders(), ", ") + "."
 }
 
 func (api *API) hasEffectiveProviderAPIKey(ctx context.Context, provider database.ChatProvider) bool {

coderd/coderd.go

@@ -50,6 +50,7 @@ import (
 	"github.com/coder/coder/v2/coderd/awsidentity"
 	"github.com/coder/coder/v2/coderd/boundaryusage"
 	"github.com/coder/coder/v2/coderd/chatd"
+	"github.com/coder/coder/v2/coderd/chatd/chatprovider"
 	"github.com/coder/coder/v2/coderd/connectionlog"
 	"github.com/coder/coder/v2/coderd/cryptokeys"
 	"github.com/coder/coder/v2/coderd/database"
@@ -592,7 +593,7 @@ func New(options *Options) *API {
 	var noopUsageChecker wsbuilder.UsageChecker = wsbuilder.NoopUsageChecker{}
 	buildUsageChecker.Store(&noopUsageChecker)
 
-	chatProviderAPIKeys := chatd.ProviderAPIKeys{
+	chatProviderAPIKeys := chatprovider.ProviderAPIKeys{
 		OpenAI:    options.DeploymentValues.AI.BridgeConfig.OpenAI.Key.Value(),
 		Anthropic: options.DeploymentValues.AI.BridgeConfig.Anthropic.Key.Value(),
 		BaseURLByProvider: map[string]string{
@@ -601,22 +602,31 @@ func New(options *Options) *API {
 		},
 	}
 
-	chatProviderAPIKeysResolver := func(ctx context.Context) (chatd.ProviderAPIKeys, error) {
+	chatProviderAPIKeysResolver := func(
+		ctx context.Context,
+	) (chatprovider.ProviderAPIKeys, error) {
 		providers, err := options.Database.GetEnabledChatProviders(ctx)
 		if err != nil {
-			return chatd.ProviderAPIKeys{}, err
+			return chatprovider.ProviderAPIKeys{}, err
 		}
 
-		configuredProviders := make([]chatd.ConfiguredProvider, 0, len(providers))
+		configuredProviders := make(
+			[]chatprovider.ConfiguredProvider,
+			0,
+			len(providers),
+		)
 		for _, provider := range providers {
-			configuredProviders = append(configuredProviders, chatd.ConfiguredProvider{
+			configuredProviders = append(configuredProviders, chatprovider.ConfiguredProvider{
 				Provider: provider.Provider,
 				APIKey:   provider.APIKey,
 				BaseURL:  provider.BaseUrl,
 			})
 		}
 
-		return chatd.MergeProviderAPIKeys(chatProviderAPIKeys, configuredProviders), nil
+		return chatprovider.MergeProviderAPIKeys(
+			chatProviderAPIKeys,
+			configuredProviders,
+		), nil
 	}
 
 	api := &API{