From 99eb46dac131b2c128a3b9aa2aa8789ed3bf3788 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 28 Apr 2026 12:45:23 +0200 Subject: [PATCH] fix(coderd/x/chatd): repair Anthropic provider tool history (#24744) ## Problem Anthropic returns HTTP 400 when an assistant message contains a `web_search_tool_result` block whose `tool_use_id` has no matching earlier `server_tool_use` block in the same assistant message. A previous fix (#24706) sanitized provider-executed tool calls without matching results, but the opposite direction, orphaned or misordered provider-executed results, could still slip through both the prompt sanitizer and the persistence path. ## Fix Tighten Anthropic provider-executed tool history handling while preserving the useful result payload as normal assistant text when the provider-tool metadata is unsafe. 1. Extract Anthropic provider-tool sanitization into `coderd/x/chatd/chatsanitize` so provider-specific repair logic is no longer spread through `chatprompt` and `chatloop`. 2. `chatsanitize.SanitizeAnthropicProviderToolHistory` removes invalid provider-executed tool structure for Anthropic prompts: orphans in either direction, result-before-call, duplicate IDs, invalid JSON inputs, empty IDs and tool names, unsupported tool names, mismatched `ProviderExecuted` flags, provider-executed blocks outside assistant messages, and web-search results without serializable Anthropic result metadata. Provider-executed result payloads are textified instead of being discarded when there is text to preserve. 3. `chatsanitize.SanitizeAnthropicProviderToolContent` mirrors the same rule at the streamed step content level. Persisted history no longer carries invalid provider-tool blocks forward, but it keeps the result text for future turns. 4. `chatsanitize.ApplyAnthropicProviderToolGuard` only repairs structurally invalid Anthropic provider-tool history. It no longer strips otherwise-valid historical `web_search` blocks just because web search is disabled for the current request. The fail-closed fallback also textifies provider results before removing provider-tool metadata. Tests cover prompt sanitization, validation reason strings, result payload textification, content-level persistence sanitization, disabled web-search history preservation, direct pre-request guard behavior, and the fallback strip path. > Mux is acting on Mike's behalf. --- coderd/x/chatd/chatd.go | 9 +- coderd/x/chatd/chatloop/chatloop.go | 81 +- coderd/x/chatd/chatloop/chatloop_test.go | 703 ++++++++ coderd/x/chatd/chatprompt/chatprompt.go | 209 +-- coderd/x/chatd/chatprompt/chatprompt_test.go | 288 +--- coderd/x/chatd/chatsanitize/anthropic.go | 1135 +++++++++++++ .../chatsanitize/anthropic_internal_test.go | 146 ++ coderd/x/chatd/chatsanitize/anthropic_test.go | 1412 +++++++++++++++++ 8 files changed, 3467 insertions(+), 516 deletions(-) create mode 100644 coderd/x/chatd/chatsanitize/anthropic.go create mode 100644 coderd/x/chatd/chatsanitize/anthropic_internal_test.go create mode 100644 coderd/x/chatd/chatsanitize/anthropic_test.go diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 215be30515..540cc92653 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -43,6 +43,7 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/coderd/x/chatd/chatretry" + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" "github.com/coder/coder/v2/coderd/x/chatd/chattool" "github.com/coder/coder/v2/coderd/x/chatd/internal/agentselect" "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" @@ -6243,8 +6244,8 @@ func (p *Server) runChat( if err := g2.Wait(); err != nil { return result, err } - prompt, sanitizeStats := chatprompt.SanitizeAnthropicProviderToolCalls(model.Provider(), prompt) - chatprompt.LogAnthropicProviderToolSanitization( + prompt, sanitizeStats := chatsanitize.SanitizeAnthropicProviderToolHistory(model.Provider(), prompt) + chatsanitize.LogAnthropicProviderToolSanitization( ctx, logger, "persisted_history_replay", model.Provider(), model.Model(), sanitizeStats, ) subagentInstruction := "" @@ -6871,8 +6872,8 @@ func (p *Server) runChat( if err != nil { return nil, xerrors.Errorf("convert reloaded messages: %w", err) } - reloadedPrompt, sanitizeStats := chatprompt.SanitizeAnthropicProviderToolCalls(model.Provider(), reloadedPrompt) - chatprompt.LogAnthropicProviderToolSanitization( + reloadedPrompt, sanitizeStats := chatsanitize.SanitizeAnthropicProviderToolHistory(model.Provider(), reloadedPrompt) + chatsanitize.LogAnthropicProviderToolSanitization( reloadCtx, logger, "reload_messages", model.Provider(), model.Model(), sanitizeStats, ) // Re-derive instruction and skills from the reloaded diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 915e79dd0d..4e367e818b 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -26,6 +26,7 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chaterror" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatretry" + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" "github.com/coder/coder/v2/coderd/x/chatd/chattool" "github.com/coder/coder/v2/codersdk" "github.com/coder/quartz" @@ -391,12 +392,15 @@ func Run(ctx context.Context, opts RunOptions) error { } prepared := make([]fantasy.Message, len(messages)) copy(prepared, messages) - prepared, sanitizeStats := chatprompt.SanitizeAnthropicProviderToolCalls(provider, prepared) - chatprompt.LogAnthropicProviderToolSanitization( + prepared, sanitizeStats := chatsanitize.SanitizeAnthropicProviderToolHistory(provider, prepared) + chatsanitize.LogAnthropicProviderToolSanitization( ctx, opts.Logger, "pre_request", provider, modelName, sanitizeStats, slog.F("step_index", step), slog.F("total_steps", totalSteps), ) + prepared = chatsanitize.ApplyAnthropicProviderToolGuard( + ctx, opts.Logger, provider, modelName, prepared, + ) if applyAnthropicCaching { addAnthropicPromptCaching(prepared) } @@ -529,7 +533,7 @@ func Run(ctx context.Context, opts RunOptions) error { opts.ContextLimitFallback, ) - result.content = sanitizeAnthropicProviderToolStepContent( + result.content = chatsanitize.SanitizeAnthropicProviderToolStepContent( ctx, opts.Logger, provider, modelName, "dynamic_tool_persist", step, result.finishReason, result.content, ) @@ -576,7 +580,7 @@ func Run(ctx context.Context, opts RunOptions) error { result.providerMetadata, opts.ContextLimitFallback, ) - result.content = sanitizeAnthropicProviderToolStepContent( + result.content = chatsanitize.SanitizeAnthropicProviderToolStepContent( ctx, opts.Logger, provider, modelName, "normal_persist", step, result.finishReason, result.content, ) @@ -734,67 +738,6 @@ func Run(ctx context.Context, opts RunOptions) error { return nil } -func sanitizeAnthropicProviderToolStepContent( - ctx context.Context, - logger slog.Logger, - provider string, - modelName string, - phase string, - step int, - finishReason fantasy.FinishReason, - content []fantasy.Content, -) []fantasy.Content { - sanitized, stats := sanitizeAnthropicProviderToolContent(provider, content) - chatprompt.LogAnthropicProviderToolSanitization( - ctx, logger, phase, provider, modelName, stats, - slog.F("step_index", step), - slog.F("finish_reason", finishReason), - ) - return sanitized -} - -func sanitizeAnthropicProviderToolContent( - provider string, - content []fantasy.Content, -) ([]fantasy.Content, chatprompt.AnthropicProviderToolSanitizationStats) { - var stats chatprompt.AnthropicProviderToolSanitizationStats - if provider != fantasyanthropic.Name || len(content) == 0 { - return content, stats - } - - matchedResultIDs := make(map[string]struct{}) - for _, block := range content { - result, ok := fantasy.AsContentType[fantasy.ToolResultContent](block) - if !ok || !result.ProviderExecuted || result.ToolCallID == "" { - continue - } - matchedResultIDs[result.ToolCallID] = struct{}{} - } - - out := make([]fantasy.Content, 0, len(content)) - for _, block := range content { - toolCall, ok := fantasy.AsContentType[fantasy.ToolCallContent](block) - if ok && isAnthropicProviderExecutedToolCall(provider, toolCall) { - if _, hasResult := matchedResultIDs[toolCall.ToolCallID]; !hasResult { - stats.RemovedToolCalls++ - continue - } - } - out = append(out, block) - } - if stats.RemovedToolCalls == 0 { - return content, stats - } - return out, stats -} - -func isAnthropicProviderExecutedToolCall( - provider string, - toolCall fantasy.ToolCallContent, -) bool { - return provider == fantasyanthropic.Name && toolCall.ProviderExecuted -} - // guardedAttempt owns an attempt-scoped context and startup guard // around a provider stream. release is idempotent and frees the // attempt-scoped timer/context. finish canonicalizes startup timeout @@ -1380,9 +1323,9 @@ func persistInterruptedStep( provider = opts.Model.Provider() modelName = opts.Model.Model() } - var sanitizeStats chatprompt.AnthropicProviderToolSanitizationStats - result.content, sanitizeStats = sanitizeAnthropicProviderToolContent(provider, result.content) - chatprompt.LogAnthropicProviderToolSanitization( + var sanitizeStats chatsanitize.AnthropicProviderToolSanitizationStats + result.content, sanitizeStats = chatsanitize.SanitizeAnthropicProviderToolContent(provider, result.content) + chatsanitize.LogAnthropicProviderToolSanitization( ctx, opts.Logger, "interrupted_persist", provider, modelName, sanitizeStats, ) @@ -1420,7 +1363,7 @@ func persistInterruptedStep( if _, exists := answeredToolCalls[tc.ToolCallID]; exists { continue } - if isAnthropicProviderExecutedToolCall(provider, tc) { + if chatsanitize.IsAnthropicProviderExecutedToolCall(provider, tc) { continue } content = append(content, fantasy.ToolResultContent{ diff --git a/coderd/x/chatd/chatloop/chatloop_test.go b/coderd/x/chatd/chatloop/chatloop_test.go index c2f63b3097..08489773ce 100644 --- a/coderd/x/chatd/chatloop/chatloop_test.go +++ b/coderd/x/chatd/chatloop/chatloop_test.go @@ -20,8 +20,10 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog/v3" + "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/x/chatd/chaterror" "github.com/coder/coder/v2/coderd/x/chatd/chatretry" + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" @@ -30,6 +32,93 @@ import ( const activeToolName = "read_file" +func validWebSearchProviderMetadataForTest() fantasy.ProviderMetadata { + return fantasy.ProviderMetadata{ + fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{ + Results: []fantasyanthropic.WebSearchResultItem{ + { + URL: "https://example.com", + Title: "Example", + EncryptedContent: "encrypted", + }, + }, + }, + } +} + +func safeToolCallContent(block fantasy.Content) (fantasy.ToolCallContent, bool) { + var zero fantasy.ToolCallContent + switch value := block.(type) { + case fantasy.ToolCallContent: + return value, true + case *fantasy.ToolCallContent: + if value == nil { + return zero, false + } + return *value, true + default: + return zero, false + } +} + +func safeToolResultContent(block fantasy.Content) (fantasy.ToolResultContent, bool) { + var zero fantasy.ToolResultContent + switch value := block.(type) { + case fantasy.ToolResultContent: + return value, true + case *fantasy.ToolResultContent: + if value == nil { + return zero, false + } + return *value, true + default: + return zero, false + } +} + +func safeToolCallPart(part fantasy.MessagePart) (fantasy.ToolCallPart, bool) { + var zero fantasy.ToolCallPart + if part == nil { + return zero, false + } + if value, ok := part.(*fantasy.ToolCallPart); ok && value == nil { + return zero, false + } + type toolCallPart = fantasy.ToolCallPart + return fantasy.AsMessagePart[toolCallPart](part) +} + +func safeToolResultPart(part fantasy.MessagePart) (fantasy.ToolResultPart, bool) { + var zero fantasy.ToolResultPart + if part == nil { + return zero, false + } + if value, ok := part.(*fantasy.ToolResultPart); ok && value == nil { + return zero, false + } + type toolResultPart = fantasy.ToolResultPart + return fantasy.AsMessagePart[toolResultPart](part) +} + +func toolCallContentToPart(toolCall fantasy.ToolCallContent) fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: toolCall.ToolCallID, + ToolName: toolCall.ToolName, + Input: toolCall.Input, + ProviderExecuted: toolCall.ProviderExecuted, + ProviderOptions: fantasy.ProviderOptions(toolCall.ProviderMetadata), + } +} + +func toolResultContentToPart(toolResult fantasy.ToolResultContent) fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: toolResult.ToolCallID, + Output: toolResult.Result, + ProviderExecuted: toolResult.ProviderExecuted, + ProviderOptions: fantasy.ProviderOptions(toolResult.ProviderMetadata), + } +} + func awaitRunResult(ctx context.Context, t *testing.T, done <-chan error) error { t.Helper() @@ -1041,6 +1130,21 @@ func requireNoProviderExecutedToolResultContent(t *testing.T, content []fantasy. } } +func requireTextPrompt(t *testing.T, prompt []fantasy.Message, text string) fantasy.TextPart { + t.Helper() + + for _, message := range prompt { + for _, part := range message.Content { + textPart, ok := fantasy.AsMessagePart[fantasy.TextPart](part) + if ok && textPart.Text == text { + return textPart + } + } + } + t.Fatalf("missing prompt text %q", text) + return fantasy.TextPart{} +} + func requireNoProviderExecutedToolCallPrompt(t *testing.T, prompt []fantasy.Message) { t.Helper() @@ -1108,6 +1212,75 @@ func requireToolResultPrompt(t *testing.T, prompt []fantasy.Message, id string) return fantasy.ToolResultPart{} } +func requireNoProviderExecutedToolResultPrompt(t *testing.T, prompt []fantasy.Message) { + t.Helper() + + for i, message := range prompt { + for j, part := range message.Content { + toolResult, ok := safeToolResultPart(part) + if ok && toolResult.ProviderExecuted { + t.Fatalf("prompt[%d].content[%d]: unexpected provider-executed result", i, j) + } + } + } +} + +func requireProviderExecutedToolCallPrompt( + t *testing.T, + prompt []fantasy.Message, + id string, +) fantasy.ToolCallPart { + t.Helper() + + for _, message := range prompt { + for _, part := range message.Content { + toolCall, ok := safeToolCallPart(part) + if ok && toolCall.ProviderExecuted && toolCall.ToolCallID == id { + return toolCall + } + } + } + t.Fatalf("missing provider-executed prompt tool call %q", id) + return fantasy.ToolCallPart{} +} + +func requireProviderExecutedToolResultPrompt( + t *testing.T, + prompt []fantasy.Message, + id string, +) fantasy.ToolResultPart { + t.Helper() + + for _, message := range prompt { + for _, part := range message.Content { + toolResult, ok := safeToolResultPart(part) + if ok && toolResult.ProviderExecuted && toolResult.ToolCallID == id { + return toolResult + } + } + } + t.Fatalf("missing provider-executed prompt tool result %q", id) + return fantasy.ToolResultPart{} +} + +func requireAnthropicProviderToolPromptSafe(t *testing.T, prompt []fantasy.Message) { + t.Helper() + + require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory(prompt)) +} + +func requireLogField(t *testing.T, entry slog.SinkEntry, name string) any { + t.Helper() + + for _, field := range entry.Fields { + if field.Name == name { + return field.Value + } + } + t.Fatalf("missing log field %q", name) + return nil +} + func containsPromptSentinel(prompt []fantasy.Message) bool { for _, message := range prompt { if message.Role != fantasy.MessageRoleUser || len(message.Content) != 1 { @@ -2027,6 +2200,7 @@ func TestRun_AnthropicKeepsPairedWebSearchBeforePersist(t *testing.T) { ID: "ws-1", ToolCallName: "web_search", ProviderExecuted: true, + ProviderMetadata: validWebSearchProviderMetadataForTest(), }, {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "search done"}, @@ -2322,6 +2496,535 @@ func TestRun_AnthropicSanitizesWebSearchBeforeContinuation(t *testing.T) { require.False(t, promptResult.ProviderExecuted) } +func TestSanitizeAnthropicProviderToolContent(t *testing.T) { + t.Parallel() + + providerCall := func(id, name, input string) fantasy.ToolCallContent { + return fantasy.ToolCallContent{ + ToolCallID: id, + ToolName: name, + Input: input, + ProviderExecuted: true, + } + } + providerResult := func(id, name string) fantasy.ToolResultContent { + return fantasy.ToolResultContent{ + ToolCallID: id, + ToolName: name, + ProviderExecuted: true, + ProviderMetadata: validWebSearchProviderMetadataForTest(), + Result: fantasy.ToolResultOutputContentText{Text: "ok"}, + } + } + localCall := func(id, name string) fantasy.ToolCallContent { + return fantasy.ToolCallContent{ + ToolCallID: id, + ToolName: name, + Input: `{}`, + } + } + localResult := func(id, name string) fantasy.ToolResultContent { + return fantasy.ToolResultContent{ + ToolCallID: id, + ToolName: name, + Result: fantasy.ToolResultOutputContentText{Text: "ok"}, + } + } + type contentSummary struct { + providerCalls []string + providerResults []string + localCalls []string + localResults []string + } + summarizeContent := func(content []fantasy.Content) contentSummary { + var summary contentSummary + for _, block := range content { + if toolCall, ok := safeToolCallContent(block); ok { + if toolCall.ProviderExecuted { + summary.providerCalls = append(summary.providerCalls, toolCall.ToolCallID) + } else { + summary.localCalls = append(summary.localCalls, toolCall.ToolCallID) + } + continue + } + if toolResult, ok := safeToolResultContent(block); ok { + if toolResult.ProviderExecuted { + summary.providerResults = append(summary.providerResults, toolResult.ToolCallID) + } else { + summary.localResults = append(summary.localResults, toolResult.ToolCallID) + } + } + } + return summary + } + assertProviderHistoryValid := func(t *testing.T, content []fantasy.Content) { + t.Helper() + + parts := make([]fantasy.MessagePart, 0) + for _, block := range content { + if toolCall, ok := safeToolCallContent(block); ok && toolCall.ProviderExecuted { + parts = append(parts, toolCallContentToPart(toolCall)) + continue + } + if toolResult, ok := safeToolResultContent(block); ok && toolResult.ProviderExecuted { + parts = append(parts, toolResultContentToPart(toolResult)) + } + } + if len(parts) == 0 { + return + } + require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory([]fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: parts, + }, + })) + } + + metadataCall := providerCall("ws-meta", "web_search", `{"query":"coder"}`) + metadataCall.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil} + metadataResult := providerResult("ws-meta", "web_search") + metadataResult.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil} + pointerCall := providerCall("ws-pointer", "web_search", `{"query":"coder"}`) + var nilToolCall *fantasy.ToolCallContent + + testCases := []struct { + name string + provider string + content []fantasy.Content + wantSummary contentSummary + wantRemovedCalls int + wantRemovedResults int + wantTexts []string + validateAnthropic bool + }{ + { + name: "orphan provider result textified", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + fantasy.TextContent{Text: "keep"}, + providerResult("ws-1", "web_search"), + }, + wantRemovedResults: 1, + wantTexts: []string{"keep", "ok"}, + validateAnthropic: true, + }, + { + name: "result before call removes both provider blocks", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerResult("ws-1", "web_search"), + providerCall("ws-1", "web_search", `{"query":"coder"}`), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "valid web search pair preserved", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("ws-1", "web_search", `{"query":"coder"}`), + providerResult("ws-1", "web_search"), + fantasy.TextContent{Text: "search done"}, + }, + wantSummary: contentSummary{ + providerCalls: []string{"ws-1"}, + providerResults: []string{"ws-1"}, + }, + wantTexts: []string{"search done"}, + validateAnthropic: true, + }, + { + name: "invalid JSON provider call drops pair", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("ws-1", "web_search", `{`), + providerResult("ws-1", "web_search"), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "empty ID provider call drops pair", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("", "web_search", `{"query":"coder"}`), + providerResult("", "web_search"), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "empty tool name provider call drops pair", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("ws-empty", "", `{"query":"coder"}`), + providerResult("ws-empty", ""), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "non web search provider pair drops through serializable helper", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("code-1", "code_execution", `{"code":"print(1)"}`), + providerResult("code-1", "code_execution"), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "mismatched provider result tool name drops pair", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("ws-mismatch", "web_search", `{"query":"coder"}`), + providerResult("ws-mismatch", "code_execution"), + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "duplicate provider IDs drop all provider content for ID", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("dup-1", "web_search", `{"query":"coder"}`), + providerResult("dup-1", "web_search"), + providerCall("dup-1", "web_search", `{"query":"coder"}`), + }, + wantRemovedCalls: 2, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "mismatched provider flags remove only provider side", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + providerCall("mix-1", "web_search", `{"query":"coder"}`), + localResult("mix-1", "web_search"), + localCall("mix-2", "read_file"), + providerResult("mix-2", "web_search"), + }, + wantSummary: contentSummary{ + localCalls: []string{"mix-2"}, + localResults: []string{"mix-1"}, + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "malformed provider metadata textifies result", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + metadataCall, + metadataResult, + }, + wantRemovedCalls: 1, + wantRemovedResults: 1, + wantTexts: []string{"ok"}, + validateAnthropic: true, + }, + { + name: "pointer and nil pointer variants are handled safely", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + nilToolCall, + &pointerCall, + providerResult("ws-pointer", "web_search"), + }, + wantSummary: contentSummary{ + providerCalls: []string{"ws-pointer"}, + providerResults: []string{"ws-pointer"}, + }, + validateAnthropic: true, + }, + { + name: "local tool content is unchanged", + provider: fantasyanthropic.Name, + content: []fantasy.Content{ + localCall("tc-1", "read_file"), + localResult("tc-1", "read_file"), + }, + wantSummary: contentSummary{ + localCalls: []string{"tc-1"}, + localResults: []string{"tc-1"}, + }, + validateAnthropic: true, + }, + { + name: "non Anthropic provider content is unchanged", + provider: "fake", + content: []fantasy.Content{ + providerCall("ws-1", "web_search", `{"query":"coder"}`), + }, + wantSummary: contentSummary{ + providerCalls: []string{"ws-1"}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + sanitized, stats := chatsanitize.SanitizeAnthropicProviderToolContent(tc.provider, tc.content) + require.Equal(t, tc.wantRemovedCalls, stats.RemovedToolCalls) + require.Equal(t, tc.wantRemovedResults, stats.RemovedToolResults) + require.Zero(t, stats.DroppedMessages) + + summary := summarizeContent(sanitized) + assert.ElementsMatch(t, tc.wantSummary.providerCalls, summary.providerCalls) + assert.ElementsMatch(t, tc.wantSummary.providerResults, summary.providerResults) + assert.ElementsMatch(t, tc.wantSummary.localCalls, summary.localCalls) + assert.ElementsMatch(t, tc.wantSummary.localResults, summary.localResults) + for _, text := range tc.wantTexts { + requireTextContent(t, sanitized, text) + } + if tc.validateAnthropic { + assertProviderHistoryValid(t, sanitized) + } + }) + } +} + +func TestRun_AnthropicProviderToolPreRequestGuard(t *testing.T) { + t.Parallel() + + webSearchTool := ProviderTool{ + Definition: fantasy.ProviderDefinedTool{ + ID: "anthropic.web_search", + Name: "web_search", + }, + } + providerPair := func(id string) []fantasy.MessagePart { + return []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: id, + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, + fantasy.ToolResultPart{ + ToolCallID: id, + Output: fantasy.ToolResultOutputContentText{Text: "ok"}, + ProviderExecuted: true, + ProviderOptions: fantasy.ProviderOptions(validWebSearchProviderMetadataForTest()), + }, + } + } + completionModel := func(capturedPrompt *[]fantasy.Message) *chattest.FakeModel { + return &chattest.FakeModel{ + ProviderName: fantasyanthropic.Name, + ModelName: "claude-test", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + *capturedPrompt = append([]fantasy.Message(nil), call.Prompt...) + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + } + } + + t.Run("allowed web search survives when provider tool is enabled", func(t *testing.T) { + t.Parallel() + + var capturedPrompt []fantasy.Message + err := Run(context.Background(), RunOptions{ + Model: completionModel(&capturedPrompt), + Messages: []fantasy.Message{ + textMessage(fantasy.MessageRoleUser, "search"), + { + Role: fantasy.MessageRoleAssistant, + Content: providerPair("ws-allowed"), + }, + textMessage(fantasy.MessageRoleUser, "continue"), + }, + ProviderTools: []ProviderTool{webSearchTool}, + MaxSteps: 1, + PersistStep: func(_ context.Context, _ PersistedStep) error { + return nil + }, + }) + require.NoError(t, err) + + toolCall := requireProviderExecutedToolCallPrompt(t, capturedPrompt, "ws-allowed") + require.Equal(t, "web_search", toolCall.ToolName) + requireProviderExecutedToolResultPrompt(t, capturedPrompt, "ws-allowed") + requireAnthropicProviderToolPromptSafe(t, capturedPrompt) + }) + + t.Run("web search history survives when provider tool is disabled", func(t *testing.T) { + t.Parallel() + + var capturedPrompt []fantasy.Message + err := Run(context.Background(), RunOptions{ + Model: completionModel(&capturedPrompt), + Messages: []fantasy.Message{ + textMessage(fantasy.MessageRoleUser, "search and read"), + { + Role: fantasy.MessageRoleAssistant, + Content: append(providerPair("ws-disabled"), fantasy.ToolCallPart{ + ToolCallID: "tc-1", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + }), + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "tc-1", + Output: fantasy.ToolResultOutputContentText{Text: "file"}, + }, + }, + }, + textMessage(fantasy.MessageRoleUser, "continue"), + }, + MaxSteps: 1, + PersistStep: func(_ context.Context, _ PersistedStep) error { + return nil + }, + }) + require.NoError(t, err) + + requireProviderExecutedToolCallPrompt(t, capturedPrompt, "ws-disabled") + requireProviderExecutedToolResultPrompt(t, capturedPrompt, "ws-disabled") + promptResult := requireToolResultPrompt(t, capturedPrompt, "tc-1") + require.False(t, promptResult.ProviderExecuted) + requireAnthropicProviderToolPromptSafe(t, capturedPrompt) + }) + + t.Run("direct guard textifies orphaned provider result", func(t *testing.T) { + t.Parallel() + + guarded := chatsanitize.ApplyAnthropicProviderToolGuard( + context.Background(), + slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}), + fantasyanthropic.Name, + "claude-test", + []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "keep"}, + fantasy.ToolResultPart{ + ToolCallID: "ws-orphan", + Output: fantasy.ToolResultOutputContentText{Text: "search result"}, + ProviderExecuted: true, + }, + }, + }, + }, + ) + + requireNoProviderExecutedToolResultPrompt(t, guarded) + requireAnthropicProviderToolPromptSafe(t, guarded) + require.Len(t, guarded, 1) + require.Len(t, guarded[0].Content, 2) + textPart, ok := fantasy.AsMessagePart[fantasy.TextPart](guarded[0].Content[0]) + require.True(t, ok) + require.Equal(t, "keep", textPart.Text) + textPart, ok = fantasy.AsMessagePart[fantasy.TextPart](guarded[0].Content[1]) + require.True(t, ok) + require.Equal(t, "search result", textPart.Text) + }) + + t.Run("direct guard leaves valid provider history unchanged", func(t *testing.T) { + t.Parallel() + + content := []fantasy.MessagePart{fantasy.TextPart{Text: "keep"}} + content = append(content, providerPair("ws-one")...) + content = append(content, providerPair("ws-two")...) + guarded := chatsanitize.ApplyAnthropicProviderToolGuard( + context.Background(), + slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}), + fantasyanthropic.Name, + "claude-test", + []fantasy.Message{{Role: fantasy.MessageRoleAssistant, Content: content}}, + ) + + requireAnthropicProviderToolPromptSafe(t, guarded) + require.Len(t, guarded, 1) + require.Len(t, guarded[0].Content, len(content)) + requireProviderExecutedToolCallPrompt(t, guarded, "ws-one") + requireProviderExecutedToolResultPrompt(t, guarded, "ws-one") + requireProviderExecutedToolCallPrompt(t, guarded, "ws-two") + requireProviderExecutedToolResultPrompt(t, guarded, "ws-two") + }) + + t.Run("direct guard leaves non Anthropic providers unchanged", func(t *testing.T) { + t.Parallel() + + prompt := []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: providerPair("ws-other-provider"), + }, + } + guarded := chatsanitize.ApplyAnthropicProviderToolGuard( + context.Background(), + slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}), + "fake", + "fake-model", + prompt, + ) + require.Equal(t, prompt, guarded) + }) + + t.Run("guard logs removals", func(t *testing.T) { + t.Parallel() + + logSink := testutil.NewFakeSink(t) + logger := logSink.Logger() + logPair := providerPair("ws-log") + guarded := chatsanitize.ApplyAnthropicProviderToolGuard( + context.Background(), + logger, + fantasyanthropic.Name, + "claude-test", + []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + logPair[1], + logPair[0], + }, + }, + }, + ) + + requireNoProviderExecutedToolCallPrompt(t, guarded) + requireNoProviderExecutedToolResultPrompt(t, guarded) + requireTextPrompt(t, guarded, "ok") + entries := logSink.Entries(func(e slog.SinkEntry) bool { + return e.Level == slog.LevelWarn && + e.Message == "removed provider-executed tool history" + }) + require.Len(t, entries, 1) + require.Equal(t, "pre_request_guard", requireLogField(t, entries[0], "phase")) + require.Equal(t, 1, requireLogField(t, entries[0], "removed_tool_calls")) + require.Equal(t, 1, requireLogField(t, entries[0], "removed_tool_results")) + }) +} + // TestRun_PersistStepInterruptedFallback verifies that when the normal // PersistStep call returns ErrInterrupted (e.g., context canceled in a // race), the step is retried via the interrupt-safe path. diff --git a/coderd/x/chatd/chatprompt/chatprompt.go b/coderd/x/chatd/chatprompt/chatprompt.go index 9b2ccdccfa..b7681a6b44 100644 --- a/coderd/x/chatd/chatprompt/chatprompt.go +++ b/coderd/x/chatd/chatprompt/chatprompt.go @@ -11,7 +11,6 @@ import ( "strings" "charm.land/fantasy" - fantasyanthropic "charm.land/fantasy/providers/anthropic" "github.com/google/uuid" "github.com/sqlc-dev/pqtype" "golang.org/x/xerrors" @@ -35,192 +34,28 @@ var toolCallIDSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_-]`) var syntheticPasteFileNamePattern = regexp.MustCompile(`^pasted-text-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}\.txt$`) -// AnthropicProviderToolSanitizationStats describes prompt changes made -// while removing unpaired Anthropic provider-executed tool calls. -type AnthropicProviderToolSanitizationStats struct { - RemovedToolCalls int - DroppedMessages int +func safeAsToolCallPart(part fantasy.MessagePart) (fantasy.ToolCallPart, bool) { + var zero fantasy.ToolCallPart + if part == nil { + return zero, false + } + if value, ok := part.(*fantasy.ToolCallPart); ok && value == nil { + return zero, false + } + type toolCallPart = fantasy.ToolCallPart + return fantasy.AsMessagePart[toolCallPart](part) } -// LogAnthropicProviderToolSanitization logs prompt changes made while removing -// unpaired Anthropic provider-executed tool calls. -func LogAnthropicProviderToolSanitization( - ctx context.Context, - logger slog.Logger, - phase string, - provider string, - modelName string, - stats AnthropicProviderToolSanitizationStats, - extra ...slog.Field, -) { - if stats.RemovedToolCalls == 0 { - return +func safeAsToolResultPart(part fantasy.MessagePart) (fantasy.ToolResultPart, bool) { + var zero fantasy.ToolResultPart + if part == nil { + return zero, false } - fields := []slog.Field{ - slog.F("phase", phase), - slog.F("tool_type", "provider_executed"), - slog.F("provider", provider), - slog.F("model", modelName), - slog.F("removed_tool_calls", stats.RemovedToolCalls), - slog.F("dropped_messages", stats.DroppedMessages), + if value, ok := part.(*fantasy.ToolResultPart); ok && value == nil { + return zero, false } - fields = append(fields, extra...) - logger.Warn(ctx, "removed unpaired provider-executed tool calls", fields...) -} - -// SanitizeAnthropicProviderToolCalls removes Anthropic provider-executed -// calls that do not have a same-message provider result. -func SanitizeAnthropicProviderToolCalls( - provider string, - messages []fantasy.Message, -) ([]fantasy.Message, AnthropicProviderToolSanitizationStats) { - var stats AnthropicProviderToolSanitizationStats - if provider != fantasyanthropic.Name || len(messages) == 0 { - return messages, stats - } - - out := make([]fantasy.Message, 0, len(messages)) - changed := false - for _, msg := range messages { - if msg.Role != fantasy.MessageRoleAssistant { - out = appendSanitizedMessage(out, msg) - continue - } - - matchedResultIDs := make(map[string]struct{}) - for _, part := range msg.Content { - result, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) - if !ok || !result.ProviderExecuted || result.ToolCallID == "" { - continue - } - matchedResultIDs[result.ToolCallID] = struct{}{} - } - - parts := make([]fantasy.MessagePart, 0, len(msg.Content)) - removedFromMessage := 0 - for _, part := range msg.Content { - toolCall, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part) - if ok && toolCall.ProviderExecuted { - if _, hasResult := matchedResultIDs[toolCall.ToolCallID]; !hasResult { - stats.RemovedToolCalls++ - removedFromMessage++ - changed = true - continue - } - } - parts = append(parts, part) - } - - if removedFromMessage > 0 { - if len(parts) == 0 { - stats.DroppedMessages++ - continue - } - msg.Content = parts - } - out = appendSanitizedMessage(out, msg) - } - if !changed { - return messages, stats - } - return out, stats -} - -func appendSanitizedMessage(out []fantasy.Message, msg fantasy.Message) []fantasy.Message { - if len(out) == 0 || out[len(out)-1].Role != msg.Role { - return append(out, msg) - } - - last := &out[len(out)-1] - lastContent := applyMessageProviderOptionsToLastPart(last.Content, last.ProviderOptions) - msgContent := applyMessageProviderOptionsToLastPart(msg.Content, msg.ProviderOptions) - content := make([]fantasy.MessagePart, 0, len(lastContent)+len(msgContent)) - content = append(content, lastContent...) - content = append(content, msgContent...) - last.Content = content - last.ProviderOptions = nil - return out -} - -func applyMessageProviderOptionsToLastPart( - parts []fantasy.MessagePart, - options fantasy.ProviderOptions, -) []fantasy.MessagePart { - if len(options) == 0 || len(parts) == 0 { - return parts - } - - out := make([]fantasy.MessagePart, len(parts)) - copy(out, parts) - lastIndex := len(out) - 1 - switch part := out[lastIndex].(type) { - case fantasy.TextPart: - part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) - out[lastIndex] = part - case *fantasy.TextPart: - if part != nil { - clone := *part - clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) - out[lastIndex] = &clone - } - case fantasy.ReasoningPart: - part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) - out[lastIndex] = part - case *fantasy.ReasoningPart: - if part != nil { - clone := *part - clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) - out[lastIndex] = &clone - } - case fantasy.FilePart: - part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) - out[lastIndex] = part - case *fantasy.FilePart: - if part != nil { - clone := *part - clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) - out[lastIndex] = &clone - } - case fantasy.ToolCallPart: - part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) - out[lastIndex] = part - case *fantasy.ToolCallPart: - if part != nil { - clone := *part - clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) - out[lastIndex] = &clone - } - case fantasy.ToolResultPart: - part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) - out[lastIndex] = part - case *fantasy.ToolResultPart: - if part != nil { - clone := *part - clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) - out[lastIndex] = &clone - } - } - return out -} - -func mergeProviderOptions(first, second fantasy.ProviderOptions) fantasy.ProviderOptions { - if len(first) == 0 { - return second - } - if len(second) == 0 { - return first - } - - merged := make(fantasy.ProviderOptions, len(first)+len(second)) - for provider, options := range first { - merged[provider] = options - } - for provider, options := range second { - if options != nil { - merged[provider] = options - } - } - return merged + type toolResultPart = fantasy.ToolResultPart + return fantasy.AsMessagePart[toolResultPart](part) } // FileData holds resolved file content for LLM prompt building. @@ -764,7 +599,7 @@ func normalizeAssistantToolCallInputs( ) []fantasy.MessagePart { normalized := make([]fantasy.MessagePart, 0, len(parts)) for _, part := range parts { - toolCall, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part) + toolCall, ok := safeAsToolCallPart(part) if !ok { normalized = append(normalized, part) continue @@ -797,7 +632,7 @@ func normalizeToolCallInput(input string) string { func ExtractToolCalls(parts []fantasy.MessagePart) []fantasy.ToolCallContent { toolCalls := make([]fantasy.ToolCallContent, 0, len(parts)) for _, part := range parts { - toolCall, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part) + toolCall, ok := safeAsToolCallPart(part) if !ok { continue } @@ -1148,7 +983,7 @@ func injectMissingToolResults(prompt []fantasy.Message) []fantasy.Message { break } for _, part := range prompt[j].Content { - tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) + tr, ok := safeAsToolResultPart(part) if !ok { continue } @@ -1205,7 +1040,7 @@ func injectMissingToolUses( allToolResults := make([]fantasy.ToolResultPart, 0, len(msg.Content)) for _, part := range msg.Content { - toolResult, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) + toolResult, ok := safeAsToolResultPart(part) if !ok { continue } diff --git a/coderd/x/chatd/chatprompt/chatprompt_test.go b/coderd/x/chatd/chatprompt/chatprompt_test.go index d9b6fc7f20..7d06ec0224 100644 --- a/coderd/x/chatd/chatprompt/chatprompt_test.go +++ b/coderd/x/chatd/chatprompt/chatprompt_test.go @@ -22,6 +22,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" "github.com/coder/coder/v2/coderd/x/chatd/chattool" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" @@ -59,243 +60,16 @@ func convertMessagesWithoutFiles(t *testing.T, messages []database.ChatMessage) return prompt } -func TestSanitizeAnthropicProviderToolCalls(t *testing.T) { - t.Parallel() +type testToolCallPart = fantasy.ToolCallPart - textPart := fantasy.TextPart{Text: "Here is a summary."} - webSearchCall := fantasy.ToolCallPart{ - ToolCallID: "srvtoolu_search", - ToolName: "web_search", - Input: `{"query":"coder"}`, - ProviderExecuted: true, - } - matchedResult := fantasy.ToolResultPart{ - ToolCallID: "srvtoolu_search", - Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, - ProviderExecuted: true, - } - codeExecutionCall := fantasy.ToolCallPart{ - ToolCallID: "srvtoolu_code", - ToolName: "code_execution", - Input: `{"code":"print(1)"}`, - ProviderExecuted: true, - } - localCall := fantasy.ToolCallPart{ - ToolCallID: "toolu_local", - ToolName: "read_file", - Input: `{"path":"main.go"}`, - } - unpairedWebSearchCall := webSearchCall - unpairedWebSearchCall.ToolCallID = "srvtoolu_unpaired" - disableParallelToolUse := true - providerOptions := fantasy.ProviderOptions{ - fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{ - DisableParallelToolUse: &disableParallelToolUse, - }, - } - enableParallelToolUse := false - providerOptionsAllowParallel := fantasy.ProviderOptions{ - fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{ - DisableParallelToolUse: &enableParallelToolUse, - }, - } +type testToolResultPart = fantasy.ToolResultPart - testCases := []struct { - name string - provider string - messages []fantasy.Message - want []fantasy.Message - wantRemoved int - wantDropped int - }{ - { - name: "removes unpaired call and keeps text", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - textPart, - webSearchCall, - }, - }}, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{textPart}, - }}, - wantRemoved: 1, - }, - { - name: "drops assistant message when only part is removed", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{webSearchCall}, - }}, - want: []fantasy.Message{}, - wantRemoved: 1, - wantDropped: 1, - }, - { - name: "coalesces adjacent roles after dropping empty message", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{ - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "search for coder"}, - }, - }, - { - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{webSearchCall}, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "now summarize"}, - }, - ProviderOptions: providerOptions, - }, - }, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "search for coder"}, - fantasy.TextPart{ - Text: "now summarize", - ProviderOptions: providerOptions, - }, - }, - }}, - wantRemoved: 1, - wantDropped: 1, - }, - { - name: "coalesces adjacent provider options without flattening boundaries", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{ - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "search for coder"}, - }, - ProviderOptions: providerOptionsAllowParallel, - }, - { - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{webSearchCall}, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "now summarize"}, - }, - ProviderOptions: providerOptions, - }, - }, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{ - Text: "search for coder", - ProviderOptions: providerOptionsAllowParallel, - }, - fantasy.TextPart{ - Text: "now summarize", - ProviderOptions: providerOptions, - }, - }, - }}, - wantRemoved: 1, - wantDropped: 1, - }, - { - name: "keeps matched call and result", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - webSearchCall, - matchedResult, - }, - }}, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - webSearchCall, - matchedResult, - }, - }}, - }, - { - name: "removes only unpaired call from mixed message", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - textPart, - webSearchCall, - matchedResult, - unpairedWebSearchCall, - }, - }}, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - textPart, - webSearchCall, - matchedResult, - }, - }}, - wantRemoved: 1, - }, - { - name: "removes unpaired provider call and keeps local call", - provider: fantasyanthropic.Name, - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - textPart, - codeExecutionCall, - localCall, - }, - }}, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - textPart, - localCall, - }, - }}, - wantRemoved: 1, - }, - { - name: "leaves other providers unchanged", - provider: "fake", - messages: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{webSearchCall}, - }}, - want: []fantasy.Message{{ - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{webSearchCall}, - }}, - }, - } +func asToolCallPartForTest(part fantasy.MessagePart) (fantasy.ToolCallPart, bool) { + return fantasy.AsMessagePart[testToolCallPart](part) +} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - sanitized, stats := chatprompt.SanitizeAnthropicProviderToolCalls( - tc.provider, - tc.messages, - ) - require.Equal(t, tc.wantRemoved, stats.RemovedToolCalls) - require.Equal(t, tc.wantDropped, stats.DroppedMessages) - require.Equal(t, tc.want, sanitized) - }) - } +func asToolResultPartForTest(part fantasy.MessagePart) (fantasy.ToolResultPart, bool) { + return fantasy.AsMessagePart[testToolResultPart](part) } func TestConvertMessagesWithFiles_NormalizesAssistantToolCallInput(t *testing.T) { @@ -742,18 +516,20 @@ func TestInjectMissingToolResults_SkipsProviderExecuted(t *testing.T) { // The tool message should have exactly one result (the local one). var resultIDs []string for _, part := range prompt[1].Content { - tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) + tr, ok := asToolResultPartForTest(part) if ok { resultIDs = append(resultIDs, tr.ToolCallID) } } require.Equal(t, []string{"toolu_local"}, resultIDs) - sanitized, sanitizeStats := chatprompt.SanitizeAnthropicProviderToolCalls( + sanitized, sanitizeStats := chatsanitize.SanitizeAnthropicProviderToolHistory( fantasyanthropic.Name, prompt, ) require.Equal(t, 1, sanitizeStats.RemovedToolCalls) + require.Equal(t, 0, sanitizeStats.RemovedToolResults) require.Len(t, sanitized, 2) + require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory(sanitized)) remainingToolCalls := chatprompt.ExtractToolCalls(sanitized[0].Content) require.Len(t, remainingToolCalls, 1) require.Equal(t, "toolu_local", remainingToolCalls[0].ToolCallID) @@ -799,7 +575,7 @@ func TestInjectMissingToolResults_SkipsProviderExecutedAndInjectsLocal(t *testin require.Equal(t, []string{"toolu_read"}, extractToolResultIDs(t, prompt[1])) require.Len(t, prompt[1].Content, 1) - toolResult, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + toolResult, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok, "expected synthetic ToolResultPart") require.Equal(t, "toolu_read", toolResult.ToolCallID) require.False(t, toolResult.ProviderExecuted) @@ -992,7 +768,7 @@ func TestInjectMissingToolUses_DropsProviderExecutedOrphans(t *testing.T) { for i, msg := range prompt { if msg.Role == fantasy.MessageRoleAssistant { for _, part := range msg.Content { - tc, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part) + tc, ok := asToolCallPartForTest(part) if ok && tc.Input == "{}" && tc.ToolCallID == "srvtoolu_C" { t.Errorf("message[%d]: unexpected synthetic tool_use for srvtoolu_C", i) } @@ -1092,12 +868,12 @@ func TestProviderExecutedResultInAssistantContent(t *testing.T) { // The assistant message must contain 3 parts: tool_call, tool_result, text. var foundToolCall, foundToolResult, foundText bool for _, part := range prompt[0].Content { - if tc, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part); ok { + if tc, ok := asToolCallPartForTest(part); ok { require.Equal(t, "srvtoolu_WS", tc.ToolCallID) require.True(t, tc.ProviderExecuted, "ToolCallPart.ProviderExecuted must be true") foundToolCall = true } - if tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part); ok { + if tr, ok := asToolResultPartForTest(part); ok { require.Equal(t, "srvtoolu_WS", tr.ToolCallID) require.True(t, tr.ProviderExecuted, "ToolResultPart.ProviderExecuted must be true") foundToolResult = true @@ -1844,7 +1620,7 @@ func TestMixedFormatConversation(t *testing.T) { // 4. Old tool: result paired with call_1. require.Equal(t, fantasy.MessageRoleTool, prompt[3].Role) require.Len(t, prompt[3].Content, 1) - toolResult, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[3].Content[0]) + toolResult, ok := asToolResultPartForTest(prompt[3].Content[0]) require.True(t, ok) assert.Equal(t, "call_1", toolResult.ToolCallID) @@ -2026,7 +1802,7 @@ func extractToolResultIDs(t *testing.T, msgs ...fantasy.Message) []string { var ids []string for _, msg := range msgs { for _, part := range msg.Content { - tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) + tr, ok := asToolResultPartForTest(part) if ok { ids = append(ids, tr.ToolCallID) } @@ -2304,11 +2080,11 @@ func TestConvertMessagesWithFiles_FiltersEmptyTextAndReasoningParts(t *testing.T require.True(t, ok, "expected ReasoningPart at index 2") require.Equal(t, "thinking deeply", reasoningPart.Text) - toolCallPart, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](resultParts[3]) + toolCallPart, ok := asToolCallPartForTest(resultParts[3]) require.True(t, ok, "expected ToolCallPart at index 3") require.Equal(t, "call-1", toolCallPart.ToolCallID) - toolResultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](resultParts[4]) + toolResultPart, ok := asToolResultPartForTest(resultParts[4]) require.True(t, ok, "expected ToolResultPart at index 4") require.Equal(t, "call-1", toolResultPart.ToolCallID) }) @@ -2342,7 +2118,7 @@ func TestConvertMessagesWithFiles_FiltersEmptyTextAndReasoningParts(t *testing.T require.True(t, ok, "expected TextPart") require.Equal(t, " reply ", textPart.Text) - tcPart, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](resultParts[1]) + tcPart, ok := asToolCallPartForTest(resultParts[1]) require.True(t, ok, "expected ToolCallPart") require.Equal(t, "tc-1", tcPart.ToolCallID) }) @@ -2739,7 +2515,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { require.Equal(t, fantasy.MessageRoleTool, toolMsg.Role) require.Len(t, toolMsg.Content, 1) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](toolMsg.Content[0]) + resultPart, ok := asToolResultPartForTest(toolMsg.Content[0]) require.True(t, ok, "expected ToolResultPart") require.Equal(t, callID, resultPart.ToolCallID) require.False(t, resultPart.ProviderExecuted) @@ -2796,7 +2572,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok, "expected ToolResultPart") mediaOutput, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -2869,7 +2645,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) require.False(t, resultPart.ProviderExecuted) @@ -2895,7 +2671,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -2921,7 +2697,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -2943,7 +2719,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -2971,7 +2747,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) errOutput, isError := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentError](resultPart.Output) @@ -3003,7 +2779,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -3032,7 +2808,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -3060,7 +2836,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) @@ -3091,7 +2867,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) { prompt := loadPrompt(t, chat) require.Len(t, prompt, 2) - resultPart, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](prompt[1].Content[0]) + resultPart, ok := asToolResultPartForTest(prompt[1].Content[0]) require.True(t, ok) _, isMedia := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](resultPart.Output) diff --git a/coderd/x/chatd/chatsanitize/anthropic.go b/coderd/x/chatd/chatsanitize/anthropic.go new file mode 100644 index 0000000000..151376a409 --- /dev/null +++ b/coderd/x/chatd/chatsanitize/anthropic.go @@ -0,0 +1,1135 @@ +package chatsanitize + +import ( + "context" + "encoding/json" + "strings" + + "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + + "cdr.dev/slog/v3" +) + +const maxAnthropicProviderToolViolationLogDetails = 32 + +// supportedAnthropicProviderToolNames is the allowlist of provider-executed +// tool names the Anthropic provider in fantasy can currently serialize. +var supportedAnthropicProviderToolNames = map[string]struct{}{ + "web_search": {}, +} + +const ( + anthropicProviderToolViolationOutsideAssistant = "provider_executed_block_outside_assistant" + anthropicProviderToolViolationOrphanCall = "provider_executed_call_without_result" + anthropicProviderToolViolationOrphanResult = "provider_executed_result_without_call" + anthropicProviderToolViolationDuplicateID = "duplicate_provider_executed_id" + anthropicProviderToolViolationResultBeforeCall = "provider_executed_result_before_call" + anthropicProviderToolViolationInvalidCall = "invalid_provider_executed_tool_call" + anthropicProviderToolViolationInvalidResult = "invalid_provider_executed_tool_result" +) + +// AnthropicProviderToolSanitizationStats describes prompt changes made +// while removing invalid Anthropic provider-executed tool history. +type AnthropicProviderToolSanitizationStats struct { + RemovedToolCalls int + RemovedToolResults int + DroppedMessages int +} + +// AnthropicProviderToolHistoryViolation describes an invalid +// provider-executed tool history block in an Anthropic prompt. +type AnthropicProviderToolHistoryViolation struct { + MessageIndex int + PartIndex int + ID string + Reason string +} + +// LogAnthropicProviderToolSanitization logs prompt changes made while +// removing invalid Anthropic provider-executed tool history. +func LogAnthropicProviderToolSanitization( + ctx context.Context, + logger slog.Logger, + phase string, + provider string, + modelName string, + stats AnthropicProviderToolSanitizationStats, + extra ...slog.Field, +) { + if stats.RemovedToolCalls == 0 && stats.RemovedToolResults == 0 { + return + } + fields := []slog.Field{ + slog.F("phase", phase), + slog.F("tool_type", "provider_executed"), + slog.F("provider", provider), + slog.F("model", modelName), + slog.F("removed_tool_calls", stats.RemovedToolCalls), + slog.F("removed_tool_results", stats.RemovedToolResults), + slog.F("dropped_messages", stats.DroppedMessages), + } + fields = append(fields, extra...) + logger.Warn(ctx, "removed provider-executed tool history", fields...) +} + +// IsSerializableAnthropicProviderToolCall reports whether part can be +// serialized as an Anthropic provider-executed tool call. +func IsSerializableAnthropicProviderToolCall(part fantasy.MessagePart) bool { + toolCall, ok := safeMessageToolCallPart(part) + if !ok || !toolCall.ProviderExecuted { + return false + } + if strings.TrimSpace(toolCall.ToolCallID) == "" || toolCall.ToolName == "" { + return false + } + if !IsAllowedAnthropicProviderToolName(toolCall.ToolName) { + return false + } + return json.Valid([]byte(strings.TrimSpace(toolCall.Input))) +} + +// IsSerializableAnthropicProviderToolResult reports whether part can be +// serialized as an Anthropic provider-executed tool result for matchedCall. +func IsSerializableAnthropicProviderToolResult( + part fantasy.MessagePart, + matchedCall fantasy.MessagePart, +) bool { + result, ok := safeMessageToolResultPart(part) + if !ok || !result.ProviderExecuted { + return false + } + if strings.TrimSpace(result.ToolCallID) == "" { + return false + } + toolCall, ok := safeMessageToolCallPart(matchedCall) + if !ok || result.ToolCallID != toolCall.ToolCallID { + return false + } + if !IsSerializableAnthropicProviderToolCall(matchedCall) { + return false + } + return hasSerializableAnthropicProviderToolResultMetadata(result, toolCall) +} + +func hasSerializableAnthropicProviderToolResultMetadata( + result fantasy.ToolResultPart, + matchedCall fantasy.ToolCallPart, +) bool { + if matchedCall.ToolName != "web_search" { + return false + } + providerMetadata := result.ProviderOptions[fantasyanthropic.Name] + metadata, ok := providerMetadata.(*fantasyanthropic.WebSearchResultMetadata) + return ok && metadata != nil +} + +// AnthropicProviderToolResultTextPart converts a provider-executed tool +// result into text so unsafe provider-tool structure can be removed without +// losing the result payload. +func AnthropicProviderToolResultTextPart( + part fantasy.MessagePart, +) (fantasy.TextPart, bool) { + var zero fantasy.TextPart + result, ok := safeMessageToolResultPart(part) + if !ok || !result.ProviderExecuted { + return zero, false + } + text := AnthropicToolResultOutputText(result.Output) + if text == "" { + return zero, false + } + return fantasy.TextPart{Text: text}, true +} + +// AnthropicToolResultOutputText converts a tool result payload into the text +// that should remain in the prompt when provider-tool metadata is unsafe. +func AnthropicToolResultOutputText(output fantasy.ToolResultOutputContent) string { + switch value := output.(type) { + case fantasy.ToolResultOutputContentText: + return value.Text + case *fantasy.ToolResultOutputContentText: + if value == nil { + return "" + } + return value.Text + case fantasy.ToolResultOutputContentError: + if value.Error == nil { + return "" + } + return value.Error.Error() + case *fantasy.ToolResultOutputContentError: + if value == nil || value.Error == nil { + return "" + } + return value.Error.Error() + case fantasy.ToolResultOutputContentMedia: + return value.Text + case *fantasy.ToolResultOutputContentMedia: + if value == nil { + return "" + } + return value.Text + } + + if output == nil { + return "" + } + encoded, err := json.Marshal(output) + if err != nil { + return "" + } + return string(encoded) +} + +// IsAllowedAnthropicProviderToolName reports whether name is an Anthropic +// provider-executed tool name we know how to serialize. +func IsAllowedAnthropicProviderToolName(name string) bool { + _, ok := supportedAnthropicProviderToolNames[name] + return ok +} + +// ValidateAnthropicProviderToolHistory returns violations found in messages +// with invalid Anthropic provider-executed tool history blocks. +func ValidateAnthropicProviderToolHistory( + messages []fantasy.Message, +) []AnthropicProviderToolHistoryViolation { + analysis := analyzeAnthropicProviderToolHistory(messages) + return analysis.violations +} + +// AnthropicProviderToolPartsToRemove returns provider-executed tool parts +// that cannot be serialized safely in a single Anthropic assistant message. +// Violation MessageIndex values refer to the synthetic assistant message, so +// they are always 0. +func AnthropicProviderToolPartsToRemove( + provider string, + parts []fantasy.MessagePart, +) (map[int]struct{}, []AnthropicProviderToolHistoryViolation) { + remove := make(map[int]struct{}) + if provider != fantasyanthropic.Name || len(parts) == 0 { + return remove, nil + } + + analysis := analyzeAnthropicProviderToolHistory([]fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: parts, + }}) + for key := range analysis.remove { + if key.messageIndex != 0 { + continue + } + remove[key.partIndex] = struct{}{} + } + + violations := make([]AnthropicProviderToolHistoryViolation, len(analysis.violations)) + copy(violations, analysis.violations) + return remove, violations +} + +// SanitizeAnthropicProviderToolHistory removes Anthropic provider-executed +// tool history that cannot be serialized safely. +func SanitizeAnthropicProviderToolHistory( + provider string, + messages []fantasy.Message, +) ([]fantasy.Message, AnthropicProviderToolSanitizationStats) { + var stats AnthropicProviderToolSanitizationStats + if provider != fantasyanthropic.Name || len(messages) == 0 { + return messages, stats + } + + current := messages + changed := false + for { + // Each pass shrinks the finite part set, so the loop terminates. + analysis := analyzeAnthropicProviderToolHistory(current) + if len(analysis.remove) == 0 { + if !changed { + return messages, stats + } + return current, stats + } + + out := make([]fantasy.Message, 0, len(current)) + for messageIndex, msg := range current { + parts := make([]fantasy.MessagePart, 0, len(msg.Content)) + removedFromMessage := 0 + for partIndex, part := range msg.Content { + key := anthropicProviderToolPartKey{ + messageIndex: messageIndex, + partIndex: partIndex, + } + if _, remove := analysis.remove[key]; remove { + countRemovedAnthropicProviderToolPart(&stats, part) + if textPart, ok := AnthropicProviderToolResultTextPart(part); ok { + parts = append(parts, textPart) + } + removedFromMessage++ + changed = true + continue + } + parts = append(parts, part) + } + + if removedFromMessage > 0 { + if len(parts) == 0 { + stats.DroppedMessages++ + continue + } + msg.Content = parts + } + out = appendSanitizedMessage(out, msg) + } + current = out + } +} + +// SanitizeAnthropicProviderToolStepContent removes invalid Anthropic +// provider-executed tool content from a streamed step and logs removals. +func SanitizeAnthropicProviderToolStepContent( + ctx context.Context, + logger slog.Logger, + provider string, + modelName string, + phase string, + step int, + finishReason fantasy.FinishReason, + content []fantasy.Content, +) []fantasy.Content { + sanitized, stats := SanitizeAnthropicProviderToolContent(provider, content) + LogAnthropicProviderToolSanitization( + ctx, logger, phase, provider, modelName, stats, + slog.F("step_index", step), + slog.F("finish_reason", finishReason), + ) + return sanitized +} + +// SanitizeAnthropicProviderToolContent removes invalid Anthropic +// provider-executed tool blocks from streamed content. +func SanitizeAnthropicProviderToolContent( + provider string, + content []fantasy.Content, +) ([]fantasy.Content, AnthropicProviderToolSanitizationStats) { + var stats AnthropicProviderToolSanitizationStats + if provider != fantasyanthropic.Name || len(content) == 0 { + return content, stats + } + + partIndexByContentIndex := make([]int, len(content)) + for index := range partIndexByContentIndex { + partIndexByContentIndex[index] = noMappedToolPartIndex + } + contentKinds := make([]mappedToolContentKind, len(content)) + parts := make([]fantasy.MessagePart, 0, len(content)) + providerCalls := make(map[string][]mappedProviderToolCall) + providerResultNames := make(map[string][]string) + for contentIndex, block := range content { + if toolCall, ok := safeToolCallContent(block); ok { + partIndex := len(parts) + parts = append(parts, toolCallContentToPart(toolCall)) + partIndexByContentIndex[contentIndex] = partIndex + contentKinds[contentIndex] = mappedToolContentCall + if toolCall.ProviderExecuted { + providerCalls[toolCall.ToolCallID] = append( + providerCalls[toolCall.ToolCallID], + mappedProviderToolCall{ + partIndex: partIndex, + toolName: toolCall.ToolName, + }, + ) + } + continue + } + if toolResult, ok := safeToolResultContent(block); ok { + partIndex := len(parts) + parts = append(parts, toolResultContentToPart(toolResult)) + partIndexByContentIndex[contentIndex] = partIndex + contentKinds[contentIndex] = mappedToolContentResult + if toolResult.ProviderExecuted { + providerResultNames[toolResult.ToolCallID] = append( + providerResultNames[toolResult.ToolCallID], + toolResult.ToolName, + ) + } + } + } + if len(parts) == 0 { + return content, stats + } + + // ToolResultContent carries ToolName, but ToolResultPart does not. Preserve + // the content sanitizer mismatch check by invalidating the synthetic call. + for id, calls := range providerCalls { + for _, call := range calls { + for _, resultToolName := range providerResultNames[id] { + if resultToolName == "" || resultToolName == call.toolName { + continue + } + toolCall, ok := parts[call.partIndex].(fantasy.ToolCallPart) + if !ok { + break + } + toolCall.ToolName = "" + parts[call.partIndex] = toolCall + break + } + } + } + + removeParts, _ := AnthropicProviderToolPartsToRemove(provider, parts) + if len(removeParts) == 0 { + return content, stats + } + + removeContent := make(map[int]struct{}, len(removeParts)) + for contentIndex, partIndex := range partIndexByContentIndex { + if partIndex == noMappedToolPartIndex { + continue + } + if _, remove := removeParts[partIndex]; remove { + removeContent[contentIndex] = struct{}{} + } + } + if len(removeContent) == 0 { + return content, stats + } + + out := make([]fantasy.Content, 0, len(content)) + for contentIndex, block := range content { + if _, remove := removeContent[contentIndex]; remove { + switch contentKinds[contentIndex] { + case mappedToolContentCall: + stats.RemovedToolCalls++ + case mappedToolContentResult: + stats.RemovedToolResults++ + if textContent, ok := anthropicProviderToolResultTextContent(block); ok { + out = append(out, textContent) + } + } + continue + } + out = append(out, block) + } + return out, stats +} + +// IsAnthropicProviderExecutedToolCall reports whether toolCall is an +// Anthropic provider-executed tool call. +func IsAnthropicProviderExecutedToolCall( + provider string, + toolCall fantasy.ToolCallContent, +) bool { + return provider == fantasyanthropic.Name && toolCall.ProviderExecuted +} + +// ApplyAnthropicProviderToolGuard fail-closes unsafe Anthropic provider-tool +// history immediately before a provider request is issued. +func ApplyAnthropicProviderToolGuard( + ctx context.Context, + logger slog.Logger, + provider string, + modelName string, + messages []fantasy.Message, +) []fantasy.Message { + if provider != fantasyanthropic.Name || len(messages) == 0 { + return messages + } + + violations := ValidateAnthropicProviderToolHistory(messages) + if len(violations) == 0 { + return messages + } + affectedMessages := messageIndexesFromAnthropicProviderToolViolations( + violations, + len(messages), + ) + guarded := sanitizeAnthropicProviderToolGuardMessages( + ctx, + logger, + provider, + modelName, + messages, + affectedMessages, + len(violations), + ) + if isSafeAnthropicProviderToolPrompt(guarded) { + return guarded + } + + fallbackViolations := ValidateAnthropicProviderToolHistory(guarded) + fallbackAffectedMessages := providerExecutedToolMessageIndexes(guarded) + guarded = sanitizeAnthropicProviderToolGuardMessages( + ctx, + logger, + provider, + modelName, + guarded, + fallbackAffectedMessages, + len(fallbackViolations), + slog.F("fallback", true), + ) + if isSafeAnthropicProviderToolPrompt(guarded) { + return guarded + } + + // The guard sanitizer should normally remove every typed provider block it + // selects. The strip path is a fail-closed backstop for analyzer and + // provider serialization drift, not a path we can drive without hooks. + preStripViolations := ValidateAnthropicProviderToolHistory(guarded) + stripMessages := messageIndexesFromAnthropicProviderToolViolations( + preStripViolations, + len(guarded), + ) + + var stripStats AnthropicProviderToolSanitizationStats + guarded, stripStats = stripAnthropicProviderToolHistoryFromMessages( + guarded, + stripMessages, + ) + var sanitizeStats AnthropicProviderToolSanitizationStats + guarded, sanitizeStats = SanitizeAnthropicProviderToolHistory( + provider, + guarded, + ) + stripStats = addAnthropicProviderToolSanitizationStats(stripStats, sanitizeStats) + + if !isSafeAnthropicProviderToolPrompt(guarded) { + guarded, sanitizeStats = stripAnthropicProviderToolHistoryFromMessages( + guarded, + providerExecutedToolMessageIndexes(guarded), + ) + stripStats = addAnthropicProviderToolSanitizationStats(stripStats, sanitizeStats) + guarded, sanitizeStats = SanitizeAnthropicProviderToolHistory( + provider, + guarded, + ) + stripStats = addAnthropicProviderToolSanitizationStats(stripStats, sanitizeStats) + if !isSafeAnthropicProviderToolPrompt(guarded) { + logger.Error( + ctx, + "anthropic provider tool guard postcondition failed: prompt still unsafe after nuclear strip", + slog.F("phase", "pre_request_guard_postcondition_failed"), + slog.F("tool_type", "provider_executed"), + slog.F("provider", provider), + slog.F("model", modelName), + ) + } + } + + details, truncated := anthropicProviderToolViolationLogDetails( + preStripViolations, + ) + LogAnthropicProviderToolSanitization( + ctx, + logger, + "pre_request_guard_fallback_strip", + provider, + modelName, + stripStats, + slog.F("validation_violations", len(preStripViolations)), + slog.F("validation_violation_details", details), + slog.F("truncated_violations", truncated), + ) + return guarded +} + +type anthropicProviderToolPartKey struct { + messageIndex int + partIndex int +} + +type anthropicProviderToolHistoryAnalysis struct { + remove map[anthropicProviderToolPartKey]struct{} + violations []AnthropicProviderToolHistoryViolation +} + +type anthropicProviderToolOccurrence struct { + partIndex int + part fantasy.MessagePart +} + +type anthropicProviderToolIDHistory struct { + calls []anthropicProviderToolOccurrence + results []anthropicProviderToolOccurrence +} + +func analyzeAnthropicProviderToolHistory( + messages []fantasy.Message, +) anthropicProviderToolHistoryAnalysis { + analysis := anthropicProviderToolHistoryAnalysis{ + remove: make(map[anthropicProviderToolPartKey]struct{}), + } + for messageIndex, msg := range messages { + if msg.Role != fantasy.MessageRoleAssistant { + for partIndex, part := range msg.Content { + id, ok := anthropicProviderExecutedToolPartID(part) + if !ok { + continue + } + analysis.addViolation( + messageIndex, + partIndex, + id, + anthropicProviderToolViolationOutsideAssistant, + ) + } + continue + } + analysis.analyzeAssistantMessage(messageIndex, msg) + } + return analysis +} + +func (a *anthropicProviderToolHistoryAnalysis) analyzeAssistantMessage( + messageIndex int, + msg fantasy.Message, +) { + histories := make(map[string]*anthropicProviderToolIDHistory) + ids := make([]string, 0) + for partIndex, part := range msg.Content { + if toolCall, ok := safeMessageToolCallPart(part); ok && toolCall.ProviderExecuted { + history := ensureAnthropicProviderToolIDHistory( + histories, + &ids, + toolCall.ToolCallID, + ) + history.calls = append(history.calls, anthropicProviderToolOccurrence{ + partIndex: partIndex, + part: part, + }) + continue + } + if result, ok := safeMessageToolResultPart(part); ok && result.ProviderExecuted { + history := ensureAnthropicProviderToolIDHistory( + histories, + &ids, + result.ToolCallID, + ) + history.results = append(history.results, anthropicProviderToolOccurrence{ + partIndex: partIndex, + part: part, + }) + } + } + + for _, id := range ids { + history := histories[id] + switch { + case len(history.calls) > 1 || len(history.results) > 1: + a.addHistoryViolations( + messageIndex, + id, + history, + anthropicProviderToolViolationDuplicateID, + ) + case len(history.calls) == 1 && len(history.results) == 0: + a.addOccurrenceViolation( + messageIndex, + id, + history.calls[0], + anthropicProviderToolViolationOrphanCall, + ) + case len(history.calls) == 0 && len(history.results) == 1: + a.addOccurrenceViolation( + messageIndex, + id, + history.results[0], + anthropicProviderToolViolationOrphanResult, + ) + case len(history.calls) == 1 && len(history.results) == 1: + call := history.calls[0] + result := history.results[0] + if call.partIndex >= result.partIndex { + a.addHistoryViolations( + messageIndex, + id, + history, + anthropicProviderToolViolationResultBeforeCall, + ) + continue + } + if !IsSerializableAnthropicProviderToolCall(call.part) { + a.addHistoryViolations( + messageIndex, + id, + history, + anthropicProviderToolViolationInvalidCall, + ) + continue + } + if !IsSerializableAnthropicProviderToolResult(result.part, call.part) { + a.addHistoryViolations( + messageIndex, + id, + history, + anthropicProviderToolViolationInvalidResult, + ) + } + } + } +} + +func ensureAnthropicProviderToolIDHistory( + histories map[string]*anthropicProviderToolIDHistory, + ids *[]string, + id string, +) *anthropicProviderToolIDHistory { + history, ok := histories[id] + if ok { + return history + } + history = &anthropicProviderToolIDHistory{} + histories[id] = history + *ids = append(*ids, id) + return history +} + +func (a *anthropicProviderToolHistoryAnalysis) addHistoryViolations( + messageIndex int, + id string, + history *anthropicProviderToolIDHistory, + reason string, +) { + for _, occurrence := range history.calls { + a.addOccurrenceViolation(messageIndex, id, occurrence, reason) + } + for _, occurrence := range history.results { + a.addOccurrenceViolation(messageIndex, id, occurrence, reason) + } +} + +func (a *anthropicProviderToolHistoryAnalysis) addOccurrenceViolation( + messageIndex int, + id string, + occurrence anthropicProviderToolOccurrence, + reason string, +) { + a.addViolation(messageIndex, occurrence.partIndex, id, reason) +} + +func (a *anthropicProviderToolHistoryAnalysis) addViolation( + messageIndex int, + partIndex int, + id string, + reason string, +) { + key := anthropicProviderToolPartKey{ + messageIndex: messageIndex, + partIndex: partIndex, + } + if _, ok := a.remove[key]; ok { + return + } + a.remove[key] = struct{}{} + a.violations = append(a.violations, AnthropicProviderToolHistoryViolation{ + MessageIndex: messageIndex, + PartIndex: partIndex, + ID: id, + Reason: reason, + }) +} + +func anthropicProviderExecutedToolPartID(part fantasy.MessagePart) (string, bool) { + if toolCall, ok := safeMessageToolCallPart(part); ok && toolCall.ProviderExecuted { + return toolCall.ToolCallID, true + } + if result, ok := safeMessageToolResultPart(part); ok && result.ProviderExecuted { + return result.ToolCallID, true + } + return "", false +} + +func countRemovedAnthropicProviderToolPart( + stats *AnthropicProviderToolSanitizationStats, + part fantasy.MessagePart, +) { + if toolCall, ok := safeMessageToolCallPart(part); ok && toolCall.ProviderExecuted { + stats.RemovedToolCalls++ + return + } + if result, ok := safeMessageToolResultPart(part); ok && result.ProviderExecuted { + stats.RemovedToolResults++ + } +} + +const noMappedToolPartIndex = -1 + +type mappedToolContentKind int + +const ( + _ mappedToolContentKind = iota + mappedToolContentCall + mappedToolContentResult +) + +type mappedProviderToolCall struct { + partIndex int + toolName string +} + +func anthropicProviderToolResultTextContent( + block fantasy.Content, +) (fantasy.TextContent, bool) { + var zero fantasy.TextContent + toolResult, ok := safeToolResultContent(block) + if !ok || !toolResult.ProviderExecuted { + return zero, false + } + text := AnthropicToolResultOutputText(toolResult.Result) + if text == "" { + return zero, false + } + return fantasy.TextContent{Text: text}, true +} + +func safeToolCallContent(block fantasy.Content) (fantasy.ToolCallContent, bool) { + var zero fantasy.ToolCallContent + switch value := block.(type) { + case fantasy.ToolCallContent: + return value, true + case *fantasy.ToolCallContent: + if value == nil { + return zero, false + } + return *value, true + default: + return zero, false + } +} + +func safeToolResultContent(block fantasy.Content) (fantasy.ToolResultContent, bool) { + var zero fantasy.ToolResultContent + switch value := block.(type) { + case fantasy.ToolResultContent: + return value, true + case *fantasy.ToolResultContent: + if value == nil { + return zero, false + } + return *value, true + default: + return zero, false + } +} + +func toolCallContentToPart(toolCall fantasy.ToolCallContent) fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: toolCall.ToolCallID, + ToolName: toolCall.ToolName, + Input: toolCall.Input, + ProviderExecuted: toolCall.ProviderExecuted, + ProviderOptions: fantasy.ProviderOptions(toolCall.ProviderMetadata), + } +} + +func toolResultContentToPart(toolResult fantasy.ToolResultContent) fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: toolResult.ToolCallID, + Output: toolResult.Result, + ProviderExecuted: toolResult.ProviderExecuted, + ProviderOptions: fantasy.ProviderOptions(toolResult.ProviderMetadata), + } +} + +func sanitizeAnthropicProviderToolGuardMessages( + ctx context.Context, + logger slog.Logger, + provider string, + modelName string, + messages []fantasy.Message, + affectedMessages map[int]struct{}, + validationViolations int, + extraFields ...slog.Field, +) []fantasy.Message { + guardPrompt := invalidateProviderExecutedToolCallsInMessages(messages, affectedMessages) + // Marking affected provider calls invalid lets the sanitizer remove the + // unsafe history while preserving result payloads as plain text. + sanitized, stats := SanitizeAnthropicProviderToolHistory(provider, guardPrompt) + extra := []slog.Field{ + slog.F("validation_violations", validationViolations), + } + extra = append(extra, extraFields...) + LogAnthropicProviderToolSanitization( + ctx, + logger, + "pre_request_guard", + provider, + modelName, + stats, + extra..., + ) + return sanitized +} + +func isSafeAnthropicProviderToolPrompt(messages []fantasy.Message) bool { + return len(ValidateAnthropicProviderToolHistory(messages)) == 0 +} + +func messageIndexesFromAnthropicProviderToolViolations( + violations []AnthropicProviderToolHistoryViolation, + messageCount int, +) map[int]struct{} { + indexes := make(map[int]struct{}) + for _, violation := range violations { + if violation.MessageIndex < 0 || violation.MessageIndex >= messageCount { + continue + } + indexes[violation.MessageIndex] = struct{}{} + } + return indexes +} + +func providerExecutedToolMessageIndexes(messages []fantasy.Message) map[int]struct{} { + indexes := make(map[int]struct{}) + for messageIndex, message := range messages { + for _, part := range message.Content { + if toolCall, ok := safeMessageToolCallPart(part); ok && toolCall.ProviderExecuted { + indexes[messageIndex] = struct{}{} + break + } + if toolResult, ok := safeMessageToolResultPart(part); ok && toolResult.ProviderExecuted { + indexes[messageIndex] = struct{}{} + break + } + } + } + return indexes +} + +func stripAnthropicProviderToolHistoryFromMessages( + messages []fantasy.Message, + affectedMessages map[int]struct{}, +) ([]fantasy.Message, AnthropicProviderToolSanitizationStats) { + var stats AnthropicProviderToolSanitizationStats + if len(affectedMessages) == 0 { + return messages, stats + } + + out := make([]fantasy.Message, 0, len(messages)) + for messageIndex, message := range messages { + if _, affected := affectedMessages[messageIndex]; !affected { + out = appendSanitizedMessage(out, message) + continue + } + + parts := make([]fantasy.MessagePart, 0, len(message.Content)) + for _, part := range message.Content { + if toolCall, ok := safeMessageToolCallPart(part); ok && toolCall.ProviderExecuted { + stats.RemovedToolCalls++ + continue + } + if toolResult, ok := safeMessageToolResultPart(part); ok && toolResult.ProviderExecuted { + stats.RemovedToolResults++ + if textPart, ok := AnthropicProviderToolResultTextPart(part); ok { + parts = append(parts, textPart) + } + continue + } + parts = append(parts, part) + } + if len(parts) == 0 { + stats.DroppedMessages++ + continue + } + message.Content = parts + out = appendSanitizedMessage(out, message) + } + return out, stats +} + +func appendSanitizedMessage(out []fantasy.Message, msg fantasy.Message) []fantasy.Message { + if len(out) == 0 || out[len(out)-1].Role != msg.Role { + return append(out, msg) + } + + last := &out[len(out)-1] + lastContent := applyMessageProviderOptionsToLastPart(last.Content, last.ProviderOptions) + msgContent := applyMessageProviderOptionsToLastPart(msg.Content, msg.ProviderOptions) + content := make([]fantasy.MessagePart, 0, len(lastContent)+len(msgContent)) + content = append(content, lastContent...) + content = append(content, msgContent...) + last.Content = content + last.ProviderOptions = nil + return out +} + +func applyMessageProviderOptionsToLastPart( + parts []fantasy.MessagePart, + options fantasy.ProviderOptions, +) []fantasy.MessagePart { + if len(options) == 0 || len(parts) == 0 { + return parts + } + + out := make([]fantasy.MessagePart, len(parts)) + copy(out, parts) + lastIndex := len(out) - 1 + switch part := out[lastIndex].(type) { + case fantasy.TextPart: + part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) + out[lastIndex] = part + case *fantasy.TextPart: + if part != nil { + clone := *part + clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) + out[lastIndex] = &clone + } + case fantasy.ReasoningPart: + part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) + out[lastIndex] = part + case *fantasy.ReasoningPart: + if part != nil { + clone := *part + clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) + out[lastIndex] = &clone + } + case fantasy.FilePart: + part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) + out[lastIndex] = part + case *fantasy.FilePart: + if part != nil { + clone := *part + clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) + out[lastIndex] = &clone + } + case fantasy.ToolCallPart: + part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) + out[lastIndex] = part + case *fantasy.ToolCallPart: + if part != nil { + clone := *part + clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) + out[lastIndex] = &clone + } + case fantasy.ToolResultPart: + part.ProviderOptions = mergeProviderOptions(part.ProviderOptions, options) + out[lastIndex] = part + case *fantasy.ToolResultPart: + if part != nil { + clone := *part + clone.ProviderOptions = mergeProviderOptions(clone.ProviderOptions, options) + out[lastIndex] = &clone + } + } + return out +} + +func mergeProviderOptions(first, second fantasy.ProviderOptions) fantasy.ProviderOptions { + if len(first) == 0 { + return second + } + if len(second) == 0 { + return first + } + + merged := make(fantasy.ProviderOptions, len(first)+len(second)) + for provider, options := range first { + merged[provider] = options + } + for provider, options := range second { + if options != nil { + merged[provider] = options + } + } + return merged +} + +func addAnthropicProviderToolSanitizationStats( + first AnthropicProviderToolSanitizationStats, + second AnthropicProviderToolSanitizationStats, +) AnthropicProviderToolSanitizationStats { + return AnthropicProviderToolSanitizationStats{ + RemovedToolCalls: first.RemovedToolCalls + second.RemovedToolCalls, + RemovedToolResults: first.RemovedToolResults + second.RemovedToolResults, + DroppedMessages: first.DroppedMessages + second.DroppedMessages, + } +} + +func anthropicProviderToolViolationLogDetails( + violations []AnthropicProviderToolHistoryViolation, +) ([]map[string]any, bool) { + count := min(len(violations), maxAnthropicProviderToolViolationLogDetails) + details := make([]map[string]any, 0, count) + for _, violation := range violations[:count] { + details = append(details, map[string]any{ + "message_index": violation.MessageIndex, + "part_index": violation.PartIndex, + "id": violation.ID, + "reason": violation.Reason, + }) + } + return details, len(violations) > maxAnthropicProviderToolViolationLogDetails +} + +func invalidateProviderExecutedToolCallsInMessages( + messages []fantasy.Message, + affectedMessages map[int]struct{}, +) []fantasy.Message { + if len(affectedMessages) == 0 { + return messages + } + out := make([]fantasy.Message, len(messages)) + copy(out, messages) + for messageIndex := range affectedMessages { + if messageIndex < 0 || messageIndex >= len(out) { + continue + } + message := out[messageIndex] + if len(message.Content) == 0 { + continue + } + parts := make([]fantasy.MessagePart, len(message.Content)) + for partIndex, part := range message.Content { + parts[partIndex] = invalidateProviderExecutedToolCallPart(part) + } + message.Content = parts + out[messageIndex] = message + } + return out +} + +func invalidateProviderExecutedToolCallPart(part fantasy.MessagePart) fantasy.MessagePart { + switch value := part.(type) { + case fantasy.ToolCallPart: + if value.ProviderExecuted { + value.ToolName = "" + } + return value + case *fantasy.ToolCallPart: + if value == nil { + return part + } + clone := *value + if clone.ProviderExecuted { + clone.ToolName = "" + } + return &clone + default: + return part + } +} + +func safeMessageToolCallPart(part fantasy.MessagePart) (fantasy.ToolCallPart, bool) { + var zero fantasy.ToolCallPart + if part == nil { + return zero, false + } + if value, ok := part.(*fantasy.ToolCallPart); ok && value == nil { + return zero, false + } + type toolCallPart = fantasy.ToolCallPart + return fantasy.AsMessagePart[toolCallPart](part) +} + +func safeMessageToolResultPart(part fantasy.MessagePart) (fantasy.ToolResultPart, bool) { + var zero fantasy.ToolResultPart + if part == nil { + return zero, false + } + if value, ok := part.(*fantasy.ToolResultPart); ok && value == nil { + return zero, false + } + type toolResultPart = fantasy.ToolResultPart + return fantasy.AsMessagePart[toolResultPart](part) +} diff --git a/coderd/x/chatd/chatsanitize/anthropic_internal_test.go b/coderd/x/chatd/chatsanitize/anthropic_internal_test.go new file mode 100644 index 0000000000..f229bf6419 --- /dev/null +++ b/coderd/x/chatd/chatsanitize/anthropic_internal_test.go @@ -0,0 +1,146 @@ +package chatsanitize + +import ( + "testing" + + "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + "github.com/stretchr/testify/require" +) + +func textMessageForTest(role fantasy.MessageRole, text string) fantasy.Message { + return fantasy.Message{ + Role: role, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: text}, + }, + } +} + +func TestProviderExecutedToolMessageIndexes(t *testing.T) { + t.Parallel() + + messages := []fantasy.Message{ + textMessageForTest(fantasy.MessageRoleUser, "plain"), + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "ws-result-only", + ProviderExecuted: true, + }, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "ws-call", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "local-call", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + }, + }, + }, + } + + require.Equal(t, map[int]struct{}{1: {}, 2: {}}, providerExecutedToolMessageIndexes(messages)) +} + +func TestAnthropicProviderToolFallbackStripHelpers(t *testing.T) { + t.Parallel() + + providerCall := fantasy.ToolCallPart{ + ToolCallID: "ws-strip", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + } + providerResult := fantasy.ToolResultPart{ + ToolCallID: "ws-strip", + Output: fantasy.ToolResultOutputContentText{Text: "ok"}, + ProviderExecuted: true, + } + messages := []fantasy.Message{ + textMessageForTest(fantasy.MessageRoleAssistant, "first"), + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall, + providerResult, + }, + }, + textMessageForTest(fantasy.MessageRoleAssistant, "second"), + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "keep"}, + fantasy.ToolResultPart{ + ToolCallID: "ws-user", + ProviderExecuted: true, + }, + }, + }, + } + + stripped, stats := stripAnthropicProviderToolHistoryFromMessages( + messages, + map[int]struct{}{1: {}, 3: {}}, + ) + require.Equal(t, 1, stats.RemovedToolCalls) + require.Equal(t, 2, stats.RemovedToolResults) + require.Zero(t, stats.DroppedMessages) + + sanitized, sanitizeStats := SanitizeAnthropicProviderToolHistory( + fantasyanthropic.Name, + stripped, + ) + require.Zero(t, sanitizeStats.RemovedToolCalls) + require.Zero(t, sanitizeStats.RemovedToolResults) + require.Empty(t, ValidateAnthropicProviderToolHistory(sanitized)) + require.Len(t, sanitized, 2) + require.Equal(t, fantasy.MessageRoleAssistant, sanitized[0].Role) + require.Len(t, sanitized[0].Content, 3) + firstText, ok := fantasy.AsMessagePart[fantasy.TextPart](sanitized[0].Content[0]) + require.True(t, ok) + require.Equal(t, "first", firstText.Text) + stripText, ok := fantasy.AsMessagePart[fantasy.TextPart](sanitized[0].Content[1]) + require.True(t, ok) + require.Equal(t, "ok", stripText.Text) + secondText, ok := fantasy.AsMessagePart[fantasy.TextPart](sanitized[0].Content[2]) + require.True(t, ok) + require.Equal(t, "second", secondText.Text) + require.Equal(t, fantasy.MessageRoleUser, sanitized[1].Role) + require.Len(t, sanitized[1].Content, 1) + keepText, ok := fantasy.AsMessagePart[fantasy.TextPart](sanitized[1].Content[0]) + require.True(t, ok) + require.Equal(t, "keep", keepText.Text) + + violations := make([]AnthropicProviderToolHistoryViolation, 33) + for i := range violations { + violations[i] = AnthropicProviderToolHistoryViolation{ + MessageIndex: i, + PartIndex: i + 1, + ID: "ws-detail", + Reason: "test_reason", + } + } + details, truncated := anthropicProviderToolViolationLogDetails(violations) + require.True(t, truncated) + require.Len(t, details, maxAnthropicProviderToolViolationLogDetails) + require.Len(t, details[0], 4) + require.Equal(t, 0, details[0]["message_index"]) + require.Equal(t, 1, details[0]["part_index"]) + require.Equal(t, "ws-detail", details[0]["id"]) + require.Equal(t, "test_reason", details[0]["reason"]) +} diff --git a/coderd/x/chatd/chatsanitize/anthropic_test.go b/coderd/x/chatd/chatsanitize/anthropic_test.go new file mode 100644 index 0000000000..456cfeb7ea --- /dev/null +++ b/coderd/x/chatd/chatsanitize/anthropic_test.go @@ -0,0 +1,1412 @@ +package chatsanitize_test + +import ( + "testing" + + "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" +) + +type testSourceMessagePart struct { + id string +} + +func (testSourceMessagePart) GetType() fantasy.ContentType { + return fantasy.ContentTypeSource +} + +func (testSourceMessagePart) Options() fantasy.ProviderOptions { + return nil +} + +type testToolResultOutput struct { + Value string `json:"value"` +} + +func (testToolResultOutput) GetType() fantasy.ToolResultContentType { + return "test" +} + +func validWebSearchProviderOptionsForTest() fantasy.ProviderOptions { + return fantasy.ProviderOptions{ + fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{ + Results: []fantasyanthropic.WebSearchResultItem{ + { + URL: "https://example.com", + Title: "Example", + EncryptedContent: "encrypted", + }, + }, + }, + } +} + +func TestSanitizeAnthropicProviderToolHistory(t *testing.T) { + t.Parallel() + + textPart := fantasy.TextPart{Text: "Here is a summary."} + sourcePart := testSourceMessagePart{id: "source-1"} + reasoningPart := fantasy.ReasoningPart{Text: "Need to search first."} + filePart := fantasy.FilePart{Data: []byte("notes"), MediaType: "text/plain"} + providerCall := func(id string) fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: id, + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + } + } + providerResult := func(id string) fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: id, + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + ProviderExecuted: true, + ProviderOptions: validWebSearchProviderOptionsForTest(), + } + } + resultText := fantasy.TextPart{Text: `{"ok":true}`} + localCall := fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_local", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + } + localResult := fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_local", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + } + disableParallelToolUse := true + providerOptions := fantasy.ProviderOptions{ + fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{ + DisableParallelToolUse: &disableParallelToolUse, + }, + } + enableParallelToolUse := false + providerOptionsAllowParallel := fantasy.ProviderOptions{ + fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{ + DisableParallelToolUse: &enableParallelToolUse, + }, + } + pointerCall := providerCall("srvtoolu_pointer") + pointerResult := providerResult("srvtoolu_pointer") + + testCases := []struct { + name string + provider string + messages []fantasy.Message + want []fantasy.Message + wantRemovedCalls int + wantRemovedResults int + wantDropped int + }{ + { + name: "removes unpaired call and keeps text", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerCall("srvtoolu_orphan_call"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{textPart}, + }}, + wantRemovedCalls: 1, + }, + { + name: "textifies result-only assistant message", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{providerResult("srvtoolu_orphan_result")}, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{resultText}, + }}, + wantRemovedResults: 1, + }, + { + name: "textifies orphan result and keeps text", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerResult("srvtoolu_orphan_result"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedResults: 1, + }, + { + name: "textifies result before matching call", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerResult("srvtoolu_search"), + providerCall("srvtoolu_search"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "keeps valid web search call and result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }}, + }, + { + name: "keeps valid pair and textifies orphan result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + providerResult("srvtoolu_orphan_result"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + resultText, + }, + }}, + wantRemovedResults: 1, + }, + { + name: "removes invalid json call and dependent result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_bad_json", + ToolName: "web_search", + Input: `{"query":`, + ProviderExecuted: true, + }, + providerResult("srvtoolu_bad_json"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "textifies result with missing provider metadata", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerCall("srvtoolu_missing_meta"), + fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_missing_meta", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + ProviderExecuted: true, + }, + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "removes empty call ID and dependent result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerCall(""), + providerResult(""), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "removes empty tool name and dependent result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_empty_name", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, + providerResult("srvtoolu_empty_name"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "removes unsupported provider tool and result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_code", + ToolName: "code_execution", + Input: `{"code":"print(1)"}`, + ProviderExecuted: true, + }, + providerResult("srvtoolu_code"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 1, + }, + { + name: "removes duplicate ID with two calls and one result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerCall("srvtoolu_duplicate"), + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + }, + }}, + wantRemovedCalls: 2, + wantRemovedResults: 1, + }, + { + name: "removes duplicate ID with one call and two results", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + resultText, + resultText, + }, + }}, + wantRemovedCalls: 1, + wantRemovedResults: 2, + }, + { + name: "textifies repeated valid-looking pairs", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + resultText, + resultText, + }, + }}, + wantRemovedCalls: 2, + wantRemovedResults: 2, + }, + { + name: "provider call plus local result removes provider call only", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_mismatch"), + fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_mismatch", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + }, + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_mismatch", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + }, + }, + }}, + wantRemovedCalls: 1, + }, + { + name: "local call plus provider result textifies provider result", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_mismatch", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + }, + providerResult("srvtoolu_mismatch"), + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_mismatch", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + }, + resultText, + }, + }}, + wantRemovedResults: 1, + }, + { + name: "textifies provider results outside assistant", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "Please summarize."}, + providerCall("srvtoolu_user_call"), + providerResult("srvtoolu_user_result"), + localResult, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + providerResult("srvtoolu_tool"), + fantasy.TextPart{Text: "local text"}, + }, + }, + }, + want: []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "Please summarize."}, + resultText, + localResult, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + resultText, + fantasy.TextPart{Text: "local text"}, + }, + }, + }, + wantRemovedCalls: 1, + wantRemovedResults: 2, + }, + { + name: "textifies non-assistant provider result message", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{providerResult("srvtoolu_tool")}, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{resultText}, + }}, + wantRemovedResults: 1, + }, + { + name: "handles pointer tool parts", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + &pointerCall, + &pointerResult, + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + &pointerCall, + &pointerResult, + }, + }}, + }, + { + name: "preserves surrounding source text reasoning and file parts", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + sourcePart, + reasoningPart, + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + filePart, + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + textPart, + sourcePart, + reasoningPart, + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + filePart, + }, + }}, + }, + { + name: "textified orphan prevents duplicate coalescing", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }, + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{providerResult("srvtoolu_orphan")}, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }, + }, + want: []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }, + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{resultText}, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + }, + }, + wantRemovedResults: 1, + }, + { + name: "keeps local srvtoolu-like IDs untouched", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + localCall, + localResult, + }, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + localCall, + localResult, + }, + }}, + }, + { + name: "coalesces adjacent roles after dropping empty message", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "search for coder"}, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{providerCall("srvtoolu_orphan_call")}, + }, + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "now summarize"}, + }, + ProviderOptions: providerOptions, + }, + }, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "search for coder"}, + fantasy.TextPart{ + Text: "now summarize", + ProviderOptions: providerOptions, + }, + }, + }}, + wantRemovedCalls: 1, + wantDropped: 1, + }, + { + name: "coalesces adjacent provider options without flattening boundaries", + provider: fantasyanthropic.Name, + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "search for coder"}, + }, + ProviderOptions: providerOptionsAllowParallel, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{providerCall("srvtoolu_orphan_call")}, + }, + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "now summarize"}, + }, + ProviderOptions: providerOptions, + }, + }, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{ + Text: "search for coder", + ProviderOptions: providerOptionsAllowParallel, + }, + fantasy.TextPart{ + Text: "now summarize", + ProviderOptions: providerOptions, + }, + }, + }}, + wantRemovedCalls: 1, + wantDropped: 1, + }, + { + name: "leaves other providers unchanged", + provider: "fake", + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{providerResult("srvtoolu_orphan_result")}, + }}, + want: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{providerResult("srvtoolu_orphan_result")}, + }}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + sanitized, stats := chatsanitize.SanitizeAnthropicProviderToolHistory( + tc.provider, + tc.messages, + ) + require.Equal(t, tc.wantRemovedCalls, stats.RemovedToolCalls) + require.Equal(t, tc.wantRemovedResults, stats.RemovedToolResults) + require.Equal(t, tc.wantDropped, stats.DroppedMessages) + require.Equal(t, tc.want, sanitized) + if tc.provider == fantasyanthropic.Name { + require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory(sanitized)) + } + }) + } +} + +func TestAnthropicProviderToolPartsToRemove(t *testing.T) { + t.Parallel() + + providerCall := func(id string) fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: id, + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + } + } + providerResult := func(id string) fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: id, + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + ProviderExecuted: true, + ProviderOptions: validWebSearchProviderOptionsForTest(), + } + } + + testCases := []struct { + name string + provider string + parts []fantasy.MessagePart + wantRemove []int + wantViolations []chatsanitize.AnthropicProviderToolHistoryViolation + }{ + { + name: "empty input", + provider: fantasyanthropic.Name, + wantRemove: []int{}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{}, + }, + { + name: "valid provider call and result", + provider: fantasyanthropic.Name, + parts: []fantasy.MessagePart{ + providerCall("srvtoolu_search"), + providerResult("srvtoolu_search"), + }, + wantRemove: []int{}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{}, + }, + { + name: "orphan provider call", + provider: fantasyanthropic.Name, + parts: []fantasy.MessagePart{ + fantasy.TextPart{Text: "keep"}, + providerCall("srvtoolu_orphan_call"), + }, + wantRemove: []int{1}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{{ + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_orphan_call", + Reason: "provider_executed_call_without_result", + }}, + }, + { + name: "orphan provider result", + provider: fantasyanthropic.Name, + parts: []fantasy.MessagePart{ + fantasy.TextPart{Text: "keep"}, + providerResult("srvtoolu_orphan_result"), + }, + wantRemove: []int{1}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{{ + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_orphan_result", + Reason: "provider_executed_result_without_call", + }}, + }, + { + name: "provider result before call", + provider: fantasyanthropic.Name, + parts: []fantasy.MessagePart{ + providerResult("srvtoolu_search"), + providerCall("srvtoolu_search"), + }, + wantRemove: []int{0, 1}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_search", + Reason: "provider_executed_result_before_call", + }, + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_search", + Reason: "provider_executed_result_before_call", + }, + }, + }, + { + name: "duplicate provider IDs", + provider: fantasyanthropic.Name, + parts: []fantasy.MessagePart{ + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + }, + wantRemove: []int{0, 1, 2}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + { + MessageIndex: 0, + PartIndex: 2, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + }, + }, + { + name: "non Anthropic provider", + provider: "fake", + parts: []fantasy.MessagePart{ + providerResult("srvtoolu_orphan_result"), + }, + wantRemove: []int{}, + wantViolations: []chatsanitize.AnthropicProviderToolHistoryViolation{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + remove, violations := chatsanitize.AnthropicProviderToolPartsToRemove( + tc.provider, + tc.parts, + ) + require.NotNil(t, remove) + + gotRemove := make([]int, 0, len(remove)) + for partIndex := range remove { + gotRemove = append(gotRemove, partIndex) + } + require.ElementsMatch(t, tc.wantRemove, gotRemove) + require.ElementsMatch(t, tc.wantViolations, violations) + }) + } +} + +func TestValidateAnthropicProviderToolHistory(t *testing.T) { + t.Parallel() + + providerCall := func(id string) fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: id, + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + } + } + providerResult := func(id string) fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: id, + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + ProviderExecuted: true, + ProviderOptions: validWebSearchProviderOptionsForTest(), + } + } + + testCases := []struct { + name string + messages []fantasy.Message + want []chatsanitize.AnthropicProviderToolHistoryViolation + }{ + { + name: "orphan result", + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "summary"}, + providerResult("srvtoolu_orphan"), + }, + }}, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{{ + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_orphan", + Reason: "provider_executed_result_without_call", + }}, + }, + { + name: "result before call", + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerResult("srvtoolu_search"), + providerCall("srvtoolu_search"), + }, + }}, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_search", + Reason: "provider_executed_result_before_call", + }, + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_search", + Reason: "provider_executed_result_before_call", + }, + }, + }, + { + name: "duplicate ID", + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + providerResult("srvtoolu_duplicate"), + }, + }}, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + { + MessageIndex: 0, + PartIndex: 2, + ID: "srvtoolu_duplicate", + Reason: "duplicate_provider_executed_id", + }, + }, + }, + { + name: "invalid call structure", + messages: []fantasy.Message{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_bad_json", + ToolName: "web_search", + Input: `{"query":`, + ProviderExecuted: true, + }, + providerResult("srvtoolu_bad_json"), + }, + }}, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_bad_json", + Reason: "invalid_provider_executed_tool_call", + }, + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_bad_json", + Reason: "invalid_provider_executed_tool_call", + }, + }, + }, + { + name: "mismatched provider flags", + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + providerCall("srvtoolu_provider_call"), + fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_provider_call", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + }, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_provider_result", + ToolName: "read_file", + Input: `{"path":"main.go"}`, + }, + providerResult("srvtoolu_provider_result"), + }, + }, + }, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 0, + ID: "srvtoolu_provider_call", + Reason: "provider_executed_call_without_result", + }, + { + MessageIndex: 1, + PartIndex: 1, + ID: "srvtoolu_provider_result", + Reason: "provider_executed_result_without_call", + }, + }, + }, + { + name: "provider blocks outside assistant", + messages: []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "search"}, + providerCall("srvtoolu_user"), + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + providerResult("srvtoolu_tool"), + }, + }, + }, + want: []chatsanitize.AnthropicProviderToolHistoryViolation{ + { + MessageIndex: 0, + PartIndex: 1, + ID: "srvtoolu_user", + Reason: "provider_executed_block_outside_assistant", + }, + { + MessageIndex: 1, + PartIndex: 0, + ID: "srvtoolu_tool", + Reason: "provider_executed_block_outside_assistant", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + violations := chatsanitize.ValidateAnthropicProviderToolHistory(tc.messages) + require.ElementsMatch(t, tc.want, violations) + }) + } +} + +func TestAnthropicProviderToolSerializationHelpers(t *testing.T) { + t.Parallel() + + validCall := func() fantasy.ToolCallPart { + return fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_search", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + } + } + validResult := func() fantasy.ToolResultPart { + return fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_search", + Output: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}, + ProviderExecuted: true, + ProviderOptions: validWebSearchProviderOptionsForTest(), + } + } + + require.True(t, chatsanitize.IsAllowedAnthropicProviderToolName("web_search")) + require.False(t, chatsanitize.IsAllowedAnthropicProviderToolName("code_execution")) + + callPointer := validCall() + var nilCall *fantasy.ToolCallPart + callTests := []struct { + name string + part fantasy.MessagePart + want bool + }{ + { + name: "valid value", + part: validCall(), + want: true, + }, + { + name: "valid pointer", + part: &callPointer, + want: true, + }, + { + name: "nil typed pointer", + part: nilCall, + }, + { + name: "unrelated concrete message part", + part: testSourceMessagePart{id: "source-1"}, + }, + { + name: "provider executed false", + part: func() fantasy.ToolCallPart { + call := validCall() + call.ProviderExecuted = false + return call + }(), + }, + { + name: "empty ID", + part: func() fantasy.ToolCallPart { + call := validCall() + call.ToolCallID = "" + return call + }(), + }, + { + name: "whitespace ID", + part: func() fantasy.ToolCallPart { + call := validCall() + call.ToolCallID = " " + return call + }(), + }, + { + name: "empty tool name", + part: func() fantasy.ToolCallPart { + call := validCall() + call.ToolName = "" + return call + }(), + }, + { + name: "unsupported tool name", + part: func() fantasy.ToolCallPart { + call := validCall() + call.ToolName = "code_execution" + return call + }(), + }, + { + name: "invalid JSON input", + part: func() fantasy.ToolCallPart { + call := validCall() + call.Input = `{"query":` + return call + }(), + }, + } + for _, tc := range callTests { + t.Run("call "+tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, chatsanitize.IsSerializableAnthropicProviderToolCall(tc.part)) + }) + } + + resultPointer := validResult() + var nilResult *fantasy.ToolResultPart + resultTests := []struct { + name string + part fantasy.MessagePart + matchedCall fantasy.MessagePart + want bool + }{ + { + name: "valid value", + part: validResult(), + matchedCall: validCall(), + want: true, + }, + { + name: "valid pointer", + part: &resultPointer, + matchedCall: &callPointer, + want: true, + }, + { + name: "nil typed pointer", + part: nilResult, + matchedCall: validCall(), + }, + { + name: "unrelated concrete message part", + part: testSourceMessagePart{id: "source-1"}, + matchedCall: validCall(), + }, + { + name: "provider executed false", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ProviderExecuted = false + return result + }(), + matchedCall: validCall(), + }, + { + name: "empty result ID", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ToolCallID = "" + return result + }(), + matchedCall: validCall(), + }, + { + name: "mismatched result ID", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ToolCallID = "srvtoolu_other" + return result + }(), + matchedCall: validCall(), + }, + { + name: "nil output with metadata", + part: func() fantasy.ToolResultPart { + result := validResult() + result.Output = nil + return result + }(), + matchedCall: validCall(), + want: true, + }, + { + name: "empty text output with metadata", + part: func() fantasy.ToolResultPart { + result := validResult() + result.Output = fantasy.ToolResultOutputContentText{} + return result + }(), + matchedCall: validCall(), + want: true, + }, + { + name: "missing metadata", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ProviderOptions = nil + return result + }(), + matchedCall: validCall(), + }, + { + name: "nil metadata", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ProviderOptions = fantasy.ProviderOptions{ + fantasyanthropic.Name: nil, + } + return result + }(), + matchedCall: validCall(), + }, + { + name: "wrong metadata type", + part: func() fantasy.ToolResultPart { + result := validResult() + result.ProviderOptions = fantasy.ProviderOptions{ + fantasyanthropic.Name: &fantasyanthropic.ProviderOptions{}, + } + return result + }(), + matchedCall: validCall(), + }, + { + name: "matched call is not serializable", + part: validResult(), + matchedCall: func() fantasy.ToolCallPart { + call := validCall() + call.Input = `{"query":` + return call + }(), + }, + { + name: "matched call is unrelated part", + part: validResult(), + matchedCall: testSourceMessagePart{id: "source-1"}, + }, + } + for _, tc := range resultTests { + t.Run("result "+tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, chatsanitize.IsSerializableAnthropicProviderToolResult(tc.part, tc.matchedCall)) + }) + } +} + +func TestAnthropicToolResultOutputText(t *testing.T) { + t.Parallel() + + textPointer := fantasy.ToolResultOutputContentText{Text: "pointer text"} + errorPointer := fantasy.ToolResultOutputContentError{Error: xerrors.New("pointer error")} + mediaPointer := fantasy.ToolResultOutputContentMedia{Text: "pointer media"} + var nilTextPointer *fantasy.ToolResultOutputContentText + var nilErrorPointer *fantasy.ToolResultOutputContentError + var nilMediaPointer *fantasy.ToolResultOutputContentMedia + + testCases := []struct { + name string + output fantasy.ToolResultOutputContent + want string + }{ + { + name: "text value", + output: fantasy.ToolResultOutputContentText{Text: "text value"}, + want: "text value", + }, + { + name: "text pointer", + output: &textPointer, + want: "pointer text", + }, + { + name: "nil text pointer", + output: nilTextPointer, + }, + { + name: "error value", + output: fantasy.ToolResultOutputContentError{Error: xerrors.New("error value")}, + want: "error value", + }, + { + name: "error pointer", + output: &errorPointer, + want: "pointer error", + }, + { + name: "nil error pointer", + output: nilErrorPointer, + }, + { + name: "error value with nil error", + output: fantasy.ToolResultOutputContentError{ + Error: nil, + }, + }, + { + name: "media value", + output: fantasy.ToolResultOutputContentMedia{Text: "media value"}, + want: "media value", + }, + { + name: "media pointer", + output: &mediaPointer, + want: "pointer media", + }, + { + name: "nil media pointer", + output: nilMediaPointer, + }, + { + name: "media value without text", + output: fantasy.ToolResultOutputContentMedia{ + Data: "base64", + MediaType: "image/png", + }, + }, + { + name: "nil output", + output: nil, + }, + { + name: "json fallback", + output: testToolResultOutput{Value: "custom"}, + want: `{"value":"custom"}`, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, chatsanitize.AnthropicToolResultOutputText(tc.output)) + }) + } +}