diff --git a/coderd/chatd/integration_test.go b/coderd/chatd/integration_test.go new file mode 100644 index 0000000000..be65ded0f9 --- /dev/null +++ b/coderd/chatd/integration_test.go @@ -0,0 +1,278 @@ +package chatd_test + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/util/ptr" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" +) + +// TestAnthropicWebSearchRoundTrip is an integration test that verifies +// provider-executed tool results (web_search) survive the full +// persist → reconstruct → re-send cycle. It sends a query that +// triggers Anthropic's web_search server tool, waits for completion, +// then sends a follow-up message. If the PE tool result was lost or +// corrupted during persistence, Anthropic rejects the second request: +// +// web_search tool use with id srvtoolu_... was found without a +// corresponding web_search_tool_result block +// +// The test requires ANTHROPIC_API_KEY to be set. +func TestAnthropicWebSearchRoundTrip(t *testing.T) { + t.Parallel() + + apiKey := os.Getenv("ANTHROPIC_API_KEY") + if apiKey == "" { + t.Skip("ANTHROPIC_API_KEY not set; skipping Anthropic integration test") + } + baseURL := os.Getenv("ANTHROPIC_BASE_URL") + + ctx := testutil.Context(t, testutil.WaitSuperLong) + + // Stand up a full coderd with the agents experiment. + deploymentValues := coderdtest.DeploymentValues(t) + deploymentValues.Experiments = []string{string(codersdk.ExperimentAgents)} + client := coderdtest.New(t, &coderdtest.Options{ + DeploymentValues: deploymentValues, + }) + _ = coderdtest.CreateFirstUser(t, client) + + // Configure an Anthropic provider with the real API key. + _, err := client.CreateChatProvider(ctx, codersdk.CreateChatProviderConfigRequest{ + Provider: "anthropic", + APIKey: apiKey, + BaseURL: baseURL, + }) + require.NoError(t, err) + + // Create a model config that enables web_search. + contextLimit := int64(200000) + isDefault := true + _, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + Provider: "anthropic", + Model: "claude-sonnet-4-20250514", + ContextLimit: &contextLimit, + IsDefault: &isDefault, + ModelConfig: &codersdk.ChatModelCallConfig{ + ProviderOptions: &codersdk.ChatModelProviderOptions{ + Anthropic: &codersdk.ChatModelAnthropicProviderOptions{ + WebSearchEnabled: ptr.Ref(true), + }, + }, + }, + }) + require.NoError(t, err) + + // --- Step 1: Send a message that triggers web_search --- + t.Log("Creating chat with web search query...") + chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "What is the current weather in San Francisco right now? Use web search to find out.", + }, + }, + }) + require.NoError(t, err) + t.Logf("Chat created: %s (status=%s)", chat.ID, chat.Status) + + // Stream events until the chat reaches a terminal status. + events, closer, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer.Close() + + waitForChatDone(ctx, t, events, "step 1") + + // Verify the chat completed and messages were persisted. + chatData, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + t.Logf("Chat status after step 1: %s, messages: %d", + chatData.Chat.Status, len(chatData.Messages)) + logMessages(t, chatData.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData.Chat.Status, + "chat should be in waiting status after step 1") + + // Find the first assistant message and verify it has the + // content parts the UI needs to render web search results: + // tool-call(PE), source, tool-result(PE), and text. + assistantMsg := findAssistantWithText(t, chatData.Messages) + require.NotNil(t, assistantMsg, + "expected an assistant message with text content after step 1") + + partTypes := partTypeSet(assistantMsg.Content) + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeToolCall, + "assistant message should contain a PE tool-call part") + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeSource, + "assistant message should contain source parts for UI citations") + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeToolResult, + "assistant message should contain a PE tool-result part") + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeText, + "assistant message should contain a text part") + + // Verify the PE tool-call is marked as provider-executed. + for _, part := range assistantMsg.Content { + if part.Type == codersdk.ChatMessagePartTypeToolCall { + require.True(t, part.ProviderExecuted, + "web_search tool-call should be provider-executed") + break + } + } + + // --- Step 2: Send a follow-up message --- + // This is the critical test: if PE tool results were lost during + // persistence, the reconstructed conversation will be rejected + // by Anthropic because server_tool_use has no matching + // web_search_tool_result. + t.Log("Sending follow-up message...") + _, err = client.CreateChatMessage(ctx, chat.ID, + codersdk.CreateChatMessageRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "Thanks! What about New York?", + }, + }, + }) + require.NoError(t, err) + + // Stream the follow-up response. + events2, closer2, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer2.Close() + + waitForChatDone(ctx, t, events2, "step 2") + + // Verify the follow-up completed and produced content. + chatData2, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + t.Logf("Chat status after step 2: %s, messages: %d", + chatData2.Chat.Status, len(chatData2.Messages)) + logMessages(t, chatData2.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData2.Chat.Status, + "chat should be in waiting status after step 2") + require.Greater(t, len(chatData2.Messages), len(chatData.Messages), + "follow-up should have added more messages") + + // The last assistant message should have text. + lastAssistant := findLastAssistantWithText(t, chatData2.Messages) + require.NotNil(t, lastAssistant, + "expected an assistant message with text in the follow-up") + + t.Log("Anthropic web_search round-trip test passed.") +} + +// waitForChatDone drains the event stream until the chat reaches +// a terminal status (waiting, completed, or error). +func waitForChatDone( + ctx context.Context, + t *testing.T, + events <-chan codersdk.ChatStreamEvent, + label string, +) { + t.Helper() + for { + select { + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for "+label+" completion") + case event, ok := <-events: + if !ok { + return + } + switch event.Type { + case codersdk.ChatStreamEventTypeError: + if event.Error != nil { + t.Logf("[%s] stream error: %s", label, event.Error.Message) + } + case codersdk.ChatStreamEventTypeStatus: + if event.Status != nil { + t.Logf("[%s] status → %s", label, event.Status.Status) + switch event.Status.Status { + case codersdk.ChatStatusWaiting, + codersdk.ChatStatusCompleted: + return + case codersdk.ChatStatusError: + require.FailNow(t, label+" ended with error status") + } + } + case codersdk.ChatStreamEventTypeMessage: + if event.Message != nil { + t.Logf("[%s] persisted message: role=%s parts=%d", + label, event.Message.Role, len(event.Message.Content)) + } + case codersdk.ChatStreamEventTypeMessagePart: + // Streaming delta — just note it. + if event.MessagePart != nil { + t.Logf("[%s] part: type=%s", + label, event.MessagePart.Part.Type) + } + } + } + } +} + +// findAssistantWithText returns the first assistant message that +// contains a non-empty text part. +func findAssistantWithText(t *testing.T, msgs []codersdk.ChatMessage) *codersdk.ChatMessage { + t.Helper() + for i := range msgs { + if msgs[i].Role != "assistant" { + continue + } + for _, part := range msgs[i].Content { + if part.Type == codersdk.ChatMessagePartTypeText && part.Text != "" { + return &msgs[i] + } + } + } + return nil +} + +// findLastAssistantWithText returns the last assistant message that +// contains a non-empty text part. +func findLastAssistantWithText(t *testing.T, msgs []codersdk.ChatMessage) *codersdk.ChatMessage { + t.Helper() + for i := len(msgs) - 1; i >= 0; i-- { + if msgs[i].Role != "assistant" { + continue + } + for _, part := range msgs[i].Content { + if part.Type == codersdk.ChatMessagePartTypeText && part.Text != "" { + return &msgs[i] + } + } + } + return nil +} + +// logMessages prints a summary of all messages for debugging. +func logMessages(t *testing.T, msgs []codersdk.ChatMessage) { + t.Helper() + for i, msg := range msgs { + types := make([]string, 0, len(msg.Content)) + for _, part := range msg.Content { + s := string(part.Type) + if part.ProviderExecuted { + s += "(PE)" + } + types = append(types, s) + } + t.Logf(" msg[%d] role=%s parts=%v", i, msg.Role, types) + } +} + +// partTypeSet returns the set of part types present in a message. +func partTypeSet(parts []codersdk.ChatMessagePart) map[codersdk.ChatMessagePartType]struct{} { + set := make(map[codersdk.ChatMessagePartType]struct{}, len(parts)) + for _, p := range parts { + set[p.Type] = struct{}{} + } + return set +} diff --git a/go.mod b/go.mod index fb6e0a7b88..84b1784c3b 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,7 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713 // 1) Adds thinking effort to Anthropic provider // 2) Downgraded to Go 1.25 due to issue with Windows CI // https://github.com/kylecarbs/fantasy/compare/main...kylecarbs:fantasy:cj/go1.25 -replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260311152049-8eff3573caec +replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260312154649-e4bbc7bb3054 replace github.com/charmbracelet/anthropic-sdk-go => github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab diff --git a/go.sum b/go.sum index 16fc5c4d77..8a82ba436b 100644 --- a/go.sum +++ b/go.sum @@ -797,8 +797,8 @@ github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab h1:5UMY github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab/go.mod h1:hqlYqR7uPKOKfnNeicUbZp0Ps0GeYFlKYtwh5HGDCx8= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3 h1:Z9/bo5PSeMutpdiKYNt/TTSfGM1Ll0naj3QzYX9VxTc= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk= -github.com/kylecarbs/fantasy v0.0.0-20260311152049-8eff3573caec h1:JBaAQYkOre8gAaQZ6SKiQZ2g+em35WxzHTWtTtFDX5M= -github.com/kylecarbs/fantasy v0.0.0-20260311152049-8eff3573caec/go.mod h1:p6cYJVG8D8AC51MgejAKCMu0myRyQ+vKLuoJQ3biaXo= +github.com/kylecarbs/fantasy v0.0.0-20260312154649-e4bbc7bb3054 h1:PlN4sfr5xfHsw5HnHEHEu6TqMZr1icbwt1I1QeqitmI= +github.com/kylecarbs/fantasy v0.0.0-20260312154649-e4bbc7bb3054/go.mod h1:p6cYJVG8D8AC51MgejAKCMu0myRyQ+vKLuoJQ3biaXo= github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e h1:OP0ZMFeZkUnOzTFRfpuK3m7Kp4fNvC6qN+exwj7aI4M= github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=