diff --git a/coderd/chatd/integration_test.go b/coderd/chatd/integration_test.go index 49e3d8df44..8c9ea9e17e 100644 --- a/coderd/chatd/integration_test.go +++ b/coderd/chatd/integration_test.go @@ -422,6 +422,160 @@ func TestOpenAIReasoningRoundTrip(t *testing.T) { t.Log("OpenAI reasoning round-trip test passed.") } +// TestOpenAIReasoningRoundTripStoreFalse is an integration test that verifies +// follow-up messages succeed when reasoning items were created with +// store: false, where OpenAI response item IDs are ephemeral and are not +// persisted on OpenAI's servers. It sends a query to a reasoning model, +// waits for completion, then sends a follow-up message to ensure chatd can +// reconstruct the conversation without relying on persisted provider item IDs. +// +// The test guards against the prior failure mode where the follow-up request +// was rejected with an error like: +// +// Item with id 'msg_xxx' not found. Items are not persisted when +// store is set to false. +// +// The test requires OPENAI_API_KEY to be set. +func TestOpenAIReasoningRoundTripStoreFalse(t *testing.T) { + t.Parallel() + + apiKey := os.Getenv("OPENAI_API_KEY") + if apiKey == "" { + t.Skip("OPENAI_API_KEY not set; skipping OpenAI integration test") + } + baseURL := os.Getenv("OPENAI_BASE_URL") + + ctx := testutil.Context(t, testutil.WaitSuperLong) + + // Stand up a full coderd with the agents experiment. + deploymentValues := coderdtest.DeploymentValues(t) + deploymentValues.Experiments = []string{string(codersdk.ExperimentAgents)} + client := coderdtest.New(t, &coderdtest.Options{ + DeploymentValues: deploymentValues, + }) + _ = coderdtest.CreateFirstUser(t, client) + + // Configure an OpenAI provider with the real API key. + _, err := client.CreateChatProvider(ctx, codersdk.CreateChatProviderConfigRequest{ + Provider: "openai", + APIKey: apiKey, + BaseURL: baseURL, + }) + require.NoError(t, err) + + // Create a model config for a reasoning model with Store: false. + // Using o4-mini because it always produces reasoning items. + contextLimit := int64(200000) + isDefault := true + reasoningSummary := "auto" + _, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + Provider: "openai", + Model: "o4-mini", + ContextLimit: &contextLimit, + IsDefault: &isDefault, + ModelConfig: &codersdk.ChatModelCallConfig{ + ProviderOptions: &codersdk.ChatModelProviderOptions{ + OpenAI: &codersdk.ChatModelOpenAIProviderOptions{ + Store: ptr.Ref(false), + ReasoningSummary: &reasoningSummary, + }, + }, + }, + }) + require.NoError(t, err) + + // --- Step 1: Send a message that triggers reasoning --- + t.Log("Creating chat with reasoning query...") + chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "What is 2+2? Be brief.", + }, + }, + }) + require.NoError(t, err) + t.Logf("Chat created: %s (status=%s)", chat.ID, chat.Status) + + // Stream events until the chat reaches a terminal status. + events, closer, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer.Close() + + waitForChatDone(ctx, t, events, "step 1") + + // Verify the chat completed and messages were persisted. + chatData, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + chatMsgs, err := client.GetChatMessages(ctx, chat.ID, nil) + require.NoError(t, err) + t.Logf("Chat status after step 1: %s, messages: %d", + chatData.Status, len(chatMsgs.Messages)) + logMessages(t, chatMsgs.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData.Status, + "chat should be in waiting status after step 1") + + // Verify the assistant message has reasoning content. + assistantMsg := findAssistantWithText(t, chatMsgs.Messages) + require.NotNil(t, assistantMsg, + "expected an assistant message with text content after step 1") + + partTypes := partTypeSet(assistantMsg.Content) + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeReasoning, + "assistant message should contain reasoning parts from o4-mini") + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeText, + "assistant message should contain a text part") + + // --- Step 2: Send a follow-up message --- + // This is the critical test: when Store is false, item IDs are + // ephemeral and cannot be looked up from OpenAI later. + t.Log("Sending follow-up message...") + _, err = client.CreateChatMessage(ctx, chat.ID, + codersdk.CreateChatMessageRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "And what is 3+3? Be brief.", + }, + }, + }) + if err != nil { + require.NotContains(t, err.Error(), + "Items are not persisted when store is set to false.", + "follow-up should reconstruct ephemeral reasoning items instead of sending stale provider item IDs") + } + require.NoError(t, err) + + // Stream the follow-up response. + events2, closer2, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer2.Close() + + waitForChatDone(ctx, t, events2, "step 2") + + // Verify the follow-up completed and produced content. + chatData2, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + chatMsgs2, err := client.GetChatMessages(ctx, chat.ID, nil) + require.NoError(t, err) + t.Logf("Chat status after step 2: %s, messages: %d", + chatData2.Status, len(chatMsgs2.Messages)) + logMessages(t, chatMsgs2.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData2.Status, + "chat should be in waiting status after step 2") + require.Greater(t, len(chatMsgs2.Messages), len(chatMsgs.Messages), + "follow-up should have added more messages") + + // The last assistant message should have text. + lastAssistant := findLastAssistantWithText(t, chatMsgs2.Messages) + require.NotNil(t, lastAssistant, + "expected an assistant message with text in the follow-up") + + t.Log("OpenAI reasoning round-trip store=false test passed.") +} + // partTypeSet returns the set of part types present in a message. func partTypeSet(parts []codersdk.ChatMessagePart) map[codersdk.ChatMessagePartType]struct{} { set := make(map[codersdk.ChatMessagePartType]struct{}, len(parts)) diff --git a/go.mod b/go.mod index ae21fcffb8..1ad6d9f5cc 100644 --- a/go.mod +++ b/go.mod @@ -76,11 +76,11 @@ replace github.com/aquasecurity/trivy => github.com/coder/trivy v0.0.0-202603091 // https://github.com/spf13/afero/pull/487 replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713-f06e86036696 -// Forked for two reasons: -// 1) Adds thinking effort to Anthropic provider -// 2) Downgraded to Go 1.25 due to issue with Windows CI -// https://github.com/kylecarbs/fantasy/compare/main...kylecarbs:fantasy:cj/go1.25 -replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4 +// Forked from kylecarbs/fantasy (cj/go1.25 branch) which adds: +// 1) Anthropic computer use + thinking effort +// 2) Go 1.25 downgrade for Windows CI compat +// 3) ibetitsmike/fantasy#4 — skip ephemeral replay items when store=false +replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a replace github.com/charmbracelet/anthropic-sdk-go => github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab diff --git a/go.sum b/go.sum index d551a1c468..17517794b5 100644 --- a/go.sum +++ b/go.sum @@ -815,8 +815,8 @@ github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab h1:5UMY github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab/go.mod h1:hqlYqR7uPKOKfnNeicUbZp0Ps0GeYFlKYtwh5HGDCx8= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3 h1:Z9/bo5PSeMutpdiKYNt/TTSfGM1Ll0naj3QzYX9VxTc= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk= -github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4 h1:DO0b5G0yfrtKkzlJofnPxEcRlS157lzQyPiUSDkzfcU= -github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk= +github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a h1:0BNb2AYtzWYH0xAscOICIRInvQylXlF2tuxOjmWhlvE= +github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk= github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae h1:xlFZNX4nnxpj9Cf6mTwD3pirXGNtBJ/6COsf9iZmsL0= github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e h1:OP0ZMFeZkUnOzTFRfpuK3m7Kp4fNvC6qN+exwj7aI4M=