diff --git a/coderd/chatd/integration_test.go b/coderd/chatd/integration_test.go index 6576677fe6..49e3d8df44 100644 --- a/coderd/chatd/integration_test.go +++ b/coderd/chatd/integration_test.go @@ -272,6 +272,156 @@ func logMessages(t *testing.T, msgs []codersdk.ChatMessage) { } } +// TestOpenAIReasoningRoundTrip is an integration test that verifies +// reasoning items from OpenAI's Responses API survive the full +// persist → reconstruct → re-send cycle when Store: true. It sends +// a query to a reasoning model, waits for completion, then sends a +// follow-up message. If reasoning items are sent back without their +// required following output item, the API rejects the second request: +// +// Item 'rs_xxx' of type 'reasoning' was provided without its +// required following item. +// +// The test requires OPENAI_API_KEY to be set. +func TestOpenAIReasoningRoundTrip(t *testing.T) { + t.Parallel() + + apiKey := os.Getenv("OPENAI_API_KEY") + if apiKey == "" { + t.Skip("OPENAI_API_KEY not set; skipping OpenAI integration test") + } + baseURL := os.Getenv("OPENAI_BASE_URL") + + ctx := testutil.Context(t, testutil.WaitSuperLong) + + // Stand up a full coderd with the agents experiment. + deploymentValues := coderdtest.DeploymentValues(t) + deploymentValues.Experiments = []string{string(codersdk.ExperimentAgents)} + client := coderdtest.New(t, &coderdtest.Options{ + DeploymentValues: deploymentValues, + }) + _ = coderdtest.CreateFirstUser(t, client) + + // Configure an OpenAI provider with the real API key. + _, err := client.CreateChatProvider(ctx, codersdk.CreateChatProviderConfigRequest{ + Provider: "openai", + APIKey: apiKey, + BaseURL: baseURL, + }) + require.NoError(t, err) + + // Create a model config for a reasoning model with Store: true + // (the default). Using o4-mini because it always produces + // reasoning items. + contextLimit := int64(200000) + isDefault := true + reasoningSummary := "auto" + _, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + Provider: "openai", + Model: "o4-mini", + ContextLimit: &contextLimit, + IsDefault: &isDefault, + ModelConfig: &codersdk.ChatModelCallConfig{ + ProviderOptions: &codersdk.ChatModelProviderOptions{ + OpenAI: &codersdk.ChatModelOpenAIProviderOptions{ + Store: ptr.Ref(true), + ReasoningSummary: &reasoningSummary, + }, + }, + }, + }) + require.NoError(t, err) + + // --- Step 1: Send a message that triggers reasoning --- + t.Log("Creating chat with reasoning query...") + chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "What is 2+2? Be brief.", + }, + }, + }) + require.NoError(t, err) + t.Logf("Chat created: %s (status=%s)", chat.ID, chat.Status) + + // Stream events until the chat reaches a terminal status. + events, closer, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer.Close() + + waitForChatDone(ctx, t, events, "step 1") + + // Verify the chat completed and messages were persisted. + chatData, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + chatMsgs, err := client.GetChatMessages(ctx, chat.ID, nil) + require.NoError(t, err) + t.Logf("Chat status after step 1: %s, messages: %d", + chatData.Status, len(chatMsgs.Messages)) + logMessages(t, chatMsgs.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData.Status, + "chat should be in waiting status after step 1") + + // Verify the assistant message has reasoning content. + assistantMsg := findAssistantWithText(t, chatMsgs.Messages) + require.NotNil(t, assistantMsg, + "expected an assistant message with text content after step 1") + + partTypes := partTypeSet(assistantMsg.Content) + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeReasoning, + "assistant message should contain reasoning parts from o4-mini") + require.Contains(t, partTypes, codersdk.ChatMessagePartTypeText, + "assistant message should contain a text part") + + // --- Step 2: Send a follow-up message --- + // This is the critical test: if reasoning items are sent back + // without their required following item, the API will reject + // the request with: + // Item 'rs_xxx' of type 'reasoning' was provided without its + // required following item. + t.Log("Sending follow-up message...") + _, err = client.CreateChatMessage(ctx, chat.ID, + codersdk.CreateChatMessageRequest{ + Content: []codersdk.ChatInputPart{ + { + Type: codersdk.ChatInputPartTypeText, + Text: "And what is 3+3? Be brief.", + }, + }, + }) + require.NoError(t, err) + + // Stream the follow-up response. + events2, closer2, err := client.StreamChat(ctx, chat.ID, nil) + require.NoError(t, err) + defer closer2.Close() + + waitForChatDone(ctx, t, events2, "step 2") + + // Verify the follow-up completed and produced content. + chatData2, err := client.GetChat(ctx, chat.ID) + require.NoError(t, err) + chatMsgs2, err := client.GetChatMessages(ctx, chat.ID, nil) + require.NoError(t, err) + t.Logf("Chat status after step 2: %s, messages: %d", + chatData2.Status, len(chatMsgs2.Messages)) + logMessages(t, chatMsgs2.Messages) + + require.Equal(t, codersdk.ChatStatusWaiting, chatData2.Status, + "chat should be in waiting status after step 2") + require.Greater(t, len(chatMsgs2.Messages), len(chatMsgs.Messages), + "follow-up should have added more messages") + + // The last assistant message should have text. + lastAssistant := findLastAssistantWithText(t, chatMsgs2.Messages) + require.NotNil(t, lastAssistant, + "expected an assistant message with text in the follow-up") + + t.Log("OpenAI reasoning round-trip test passed.") +} + // partTypeSet returns the set of part types present in a message. func partTypeSet(parts []codersdk.ChatMessagePart) map[codersdk.ChatMessagePartType]struct{} { set := make(map[codersdk.ChatMessagePartType]struct{}, len(parts)) diff --git a/go.mod b/go.mod index 105aae77dd..f13c4fd748 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713 // 1) Adds thinking effort to Anthropic provider // 2) Downgraded to Go 1.25 due to issue with Windows CI // https://github.com/kylecarbs/fantasy/compare/main...kylecarbs:fantasy:cj/go1.25 -replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260319114014-12345ae15482 +replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4 replace github.com/charmbracelet/anthropic-sdk-go => github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab diff --git a/go.sum b/go.sum index 448481850d..492856ad9c 100644 --- a/go.sum +++ b/go.sum @@ -815,8 +815,8 @@ github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab h1:5UMY github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab/go.mod h1:hqlYqR7uPKOKfnNeicUbZp0Ps0GeYFlKYtwh5HGDCx8= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3 h1:Z9/bo5PSeMutpdiKYNt/TTSfGM1Ll0naj3QzYX9VxTc= github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk= -github.com/kylecarbs/fantasy v0.0.0-20260319114014-12345ae15482 h1:7e3WA19EH4UCT7A1hEmSYEYMntIsqMIQ1RvNCg5WLaA= -github.com/kylecarbs/fantasy v0.0.0-20260319114014-12345ae15482/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk= +github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4 h1:DO0b5G0yfrtKkzlJofnPxEcRlS157lzQyPiUSDkzfcU= +github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk= github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae h1:xlFZNX4nnxpj9Cf6mTwD3pirXGNtBJ/6COsf9iZmsL0= github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e h1:OP0ZMFeZkUnOzTFRfpuK3m7Kp4fNvC6qN+exwj7aI4M=