mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix: use upstream fantasy fix for store=false replay (#23368)
This commit is contained in:
Michael Suchacz
@@ -422,6 +422,160 @@ func TestOpenAIReasoningRoundTrip(t *testing.T) {
|
||||
t.Log("OpenAI reasoning round-trip test passed.")
|
||||
}
|
||||
|
||||
// TestOpenAIReasoningRoundTripStoreFalse is an integration test that verifies
|
||||
// follow-up messages succeed when reasoning items were created with
|
||||
// store: false, where OpenAI response item IDs are ephemeral and are not
|
||||
// persisted on OpenAI's servers. It sends a query to a reasoning model,
|
||||
// waits for completion, then sends a follow-up message to ensure chatd can
|
||||
// reconstruct the conversation without relying on persisted provider item IDs.
|
||||
//
|
||||
// The test guards against the prior failure mode where the follow-up request
|
||||
// was rejected with an error like:
|
||||
//
|
||||
// Item with id 'msg_xxx' not found. Items are not persisted when
|
||||
// store is set to false.
|
||||
//
|
||||
// The test requires OPENAI_API_KEY to be set.
|
||||
func TestOpenAIReasoningRoundTripStoreFalse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
apiKey := os.Getenv("OPENAI_API_KEY")
|
||||
if apiKey == "" {
|
||||
t.Skip("OPENAI_API_KEY not set; skipping OpenAI integration test")
|
||||
}
|
||||
baseURL := os.Getenv("OPENAI_BASE_URL")
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitSuperLong)
|
||||
|
||||
// Stand up a full coderd with the agents experiment.
|
||||
deploymentValues := coderdtest.DeploymentValues(t)
|
||||
deploymentValues.Experiments = []string{string(codersdk.ExperimentAgents)}
|
||||
client := coderdtest.New(t, &coderdtest.Options{
|
||||
DeploymentValues: deploymentValues,
|
||||
})
|
||||
_ = coderdtest.CreateFirstUser(t, client)
|
||||
|
||||
// Configure an OpenAI provider with the real API key.
|
||||
_, err := client.CreateChatProvider(ctx, codersdk.CreateChatProviderConfigRequest{
|
||||
Provider: "openai",
|
||||
APIKey: apiKey,
|
||||
BaseURL: baseURL,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a model config for a reasoning model with Store: false.
|
||||
// Using o4-mini because it always produces reasoning items.
|
||||
contextLimit := int64(200000)
|
||||
isDefault := true
|
||||
reasoningSummary := "auto"
|
||||
_, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
Provider: "openai",
|
||||
Model: "o4-mini",
|
||||
ContextLimit: &contextLimit,
|
||||
IsDefault: &isDefault,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ProviderOptions: &codersdk.ChatModelProviderOptions{
|
||||
OpenAI: &codersdk.ChatModelOpenAIProviderOptions{
|
||||
Store: ptr.Ref(false),
|
||||
ReasoningSummary: &reasoningSummary,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// --- Step 1: Send a message that triggers reasoning ---
|
||||
t.Log("Creating chat with reasoning query...")
|
||||
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
|
||||
Content: []codersdk.ChatInputPart{
|
||||
{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "What is 2+2? Be brief.",
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Logf("Chat created: %s (status=%s)", chat.ID, chat.Status)
|
||||
|
||||
// Stream events until the chat reaches a terminal status.
|
||||
events, closer, err := client.StreamChat(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
defer closer.Close()
|
||||
|
||||
waitForChatDone(ctx, t, events, "step 1")
|
||||
|
||||
// Verify the chat completed and messages were persisted.
|
||||
chatData, err := client.GetChat(ctx, chat.ID)
|
||||
require.NoError(t, err)
|
||||
chatMsgs, err := client.GetChatMessages(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
t.Logf("Chat status after step 1: %s, messages: %d",
|
||||
chatData.Status, len(chatMsgs.Messages))
|
||||
logMessages(t, chatMsgs.Messages)
|
||||
|
||||
require.Equal(t, codersdk.ChatStatusWaiting, chatData.Status,
|
||||
"chat should be in waiting status after step 1")
|
||||
|
||||
// Verify the assistant message has reasoning content.
|
||||
assistantMsg := findAssistantWithText(t, chatMsgs.Messages)
|
||||
require.NotNil(t, assistantMsg,
|
||||
"expected an assistant message with text content after step 1")
|
||||
|
||||
partTypes := partTypeSet(assistantMsg.Content)
|
||||
require.Contains(t, partTypes, codersdk.ChatMessagePartTypeReasoning,
|
||||
"assistant message should contain reasoning parts from o4-mini")
|
||||
require.Contains(t, partTypes, codersdk.ChatMessagePartTypeText,
|
||||
"assistant message should contain a text part")
|
||||
|
||||
// --- Step 2: Send a follow-up message ---
|
||||
// This is the critical test: when Store is false, item IDs are
|
||||
// ephemeral and cannot be looked up from OpenAI later.
|
||||
t.Log("Sending follow-up message...")
|
||||
_, err = client.CreateChatMessage(ctx, chat.ID,
|
||||
codersdk.CreateChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{
|
||||
{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "And what is 3+3? Be brief.",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
require.NotContains(t, err.Error(),
|
||||
"Items are not persisted when store is set to false.",
|
||||
"follow-up should reconstruct ephemeral reasoning items instead of sending stale provider item IDs")
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// Stream the follow-up response.
|
||||
events2, closer2, err := client.StreamChat(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
defer closer2.Close()
|
||||
|
||||
waitForChatDone(ctx, t, events2, "step 2")
|
||||
|
||||
// Verify the follow-up completed and produced content.
|
||||
chatData2, err := client.GetChat(ctx, chat.ID)
|
||||
require.NoError(t, err)
|
||||
chatMsgs2, err := client.GetChatMessages(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
t.Logf("Chat status after step 2: %s, messages: %d",
|
||||
chatData2.Status, len(chatMsgs2.Messages))
|
||||
logMessages(t, chatMsgs2.Messages)
|
||||
|
||||
require.Equal(t, codersdk.ChatStatusWaiting, chatData2.Status,
|
||||
"chat should be in waiting status after step 2")
|
||||
require.Greater(t, len(chatMsgs2.Messages), len(chatMsgs.Messages),
|
||||
"follow-up should have added more messages")
|
||||
|
||||
// The last assistant message should have text.
|
||||
lastAssistant := findLastAssistantWithText(t, chatMsgs2.Messages)
|
||||
require.NotNil(t, lastAssistant,
|
||||
"expected an assistant message with text in the follow-up")
|
||||
|
||||
t.Log("OpenAI reasoning round-trip store=false test passed.")
|
||||
}
|
||||
|
||||
// partTypeSet returns the set of part types present in a message.
|
||||
func partTypeSet(parts []codersdk.ChatMessagePart) map[codersdk.ChatMessagePartType]struct{} {
|
||||
set := make(map[codersdk.ChatMessagePartType]struct{}, len(parts))
|
||||
|
||||
@@ -76,11 +76,11 @@ replace github.com/aquasecurity/trivy => github.com/coder/trivy v0.0.0-202603091
|
||||
// https://github.com/spf13/afero/pull/487
|
||||
replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713-f06e86036696
|
||||
|
||||
// Forked for two reasons:
|
||||
// 1) Adds thinking effort to Anthropic provider
|
||||
// 2) Downgraded to Go 1.25 due to issue with Windows CI
|
||||
// https://github.com/kylecarbs/fantasy/compare/main...kylecarbs:fantasy:cj/go1.25
|
||||
replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4
|
||||
// Forked from kylecarbs/fantasy (cj/go1.25 branch) which adds:
|
||||
// 1) Anthropic computer use + thinking effort
|
||||
// 2) Go 1.25 downgrade for Windows CI compat
|
||||
// 3) ibetitsmike/fantasy#4 — skip ephemeral replay items when store=false
|
||||
replace charm.land/fantasy => github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a
|
||||
|
||||
replace github.com/charmbracelet/anthropic-sdk-go => github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab
|
||||
|
||||
|
||||
@@ -815,8 +815,8 @@ github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab h1:5UMY
|
||||
github.com/kylecarbs/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab/go.mod h1:hqlYqR7uPKOKfnNeicUbZp0Ps0GeYFlKYtwh5HGDCx8=
|
||||
github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3 h1:Z9/bo5PSeMutpdiKYNt/TTSfGM1Ll0naj3QzYX9VxTc=
|
||||
github.com/kylecarbs/chroma/v2 v2.0.0-20240401211003-9e036e0631f3/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
|
||||
github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4 h1:DO0b5G0yfrtKkzlJofnPxEcRlS157lzQyPiUSDkzfcU=
|
||||
github.com/kylecarbs/fantasy v0.0.0-20260319151840-18e18e661ed4/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk=
|
||||
github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a h1:0BNb2AYtzWYH0xAscOICIRInvQylXlF2tuxOjmWhlvE=
|
||||
github.com/kylecarbs/fantasy v0.0.0-20260320175225-7bcfc3d2021a/go.mod h1:I/i6LkVAWnSVdFZ37SbcR0IZz6eBhu4P9IK3XHTX6Gk=
|
||||
github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae h1:xlFZNX4nnxpj9Cf6mTwD3pirXGNtBJ/6COsf9iZmsL0=
|
||||
github.com/kylecarbs/openai-go/v3 v3.0.0-20260319113850-9477dcaedcae/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
|
||||
github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e h1:OP0ZMFeZkUnOzTFRfpuK3m7Kp4fNvC6qN+exwj7aI4M=
|
||||
|
||||
Reference in New Issue
Block a user