mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: add configurable reasoning effort to Coder agents (#26974)
This commit is contained in:
+656
-27
@@ -157,14 +157,29 @@ func insertTestChatQueuedMessage(
|
||||
apiKeyID string,
|
||||
) database.ChatQueuedMessage {
|
||||
t.Helper()
|
||||
return insertTestChatQueuedMessageWithReasoningEffort(ctx, t, db, chatID, content, modelConfigID, apiKeyID, "")
|
||||
}
|
||||
|
||||
func insertTestChatQueuedMessageWithReasoningEffort(
|
||||
ctx context.Context,
|
||||
t testing.TB,
|
||||
db database.Store,
|
||||
chatID uuid.UUID,
|
||||
content json.RawMessage,
|
||||
modelConfigID uuid.UUID,
|
||||
apiKeyID string,
|
||||
reasoningEffort string,
|
||||
) database.ChatQueuedMessage {
|
||||
t.Helper()
|
||||
|
||||
queued, err := db.InsertChatQueuedMessage(
|
||||
dbauthz.AsSystemRestricted(ctx),
|
||||
database.InsertChatQueuedMessageParams{
|
||||
ChatID: chatID,
|
||||
Content: content,
|
||||
ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: modelConfigID != uuid.Nil},
|
||||
APIKeyID: sql.NullString{String: apiKeyID, Valid: apiKeyID != ""},
|
||||
ChatID: chatID,
|
||||
Content: content,
|
||||
ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: modelConfigID != uuid.Nil},
|
||||
ReasoningEffort: database.NullChatReasoningEffort{ChatReasoningEffort: database.ChatReasoningEffort(reasoningEffort), Valid: reasoningEffort != ""},
|
||||
APIKeyID: sql.NullString{String: apiKeyID, Valid: apiKeyID != ""},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
@@ -416,6 +431,65 @@ func TestPostChats(t *testing.T) {
|
||||
requireSDKError(t, err, http.StatusForbidden)
|
||||
})
|
||||
|
||||
t.Run("WithReasoningEffort", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := newChatClientWithDatabase(t)
|
||||
user := coderdtest.CreateFirstUser(t, client.Client)
|
||||
_ = createChatModelConfig(t, client)
|
||||
|
||||
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
|
||||
OrganizationID: user.OrganizationID,
|
||||
Content: []codersdk.ChatInputPart{
|
||||
{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "think hard from the start",
|
||||
},
|
||||
},
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, storedChat.LastReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
messages, err := db.GetChatMessagesByChatID(dbauthz.AsSystemRestricted(ctx), database.GetChatMessagesByChatIDParams{
|
||||
ChatID: chat.ID,
|
||||
AfterID: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
userMsg := findUserMessage(t, messages)
|
||||
require.True(t, userMsg.ReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, userMsg.ReasoningEffort.ChatReasoningEffort)
|
||||
})
|
||||
|
||||
t.Run("RejectsInvalidReasoningEffort", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
user := coderdtest.CreateFirstUser(t, client.Client)
|
||||
_ = createChatModelConfig(t, client)
|
||||
|
||||
_, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
|
||||
OrganizationID: user.OrganizationID,
|
||||
Content: []codersdk.ChatInputPart{
|
||||
{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "hello",
|
||||
},
|
||||
},
|
||||
ReasoningEffort: ptr.Ref(" HIGH "),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
|
||||
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
|
||||
})
|
||||
|
||||
t.Run("HidesSystemPromptMessages", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -3744,6 +3818,118 @@ func TestCreateChatModelConfig(t *testing.T) {
|
||||
)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortStored", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
_ = coderdtest.CreateFirstUser(t, client.Client)
|
||||
|
||||
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
|
||||
|
||||
contextLimit := int64(4096)
|
||||
modelConfig, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &aiProvider.ID,
|
||||
Model: "gpt-4o-mini",
|
||||
ContextLimit: &contextLimit,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref("medium"),
|
||||
Max: ptr.Ref("xhigh"),
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, modelConfig.ModelConfig)
|
||||
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort)
|
||||
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort.Default)
|
||||
require.Equal(t, "medium", *modelConfig.ModelConfig.ReasoningEffort.Default)
|
||||
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort.Max)
|
||||
require.Equal(t, "xhigh", *modelConfig.ModelConfig.ReasoningEffort.Max)
|
||||
require.Equal(t, []string{"none", "minimal", "low", "medium", "high", "xhigh"}, modelConfig.ReasoningEfforts)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortRejectsSingleValue", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
_ = coderdtest.CreateFirstUser(t, client.Client)
|
||||
|
||||
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
|
||||
|
||||
contextLimit := int64(4096)
|
||||
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &aiProvider.ID,
|
||||
Model: "gpt-4o-mini",
|
||||
ContextLimit: &contextLimit,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref("high"),
|
||||
},
|
||||
},
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid model config.", sdkErr.Message)
|
||||
require.Equal(t, "reasoning_effort.default and reasoning_effort.max must both be set", sdkErr.Detail)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortRejectsInvalidValue", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
_ = coderdtest.CreateFirstUser(t, client.Client)
|
||||
|
||||
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
|
||||
|
||||
contextLimit := int64(4096)
|
||||
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &aiProvider.ID,
|
||||
Model: "gpt-4o-mini",
|
||||
ContextLimit: &contextLimit,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref(" HIGH "),
|
||||
Max: ptr.Ref("high"),
|
||||
},
|
||||
},
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid model config.", sdkErr.Message)
|
||||
require.Equal(
|
||||
t,
|
||||
`reasoning_effort.default " HIGH " must be one of none, minimal, low, medium, high, xhigh, max`,
|
||||
sdkErr.Detail,
|
||||
)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortRejectsDefaultAboveMax", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
_ = coderdtest.CreateFirstUser(t, client.Client)
|
||||
|
||||
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
|
||||
|
||||
contextLimit := int64(4096)
|
||||
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &aiProvider.ID,
|
||||
Model: "gpt-4o-mini",
|
||||
ContextLimit: &contextLimit,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref("xhigh"),
|
||||
Max: ptr.Ref("low"),
|
||||
},
|
||||
},
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid model config.", sdkErr.Message)
|
||||
require.Contains(t, sdkErr.Detail, "must not exceed")
|
||||
})
|
||||
|
||||
t.Run("MissingContextLimit", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -6684,6 +6870,137 @@ func TestSendMessageWithModelOverrideUpdatesLastModelConfigID(t *testing.T) {
|
||||
require.Equal(t, modelConfigB.ID, userMsg.ModelConfigID.UUID)
|
||||
}
|
||||
|
||||
func TestSendMessageWithReasoningEffortUpdatesLastReasoningEffort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := newChatClientWithDatabase(t)
|
||||
user := coderdtest.CreateFirstUser(t, client.Client)
|
||||
modelConfig := createChatModelConfig(t, client)
|
||||
|
||||
chat := dbgen.Chat(t, db, database.Chat{
|
||||
OrganizationID: user.OrganizationID,
|
||||
OwnerID: user.UserID,
|
||||
LastModelConfigID: modelConfig.ID,
|
||||
Title: "per-turn reasoning effort",
|
||||
})
|
||||
|
||||
resp, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "think hard about this",
|
||||
}},
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.Queued)
|
||||
|
||||
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, storedChat.LastReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
messages, err := db.GetChatMessagesByChatID(dbauthz.AsSystemRestricted(ctx), database.GetChatMessagesByChatIDParams{
|
||||
ChatID: chat.ID,
|
||||
AfterID: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
userMsg := findUserMessage(t, messages)
|
||||
require.True(t, userMsg.ReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, userMsg.ReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
// A follow-up message without a reasoning effort leaves the chat's
|
||||
// last effort unchanged.
|
||||
_, err = client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "and another thing",
|
||||
}},
|
||||
BusyBehavior: codersdk.ChatBusyBehaviorInterrupt,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
storedChat, err = db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, storedChat.LastReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
|
||||
}
|
||||
|
||||
func TestSendMessageRejectsInvalidReasoningEffort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := newChatClientWithDatabase(t)
|
||||
user := coderdtest.CreateFirstUser(t, client.Client)
|
||||
modelConfig := createChatModelConfig(t, client)
|
||||
|
||||
chat := dbgen.Chat(t, db, database.Chat{
|
||||
OrganizationID: user.OrganizationID,
|
||||
OwnerID: user.UserID,
|
||||
LastModelConfigID: modelConfig.ID,
|
||||
Title: "invalid reasoning effort",
|
||||
})
|
||||
|
||||
_, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "hello",
|
||||
}},
|
||||
ReasoningEffort: ptr.Ref(" HIGH "),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
|
||||
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
|
||||
}
|
||||
|
||||
func TestSendMessageQueuesReasoningEffort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := newChatClientWithDatabase(t)
|
||||
user := coderdtest.CreateFirstUser(t, client.Client)
|
||||
modelConfig := createChatModelConfig(t, client)
|
||||
|
||||
chat := dbgen.Chat(t, db, database.Chat{
|
||||
OrganizationID: user.OrganizationID,
|
||||
OwnerID: user.UserID,
|
||||
LastModelConfigID: modelConfig.ID,
|
||||
Title: "queued reasoning effort",
|
||||
})
|
||||
|
||||
_, err := db.UpdateChatStatus(dbauthz.AsSystemRestricted(ctx), database.UpdateChatStatusParams{
|
||||
ID: chat.ID,
|
||||
Status: database.ChatStatusRunning,
|
||||
WorkerID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
|
||||
StartedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
HeartbeatAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
LastError: pqtype.NullRawMessage{},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "queue this with effort",
|
||||
}},
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.Queued)
|
||||
require.NotNil(t, resp.QueuedMessage)
|
||||
queuedMessages, err := db.GetChatQueuedMessages(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, queuedMessages, 1)
|
||||
require.True(t, queuedMessages[0].ReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, queuedMessages[0].ReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.False(t, storedChat.LastReasoningEffort.Valid)
|
||||
}
|
||||
|
||||
func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -6715,8 +7032,9 @@ func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "queue this with model b",
|
||||
}},
|
||||
ModelConfigID: ptr.Ref(modelConfigB.ID),
|
||||
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
|
||||
ModelConfigID: ptr.Ref(modelConfigB.ID),
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.Queued)
|
||||
@@ -6729,10 +7047,13 @@ func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
|
||||
require.Len(t, queuedMessages, 1)
|
||||
require.True(t, queuedMessages[0].ModelConfigID.Valid)
|
||||
require.Equal(t, modelConfigB.ID, queuedMessages[0].ModelConfigID.UUID)
|
||||
require.True(t, queuedMessages[0].ReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffortHigh, queuedMessages[0].ReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, modelConfigA.ID, storedChat.LastModelConfigID)
|
||||
require.False(t, storedChat.LastReasoningEffort.Valid)
|
||||
}
|
||||
|
||||
func TestQueuedMessageWithoutOverrideCapturesEnqueueTimeModel(t *testing.T) {
|
||||
@@ -7854,6 +8175,94 @@ func TestPatchChatMessage(t *testing.T) {
|
||||
require.False(t, foundOriginalInChat)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffort", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
requested *string
|
||||
want string
|
||||
}{
|
||||
{name: "PreservesByDefault", want: "low"},
|
||||
{name: "Overrides", requested: ptr.Ref("high"), want: "high"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := newChatClientWithDatabase(t)
|
||||
firstUser := coderdtest.CreateFirstUser(t, client.Client)
|
||||
_ = createChatModelConfig(t, client)
|
||||
|
||||
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
|
||||
OrganizationID: firstUser.OrganizationID,
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "before edit effort",
|
||||
}},
|
||||
ReasoningEffort: ptr.Ref("low"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
messagesResult, err := client.GetChatMessages(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
userMessageID := messagesResult.Messages[0].ID
|
||||
|
||||
edited, err := client.EditChatMessage(ctx, chat.ID, userMessageID, codersdk.EditChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "after edit effort",
|
||||
}},
|
||||
ReasoningEffort: tc.requested,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
storedMessage, err := db.GetChatMessageByID(dbauthz.AsSystemRestricted(ctx), edited.Message.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, storedMessage.ReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffort(tc.want), storedMessage.ReasoningEffort.ChatReasoningEffort)
|
||||
|
||||
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, storedChat.LastReasoningEffort.Valid)
|
||||
require.Equal(t, database.ChatReasoningEffort(tc.want), storedChat.LastReasoningEffort.ChatReasoningEffort)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("RejectsInvalidReasoningEffort", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client := newChatClient(t)
|
||||
firstUser := coderdtest.CreateFirstUser(t, client.Client)
|
||||
_ = createChatModelConfig(t, client)
|
||||
|
||||
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
|
||||
OrganizationID: firstUser.OrganizationID,
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "before invalid effort edit",
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
messagesResult, err := client.GetChatMessages(ctx, chat.ID, nil)
|
||||
require.NoError(t, err)
|
||||
userMessageID := messagesResult.Messages[0].ID
|
||||
|
||||
_, err = client.EditChatMessage(ctx, chat.ID, userMessageID, codersdk.EditChatMessageRequest{
|
||||
Content: []codersdk.ChatInputPart{{
|
||||
Type: codersdk.ChatInputPartTypeText,
|
||||
Text: "after invalid effort edit",
|
||||
}},
|
||||
ReasoningEffort: ptr.Ref(" HIGH "),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
|
||||
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
|
||||
})
|
||||
|
||||
t.Run("PreservesFileID", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -11216,6 +11625,34 @@ func createAdditionalChatModelConfig(
|
||||
model string,
|
||||
) codersdk.ChatModelConfig {
|
||||
t.Helper()
|
||||
return createAdditionalChatModelConfigWithModelConfig(t, client, provider, model, nil)
|
||||
}
|
||||
|
||||
func createAdditionalChatModelConfigWithReasoningEffort(
|
||||
t *testing.T,
|
||||
client *codersdk.ExperimentalClient,
|
||||
provider string,
|
||||
model string,
|
||||
defaultEffort string,
|
||||
maxEffort string,
|
||||
) codersdk.ChatModelConfig {
|
||||
t.Helper()
|
||||
return createAdditionalChatModelConfigWithModelConfig(t, client, provider, model, &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref(defaultEffort),
|
||||
Max: ptr.Ref(maxEffort),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func createAdditionalChatModelConfigWithModelConfig(
|
||||
t *testing.T,
|
||||
client *codersdk.ExperimentalClient,
|
||||
provider string,
|
||||
model string,
|
||||
modelCallConfig *codersdk.ChatModelCallConfig,
|
||||
) codersdk.ChatModelConfig {
|
||||
t.Helper()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
aiProvider := createAIProviderForTest(t, client, provider, "test-api-key")
|
||||
@@ -11226,6 +11663,7 @@ func createAdditionalChatModelConfig(
|
||||
Model: model,
|
||||
ContextLimit: &contextLimit,
|
||||
IsDefault: &isDefault,
|
||||
ModelConfig: modelCallConfig,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return modelConfig
|
||||
@@ -11791,9 +12229,10 @@ func TestChatModelOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
type overrideResponse struct {
|
||||
context codersdk.ChatModelOverrideContext
|
||||
modelConfigID string
|
||||
isMalformed bool
|
||||
context codersdk.ChatModelOverrideContext
|
||||
modelConfigID string
|
||||
reasoningEffort *string
|
||||
isMalformed bool
|
||||
}
|
||||
|
||||
type settingTest struct {
|
||||
@@ -11817,23 +12256,36 @@ func TestChatModelOverrides(t *testing.T) {
|
||||
return overrideResponse{}, err
|
||||
}
|
||||
return overrideResponse{
|
||||
context: resp.Context,
|
||||
modelConfigID: resp.ModelConfigID,
|
||||
isMalformed: resp.IsMalformed,
|
||||
context: resp.Context,
|
||||
modelConfigID: resp.ModelConfigID,
|
||||
reasoningEffort: resp.ReasoningEffort,
|
||||
isMalformed: resp.IsMalformed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
putOverrideWithEffort := func(
|
||||
ctx context.Context,
|
||||
client *codersdk.ExperimentalClient,
|
||||
overrideContext codersdk.ChatModelOverrideContext,
|
||||
modelConfigID string,
|
||||
reasoningEffort *string,
|
||||
) error {
|
||||
return client.UpdateChatModelOverride(
|
||||
ctx,
|
||||
overrideContext,
|
||||
codersdk.UpdateChatModelOverrideRequest{
|
||||
ModelConfigID: modelConfigID,
|
||||
ReasoningEffort: reasoningEffort,
|
||||
},
|
||||
)
|
||||
}
|
||||
putOverride := func(
|
||||
ctx context.Context,
|
||||
client *codersdk.ExperimentalClient,
|
||||
overrideContext codersdk.ChatModelOverrideContext,
|
||||
modelConfigID string,
|
||||
) error {
|
||||
return client.UpdateChatModelOverride(
|
||||
ctx,
|
||||
overrideContext,
|
||||
codersdk.UpdateChatModelOverrideRequest{ModelConfigID: modelConfigID},
|
||||
)
|
||||
return putOverrideWithEffort(ctx, client, overrideContext, modelConfigID, nil)
|
||||
}
|
||||
|
||||
settings := []settingTest{
|
||||
@@ -11880,6 +12332,14 @@ func TestChatModelOverrides(t *testing.T) {
|
||||
coderdtest.TestChatProviderOpenAICompat,
|
||||
"gpt-4.1-mini-"+string(setting.context),
|
||||
)
|
||||
reasoningModel := createAdditionalChatModelConfigWithReasoningEffort(
|
||||
t,
|
||||
adminClient,
|
||||
coderdtest.TestChatProviderOpenAICompat,
|
||||
"gpt-4.1-reasoning-"+string(setting.context),
|
||||
"medium",
|
||||
"high",
|
||||
)
|
||||
disabledModel := createDisabledChatModelConfig(
|
||||
t,
|
||||
adminClient,
|
||||
@@ -11933,6 +12393,62 @@ func TestChatModelOverrides(t *testing.T) {
|
||||
require.False(t, resp.isMalformed)
|
||||
})
|
||||
|
||||
t.Run("AdminCanSetReasoningEffort", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
err := putOverrideWithEffort(ctx, adminClient, setting.context, reasoningModel.ID.String(), ptr.Ref("high"))
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := setting.dbGet(ctx, db)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, reasoningModel.ID.String()+":high", raw)
|
||||
|
||||
resp, err := getOverride(ctx, adminClient, setting.context)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, reasoningModel.ID.String(), resp.modelConfigID)
|
||||
require.Equal(t, ptr.Ref("high"), resp.reasoningEffort)
|
||||
require.False(t, resp.isMalformed)
|
||||
|
||||
err = putOverride(ctx, adminClient, setting.context, "")
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("PUTRejectsEncodedModelConfigID", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
encodedModelConfigID := reasoningModel.ID.String() + ":high"
|
||||
err := putOverride(ctx, adminClient, setting.context, encodedModelConfigID)
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid model_config_id.", sdkErr.Message)
|
||||
require.Equal(t, "Value "+strconv.Quote(encodedModelConfigID)+" is not a valid UUID.", sdkErr.Detail)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortRequiresModel", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
err := putOverrideWithEffort(ctx, adminClient, setting.context, "", ptr.Ref("high"))
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "reasoning_effort requires model_config_id.", sdkErr.Message)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortMustBeSelectable", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
err := putOverrideWithEffort(ctx, adminClient, setting.context, reasoningModel.ID.String(), ptr.Ref("xhigh"))
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Equal(t, "Must be one of none, minimal, low, medium, high.", sdkErr.Detail)
|
||||
})
|
||||
|
||||
t.Run("ReasoningEffortUnsupportedModel", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
err := putOverrideWithEffort(ctx, adminClient, setting.context, openAIModel.ID.String(), ptr.Ref("high"))
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Equal(t, "This model does not support reasoning effort.", sdkErr.Detail)
|
||||
})
|
||||
|
||||
t.Run("MalformedStoredOverrideIsReportedAndCanBeCleared", func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
@@ -12102,11 +12618,21 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
contextLimit := int64(4096)
|
||||
modelConfig, err := adminClient.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
modelConfigRequest := codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &modelProvider.ID,
|
||||
Model: "claude-personal-" + uuid.NewString(),
|
||||
ContextLimit: &contextLimit,
|
||||
})
|
||||
}
|
||||
modelConfig, err := adminClient.CreateChatModelConfig(ctx, modelConfigRequest)
|
||||
require.NoError(t, err)
|
||||
modelConfigRequest.Model = "claude-personal-reasoning-" + uuid.NewString()
|
||||
modelConfigRequest.ModelConfig = &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref("medium"),
|
||||
Max: ptr.Ref("high"),
|
||||
},
|
||||
}
|
||||
reasoningModelConfig, err := adminClient.CreateChatModelConfig(ctx, modelConfigRequest)
|
||||
require.NoError(t, err)
|
||||
err = adminClient.UpdateChatModelOverride(ctx, codersdk.ChatModelOverrideContextGeneral, codersdk.UpdateChatModelOverrideRequest{
|
||||
ModelConfigID: modelConfig.ID.String(),
|
||||
@@ -12156,6 +12682,24 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
return codersdk.ChatPersonalModelOverride{}
|
||||
}
|
||||
}
|
||||
assertOverrideWithEffort := func(
|
||||
resp codersdk.UserChatPersonalModelOverridesResponse,
|
||||
overrideContext codersdk.ChatPersonalModelOverrideContext,
|
||||
mode codersdk.ChatPersonalModelOverrideMode,
|
||||
modelConfigID string,
|
||||
reasoningEffort *string,
|
||||
isSet bool,
|
||||
isMalformed bool,
|
||||
) {
|
||||
t.Helper()
|
||||
override := personalOverride(resp, overrideContext)
|
||||
require.Equal(t, overrideContext, override.Context)
|
||||
require.Equal(t, mode, override.Mode)
|
||||
require.Equal(t, modelConfigID, override.ModelConfigID)
|
||||
require.Equal(t, reasoningEffort, override.ReasoningEffort)
|
||||
require.Equal(t, isSet, override.IsSet)
|
||||
require.Equal(t, isMalformed, override.IsMalformed)
|
||||
}
|
||||
assertOverride := func(
|
||||
resp codersdk.UserChatPersonalModelOverridesResponse,
|
||||
overrideContext codersdk.ChatPersonalModelOverrideContext,
|
||||
@@ -12165,17 +12709,13 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
isMalformed bool,
|
||||
) {
|
||||
t.Helper()
|
||||
override := personalOverride(resp, overrideContext)
|
||||
require.Equal(t, overrideContext, override.Context)
|
||||
require.Equal(t, mode, override.Mode)
|
||||
require.Equal(t, modelConfigID, override.ModelConfigID)
|
||||
require.Equal(t, isSet, override.IsSet)
|
||||
require.Equal(t, isMalformed, override.IsMalformed)
|
||||
assertOverrideWithEffort(resp, overrideContext, mode, modelConfigID, nil, isSet, isMalformed)
|
||||
}
|
||||
assertDeploymentDefault := func(
|
||||
resp codersdk.UserChatPersonalModelOverridesResponse,
|
||||
overrideContext codersdk.ChatModelOverrideContext,
|
||||
modelConfigID string,
|
||||
reasoningEffort *string,
|
||||
isMalformed bool,
|
||||
) {
|
||||
t.Helper()
|
||||
@@ -12190,6 +12730,7 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
}
|
||||
require.Equal(t, overrideContext, override.Context)
|
||||
require.Equal(t, modelConfigID, override.ModelConfigID)
|
||||
require.Equal(t, reasoningEffort, override.ReasoningEffort)
|
||||
require.Equal(t, isMalformed, override.IsMalformed)
|
||||
}
|
||||
upsertRaw := func(
|
||||
@@ -12246,8 +12787,20 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
t.Run("GETIncludesDeploymentDefaults", func(t *testing.T) {
|
||||
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
|
||||
require.NoError(t, err)
|
||||
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, modelConfig.ID.String(), false)
|
||||
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextExplore, defaultModelConfig.ID.String(), false)
|
||||
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, modelConfig.ID.String(), nil, false)
|
||||
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextExplore, defaultModelConfig.ID.String(), nil, false)
|
||||
})
|
||||
|
||||
t.Run("GETIncludesDeploymentDefaultReasoningEffort", func(t *testing.T) {
|
||||
err := adminClient.UpdateChatModelOverride(ctx, codersdk.ChatModelOverrideContextGeneral, codersdk.UpdateChatModelOverrideRequest{
|
||||
ModelConfigID: reasoningModelConfig.ID.String(),
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
|
||||
require.NoError(t, err)
|
||||
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, reasoningModelConfig.ID.String(), ptr.Ref("high"), false)
|
||||
})
|
||||
|
||||
t.Run("PUTDisabledReturns403AndPreservesRows", func(t *testing.T) {
|
||||
@@ -12358,6 +12911,57 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("PUTModelRoundTripsReasoningEffort", func(t *testing.T) {
|
||||
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
|
||||
Mode: codersdk.ChatPersonalModelOverrideModeModel,
|
||||
ModelConfigID: reasoningModelConfig.ID.String(),
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "model:"+reasoningModelConfig.ID.String()+":high", getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
|
||||
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
|
||||
require.NoError(t, err)
|
||||
assertOverrideWithEffort(resp, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.ChatPersonalModelOverrideModeModel, reasoningModelConfig.ID.String(), ptr.Ref("high"), true, false)
|
||||
})
|
||||
|
||||
t.Run("PUTReasoningEffortRejectsNonModelMode", func(t *testing.T) {
|
||||
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
|
||||
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
|
||||
Mode: codersdk.ChatPersonalModelOverrideModeDeploymentDefault,
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "reasoning_effort requires mode model.", sdkErr.Message)
|
||||
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
|
||||
})
|
||||
|
||||
t.Run("PUTReasoningEffortMustBeSelectable", func(t *testing.T) {
|
||||
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
|
||||
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
|
||||
Mode: codersdk.ChatPersonalModelOverrideModeModel,
|
||||
ModelConfigID: reasoningModelConfig.ID.String(),
|
||||
ReasoningEffort: ptr.Ref("xhigh"),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Equal(t, "Must be one of none, minimal, low, medium, high.", sdkErr.Detail)
|
||||
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
|
||||
})
|
||||
|
||||
t.Run("PUTReasoningEffortUnsupportedModel", func(t *testing.T) {
|
||||
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
|
||||
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
|
||||
Mode: codersdk.ChatPersonalModelOverrideModeModel,
|
||||
ModelConfigID: modelConfig.ID.String(),
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
|
||||
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
|
||||
require.Equal(t, "This model does not support reasoning effort.", sdkErr.Detail)
|
||||
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
|
||||
})
|
||||
|
||||
t.Run("PUTModelRejectsInvalidModels", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -12575,6 +13179,31 @@ func TestCreateChatPersonalModelOverrideRoot(t *testing.T) {
|
||||
require.Equal(t, overrideModel.ID, chat.LastModelConfigID)
|
||||
})
|
||||
|
||||
t.Run("RootModelOverrideUsesSavedReasoningEffort", func(t *testing.T) {
|
||||
reasoningModel, err := adminClient.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
|
||||
AIProviderID: &overrideProvider.ID,
|
||||
Model: "claude-root-personal-reasoning-" + uuid.NewString(),
|
||||
ContextLimit: &contextLimit,
|
||||
ModelConfig: &codersdk.ChatModelCallConfig{
|
||||
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
|
||||
Default: ptr.Ref("medium"),
|
||||
Max: ptr.Ref("high"),
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = adminClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextRoot, codersdk.UpdateUserChatPersonalModelOverrideRequest{
|
||||
Mode: codersdk.ChatPersonalModelOverrideModeModel,
|
||||
ModelConfigID: reasoningModel.ID.String(),
|
||||
ReasoningEffort: ptr.Ref("high"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
chat := createChat(adminClient, "root model override uses saved reasoning effort", nil)
|
||||
require.Equal(t, reasoningModel.ID, chat.LastModelConfigID)
|
||||
require.Equal(t, ptr.Ref("high"), chat.LastReasoningEffort)
|
||||
})
|
||||
|
||||
t.Run("UnavailableRootModelFallsBackToDefault", func(t *testing.T) {
|
||||
upsertRootRaw(firstUser.UserID, "model:"+disabledModel.ID.String())
|
||||
chat := createChat(adminClient, "disabled root model falls back", nil)
|
||||
|
||||
Reference in New Issue
Block a user