feat: add configurable reasoning effort to Coder agents (#26974)

This commit is contained in:
Danielle Maywood
2026-07-09 23:35:12 +01:00
committed by GitHub
parent 5fed583a46
commit d66e4d794f
72 changed files with 4389 additions and 1275 deletions
+656 -27
View File
@@ -157,14 +157,29 @@ func insertTestChatQueuedMessage(
apiKeyID string,
) database.ChatQueuedMessage {
t.Helper()
return insertTestChatQueuedMessageWithReasoningEffort(ctx, t, db, chatID, content, modelConfigID, apiKeyID, "")
}
func insertTestChatQueuedMessageWithReasoningEffort(
ctx context.Context,
t testing.TB,
db database.Store,
chatID uuid.UUID,
content json.RawMessage,
modelConfigID uuid.UUID,
apiKeyID string,
reasoningEffort string,
) database.ChatQueuedMessage {
t.Helper()
queued, err := db.InsertChatQueuedMessage(
dbauthz.AsSystemRestricted(ctx),
database.InsertChatQueuedMessageParams{
ChatID: chatID,
Content: content,
ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: modelConfigID != uuid.Nil},
APIKeyID: sql.NullString{String: apiKeyID, Valid: apiKeyID != ""},
ChatID: chatID,
Content: content,
ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: modelConfigID != uuid.Nil},
ReasoningEffort: database.NullChatReasoningEffort{ChatReasoningEffort: database.ChatReasoningEffort(reasoningEffort), Valid: reasoningEffort != ""},
APIKeyID: sql.NullString{String: apiKeyID, Valid: apiKeyID != ""},
},
)
require.NoError(t, err)
@@ -416,6 +431,65 @@ func TestPostChats(t *testing.T) {
requireSDKError(t, err, http.StatusForbidden)
})
t.Run("WithReasoningEffort", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := newChatClientWithDatabase(t)
user := coderdtest.CreateFirstUser(t, client.Client)
_ = createChatModelConfig(t, client)
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
OrganizationID: user.OrganizationID,
Content: []codersdk.ChatInputPart{
{
Type: codersdk.ChatInputPartTypeText,
Text: "think hard from the start",
},
},
ReasoningEffort: ptr.Ref("high"),
})
require.NoError(t, err)
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.True(t, storedChat.LastReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
messages, err := db.GetChatMessagesByChatID(dbauthz.AsSystemRestricted(ctx), database.GetChatMessagesByChatIDParams{
ChatID: chat.ID,
AfterID: 0,
})
require.NoError(t, err)
userMsg := findUserMessage(t, messages)
require.True(t, userMsg.ReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, userMsg.ReasoningEffort.ChatReasoningEffort)
})
t.Run("RejectsInvalidReasoningEffort", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
user := coderdtest.CreateFirstUser(t, client.Client)
_ = createChatModelConfig(t, client)
_, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
OrganizationID: user.OrganizationID,
Content: []codersdk.ChatInputPart{
{
Type: codersdk.ChatInputPartTypeText,
Text: "hello",
},
},
ReasoningEffort: ptr.Ref(" HIGH "),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
})
t.Run("HidesSystemPromptMessages", func(t *testing.T) {
t.Parallel()
@@ -3744,6 +3818,118 @@ func TestCreateChatModelConfig(t *testing.T) {
)
})
t.Run("ReasoningEffortStored", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
_ = coderdtest.CreateFirstUser(t, client.Client)
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
contextLimit := int64(4096)
modelConfig, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
AIProviderID: &aiProvider.ID,
Model: "gpt-4o-mini",
ContextLimit: &contextLimit,
ModelConfig: &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref("medium"),
Max: ptr.Ref("xhigh"),
},
},
})
require.NoError(t, err)
require.NotNil(t, modelConfig.ModelConfig)
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort)
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort.Default)
require.Equal(t, "medium", *modelConfig.ModelConfig.ReasoningEffort.Default)
require.NotNil(t, modelConfig.ModelConfig.ReasoningEffort.Max)
require.Equal(t, "xhigh", *modelConfig.ModelConfig.ReasoningEffort.Max)
require.Equal(t, []string{"none", "minimal", "low", "medium", "high", "xhigh"}, modelConfig.ReasoningEfforts)
})
t.Run("ReasoningEffortRejectsSingleValue", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
_ = coderdtest.CreateFirstUser(t, client.Client)
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
contextLimit := int64(4096)
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
AIProviderID: &aiProvider.ID,
Model: "gpt-4o-mini",
ContextLimit: &contextLimit,
ModelConfig: &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref("high"),
},
},
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid model config.", sdkErr.Message)
require.Equal(t, "reasoning_effort.default and reasoning_effort.max must both be set", sdkErr.Detail)
})
t.Run("ReasoningEffortRejectsInvalidValue", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
_ = coderdtest.CreateFirstUser(t, client.Client)
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
contextLimit := int64(4096)
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
AIProviderID: &aiProvider.ID,
Model: "gpt-4o-mini",
ContextLimit: &contextLimit,
ModelConfig: &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref(" HIGH "),
Max: ptr.Ref("high"),
},
},
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid model config.", sdkErr.Message)
require.Equal(
t,
`reasoning_effort.default " HIGH " must be one of none, minimal, low, medium, high, xhigh, max`,
sdkErr.Detail,
)
})
t.Run("ReasoningEffortRejectsDefaultAboveMax", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
_ = coderdtest.CreateFirstUser(t, client.Client)
aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key")
contextLimit := int64(4096)
_, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
AIProviderID: &aiProvider.ID,
Model: "gpt-4o-mini",
ContextLimit: &contextLimit,
ModelConfig: &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref("xhigh"),
Max: ptr.Ref("low"),
},
},
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid model config.", sdkErr.Message)
require.Contains(t, sdkErr.Detail, "must not exceed")
})
t.Run("MissingContextLimit", func(t *testing.T) {
t.Parallel()
@@ -6684,6 +6870,137 @@ func TestSendMessageWithModelOverrideUpdatesLastModelConfigID(t *testing.T) {
require.Equal(t, modelConfigB.ID, userMsg.ModelConfigID.UUID)
}
func TestSendMessageWithReasoningEffortUpdatesLastReasoningEffort(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := newChatClientWithDatabase(t)
user := coderdtest.CreateFirstUser(t, client.Client)
modelConfig := createChatModelConfig(t, client)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
LastModelConfigID: modelConfig.ID,
Title: "per-turn reasoning effort",
})
resp, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "think hard about this",
}},
ReasoningEffort: ptr.Ref("high"),
})
require.NoError(t, err)
require.False(t, resp.Queued)
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.True(t, storedChat.LastReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
messages, err := db.GetChatMessagesByChatID(dbauthz.AsSystemRestricted(ctx), database.GetChatMessagesByChatIDParams{
ChatID: chat.ID,
AfterID: 0,
})
require.NoError(t, err)
userMsg := findUserMessage(t, messages)
require.True(t, userMsg.ReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, userMsg.ReasoningEffort.ChatReasoningEffort)
// A follow-up message without a reasoning effort leaves the chat's
// last effort unchanged.
_, err = client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "and another thing",
}},
BusyBehavior: codersdk.ChatBusyBehaviorInterrupt,
})
require.NoError(t, err)
storedChat, err = db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.True(t, storedChat.LastReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, storedChat.LastReasoningEffort.ChatReasoningEffort)
}
func TestSendMessageRejectsInvalidReasoningEffort(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := newChatClientWithDatabase(t)
user := coderdtest.CreateFirstUser(t, client.Client)
modelConfig := createChatModelConfig(t, client)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
LastModelConfigID: modelConfig.ID,
Title: "invalid reasoning effort",
})
_, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "hello",
}},
ReasoningEffort: ptr.Ref(" HIGH "),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
}
func TestSendMessageQueuesReasoningEffort(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := newChatClientWithDatabase(t)
user := coderdtest.CreateFirstUser(t, client.Client)
modelConfig := createChatModelConfig(t, client)
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
LastModelConfigID: modelConfig.ID,
Title: "queued reasoning effort",
})
_, err := db.UpdateChatStatus(dbauthz.AsSystemRestricted(ctx), database.UpdateChatStatusParams{
ID: chat.ID,
Status: database.ChatStatusRunning,
WorkerID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
StartedAt: sql.NullTime{Time: time.Now(), Valid: true},
HeartbeatAt: sql.NullTime{Time: time.Now(), Valid: true},
LastError: pqtype.NullRawMessage{},
})
require.NoError(t, err)
resp, err := client.CreateChatMessage(ctx, chat.ID, codersdk.CreateChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "queue this with effort",
}},
ReasoningEffort: ptr.Ref("high"),
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
})
require.NoError(t, err)
require.True(t, resp.Queued)
require.NotNil(t, resp.QueuedMessage)
queuedMessages, err := db.GetChatQueuedMessages(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.Len(t, queuedMessages, 1)
require.True(t, queuedMessages[0].ReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, queuedMessages[0].ReasoningEffort.ChatReasoningEffort)
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.False(t, storedChat.LastReasoningEffort.Valid)
}
func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
t.Parallel()
@@ -6715,8 +7032,9 @@ func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
Type: codersdk.ChatInputPartTypeText,
Text: "queue this with model b",
}},
ModelConfigID: ptr.Ref(modelConfigB.ID),
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
ModelConfigID: ptr.Ref(modelConfigB.ID),
ReasoningEffort: ptr.Ref("high"),
BusyBehavior: codersdk.ChatBusyBehaviorQueue,
})
require.NoError(t, err)
require.True(t, resp.Queued)
@@ -6729,10 +7047,13 @@ func TestSendMessageQueuesEffectiveModelConfigID(t *testing.T) {
require.Len(t, queuedMessages, 1)
require.True(t, queuedMessages[0].ModelConfigID.Valid)
require.Equal(t, modelConfigB.ID, queuedMessages[0].ModelConfigID.UUID)
require.True(t, queuedMessages[0].ReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffortHigh, queuedMessages[0].ReasoningEffort.ChatReasoningEffort)
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.Equal(t, modelConfigA.ID, storedChat.LastModelConfigID)
require.False(t, storedChat.LastReasoningEffort.Valid)
}
func TestQueuedMessageWithoutOverrideCapturesEnqueueTimeModel(t *testing.T) {
@@ -7854,6 +8175,94 @@ func TestPatchChatMessage(t *testing.T) {
require.False(t, foundOriginalInChat)
})
t.Run("ReasoningEffort", func(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
requested *string
want string
}{
{name: "PreservesByDefault", want: "low"},
{name: "Overrides", requested: ptr.Ref("high"), want: "high"},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client, db := newChatClientWithDatabase(t)
firstUser := coderdtest.CreateFirstUser(t, client.Client)
_ = createChatModelConfig(t, client)
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
OrganizationID: firstUser.OrganizationID,
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "before edit effort",
}},
ReasoningEffort: ptr.Ref("low"),
})
require.NoError(t, err)
messagesResult, err := client.GetChatMessages(ctx, chat.ID, nil)
require.NoError(t, err)
userMessageID := messagesResult.Messages[0].ID
edited, err := client.EditChatMessage(ctx, chat.ID, userMessageID, codersdk.EditChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "after edit effort",
}},
ReasoningEffort: tc.requested,
})
require.NoError(t, err)
storedMessage, err := db.GetChatMessageByID(dbauthz.AsSystemRestricted(ctx), edited.Message.ID)
require.NoError(t, err)
require.True(t, storedMessage.ReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffort(tc.want), storedMessage.ReasoningEffort.ChatReasoningEffort)
storedChat, err := db.GetChatByID(dbauthz.AsSystemRestricted(ctx), chat.ID)
require.NoError(t, err)
require.True(t, storedChat.LastReasoningEffort.Valid)
require.Equal(t, database.ChatReasoningEffort(tc.want), storedChat.LastReasoningEffort.ChatReasoningEffort)
})
}
})
t.Run("RejectsInvalidReasoningEffort", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
client := newChatClient(t)
firstUser := coderdtest.CreateFirstUser(t, client.Client)
_ = createChatModelConfig(t, client)
chat, err := client.CreateChat(ctx, codersdk.CreateChatRequest{
OrganizationID: firstUser.OrganizationID,
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "before invalid effort edit",
}},
})
require.NoError(t, err)
messagesResult, err := client.GetChatMessages(ctx, chat.ID, nil)
require.NoError(t, err)
userMessageID := messagesResult.Messages[0].ID
_, err = client.EditChatMessage(ctx, chat.ID, userMessageID, codersdk.EditChatMessageRequest{
Content: []codersdk.ChatInputPart{{
Type: codersdk.ChatInputPartTypeText,
Text: "after invalid effort edit",
}},
ReasoningEffort: ptr.Ref(" HIGH "),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Contains(t, sdkErr.Detail, `Invalid value " HIGH "`)
require.Contains(t, sdkErr.Detail, "must be one of none, minimal, low, medium, high, xhigh, max")
})
t.Run("PreservesFileID", func(t *testing.T) {
t.Parallel()
@@ -11216,6 +11625,34 @@ func createAdditionalChatModelConfig(
model string,
) codersdk.ChatModelConfig {
t.Helper()
return createAdditionalChatModelConfigWithModelConfig(t, client, provider, model, nil)
}
func createAdditionalChatModelConfigWithReasoningEffort(
t *testing.T,
client *codersdk.ExperimentalClient,
provider string,
model string,
defaultEffort string,
maxEffort string,
) codersdk.ChatModelConfig {
t.Helper()
return createAdditionalChatModelConfigWithModelConfig(t, client, provider, model, &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref(defaultEffort),
Max: ptr.Ref(maxEffort),
},
})
}
func createAdditionalChatModelConfigWithModelConfig(
t *testing.T,
client *codersdk.ExperimentalClient,
provider string,
model string,
modelCallConfig *codersdk.ChatModelCallConfig,
) codersdk.ChatModelConfig {
t.Helper()
ctx := testutil.Context(t, testutil.WaitLong)
aiProvider := createAIProviderForTest(t, client, provider, "test-api-key")
@@ -11226,6 +11663,7 @@ func createAdditionalChatModelConfig(
Model: model,
ContextLimit: &contextLimit,
IsDefault: &isDefault,
ModelConfig: modelCallConfig,
})
require.NoError(t, err)
return modelConfig
@@ -11791,9 +12229,10 @@ func TestChatModelOverrides(t *testing.T) {
t.Parallel()
type overrideResponse struct {
context codersdk.ChatModelOverrideContext
modelConfigID string
isMalformed bool
context codersdk.ChatModelOverrideContext
modelConfigID string
reasoningEffort *string
isMalformed bool
}
type settingTest struct {
@@ -11817,23 +12256,36 @@ func TestChatModelOverrides(t *testing.T) {
return overrideResponse{}, err
}
return overrideResponse{
context: resp.Context,
modelConfigID: resp.ModelConfigID,
isMalformed: resp.IsMalformed,
context: resp.Context,
modelConfigID: resp.ModelConfigID,
reasoningEffort: resp.ReasoningEffort,
isMalformed: resp.IsMalformed,
}, nil
}
putOverrideWithEffort := func(
ctx context.Context,
client *codersdk.ExperimentalClient,
overrideContext codersdk.ChatModelOverrideContext,
modelConfigID string,
reasoningEffort *string,
) error {
return client.UpdateChatModelOverride(
ctx,
overrideContext,
codersdk.UpdateChatModelOverrideRequest{
ModelConfigID: modelConfigID,
ReasoningEffort: reasoningEffort,
},
)
}
putOverride := func(
ctx context.Context,
client *codersdk.ExperimentalClient,
overrideContext codersdk.ChatModelOverrideContext,
modelConfigID string,
) error {
return client.UpdateChatModelOverride(
ctx,
overrideContext,
codersdk.UpdateChatModelOverrideRequest{ModelConfigID: modelConfigID},
)
return putOverrideWithEffort(ctx, client, overrideContext, modelConfigID, nil)
}
settings := []settingTest{
@@ -11880,6 +12332,14 @@ func TestChatModelOverrides(t *testing.T) {
coderdtest.TestChatProviderOpenAICompat,
"gpt-4.1-mini-"+string(setting.context),
)
reasoningModel := createAdditionalChatModelConfigWithReasoningEffort(
t,
adminClient,
coderdtest.TestChatProviderOpenAICompat,
"gpt-4.1-reasoning-"+string(setting.context),
"medium",
"high",
)
disabledModel := createDisabledChatModelConfig(
t,
adminClient,
@@ -11933,6 +12393,62 @@ func TestChatModelOverrides(t *testing.T) {
require.False(t, resp.isMalformed)
})
t.Run("AdminCanSetReasoningEffort", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
err := putOverrideWithEffort(ctx, adminClient, setting.context, reasoningModel.ID.String(), ptr.Ref("high"))
require.NoError(t, err)
raw, err := setting.dbGet(ctx, db)
require.NoError(t, err)
require.Equal(t, reasoningModel.ID.String()+":high", raw)
resp, err := getOverride(ctx, adminClient, setting.context)
require.NoError(t, err)
require.Equal(t, reasoningModel.ID.String(), resp.modelConfigID)
require.Equal(t, ptr.Ref("high"), resp.reasoningEffort)
require.False(t, resp.isMalformed)
err = putOverride(ctx, adminClient, setting.context, "")
require.NoError(t, err)
})
t.Run("PUTRejectsEncodedModelConfigID", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
encodedModelConfigID := reasoningModel.ID.String() + ":high"
err := putOverride(ctx, adminClient, setting.context, encodedModelConfigID)
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid model_config_id.", sdkErr.Message)
require.Equal(t, "Value "+strconv.Quote(encodedModelConfigID)+" is not a valid UUID.", sdkErr.Detail)
})
t.Run("ReasoningEffortRequiresModel", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
err := putOverrideWithEffort(ctx, adminClient, setting.context, "", ptr.Ref("high"))
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "reasoning_effort requires model_config_id.", sdkErr.Message)
})
t.Run("ReasoningEffortMustBeSelectable", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
err := putOverrideWithEffort(ctx, adminClient, setting.context, reasoningModel.ID.String(), ptr.Ref("xhigh"))
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Equal(t, "Must be one of none, minimal, low, medium, high.", sdkErr.Detail)
})
t.Run("ReasoningEffortUnsupportedModel", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
err := putOverrideWithEffort(ctx, adminClient, setting.context, openAIModel.ID.String(), ptr.Ref("high"))
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Equal(t, "This model does not support reasoning effort.", sdkErr.Detail)
})
t.Run("MalformedStoredOverrideIsReportedAndCanBeCleared", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
@@ -12102,11 +12618,21 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
})
require.NoError(t, err)
contextLimit := int64(4096)
modelConfig, err := adminClient.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
modelConfigRequest := codersdk.CreateChatModelConfigRequest{
AIProviderID: &modelProvider.ID,
Model: "claude-personal-" + uuid.NewString(),
ContextLimit: &contextLimit,
})
}
modelConfig, err := adminClient.CreateChatModelConfig(ctx, modelConfigRequest)
require.NoError(t, err)
modelConfigRequest.Model = "claude-personal-reasoning-" + uuid.NewString()
modelConfigRequest.ModelConfig = &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref("medium"),
Max: ptr.Ref("high"),
},
}
reasoningModelConfig, err := adminClient.CreateChatModelConfig(ctx, modelConfigRequest)
require.NoError(t, err)
err = adminClient.UpdateChatModelOverride(ctx, codersdk.ChatModelOverrideContextGeneral, codersdk.UpdateChatModelOverrideRequest{
ModelConfigID: modelConfig.ID.String(),
@@ -12156,6 +12682,24 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
return codersdk.ChatPersonalModelOverride{}
}
}
assertOverrideWithEffort := func(
resp codersdk.UserChatPersonalModelOverridesResponse,
overrideContext codersdk.ChatPersonalModelOverrideContext,
mode codersdk.ChatPersonalModelOverrideMode,
modelConfigID string,
reasoningEffort *string,
isSet bool,
isMalformed bool,
) {
t.Helper()
override := personalOverride(resp, overrideContext)
require.Equal(t, overrideContext, override.Context)
require.Equal(t, mode, override.Mode)
require.Equal(t, modelConfigID, override.ModelConfigID)
require.Equal(t, reasoningEffort, override.ReasoningEffort)
require.Equal(t, isSet, override.IsSet)
require.Equal(t, isMalformed, override.IsMalformed)
}
assertOverride := func(
resp codersdk.UserChatPersonalModelOverridesResponse,
overrideContext codersdk.ChatPersonalModelOverrideContext,
@@ -12165,17 +12709,13 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
isMalformed bool,
) {
t.Helper()
override := personalOverride(resp, overrideContext)
require.Equal(t, overrideContext, override.Context)
require.Equal(t, mode, override.Mode)
require.Equal(t, modelConfigID, override.ModelConfigID)
require.Equal(t, isSet, override.IsSet)
require.Equal(t, isMalformed, override.IsMalformed)
assertOverrideWithEffort(resp, overrideContext, mode, modelConfigID, nil, isSet, isMalformed)
}
assertDeploymentDefault := func(
resp codersdk.UserChatPersonalModelOverridesResponse,
overrideContext codersdk.ChatModelOverrideContext,
modelConfigID string,
reasoningEffort *string,
isMalformed bool,
) {
t.Helper()
@@ -12190,6 +12730,7 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
}
require.Equal(t, overrideContext, override.Context)
require.Equal(t, modelConfigID, override.ModelConfigID)
require.Equal(t, reasoningEffort, override.ReasoningEffort)
require.Equal(t, isMalformed, override.IsMalformed)
}
upsertRaw := func(
@@ -12246,8 +12787,20 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
t.Run("GETIncludesDeploymentDefaults", func(t *testing.T) {
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
require.NoError(t, err)
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, modelConfig.ID.String(), false)
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextExplore, defaultModelConfig.ID.String(), false)
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, modelConfig.ID.String(), nil, false)
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextExplore, defaultModelConfig.ID.String(), nil, false)
})
t.Run("GETIncludesDeploymentDefaultReasoningEffort", func(t *testing.T) {
err := adminClient.UpdateChatModelOverride(ctx, codersdk.ChatModelOverrideContextGeneral, codersdk.UpdateChatModelOverrideRequest{
ModelConfigID: reasoningModelConfig.ID.String(),
ReasoningEffort: ptr.Ref("high"),
})
require.NoError(t, err)
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
require.NoError(t, err)
assertDeploymentDefault(resp, codersdk.ChatModelOverrideContextGeneral, reasoningModelConfig.ID.String(), ptr.Ref("high"), false)
})
t.Run("PUTDisabledReturns403AndPreservesRows", func(t *testing.T) {
@@ -12358,6 +12911,57 @@ func TestUserChatPersonalModelOverrides(t *testing.T) {
}
})
t.Run("PUTModelRoundTripsReasoningEffort", func(t *testing.T) {
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
Mode: codersdk.ChatPersonalModelOverrideModeModel,
ModelConfigID: reasoningModelConfig.ID.String(),
ReasoningEffort: ptr.Ref("high"),
})
require.NoError(t, err)
require.Equal(t, "model:"+reasoningModelConfig.ID.String()+":high", getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
resp, err := memberClient.GetUserChatPersonalModelOverrides(ctx)
require.NoError(t, err)
assertOverrideWithEffort(resp, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.ChatPersonalModelOverrideModeModel, reasoningModelConfig.ID.String(), ptr.Ref("high"), true, false)
})
t.Run("PUTReasoningEffortRejectsNonModelMode", func(t *testing.T) {
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
Mode: codersdk.ChatPersonalModelOverrideModeDeploymentDefault,
ReasoningEffort: ptr.Ref("high"),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "reasoning_effort requires mode model.", sdkErr.Message)
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
})
t.Run("PUTReasoningEffortMustBeSelectable", func(t *testing.T) {
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
Mode: codersdk.ChatPersonalModelOverrideModeModel,
ModelConfigID: reasoningModelConfig.ID.String(),
ReasoningEffort: ptr.Ref("xhigh"),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Equal(t, "Must be one of none, minimal, low, medium, high.", sdkErr.Detail)
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
})
t.Run("PUTReasoningEffortUnsupportedModel", func(t *testing.T) {
rawBefore := getRaw(codersdk.ChatPersonalModelOverrideContextGeneral)
err := memberClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextGeneral, codersdk.UpdateUserChatPersonalModelOverrideRequest{
Mode: codersdk.ChatPersonalModelOverrideModeModel,
ModelConfigID: modelConfig.ID.String(),
ReasoningEffort: ptr.Ref("high"),
})
sdkErr := requireSDKError(t, err, http.StatusBadRequest)
require.Equal(t, "Invalid reasoning_effort value.", sdkErr.Message)
require.Equal(t, "This model does not support reasoning effort.", sdkErr.Detail)
require.Equal(t, rawBefore, getRaw(codersdk.ChatPersonalModelOverrideContextGeneral))
})
t.Run("PUTModelRejectsInvalidModels", func(t *testing.T) {
cases := []struct {
name string
@@ -12575,6 +13179,31 @@ func TestCreateChatPersonalModelOverrideRoot(t *testing.T) {
require.Equal(t, overrideModel.ID, chat.LastModelConfigID)
})
t.Run("RootModelOverrideUsesSavedReasoningEffort", func(t *testing.T) {
reasoningModel, err := adminClient.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{
AIProviderID: &overrideProvider.ID,
Model: "claude-root-personal-reasoning-" + uuid.NewString(),
ContextLimit: &contextLimit,
ModelConfig: &codersdk.ChatModelCallConfig{
ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{
Default: ptr.Ref("medium"),
Max: ptr.Ref("high"),
},
},
})
require.NoError(t, err)
err = adminClient.UpdateUserChatPersonalModelOverride(ctx, codersdk.ChatPersonalModelOverrideContextRoot, codersdk.UpdateUserChatPersonalModelOverrideRequest{
Mode: codersdk.ChatPersonalModelOverrideModeModel,
ModelConfigID: reasoningModel.ID.String(),
ReasoningEffort: ptr.Ref("high"),
})
require.NoError(t, err)
chat := createChat(adminClient, "root model override uses saved reasoning effort", nil)
require.Equal(t, reasoningModel.ID, chat.LastModelConfigID)
require.Equal(t, ptr.Ref("high"), chat.LastReasoningEffort)
})
t.Run("UnavailableRootModelFallsBackToDefault", func(t *testing.T) {
upsertRootRaw(firstUser.UserID, "model:"+disabledModel.ID.String())
chat := createChat(adminClient, "disabled root model falls back", nil)