opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: update scaletest chat provider bootstrap (#25948)

Browse Source 
`coder exp scaletest chat` now bootstraps its mock LLM using the,
post-gateway unification, AI provider API instead of the removed
experimental chat-provider API, and creates or reuses a chat model
config linked to that provider. When the mock provider is created or
updated, the command waits a flat, hidden `--provider-propagation-wait`
(default 15s) before starting the scale run, since each coderd replica
caches provider config with a 10s TTL and only expiry guarantees every
replica sees the change. The command also runs without any scaletest
workspaces, creating chats with no workspace context. The integration
test covers the CLI path against `llmmock` with a near-zero propagation
wait, verifies the provider/model config setup, and asserts the
generated chat records user and assistant messages.



Relates to CODAGT-307

Relates to GRU-48
This commit is contained in:
Ethan
2026-06-12 14:41:58 +10:00
committed by GitHub
parent f13c858482
commit b1c6010eb9
7 changed files with 300 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/exp_scaletest.go
+4
View File
@@ -396,6 +396,7 @@ type workspaceTargetFlags struct {
template string
targetWorkspaces string
useHostLogin bool
allowEmpty bool
}
// attach adds the workspace target flags to the given options set.
@@ -463,6 +464,9 @@ func (f *workspaceTargetFlags) getTargetedWorkspaces(ctx context.Context, client
// Validate range
if len(workspaces) == 0 {
if f.allowEmpty {
return nil, nil
}
return nil, xerrors.Errorf("no scaletest workspaces exist")
}
if targetEnd > len(workspaces) {
cli/exp_scaletest_chat.go
+27 -16
View File
@@ -11,8 +11,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/sloghuman"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/chat"
"github.com/coder/coder/v2/scaletest/harness"
@@ -22,17 +20,18 @@ import (
func (r *RootCmd) scaletestChat() *serpent.Command {
var (
chatsPerWorkspace int64
prompt string
turns int64
turnStartDelay time.Duration
llmMockURL string
targetFlags = &workspaceTargetFlags{}
tracingFlags = &scaletestTracingFlags{}
prometheusFlags = &scaletestPrometheusFlags{}
timeoutStrategy = &timeoutFlags{}
cleanupStrategy = newScaletestCleanupStrategy()
output = &scaletestOutputFlags{}
chatsPerWorkspace int64
prompt string
turns int64
turnStartDelay time.Duration
llmMockURL string
providerPropagationWait time.Duration
targetFlags = &workspaceTargetFlags{allowEmpty: true}
tracingFlags = &scaletestTracingFlags{}
prometheusFlags = &scaletestPrometheusFlags{}
timeoutStrategy = &timeoutFlags{}
cleanupStrategy = newScaletestCleanupStrategy()
output = &scaletestOutputFlags{}
)
cmd := &serpent.Command{
@@ -72,8 +71,13 @@ func (r *RootCmd) scaletestChat() *serpent.Command {
return err
}
logger := slog.Make(sloghuman.Sink(inv.Stderr)).Leveled(slog.LevelDebug)
modelConfigID, err := chat.EnsureScaletestModelConfig(ctx, codersdk.NewExperimentalClient(client), logger, llmMockURL)
if len(workspaces) == 0 {
workspaces = append(workspaces, codersdk.Workspace{OrganizationID: me.OrganizationIDs[0]})
_, _ = fmt.Fprintln(inv.Stderr, "No scaletest workspaces found; running chats without workspace context.")
}
logger := inv.Logger
modelConfigID, err := chat.EnsureScaletestModelConfig(ctx, client, logger, llmMockURL, providerPropagationWait)
if err != nil {
return err
}
@@ -154,7 +158,7 @@ func (r *RootCmd) scaletestChat() *serpent.Command {
// Run the chat harness in the background so the CLI can release the
// follow-up turns after every runner finishes its initial turn.
totalChats := int64(len(workspaces)) * chatsPerWorkspace
_, _ = fmt.Fprintf(inv.Stderr, "Starting chat scale test with %d chats across %d workspaces...\n", totalChats, len(workspaces))
_, _ = fmt.Fprintf(inv.Stderr, "Starting chat scale test with %d chats across %d targets...\n", totalChats, len(workspaces))
testCtx, testCancel := timeoutStrategy.toContext(ctx)
defer testCancel()
testDone := make(chan error, 1)
@@ -243,6 +247,13 @@ func (r *RootCmd) scaletestChat() *serpent.Command {
Value: serpent.StringOf(&llmMockURL),
Required: true,
},
{
Flag: "provider-propagation-wait",
Description: "Time to wait after creating or updating the mock LLM provider so every coderd replica's cached provider config expires. The default exceeds the server-side cache TTL.",
Default: chat.DefaultProviderPropagationWait.String(),
Value: serpent.DurationOf(&providerPropagationWait),
Hidden: true,
},
}
targetFlags.attach(&cmd.Options)
output.attach(&cmd.Options)
cli/exp_scaletest_chat_test.go
+141
View File
@@ -0,0 +1,141 @@
//go:build !slim
package cli_test
import (
"bytes"
"context"
"io"
"strings"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/sloghuman"
"github.com/coder/coder/v2/cli/clitest"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/llmmock"
"github.com/coder/coder/v2/testutil"
)
const scaletestChatPrompt = "Reply with one short sentence from the scaletest."
func TestScaleTestChat(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
values := coderdtest.DeploymentValues(t, func(dv *codersdk.DeploymentValues) {
require.NoError(t, dv.AI.BridgeConfig.Enabled.Set("true"))
// Keep AI Gateway routing disabled so the chat uses the direct model
// route to the mock provider, avoiding the need for an aibridged daemon.
require.NoError(t, dv.AI.Chat.AIGatewayRoutingEnabled.Set("false"))
})
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: values,
})
coderdtest.CreateFirstUser(t, client)
server := new(llmmock.Server)
require.NoError(t, server.Start(context.Background(), llmmock.Config{
Address: "127.0.0.1:0",
Logger: slog.Make(sloghuman.Sink(io.Discard)).Leveled(slog.LevelDebug),
}))
t.Cleanup(func() {
require.NoError(t, server.Stop())
})
mockURL := server.APIAddress() + "/v1"
inv, root := clitest.New(t,
"exp", "scaletest", "chat",
"--chats-per-workspace", "1",
"--turns", "1",
"--prompt", scaletestChatPrompt,
"--timeout", "30s",
"--job-timeout", "30s",
"--cleanup-timeout", "30s",
"--cleanup-job-timeout", "30s",
"--scaletest-prometheus-address", "127.0.0.1:0",
"--scaletest-prometheus-wait", "0s",
"--provider-propagation-wait", "10ms",
"--llm-mock-url", mockURL,
)
//nolint:gocritic // The scaletest chat command requires an admin client.
clitest.SetupConfig(t, client, root)
var stderr bytes.Buffer
inv.Stdout = io.Discard
inv.Stderr = &stderr
err := inv.WithContext(ctx).Run()
require.NoError(t, err, stderr.String())
require.Contains(t, stderr.String(), "Scale test passed: 1/1 runs succeeded")
provider, err := client.AIProvider(ctx, "coder-scaletest-mock")
require.NoError(t, err)
require.Equal(t, mockURL, provider.BaseURL)
expClient := codersdk.NewExperimentalClient(client)
configs, err := expClient.ListChatModelConfigs(ctx)
require.NoError(t, err)
matchingConfigs := scaletestModelConfigsForProvider(configs, provider.ID)
require.Len(t, matchingConfigs, 1)
require.True(t, matchingConfigs[0].Enabled)
chats, err := expClient.ListChats(ctx, &codersdk.ListChatsOptions{Query: "archived:true"})
require.NoError(t, err)
var scaletestMessages []codersdk.ChatMessage
for _, chat := range chats {
resp, err := expClient.GetChatMessages(ctx, chat.ID, nil)
require.NoError(t, err)
if userText, ok := chatMessageText(resp.Messages, codersdk.ChatMessageRoleUser); ok &&
strings.Contains(userText, scaletestChatPrompt) {
scaletestMessages = resp.Messages
break
}
}
require.NotEmpty(t, scaletestMessages)
assistantText, ok := chatMessageText(scaletestMessages, codersdk.ChatMessageRoleAssistant)
require.True(t, ok, "expected an assistant reply in the scaletest chat")
require.NotEmpty(t, assistantText)
}
// chatMessageText concatenates the text parts of every message with the given
// role, reporting whether any such message was found. It aggregates across
// messages because the API returns them newest-first and a turn can produce
// more than one message per role.
func chatMessageText(messages []codersdk.ChatMessage, role codersdk.ChatMessageRole) (string, bool) {
var (
b strings.Builder
found bool
)
for _, msg := range messages {
if msg.Role != role {
continue
}
found = true
for _, part := range msg.Content {
if part.Type == codersdk.ChatMessagePartTypeText {
_, _ = b.WriteString(part.Text)
}
}
}
return b.String(), found
}
func scaletestModelConfigsForProvider(configs []codersdk.ChatModelConfig, providerID uuid.UUID) []codersdk.ChatModelConfig {
matches := make([]codersdk.ChatModelConfig, 0, 1)
for _, config := range configs {
if config.AIProviderID == nil || *config.AIProviderID != providerID {
continue
}
if config.Model != "scaletest-model" {
continue
}
matches = append(matches, config)
}
return matches
}