feat: show agent turn summary in agents sidebar (#24942)

Persists the agent-generated turn-end summary on `chats` and shows it as
the Agents sidebar subtitle when present, falling back to the model
name. Errors still take precedence.

> Mux is acting on Mike's behalf.

## What changes

**Storage.** New nullable `last_turn_summary` column on `chats`
(migration `000486`). New `UpdateChatLastTurnSummary` query normalizes
blank/whitespace input to `NULL`, preserves `updated_at` (so the chat
does not jump to the top of the sidebar on summary writes), and uses an
`expected_updated_at` stale-write guard so an older async summary cannot
overwrite a newer turn.

**Backend.** `coderd/x/chatd/chatd.go` decouples summary generation from
webpush. Generated summaries persist for completed parent turns even
when webpush is unconfigured or has no subscriptions. The same generated
text is reused as the webpush body when webpush is configured, so the
summary model is not called twice. Generic fallback push text is no
longer persisted; it clears any stale summary instead.
Error/interrupt/pending-action terminal paths clear `last_turn_summary`
for the latest turn.

**Frontend.** `AgentsSidebar.tsx` subtitle priority is now `errorReason
|| lastTurnSummary || modelName`, normalized via the existing
`asNonEmptyString` helper from `blockUtils.ts`.

## Tests

- `TestUpdateChatLastTurnSummary` (database): success,
whitespace-to-NULL, stale guard rejects, `updated_at` preserved.
- `TestUpdateLastTurnSummaryRejectsStaleWrites` (chatd internal): direct
stale-`expected_updated_at` test.
- `TestSuccessfulChatPersistsTurnSummaryWithoutWebPush`: persistence
works without webpush subscriptions.
- `TestSuccessfulChatSendsWebPushWithSummary`: same generated text
drives both DB and push body.
-
`TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText`:
fallback text is not persisted.
- `TestErroredChatClearsLastTurnSummaryAndSendsWebPush`: error path
clears the field.
- `TestInterruptChatDoesNotSendWebPushNotification`: interrupt path
clears the field, no push fires.
- `AgentsSidebar.test.tsx`: subtitle priority for summary-present,
error-wins, no-summary fallback, whitespace fallback.
- `AgentsSidebar.stories.tsx`: `ChatWithTurnSummary` and
`ChatWithTurnSummaryAndError`.

## Notes

- No backfill. Existing chats keep showing the model name until their
next turn completes.
- Parent chats only in this iteration; the field is rendered on any
`Chat` if a future change extends generation to children.
- Decoupling generation from webpush adds quickgen model calls for
completed parent turns that previously skipped generation when no
subscriptions existed. Existing parent-only, assistant-text-present,
`PushSummaryModel` configured, and bounded-timeout gates keep this
behavior bounded.
This commit is contained in:
Michael Suchacz
2026-05-06 16:43:35 +02:00
committed by GitHub
parent 369a191972
commit 0bfb9f6f13
40 changed files with 1313 additions and 115 deletions
+196 -48
View File
@@ -67,6 +67,7 @@ const (
instructionCacheTTL = 5 * time.Minute
workspaceDialValidationDelay = 5 * time.Second
workspaceMCPDiscoveryTimeout = 5 * time.Second
turnSummaryWriteTimeout = 5 * time.Second
// defaultDialTimeout matches the timeout used by ~8 other
// server-side AgentConn callers.
defaultDialTimeout = 30 * time.Second
@@ -5509,12 +5510,21 @@ func (p *Server) processChat(ctx context.Context, chat database.Chat) {
})
p.publishChatActionRequired(finishResult.updatedChat, runResult.PendingDynamicToolCalls)
}
if !wasInterrupted {
if wasInterrupted {
p.maybeClearLastTurnSummaryAsync(cleanupCtx, finishResult.updatedChat, logger)
} else {
lastErrorMessage := ""
if lastErrorPayload != nil {
lastErrorMessage = lastErrorPayload.Message
}
p.maybeSendPushNotification(cleanupCtx, finishResult.updatedChat, status, lastErrorMessage, runResult, logger)
p.maybeFinalizeTurnSummaryAndPush(
cleanupCtx,
finishResult.updatedChat,
status,
lastErrorMessage,
runResult,
logger,
)
}
}()
@@ -5537,6 +5547,7 @@ func (p *Server) processChat(ctx context.Context, chat database.Chat) {
logger.Info(ctx, "chat canceled during shutdown; returning to pending")
status = database.ChatStatusPending
lastErrorPayload = nil
wasInterrupted = true
return
}
logger.Error(ctx, "failed to process chat", slog.Error(err))
@@ -5567,6 +5578,7 @@ func (p *Server) processChat(ctx context.Context, chat database.Chat) {
logger.Info(ctx, "chat completed during shutdown; returning to pending")
status = database.ChatStatusPending
lastErrorPayload = nil
wasInterrupted = true
return
}
}
@@ -8251,12 +8263,9 @@ func parseDynamicToolNames(raw pqtype.NullRawMessage) (map[string]bool, error) {
return names, nil
}
// maybeSendPushNotification sends a web push notification when an
// agent chat reaches a terminal state. For errors it dispatches
// synchronously; for successful completions it spawns a goroutine
// that generates a short LLM summary before dispatching. The caller
// is responsible for skipping interrupted chats.
func (p *Server) maybeSendPushNotification(
// maybeFinalizeTurnSummaryAndPush updates the cached turn summary for
// parent chats and optionally sends a web push notification.
func (p *Server) maybeFinalizeTurnSummaryAndPush(
ctx context.Context,
chat database.Chat,
status database.ChatStatus,
@@ -8264,56 +8273,195 @@ func (p *Server) maybeSendPushNotification(
runResult runChatResult,
logger slog.Logger,
) {
if p.webpushDispatcher == nil || p.webpushDispatcher.PublicKey() == "" {
return
}
if chat.ParentChatID.Valid {
return
}
switch status {
case database.ChatStatusError:
pushBody := "Agent encountered an error."
if lastError != "" {
pushBody = lastError
}
p.dispatchPush(ctx, chat, pushBody, status, logger)
case database.ChatStatusWaiting:
// Generate a push notification summary asynchronously
// using a cheap LLM model. This avoids blocking the
// deferred cleanup path while still providing a
// meaningful notification body.
debugSvc := p.existingDebugService()
p.inflight.Add(1)
go func() {
defer p.inflight.Done()
pushCtx := context.WithoutCancel(ctx)
pushBody := "Agent has finished running."
assistantText := strings.TrimSpace(runResult.FinalAssistantText)
if assistantText != "" && runResult.PushSummaryModel != nil {
if summary := generatePushSummary(
pushCtx,
chat,
assistantText,
runResult.FallbackProvider,
runResult.FallbackModel,
runResult.PushSummaryModel,
runResult.ProviderKeys,
logger,
debugSvc,
runResult.TriggerMessageID,
runResult.HistoryTipMessageID,
); summary != "" {
pushBody = summary
}
}
p.finalizeSuccessfulTurnSummaryAndPush(ctx, chat, runResult, logger)
p.dispatchPush(pushCtx, chat, pushBody, status, logger)
}()
case database.ChatStatusPending:
p.finalizeSuccessfulTurnSummary(ctx, chat, runResult, logger)
case database.ChatStatusError:
p.clearLastTurnSummaryAsync(ctx, chat, logger)
if p.webpushConfigured() {
pushBody := "Agent encountered an error."
if lastError != "" {
pushBody = lastError
}
p.dispatchPush(ctx, chat, pushBody, status, logger)
}
case database.ChatStatusRequiresAction:
p.clearLastTurnSummaryAsync(ctx, chat, logger)
default:
// New statuses must be classified before they can safely
// preserve or finalize a cached turn summary.
p.clearLastTurnSummaryAsync(ctx, chat, logger)
}
}
func (p *Server) finalizeSuccessfulTurnSummary(
ctx context.Context,
chat database.Chat,
runResult runChatResult,
logger slog.Logger,
) {
p.finalizeSuccessfulTurnSummaryWithAfterFunc(ctx, chat, runResult, logger, func(context.Context, string) {})
}
func (p *Server) finalizeSuccessfulTurnSummaryAndPush(
ctx context.Context,
chat database.Chat,
runResult runChatResult,
logger slog.Logger,
) {
p.finalizeSuccessfulTurnSummaryWithAfterFunc(ctx, chat, runResult, logger, func(finalizeCtx context.Context, summary string) {
p.dispatchSuccessfulTurnPush(finalizeCtx, chat, summary, logger)
})
}
func (p *Server) finalizeSuccessfulTurnSummaryWithAfterFunc(
ctx context.Context,
chat database.Chat,
runResult runChatResult,
logger slog.Logger,
afterFinalize func(context.Context, string),
) {
debugSvc := p.existingDebugService()
// This helper runs during processChat cleanup, while processChat is
// still counted in p.inflight. Do not take inflightMu here because
// drainInflight holds it while waiting.
p.inflight.Go(func() {
finalizeCtx := context.WithoutCancel(ctx)
summary := ""
assistantText := strings.TrimSpace(runResult.FinalAssistantText)
if assistantText != "" && runResult.PushSummaryModel != nil {
summary = strings.TrimSpace(generatePushSummary(
finalizeCtx,
chat,
assistantText,
runResult.FallbackProvider,
runResult.FallbackModel,
runResult.PushSummaryModel,
runResult.ProviderKeys,
logger,
debugSvc,
runResult.TriggerMessageID,
runResult.HistoryTipMessageID,
))
}
shouldPersistSummary := summary != "" || chat.LastTurnSummary.Valid
if shouldPersistSummary {
p.updateLastTurnSummary(finalizeCtx, chat, chat.UpdatedAt, summary, logger)
}
afterFinalize(finalizeCtx, summary)
})
}
func (p *Server) dispatchSuccessfulTurnPush(
ctx context.Context,
chat database.Chat,
summary string,
logger slog.Logger,
) {
if !p.webpushConfigured() {
return
}
pushBody := "Agent has finished running."
if summary != "" {
pushBody = summary
}
p.dispatchPush(ctx, chat, pushBody, database.ChatStatusWaiting, logger)
}
func (p *Server) maybeClearLastTurnSummaryAsync(
ctx context.Context,
chat database.Chat,
logger slog.Logger,
) {
if chat.ParentChatID.Valid {
return
}
p.clearLastTurnSummaryAsync(ctx, chat, logger)
}
func (p *Server) clearLastTurnSummaryAsync(
ctx context.Context,
chat database.Chat,
logger slog.Logger,
) {
if !chat.LastTurnSummary.Valid {
return
}
// This helper runs during processChat cleanup, while processChat is
// still counted in p.inflight. Do not take inflightMu here because
// drainInflight holds it while waiting.
p.inflight.Go(func() {
p.updateLastTurnSummary(context.WithoutCancel(ctx), chat, chat.UpdatedAt, "", logger)
})
}
// updateLastTurnSummary writes the cached sidebar summary for a chat.
// Callers should pass a detached context because this method is used for
// best-effort background cache writes.
func (p *Server) updateLastTurnSummary(
ctx context.Context,
chat database.Chat,
expectedUpdatedAt time.Time,
summary string,
logger slog.Logger,
) {
summary = strings.TrimSpace(summary)
lastTurnSummary := sql.NullString{String: summary, Valid: summary != ""}
//nolint:gocritic // Narrow daemon access for best-effort summary cache writes.
updateCtx := dbauthz.AsChatd(ctx)
updateCtx, cancel := context.WithTimeout(updateCtx, turnSummaryWriteTimeout)
defer cancel()
affected, err := p.db.UpdateChatLastTurnSummary(updateCtx, database.UpdateChatLastTurnSummaryParams{
ID: chat.ID,
ExpectedUpdatedAt: expectedUpdatedAt,
LastTurnSummary: lastTurnSummary,
})
if err != nil {
logger.Warn(updateCtx, "failed to update chat turn summary",
slog.F("chat_id", chat.ID),
slog.Error(err),
)
return
}
if affected == 0 {
if summary != "" {
logger.Info(updateCtx, "skipped stale chat turn summary update with non-empty summary",
slog.F("chat_id", chat.ID),
slog.F("summary_length", len(summary)),
slog.F("expected_updated_at", expectedUpdatedAt),
)
return
}
logger.Debug(updateCtx, "skipped stale chat turn summary update",
slog.F("chat_id", chat.ID),
slog.F("expected_updated_at", expectedUpdatedAt),
)
return
}
updatedChat := chat
updatedChat.LastTurnSummary = lastTurnSummary
p.publishChatPubsubEvent(updatedChat, codersdk.ChatWatchEventKindSummaryChange, nil)
}
func (p *Server) webpushConfigured() bool {
return p.webpushDispatcher != nil && p.webpushDispatcher.PublicKey() != ""
}
func (p *Server) dispatchPush(
ctx context.Context,
chat database.Chat,
+9 -1
View File
@@ -3442,7 +3442,11 @@ func TestProcessChat_IgnoresStaleControlNotification(t *testing.T) {
db.EXPECT().UpdateChatStatus(gomock.Any(), gomock.Any()).DoAndReturn(
func(_ context.Context, params database.UpdateChatStatusParams) (database.Chat, error) {
finalStatus = params.Status
return database.Chat{ID: chatID, Status: params.Status}, nil
return database.Chat{
ID: chatID,
Status: params.Status,
LastTurnSummary: sql.NullString{String: "previous summary", Valid: true},
}, nil
},
)
db.EXPECT().GetChatByID(gomock.Any(), chatID).Return(
@@ -3450,6 +3454,8 @@ func TestProcessChat_IgnoresStaleControlNotification(t *testing.T) {
nil,
)
db.EXPECT().UpdateChatLastTurnSummary(gomock.Any(), gomock.Any()).Return(int64(1), nil)
// resolveChatModel fails immediately — that's fine, we only
// need processChat to get past initialization without being
// interrupted by the stale notification.
@@ -3475,6 +3481,8 @@ func TestProcessChat_IgnoresStaleControlNotification(t *testing.T) {
// the status update itself races test teardown.
testutil.TryReceive(ctx, t, done)
WaitUntilIdleForTest(server)
// If the stale notification interrupted us, status would be
// "waiting" (the ErrInterrupted path). Since the gate blocked
// it, processChat reached runChat, which failed on model
+267 -17
View File
@@ -4047,6 +4047,93 @@ func TestPersistToolResultWithBinaryData(t *testing.T) {
require.True(t, foundToolResultInSecondCall, "expected second streamed model call to include execute tool output")
}
func TestRequiresActionChatClearsLastTurnSummary(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if !req.Stream {
return chattest.OpenAINonStreamingResponse("Dynamic tool test")
}
return chattest.OpenAIStreamingResponse(
chattest.OpenAIToolCallChunk(
"my_dynamic_tool",
`{"input":"hello world"}`,
),
)
})
mockPush := &mockWebpushDispatcher{}
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server := chatd.New(chatd.Config{
Logger: logger,
Database: db,
ReplicaID: uuid.New(),
Pubsub: ps,
PendingChatAcquireInterval: 10 * time.Millisecond,
InFlightChatStaleAfter: testutil.WaitSuperLong,
WebpushDispatcher: mockPush,
})
t.Cleanup(func() {
require.NoError(t, server.Close())
})
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
dynamicToolsJSON, err := json.Marshal([]mcpgo.Tool{{
Name: "my_dynamic_tool",
Description: "A test dynamic tool.",
InputSchema: mcpgo.ToolInputSchema{
Type: "object",
Properties: map[string]any{
"input": map[string]any{"type": "string"},
},
Required: []string{"input"},
},
}})
require.NoError(t, err)
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "requires-action-summary-clear",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{
codersdk.ChatMessageText("Please call the dynamic tool."),
},
DynamicTools: dynamicToolsJSON,
})
require.NoError(t, err)
seedLastTurnSummary(ctx, t, db, chat, "previous summary")
server.Start()
var fromDB database.Chat
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
got, dbErr := db.GetChatByID(ctx, chat.ID)
if dbErr != nil {
return false
}
fromDB = got
if got.Status == database.ChatStatusError {
return true
}
return got.Status == database.ChatStatusRequiresAction &&
!got.LastTurnSummary.Valid
}, testutil.IntervalFast)
chatd.WaitUntilIdleForTest(server)
require.Equal(t, database.ChatStatusRequiresAction, fromDB.Status,
"expected requires_action, got %s (last_error=%q)",
fromDB.Status, string(fromDB.LastError.RawMessage))
require.False(t, fromDB.LastTurnSummary.Valid,
"requires action chats should clear cached turn summaries")
require.Equal(t, int32(0), mockPush.dispatchCount.Load(),
"expected no web push dispatch for a requires_action chat")
}
func TestDynamicToolCallPausesAndResumes(t *testing.T) {
t.Parallel()
@@ -5907,6 +5994,24 @@ func seedChatDependenciesWithProviderPolicy(
return user, org, providerConfig, model
}
func seedLastTurnSummary(
ctx context.Context,
t *testing.T,
db database.Store,
chat database.Chat,
summary string,
) {
t.Helper()
affected, err := db.UpdateChatLastTurnSummary(ctx, database.UpdateChatLastTurnSummaryParams{
ID: chat.ID,
ExpectedUpdatedAt: chat.UpdatedAt,
LastTurnSummary: sql.NullString{String: summary, Valid: true},
})
require.NoError(t, err)
require.Equal(t, int64(1), affected)
}
func waitForTerminalChatStatusEvent(
ctx context.Context,
t *testing.T,
@@ -6121,7 +6226,6 @@ func TestInterruptChatDoesNotSendWebPushNotification(t *testing.T) {
InFlightChatStaleAfter: testutil.WaitSuperLong,
WebpushDispatcher: mockPush,
})
server.Start()
t.Cleanup(func() {
require.NoError(t, server.Close())
})
@@ -6137,6 +6241,9 @@ func TestInterruptChatDoesNotSendWebPushNotification(t *testing.T) {
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("hello")},
})
require.NoError(t, err)
seedLastTurnSummary(ctx, t, db, chat, "previous summary")
server.Start()
// Wait for the chat to be picked up and start streaming.
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
@@ -6168,6 +6275,12 @@ func TestInterruptChatDoesNotSendWebPushNotification(t *testing.T) {
}
return fromDB.Status == database.ChatStatusWaiting && !fromDB.WorkerID.Valid
}, testutil.IntervalFast)
chatd.WaitUntilIdleForTest(server)
fromDB, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.False(t, fromDB.LastTurnSummary.Valid,
"interrupted chats should clear cached turn summaries")
// Verify no web push notification was dispatched.
require.Equal(t, int32(0), mockPush.dispatchCount.Load(),
@@ -6435,7 +6548,7 @@ func TestSuccessfulChatSendsWebPushWithSummary(t *testing.T) {
user, org, model := seedChatDependencies(t, db)
setOpenAIProviderBaseURL(ctx, t, db, openAIURL)
_, err := server.CreateChat(ctx, chatd.CreateOptions{
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "summary-push-test",
@@ -6447,19 +6560,71 @@ func TestSuccessfulChatSendsWebPushWithSummary(t *testing.T) {
// The push notification is dispatched asynchronously after the
// chat finishes, so we poll for it rather than checking
// immediately after the status transitions to waiting.
var fromDB database.Chat
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
return mockPush.dispatchCount.Load() >= 1
var dbErr error
fromDB, dbErr = db.GetChatByID(ctx, chat.ID)
return dbErr == nil && mockPush.dispatchCount.Load() >= 1 && fromDB.LastTurnSummary.Valid
}, testutil.IntervalFast)
msg := mockPush.getLastMessage()
require.Equal(t, summaryText, msg.Body,
"push body should be the LLM-generated summary")
require.Equal(t, summaryText, fromDB.LastTurnSummary.String,
"last turn summary should be the LLM-generated summary")
require.Equal(t, fromDB.LastTurnSummary.String, msg.Body,
"push body should reuse the persisted generated summary")
require.NotEqual(t, "Agent has finished running.", msg.Body,
"push body should not use the default fallback text")
require.Equal(t, int32(1), nonStreamingRequests.Load(),
"expected exactly one non-streaming request for push summary generation")
}
func TestSuccessfulChatPersistsTurnSummaryWithoutWebPush(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
const assistantText = "I fixed the bug and added regression coverage."
const summaryText = "Fixed the bug and added regression coverage."
var nonStreamingRequests atomic.Int32
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if !req.Stream {
nonStreamingRequests.Add(1)
return chattest.OpenAINonStreamingResponse(summaryText)
}
return chattest.OpenAIStreamingResponse(
chattest.OpenAITextChunks(assistantText)...,
)
})
server := newActiveTestServer(t, db, ps)
user, org, model := seedChatDependencies(t, db)
setOpenAIProviderBaseURL(ctx, t, db, openAIURL)
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "summary-no-webpush-test",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("do the thing")},
})
require.NoError(t, err)
var fromDB database.Chat
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
var dbErr error
fromDB, dbErr = db.GetChatByID(ctx, chat.ID)
return dbErr == nil && fromDB.LastTurnSummary.Valid
}, testutil.IntervalFast)
require.Equal(t, summaryText, fromDB.LastTurnSummary.String,
"summary should persist even when web push is unavailable")
require.Equal(t, int32(1), nonStreamingRequests.Load(),
"expected exactly one non-streaming request for summary generation")
}
func TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText(t *testing.T) {
t.Parallel()
@@ -6489,7 +6654,6 @@ func TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText(t
InFlightChatStaleAfter: testutil.WaitSuperLong,
WebpushDispatcher: mockPush,
})
server.Start()
t.Cleanup(func() {
require.NoError(t, server.Close())
})
@@ -6497,7 +6661,7 @@ func TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText(t
user, org, model := seedChatDependencies(t, db)
setOpenAIProviderBaseURL(ctx, t, db, openAIURL)
_, err := server.CreateChat(ctx, chatd.CreateOptions{
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "empty-summary-push-test",
@@ -6505,11 +6669,19 @@ func TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText(t
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("do the thing")},
})
require.NoError(t, err)
seedLastTurnSummary(ctx, t, db, chat, "previous summary")
server.Start()
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
return mockPush.dispatchCount.Load() >= 1
}, testutil.IntervalFast)
fromDB, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.False(t, fromDB.LastTurnSummary.Valid,
"fallback push text should not be persisted")
msg := mockPush.getLastMessage()
require.Equal(t, "Agent has finished running.", msg.Body,
"push body should fall back when the final assistant text is empty")
@@ -6517,6 +6689,68 @@ func TestSuccessfulChatSendsWebPushFallbackWithoutSummaryForEmptyAssistantText(t
"push summary should not be requested when final assistant text has no usable text")
}
func TestErroredChatClearsLastTurnSummaryAndSendsWebPush(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitLong)
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if !req.Stream {
return chattest.OpenAINonStreamingResponse("title")
}
return chattest.OpenAIErrorResponse(http.StatusBadRequest, "invalid_request_error", "Bad request")
})
mockPush := &mockWebpushDispatcher{}
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server := chatd.New(chatd.Config{
Logger: logger,
Database: db,
ReplicaID: uuid.New(),
Pubsub: ps,
PendingChatAcquireInterval: 10 * time.Millisecond,
InFlightChatStaleAfter: testutil.WaitSuperLong,
WebpushDispatcher: mockPush,
})
t.Cleanup(func() {
require.NoError(t, server.Close())
})
user, org, model := seedChatDependencies(t, db)
setOpenAIProviderBaseURL(ctx, t, db, openAIURL)
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
Title: "error-summary-clear-test",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText("do the thing")},
})
require.NoError(t, err)
seedLastTurnSummary(ctx, t, db, chat, "previous summary")
server.Start()
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
fromDB, dbErr := db.GetChatByID(ctx, chat.ID)
return dbErr == nil &&
fromDB.Status == database.ChatStatusError &&
mockPush.dispatchCount.Load() >= 1
}, testutil.IntervalFast)
chatd.WaitUntilIdleForTest(server)
fromDB, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.False(t, fromDB.LastTurnSummary.Valid,
"errored chats should clear cached turn summaries")
msg := mockPush.getLastMessage()
require.NotEqual(t, "Agent encountered an error.", msg.Body)
require.Contains(t, msg.Body, "OpenAI returned an unexpected error")
}
func TestComputerUseSubagentToolsAndModel(t *testing.T) {
t.Parallel()
@@ -6531,8 +6765,9 @@ func TestComputerUseSubagentToolsAndModel(t *testing.T) {
// computer use child chat). We use a raw HTTP handler because
// the chattest AnthropicRequest struct does not capture tools.
type anthropicCall struct {
Model string
Tools []string
Model string
Tools []string
Stream bool
}
var anthropicMu sync.Mutex
var anthropicCalls []anthropicCall
@@ -6563,8 +6798,9 @@ func TestComputerUseSubagentToolsAndModel(t *testing.T) {
}
anthropicMu.Lock()
anthropicCalls = append(anthropicCalls, anthropicCall{
Model: req.Model,
Tools: names,
Model: req.Model,
Tools: names,
Stream: req.Stream,
})
anthropicMu.Unlock()
@@ -6737,11 +6973,15 @@ func TestComputerUseSubagentToolsAndModel(t *testing.T) {
got.Status != database.ChatStatusError {
return false
}
// Ensure the Anthropic mock received at least one call.
// Ensure the Anthropic mock received the child streaming call.
anthropicMu.Lock()
n := len(anthropicCalls)
anthropicMu.Unlock()
return n >= 1
defer anthropicMu.Unlock()
for _, call := range anthropicCalls {
if call.Stream {
return true
}
}
return false
}, testutil.WaitLong, testutil.IntervalFast)
anthropicMu.Lock()
@@ -6751,8 +6991,18 @@ func TestComputerUseSubagentToolsAndModel(t *testing.T) {
require.NotEmpty(t, calls,
"expected at least one Anthropic LLM call")
childModel := calls[0].Model
childTools := calls[0].Tools
var childCall anthropicCall
for _, call := range calls {
if call.Stream {
childCall = call
break
}
}
require.True(t, childCall.Stream,
"expected at least one streaming Anthropic child LLM call")
childModel := childCall.Model
childTools := childCall.Tools
// 1. Verify the model is the computer use model.
require.Equal(t, computerUseModelName, childModel,
+1 -1
View File
@@ -248,7 +248,7 @@ func (p *Server) maybeGenerateChatTitle(
return
}
_, err = p.db.UpdateChatByID(ctx, database.UpdateChatByIDParams{
_, err = p.db.UpdateChatTitleByID(ctx, database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: title,
})
+94
View File
@@ -11,9 +11,14 @@ import (
"github.com/sqlc-dev/pqtype"
"github.com/stretchr/testify/require"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func Test_extractManualTitleTurns(t *testing.T) {
@@ -354,6 +359,95 @@ func Test_renderManualTitlePrompt(t *testing.T) {
}
}
func TestMaybeGenerateChatTitlePreservesUpdatedAt(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
owner := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: owner.ID,
OrganizationID: org.ID,
})
dbgen.ChatProvider(t, db, database.ChatProvider{
Provider: "openai",
DisplayName: "OpenAI",
APIKey: "test-key",
Enabled: true,
CentralApiKeyEnabled: true,
})
modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
Provider: "openai",
Model: "test-model",
})
userPrompt := "summarize failed workspace build logs"
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: owner.ID,
LastModelConfigID: modelConfig.ID,
Title: fallbackChatTitle(userPrompt),
Status: database.ChatStatusWaiting,
ClientType: database.ChatClientTypeUi,
})
expectedUpdatedAt := time.Date(2024, time.January, 2, 3, 4, 5, 0, time.UTC)
chat, err := db.UpdateChatStatusPreserveUpdatedAt(ctx, database.UpdateChatStatusPreserveUpdatedAtParams{
ID: chat.ID,
Status: chat.Status,
UpdatedAt: expectedUpdatedAt,
})
require.NoError(t, err)
const wantTitle = "Failed workspace logs"
model := &chattest.FakeModel{
GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
require.Equal(t, "propose_title", call.SchemaName)
return &fantasy.ObjectResponse{
Object: map[string]any{"title": wantTitle},
}, nil
},
}
message := mustChatMessage(
t,
database.ChatMessageRoleUser,
database.ChatMessageVisibilityBoth,
codersdk.ChatMessageText(userPrompt),
)
message.ID = 1
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
generated := &generatedChatTitle{}
server := &Server{db: db}
server.maybeGenerateChatTitle(
ctx,
chat,
[]database.ChatMessage{message},
"openai",
"test-model",
model,
chatprovider.ProviderAPIKeys{},
generated,
logger,
nil,
)
fetched, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, wantTitle, fetched.Title)
require.True(t, fetched.UpdatedAt.Equal(expectedUpdatedAt),
"updated_at = %s, want same instant as %s",
fetched.UpdatedAt,
expectedUpdatedAt,
)
gotTitle, ok := generated.Load()
require.True(t, ok)
require.Equal(t, wantTitle, gotTitle)
}
func Test_titleGenerationPrompt_UsesSlimRules(t *testing.T) {
t.Parallel()
+5 -5
View File
@@ -51,7 +51,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideUnset(t *testing.T) {
}
db.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return("", nil)
db.EXPECT().UpdateChatByID(gomock.Any(), database.UpdateChatByIDParams{
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
@@ -98,7 +98,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideUnset(t *testing.T) {
}
db.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return("", nil)
db.EXPECT().UpdateChatByID(gomock.Any(), database.UpdateChatByIDParams{
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
@@ -146,7 +146,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideReadDBError(t *testing.T)
}
db.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return("", sql.ErrConnDone)
db.EXPECT().UpdateChatByID(gomock.Any(), database.UpdateChatByIDParams{
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
@@ -193,7 +193,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideMalformedFallsThrough(t *
}
db.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return("not-a-uuid", nil)
db.EXPECT().UpdateChatByID(gomock.Any(), database.UpdateChatByIDParams{
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
@@ -247,7 +247,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUsable(t *testing.T) {
db.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return(overrideConfig.ID.String(), nil)
db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil)
db.EXPECT().GetEnabledChatProviders(gomock.Any()).Return([]database.ChatProvider{{Provider: "openai"}}, nil)
db.EXPECT().UpdateChatByID(gomock.Any(), database.UpdateChatByIDParams{
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
@@ -0,0 +1,194 @@
package chatd
import (
"context"
"database/sql"
"encoding/json"
"sync/atomic"
"testing"
"time"
"charm.land/fantasy"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func TestUpdateLastTurnSummaryRejectsStaleWrites(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
owner := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: owner.ID,
OrganizationID: org.ID,
})
_, err := db.InsertChatProvider(ctx, database.InsertChatProviderParams{
Provider: "openai",
DisplayName: "OpenAI",
APIKey: "test-key",
Enabled: true,
CentralApiKeyEnabled: true,
})
require.NoError(t, err)
modelCfg, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{
Provider: "openai",
Model: "test-model",
DisplayName: "Test Model",
CreatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true},
UpdatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true},
Enabled: true,
IsDefault: true,
ContextLimit: 128000,
CompressionThreshold: 80,
Options: json.RawMessage(`{}`),
})
require.NoError(t, err)
chat, err := db.InsertChat(ctx, database.InsertChatParams{
OrganizationID: org.ID,
Status: database.ChatStatusWaiting,
ClientType: database.ChatClientTypeUi,
OwnerID: owner.ID,
LastModelConfigID: modelCfg.ID,
Title: "summary-chat",
})
require.NoError(t, err)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server := &Server{db: db}
server.updateLastTurnSummary(ctx, chat, chat.UpdatedAt, "fresh summary", logger)
fetched, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, sql.NullString{String: "fresh summary", Valid: true}, fetched.LastTurnSummary)
advancedUpdatedAt := chat.UpdatedAt.Add(time.Second)
_, err = db.UpdateChatStatusPreserveUpdatedAt(ctx, database.UpdateChatStatusPreserveUpdatedAtParams{
ID: chat.ID,
Status: database.ChatStatusRunning,
UpdatedAt: advancedUpdatedAt,
})
require.NoError(t, err)
server.updateLastTurnSummary(context.WithoutCancel(ctx), chat, chat.UpdatedAt, "stale summary", logger)
fetched, err = db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, sql.NullString{String: "fresh summary", Valid: true}, fetched.LastTurnSummary)
require.Equal(t, advancedUpdatedAt, fetched.UpdatedAt)
}
func TestPendingChatPersistsSummaryButSkipsWebPush(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
owner := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: owner.ID,
OrganizationID: org.ID,
})
_, err := db.InsertChatProvider(ctx, database.InsertChatProviderParams{
Provider: "openai",
DisplayName: "OpenAI",
APIKey: "test-key",
Enabled: true,
CentralApiKeyEnabled: true,
})
require.NoError(t, err)
modelCfg, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{
Provider: "openai",
Model: "test-model",
DisplayName: "Test Model",
CreatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true},
UpdatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true},
Enabled: true,
IsDefault: true,
ContextLimit: 128000,
CompressionThreshold: 80,
Options: json.RawMessage(`{}`),
})
require.NoError(t, err)
chat, err := db.InsertChat(ctx, database.InsertChatParams{
OrganizationID: org.ID,
Status: database.ChatStatusPending,
ClientType: database.ChatClientTypeUi,
OwnerID: owner.ID,
LastModelConfigID: modelCfg.ID,
Title: "summary-pending-chat",
})
require.NoError(t, err)
const summary = "Finished the queued turn."
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) {
return &fantasy.Response{
Content: fantasy.ResponseContent{
fantasy.TextContent{Text: summary},
},
}, nil
},
}
dispatcher := &recordingWebpushDispatcher{}
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server := &Server{db: db, webpushDispatcher: dispatcher}
server.maybeFinalizeTurnSummaryAndPush(
context.WithoutCancel(ctx),
chat,
database.ChatStatusPending,
"",
runChatResult{
FinalAssistantText: "I finished the queued turn.",
PushSummaryModel: model,
FallbackProvider: model.Provider(),
FallbackModel: model.Model(),
},
logger,
)
server.drainInflight()
fetched, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Equal(t, sql.NullString{String: summary, Valid: true}, fetched.LastTurnSummary)
require.Equal(t, int32(0), dispatcher.dispatchCount.Load())
}
type recordingWebpushDispatcher struct {
dispatchCount atomic.Int32
}
func (d *recordingWebpushDispatcher) Dispatch(
_ context.Context,
_ uuid.UUID,
_ codersdk.WebpushMessage,
) error {
d.dispatchCount.Add(1)
return nil
}
func (*recordingWebpushDispatcher) Test(_ context.Context, _ codersdk.WebpushSubscription) error {
return nil
}
func (*recordingWebpushDispatcher) PublicKey() string {
return "test-vapid-public-key"
}