feat(chatd): use last assistant message as push notification summary (#22671)

Instead of the static 'Agent has finished running.' text, extract a
summary from the last assistant message to give users meaningful context
about what the agent accomplished. Falls back to the static text if no
suitable message is found.

Co-authored-by: Kyle Carberry <kyle@carberry.com>

coderd/chatd/chatd.go

@@ -2022,33 +2022,8 @@ func (p *Server) processChat(ctx context.Context, chat database.Chat) {
 		chat.Status = status
 		p.publishChatPubsubEvent(chat, coderdpubsub.ChatEventKindStatusChange)
 
-		// Send a web push notification when the agent finishes
-		// processing. We only notify for terminal states (waiting
-		// = success, error = failure) and skip sub-agent chats
-		// and user-interrupted chats to avoid unnecessary
-		// notifications.
-		if p.webpushDispatcher != nil && p.webpushDispatcher.PublicKey() != "" && !chat.ParentChatID.Valid && !wasInterrupted {
-			if status == database.ChatStatusWaiting || status == database.ChatStatusError {
-				pushMsg := codersdk.WebpushMessage{
-					Title: chat.Title,
-					Body:  "Agent has finished running.",
-					Icon:  "/favicon.ico",
-					Data:  map[string]string{"url": fmt.Sprintf("/agents/%s", chat.ID)},
-				}
-				if status == database.ChatStatusError {
-					pushMsg.Body = "Agent encountered an error."
-					if lastError != "" {
-						pushMsg.Body = lastError
-					}
-				}
-				if err := p.webpushDispatcher.Dispatch(cleanupCtx, chat.OwnerID, pushMsg); err != nil {
-					logger.Warn(cleanupCtx, "failed to send chat completion web push",
-						slog.F("chat_id", chat.ID),
-						slog.F("status", status),
-						slog.Error(err),
-					)
-				}
-			}
+		if !wasInterrupted {
+			p.maybeSendPushNotification(cleanupCtx, chat, status, lastError, logger)
 		}
 	}()
 
@@ -2977,6 +2952,90 @@ func (p *Server) recoverStaleChats(ctx context.Context) {
 	}
 }
 
+// maybeSendPushNotification sends a web push notification when an
+// agent chat reaches a terminal state. For errors it dispatches
+// synchronously; for successful completions it spawns a goroutine
+// that generates a short LLM summary before dispatching. The caller
+// is responsible for skipping interrupted chats.
+func (p *Server) maybeSendPushNotification(
+	ctx context.Context,
+	chat database.Chat,
+	status database.ChatStatus,
+	lastError string,
+	logger slog.Logger,
+) {
+	if p.webpushDispatcher == nil || p.webpushDispatcher.PublicKey() == "" {
+		return
+	}
+	if chat.ParentChatID.Valid {
+		return
+	}
+
+	switch status {
+	case database.ChatStatusError:
+		pushBody := "Agent encountered an error."
+		if lastError != "" {
+			pushBody = lastError
+		}
+		p.dispatchPush(ctx, chat, pushBody, status, logger)
+
+	case database.ChatStatusWaiting:
+		// Generate a push notification summary asynchronously
+		// using a cheap LLM model. This avoids blocking the
+		// deferred cleanup path while still providing a
+		// meaningful notification body.
+		p.inflight.Add(1)
+		go func() {
+			defer p.inflight.Done()
+			pushCtx := context.WithoutCancel(ctx)
+			pushBody := "Agent has finished running."
+
+			msg, err := p.db.GetLastChatMessageByRole(pushCtx, database.GetLastChatMessageByRoleParams{
+				ChatID: chat.ID,
+				Role:   "assistant",
+			})
+			if err == nil {
+				content, parseErr := chatprompt.ParseContent(msg.Role, msg.Content)
+				if parseErr == nil {
+					assistantText := strings.TrimSpace(contentBlocksToText(content))
+					if assistantText != "" {
+						model, _, keys, resolveErr := p.resolveChatModel(pushCtx, chat)
+						if resolveErr == nil {
+							if summary := generatePushSummary(pushCtx, chat.Title, assistantText, model, keys, logger); summary != "" {
+								pushBody = summary
+							}
+						}
+					}
+				}
+			}
+
+			p.dispatchPush(pushCtx, chat, pushBody, status, logger)
+		}()
+	}
+}
+
+func (p *Server) dispatchPush(
+	ctx context.Context,
+	chat database.Chat,
+	body string,
+	status database.ChatStatus,
+	logger slog.Logger,
+) {
+	pushMsg := codersdk.WebpushMessage{
+		Title: chat.Title,
+		Body:  body,
+		Icon:  "/favicon.ico",
+		Data:  map[string]string{"url": fmt.Sprintf("/agents/%s", chat.ID)},
+	}
+	if err := p.webpushDispatcher.Dispatch(ctx, chat.OwnerID, pushMsg); err != nil {
+		logger.Warn(ctx, "failed to send chat completion web push",
+			slog.F("chat_id", chat.ID),
+			slog.F("status", status),
+			slog.Error(err),
+		)
+	}
+}
+
 // Close stops the processor and waits for it to finish.
 func (p *Server) Close() error {
 	p.cancel()

coderd/chatd/chatd_test.go

@@ -1482,6 +1482,12 @@ func (m *mockWebpushDispatcher) Dispatch(_ context.Context, userID uuid.UUID, ms
 	return nil
 }
 
+func (m *mockWebpushDispatcher) getLastMessage() codersdk.WebpushMessage {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.lastMessage
+}
+
 func (*mockWebpushDispatcher) Test(_ context.Context, _ codersdk.WebpushSubscription) error {
 	return nil
 }
@@ -1674,3 +1680,66 @@ func TestCloseDuringShutdownContextCanceledShouldRetryOnNewReplica(t *testing.T)
 			!fromDB.LastError.Valid
 	}, testutil.WaitMedium, testutil.IntervalFast)
 }
+
+func TestSuccessfulChatSendsWebPushWithSummary(t *testing.T) {
+	t.Parallel()
+
+	db, ps := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	const assistantText = "I have completed the task successfully and all tests are passing now."
+	const summaryText = "Completed task and verified all tests pass."
+
+	openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
+		if !req.Stream {
+			// Non-streaming calls are used for title
+			// generation and push summary generation.
+			// Return the summary text for both — the title
+			// result is irrelevant to this test.
+			return chattest.OpenAINonStreamingResponse(summaryText)
+		}
+		return chattest.OpenAIStreamingResponse(
+			chattest.OpenAITextChunks(assistantText)...,
+		)
+	})
+
+	mockPush := &mockWebpushDispatcher{}
+
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	server := chatd.New(chatd.Config{
+		Logger:                     logger,
+		Database:                   db,
+		ReplicaID:                  uuid.New(),
+		Pubsub:                     ps,
+		PendingChatAcquireInterval: 10 * time.Millisecond,
+		InFlightChatStaleAfter:     testutil.WaitSuperLong,
+		WebpushDispatcher:          mockPush,
+	})
+	t.Cleanup(func() {
+		require.NoError(t, server.Close())
+	})
+
+	user, model := seedChatDependencies(ctx, t, db)
+	setOpenAIProviderBaseURL(ctx, t, db, openAIURL)
+
+	_, err := server.CreateChat(ctx, chatd.CreateOptions{
+		OwnerID:            user.ID,
+		Title:              "summary-push-test",
+		ModelConfigID:      model.ID,
+		InitialUserContent: []fantasy.Content{fantasy.TextContent{Text: "do the thing"}},
+	})
+	require.NoError(t, err)
+
+	// The push notification is dispatched asynchronously after the
+	// chat finishes, so we poll for it rather than checking
+	// immediately after the status transitions to waiting.
+	testutil.Eventually(ctx, t, func(ctx context.Context) bool {
+		return mockPush.dispatchCount.Load() >= 1
+	}, testutil.IntervalFast)
+
+	msg := mockPush.getLastMessage()
+	require.Equal(t, summaryText, msg.Body,
+		"push body should be the LLM-generated summary")
+	require.NotEqual(t, "Agent has finished running.", msg.Body,
+		"push body should not use the default fallback text")
+}

coderd/chatd/quickgen.go

@@ -128,37 +128,11 @@ func generateTitle(
 	model fantasy.LanguageModel,
 	input string,
 ) (string, error) {
-	prompt := []fantasy.Message{
-		{
-			Role: fantasy.MessageRoleSystem,
-			Content: []fantasy.MessagePart{
-				fantasy.TextPart{Text: titleGenerationPrompt},
-			},
-		},
-		{
-			Role: fantasy.MessageRoleUser,
-			Content: []fantasy.MessagePart{
-				fantasy.TextPart{Text: input},
-			},
-		},
-	}
-
-	var maxOutputTokens int64 = 256
-
-	var response *fantasy.Response
-	err := chatretry.Retry(ctx, func(retryCtx context.Context) error {
-		var genErr error
-		response, genErr = model.Generate(retryCtx, fantasy.Call{
-			Prompt:          prompt,
-			MaxOutputTokens: &maxOutputTokens,
-		})
-		return genErr
-	}, nil)
+	title, err := generateShortText(ctx, model, titleGenerationPrompt, input)
 	if err != nil {
-		return "", xerrors.Errorf("generate title text: %w", err)
+		return "", err
 	}
-
-	title := normalizeTitleOutput(contentBlocksToText(response.Content))
+	title = normalizeTitleOutput(title)
 	if title == "" {
 		return "", xerrors.New("generated title was empty")
 	}
@@ -278,3 +252,96 @@ func truncateRunes(value string, maxLen int) string {
 	}
 	return string(runes[:maxLen])
 }
+
+const pushSummaryPrompt = "You are a notification assistant. Given a chat title " +
+	"and the agent's last message, write a single short sentence (under 100 characters) " +
+	"summarizing what the agent did. This will be shown as a push notification body. " +
+	"Return plain text only — no quotes, no emoji, no markdown."
+
+// generatePushSummary calls a cheap model to produce a short push
+// notification body from the chat title and the last assistant
+// message text. It follows the same candidate-selection strategy
+// as title generation: try preferred lightweight models first, then
+// fall back to the provided model. Returns "" on any failure.
+func generatePushSummary(
+	ctx context.Context,
+	chatTitle string,
+	assistantText string,
+	fallbackModel fantasy.LanguageModel,
+	keys chatprovider.ProviderAPIKeys,
+	logger slog.Logger,
+) string {
+	summaryCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	input := "Chat title: " + chatTitle + "\n\nAgent's last message:\n" + assistantText
+
+	candidates := make([]fantasy.LanguageModel, 0, len(preferredTitleModels)+1)
+	for _, c := range preferredTitleModels {
+		m, err := chatprovider.ModelFromConfig(
+			c.provider, c.model, keys,
+		)
+		if err == nil {
+			candidates = append(candidates, m)
+		}
+	}
+	candidates = append(candidates, fallbackModel)
+
+	for _, model := range candidates {
+		summary, err := generateShortText(summaryCtx, model, pushSummaryPrompt, input)
+		if err != nil {
+			logger.Debug(ctx, "push summary model candidate failed",
+				slog.Error(err),
+			)
+			continue
+		}
+		if summary != "" {
+			return summary
+		}
+	}
+	return ""
+}
+
+// generateShortText calls a model with a system prompt and user
+// input, returning a cleaned-up short text response. It reuses the
+// same retry logic as title generation.
+func generateShortText(
+	ctx context.Context,
+	model fantasy.LanguageModel,
+	systemPrompt string,
+	userInput string,
+) (string, error) {
+	prompt := []fantasy.Message{
+		{
+			Role: fantasy.MessageRoleSystem,
+			Content: []fantasy.MessagePart{
+				fantasy.TextPart{Text: systemPrompt},
+			},
+		},
+		{
+			Role: fantasy.MessageRoleUser,
+			Content: []fantasy.MessagePart{
+				fantasy.TextPart{Text: userInput},
+			},
+		},
+	}
+
+	var maxOutputTokens int64 = 256
+
+	var response *fantasy.Response
+	err := chatretry.Retry(ctx, func(retryCtx context.Context) error {
+		var genErr error
+		response, genErr = model.Generate(retryCtx, fantasy.Call{
+			Prompt:          prompt,
+			MaxOutputTokens: &maxOutputTokens,
+		})
+		return genErr
+	}, nil)
+	if err != nil {
+		return "", xerrors.Errorf("generate short text: %w", err)
+	}
+
+	text := strings.TrimSpace(contentBlocksToText(response.Content))
+	text = strings.Trim(text, "\"'`")
+	return text, nil
+}