feat(coderd/chatd): unify chat storage on SDK parts and fix file-reference rendering (#22958)

File-reference parts in user messages were flattened to `TextContent` at
write time because fantasy has no file-reference content type. The
frontend never saw them as structured parts.

This moves all write paths (user, assistant, tool) from fantasy envelope
format to `codersdk.ChatMessagePart`. The streaming layer (`chatloop`)
is untouched, the conversion happens at the serialization boundary in
`persistStep`.

Old rows are still readable. `ParseContent` uses a structural heuristic
(`isFantasyEnvelopeFormat`) to distinguish legacy envelopes from SDK
parts. We chose this over try/fallback because fantasy envelopes
partially unmarshal into `ChatMessagePart` (the `type` field matches)
while silently losing content. A guard test enforces that no SDK part
can produce the envelope shape.

This is forward-only: new rows are unreadable by old code. Chat is
behind a feature flag so rollback risk is contained.

Also adds a typed `ChatMessageRole` to replace raw strings and
`fantasy.MessageRole*` casts at the persistence boundary. The type
covers `ChatMessage.Role`, `ChatStreamMessagePart.Role`, the
`PublishMessagePart` callback chain, and all DB write sites.
`fantasy.MessageRole*` remains only where we build `fantasy.Message`
structs for LLM dispatch.

Separately, `ProviderMetadata` was leaking to SSE clients via
`publishMessagePart`. `StripInternal` now runs on both the SSE and REST
paths, covering this.

Other cleanup:

- Old `db2sdk.contentBlockToPart` silently dropped metadata on
text/reasoning/tool-call content. New code preserves it.
- `providerMetadataToOptions` now logs warnings instead of silently
returning nil.
- `db2sdk` shrinks from ~250 lines of parallel conversion to ~15 lines
delegating to `chatprompt.ParseContent()`, removing the `fantasy` import
entirely.

Refs #22821

coderd/chats.go

@@ -18,7 +18,6 @@ import (
 	"sync"
 	"time"
 
-	"charm.land/fantasy"
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
 	"golang.org/x/xerrors"
@@ -222,7 +221,7 @@ func (api *API) postChats(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	contentBlocks, contentFileIDs, titleSource, inputError := createChatInputFromRequest(ctx, api.Database, req)
+	contentBlocks, titleSource, inputError := createChatInputFromRequest(ctx, api.Database, req)
 	if inputError != nil {
 		httpapi.Write(ctx, rw, http.StatusBadRequest, *inputError)
 		return
@@ -257,7 +256,6 @@ func (api *API) postChats(rw http.ResponseWriter, r *http.Request) {
 		ModelConfigID:      modelConfigID,
 		SystemPrompt:       api.resolvedChatSystemPrompt(ctx),
 		InitialUserContent: contentBlocks,
-		ContentFileIDs:     contentFileIDs,
 	})
 	if err != nil {
 		if database.IsForeignKeyViolation(
@@ -630,7 +628,7 @@ func (api *API) postChatMessages(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	contentBlocks, contentFileIDs, _, inputError := createChatInputFromParts(ctx, api.Database, req.Content, "content")
+	contentBlocks, _, inputError := createChatInputFromParts(ctx, api.Database, req.Content, "content")
 	if inputError != nil {
 		httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
 			Message: inputError.Message,
@@ -642,12 +640,11 @@ func (api *API) postChatMessages(rw http.ResponseWriter, r *http.Request) {
 	sendResult, sendErr := api.chatDaemon.SendMessage(
 		ctx,
 		chatd.SendMessageOptions{
-			ChatID:         chatID,
-			CreatedBy:      apiKey.UserID,
-			Content:        contentBlocks,
-			ContentFileIDs: contentFileIDs,
-			ModelConfigID:  req.ModelConfigID,
-			BusyBehavior:   chatd.SendMessageBusyBehaviorQueue,
+			ChatID:        chatID,
+			CreatedBy:     apiKey.UserID,
+			Content:       contentBlocks,
+			ModelConfigID: req.ModelConfigID,
+			BusyBehavior:  chatd.SendMessageBusyBehaviorQueue,
 		},
 	)
 	if sendErr != nil {
@@ -707,7 +704,7 @@ func (api *API) patchChatMessage(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	contentBlocks, contentFileIDs, _, inputError := createChatInputFromParts(ctx, api.Database, req.Content, "content")
+	contentBlocks, _, inputError := createChatInputFromParts(ctx, api.Database, req.Content, "content")
 	if inputError != nil {
 		httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
 			Message: inputError.Message,
@@ -721,7 +718,6 @@ func (api *API) patchChatMessage(rw http.ResponseWriter, r *http.Request) {
 		CreatedBy:       apiKey.UserID,
 		EditedMessageID: messageID,
 		Content:         contentBlocks,
-		ContentFileIDs:  contentFileIDs,
 	})
 	if editErr != nil {
 		switch {
@@ -1985,8 +1981,7 @@ func (api *API) chatFileByID(rw http.ResponseWriter, r *http.Request) {
 }
 
 func createChatInputFromRequest(ctx context.Context, db database.Store, req codersdk.CreateChatRequest) (
-	[]fantasy.Content,
-	map[int]uuid.UUID,
+	[]codersdk.ChatMessagePart,
 	string,
 	*codersdk.Response,
 ) {
@@ -1998,32 +1993,31 @@ func createChatInputFromParts(
 	db database.Store,
 	parts []codersdk.ChatInputPart,
 	fieldName string,
-) ([]fantasy.Content, map[int]uuid.UUID, string, *codersdk.Response) {
+) ([]codersdk.ChatMessagePart, string, *codersdk.Response) {
 	if len(parts) == 0 {
-		return nil, nil, "", &codersdk.Response{
+		return nil, "", &codersdk.Response{
 			Message: "Content is required.",
 			Detail:  "Content cannot be empty.",
 		}
 	}
 
-	content := make([]fantasy.Content, 0, len(parts))
-	fileIDs := make(map[int]uuid.UUID)
+	content := make([]codersdk.ChatMessagePart, 0, len(parts))
 	textParts := make([]string, 0, len(parts))
 	for i, part := range parts {
 		switch strings.ToLower(strings.TrimSpace(string(part.Type))) {
 		case string(codersdk.ChatInputPartTypeText):
 			text := strings.TrimSpace(part.Text)
 			if text == "" {
-				return nil, nil, "", &codersdk.Response{
+				return nil, "", &codersdk.Response{
 					Message: "Invalid input part.",
 					Detail:  fmt.Sprintf("%s[%d].text cannot be empty.", fieldName, i),
 				}
 			}
-			content = append(content, fantasy.TextContent{Text: text})
+			content = append(content, codersdk.ChatMessageText(text))
 			textParts = append(textParts, text)
 		case string(codersdk.ChatInputPartTypeFile):
 			if part.FileID == uuid.Nil {
-				return nil, nil, "", &codersdk.Response{
+				return nil, "", &codersdk.Response{
 					Message: "Invalid input part.",
 					Detail:  fmt.Sprintf("%s[%d].file_id is required for file parts.", fieldName, i),
 				}
@@ -2034,27 +2028,26 @@ func createChatInputFromParts(
 			chatFile, err := db.GetChatFileByID(ctx, part.FileID)
 			if err != nil {
 				if httpapi.Is404Error(err) {
-					return nil, nil, "", &codersdk.Response{
+					return nil, "", &codersdk.Response{
 						Message: "Invalid input part.",
 						Detail:  fmt.Sprintf("%s[%d].file_id references a file that does not exist.", fieldName, i),
 					}
 				}
-				return nil, nil, "", &codersdk.Response{
+				return nil, "", &codersdk.Response{
 					Message: "Internal error.",
 					Detail:  fmt.Sprintf("Failed to retrieve file for %s[%d].", fieldName, i),
 				}
 			}
-			content = append(content, fantasy.FileContent{
-				MediaType: chatFile.Mimetype,
-			})
-			fileIDs[len(content)-1] = part.FileID
+			content = append(content, codersdk.ChatMessageFile(part.FileID, chatFile.Mimetype))
 		case string(codersdk.ChatInputPartTypeFileReference):
 			if part.FileName == "" {
-				return nil, nil, "", &codersdk.Response{
+				return nil, "", &codersdk.Response{
 					Message: "Invalid input part.",
 					Detail:  fmt.Sprintf("%s[%d].file_name cannot be empty for file-reference.", fieldName, i),
 				}
 			}
+			content = append(content, codersdk.ChatMessageFileReference(part.FileName, part.StartLine, part.EndLine, part.Content))
+			// Build text representation for title generation.
 			lineRange := fmt.Sprintf("%d", part.StartLine)
 			if part.StartLine != part.EndLine {
 				lineRange = fmt.Sprintf("%d-%d", part.StartLine, part.EndLine)
@@ -2064,11 +2057,9 @@ func createChatInputFromParts(
 			if strings.TrimSpace(part.Content) != "" {
 				_, _ = fmt.Fprintf(&sb, "\n```%s\n%s\n```", part.FileName, strings.TrimSpace(part.Content))
 			}
-			text := sb.String()
-			content = append(content, fantasy.TextContent{Text: text})
-			textParts = append(textParts, text)
+			textParts = append(textParts, sb.String())
 		default:
-			return nil, nil, "", &codersdk.Response{
+			return nil, "", &codersdk.Response{
 				Message: "Invalid input part.",
 				Detail: fmt.Sprintf(
 					"%s[%d].type %q is not supported.",
@@ -2083,13 +2074,13 @@ func createChatInputFromParts(
 	// Allow file-only messages. The titleSource may be empty
 	// when only file parts are provided, callers handle this.
 	if len(content) == 0 {
-		return nil, nil, "", &codersdk.Response{
+		return nil, "", &codersdk.Response{
 			Message: "Content is required.",
 			Detail:  fmt.Sprintf("%s must include at least one text or file part.", fieldName),
 		}
 	}
 	titleSource := strings.TrimSpace(strings.Join(textParts, " "))
-	return content, fileIDs, titleSource, nil
+	return content, titleSource, nil
 }
 
 func chatTitleFromMessage(message string) string {