mirror of
https://github.com/coder/coder.git
synced 2026-09-21 20:51:01 +08:00
fix(coderd/x/chatd): convert file attachment that would otherwise be dropped (#26556)
fix(coderd/x/chatd): inline text attachments that providers would drop Text-family file attachments (e.g. application/json) sent to providers that reject them as file parts were silently dropped with a CallWarning the user never saw. Convert them to TextPart at prompt build when the target provider would drop that media type, so the model sees the content while the stored file part (chip, download, history) is unchanged. Provider acceptance is keyed on model.Provider() (the fantasy transport identity) to correctly handle aibridge routing remapping. OpenAI distinguishes Responses vs Chat Completions via IsResponsesModel. Only text/plain, text/markdown, text/csv, and application/json are ever decoded; binary content is never touched. Inlined content is sent in full with no truncation, matching how a provider that accepts the media type natively would receive the file.
This commit is contained in:
@@ -32,6 +32,8 @@ var syntheticPasteTruncationWarning = fmt.Sprintf(
|
||||
syntheticPasteInlineBudget,
|
||||
)
|
||||
|
||||
const inlinedFilePrefix = "[inlined-file] The user uploaded a file attachment. The target provider cannot accept this file type as a native attachment, so its full content is inlined below for direct model consumption.\n\n"
|
||||
|
||||
var toolCallIDSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
|
||||
|
||||
var syntheticPasteFileNamePattern = regexp.MustCompile(`^pasted-text-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}\.txt$`)
|
||||
@@ -96,12 +98,15 @@ func ExtractFileID(raw json.RawMessage) (uuid.UUID, error) {
|
||||
// prompt messages, resolving user file references via the provided
|
||||
// resolver. Missing-data placeholders are emitted only for replayed
|
||||
// user uploads; assistant-side and tool-side file metadata without
|
||||
// bytes is dropped from later model turns.
|
||||
// bytes is dropped from later model turns. acceptsFilePart, when
|
||||
// non-nil, gates whether text-family file parts are inlined as text for
|
||||
// providers that would drop them; nil preserves them as FilePart.
|
||||
func ConvertMessagesWithFiles(
|
||||
ctx context.Context,
|
||||
messages []database.ChatMessage,
|
||||
resolver FileResolver,
|
||||
logger slog.Logger,
|
||||
acceptsFilePart func(mediaType string) bool,
|
||||
) ([]fantasy.Message, error) {
|
||||
// Phase 1: Parse all messages via ParseContent (→ SDK parts)
|
||||
// and collect file_id references from user messages for batch
|
||||
@@ -183,6 +188,7 @@ func ConvertMessagesWithFiles(
|
||||
pm.parts,
|
||||
resolved,
|
||||
userMissingFilePolicy,
|
||||
acceptsFilePart,
|
||||
)
|
||||
if len(userParts) == 0 {
|
||||
continue
|
||||
@@ -193,7 +199,7 @@ func ConvertMessagesWithFiles(
|
||||
})
|
||||
case codersdk.ChatMessageRoleAssistant:
|
||||
fantasyParts := normalizeAssistantToolCallInputs(
|
||||
partsToMessageParts(ctx, logger, pm.parts, nil, dropMissingFiles),
|
||||
partsToMessageParts(ctx, logger, pm.parts, nil, dropMissingFiles, nil),
|
||||
)
|
||||
for _, toolCall := range ExtractToolCalls(fantasyParts) {
|
||||
if toolCall.ToolCallID == "" || strings.TrimSpace(toolCall.ToolName) == "" {
|
||||
@@ -217,7 +223,7 @@ func ConvertMessagesWithFiles(
|
||||
}
|
||||
}
|
||||
}
|
||||
toolParts := partsToMessageParts(ctx, logger, pm.parts, nil, dropMissingFiles)
|
||||
toolParts := partsToMessageParts(ctx, logger, pm.parts, nil, dropMissingFiles, nil)
|
||||
if len(toolParts) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -1278,6 +1284,41 @@ func formatSyntheticPasteText(name string, body []byte) string {
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// isInlinableTextMediaType reports whether mediaType is a text-family
|
||||
// type whose bytes may be decoded and inlined as prompt text. The set
|
||||
// is deliberately narrow so binary or unknown content is never decoded.
|
||||
// Any new text type added to codersdk.AllChatAttachmentMediaTypes must
|
||||
// also be added here, or it will be silently dropped on providers that
|
||||
// reject it as a file part.
|
||||
func isInlinableTextMediaType(mediaType string) bool {
|
||||
if parsed, _, err := mime.ParseMediaType(mediaType); err == nil {
|
||||
mediaType = parsed
|
||||
}
|
||||
switch mediaType {
|
||||
case "text/plain", "text/markdown", "text/csv", "application/json":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// formatInlinedFileText renders a file's full content as prompt text
|
||||
// for providers that would drop the file part. Unlike the
|
||||
// synthetic-paste path, no truncation is applied.
|
||||
func formatInlinedFileText(name string, body []byte) string {
|
||||
const fileNameLabel = "Attachment filename: "
|
||||
const fileNameSuffix = "\n\n"
|
||||
|
||||
var sb strings.Builder
|
||||
sb.Grow(len(inlinedFilePrefix) + len(fileNameLabel) + len(name) + len(fileNameSuffix) + len(body))
|
||||
_, _ = sb.WriteString(inlinedFilePrefix)
|
||||
if name != "" {
|
||||
_, _ = fmt.Fprintf(&sb, "%s%s%s", fileNameLabel, name, fileNameSuffix)
|
||||
}
|
||||
_, _ = sb.Write(body)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func formatMissingAttachmentText(mediaType string) string {
|
||||
const missingAttachmentBody = "[missing-attachment] The user attached a file here, but the content has expired and is no longer available."
|
||||
const missingAttachmentAction = " If you need to inspect it, ask the user to re-upload."
|
||||
@@ -1443,14 +1484,17 @@ const (
|
||||
|
||||
// partsToMessageParts converts SDK chat message parts into fantasy
|
||||
// message parts for LLM dispatch. resolved is a lookup map for file
|
||||
// bytes, and policy controls whether missing file-backed parts are
|
||||
// dropped or replaced with text placeholders.
|
||||
// bytes, policy controls whether missing file-backed parts are dropped
|
||||
// or replaced with text placeholders, and acceptsFilePart (when non-nil)
|
||||
// gates whether text-family file parts are inlined as text for providers
|
||||
// that would drop them.
|
||||
func partsToMessageParts(
|
||||
ctx context.Context,
|
||||
logger slog.Logger,
|
||||
parts []codersdk.ChatMessagePart,
|
||||
resolved map[uuid.UUID]FileData,
|
||||
policy missingFilePolicy,
|
||||
acceptsFilePart func(mediaType string) bool,
|
||||
) []fantasy.MessagePart {
|
||||
result := make([]fantasy.MessagePart, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
@@ -1536,6 +1580,26 @@ func partsToMessageParts(
|
||||
// not look expired.
|
||||
continue
|
||||
}
|
||||
// When the target provider would drop a text-family file part,
|
||||
// inline the content as text so the model still sees it.
|
||||
//
|
||||
// This must run after the isSyntheticPaste check above;
|
||||
// synthetic pastes use a truncating path and must not fall
|
||||
// through to the non-truncating inline path.
|
||||
if acceptsFilePart != nil &&
|
||||
isInlinableTextMediaType(mediaType) &&
|
||||
!acceptsFilePart(mediaType) {
|
||||
logger.Info(ctx,
|
||||
"inlining text-family file part as text for provider that would drop it",
|
||||
slog.F("file_name", name),
|
||||
slog.F("media_type", mediaType),
|
||||
)
|
||||
result = append(result, fantasy.TextPart{
|
||||
Text: formatInlinedFileText(name, data),
|
||||
ProviderOptions: opts,
|
||||
})
|
||||
continue
|
||||
}
|
||||
result = append(result, fantasy.FilePart{
|
||||
Filename: name,
|
||||
Data: data,
|
||||
|
||||
@@ -63,7 +63,7 @@ func TestConvertMessagesWithFilesPreservesEmptyRedactedReasoning(t *testing.T) {
|
||||
Content: content,
|
||||
ContentVersion: chatprompt.CurrentContentVersion,
|
||||
},
|
||||
}, nil, slogtest.Make(t, nil))
|
||||
}, nil, slogtest.Make(t, nil), nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
require.Len(t, prompt[0].Content, 2)
|
||||
@@ -146,7 +146,7 @@ func TestConvertMessagesWithFilesRoundTripsAnthropicInterleavedWebSearch(t *test
|
||||
Content: storedContent,
|
||||
ContentVersion: chatprompt.CurrentContentVersion,
|
||||
},
|
||||
}, nil, slogtest.Make(t, nil))
|
||||
}, nil, slogtest.Make(t, nil), nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
require.Len(t, prompt[0].Content, 5)
|
||||
@@ -198,6 +198,7 @@ func convertMessagesWithoutFiles(t *testing.T, messages []database.ChatMessage)
|
||||
messages,
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return prompt
|
||||
@@ -337,6 +338,7 @@ func TestConvertMessagesWithFiles_ResolvesFileData(t *testing.T) {
|
||||
},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -395,6 +397,7 @@ func TestConvertMessagesWithFiles_MissingFileBackedAttachmentBecomesTextPart(t *
|
||||
}},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -442,6 +445,7 @@ func TestConvertMessagesWithFiles_ResolvedZeroByteFileIsDropped(t *testing.T) {
|
||||
}},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, prompt)
|
||||
@@ -494,6 +498,7 @@ func TestConvertMessagesWithFiles_MixedResolvedAndMissingFilePartsInSingleMessag
|
||||
}},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -560,6 +565,7 @@ func TestConvertMessagesWithFiles_BackwardCompat(t *testing.T) {
|
||||
},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -1485,6 +1491,7 @@ func TestProviderMetadataRoundTrip(t *testing.T) {
|
||||
}},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -1533,6 +1540,7 @@ func TestFileReferencePreservation(t *testing.T) {
|
||||
}},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -1588,6 +1596,7 @@ func TestAssistantWriteRoundTrip(t *testing.T) {
|
||||
}},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -1732,6 +1741,7 @@ func TestMixedFormatConversation(t *testing.T) {
|
||||
|
||||
prompt, err := chatprompt.ConvertMessagesWithFiles(
|
||||
context.Background(), messages, resolver, slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 6, "all 6 messages should produce prompt entries")
|
||||
@@ -1852,6 +1862,7 @@ func TestQueuedMessageRoundTrip(t *testing.T) {
|
||||
}},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -2167,6 +2178,7 @@ func TestConvertMessagesWithFiles_FiltersEmptyTextAndReasoningParts(t *testing.T
|
||||
[]database.ChatMessage{makeMsg(t, database.ChatMessageRoleUser, parts)},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
@@ -2213,6 +2225,7 @@ func TestConvertMessagesWithFiles_FiltersEmptyTextAndReasoningParts(t *testing.T
|
||||
[]database.ChatMessage{makeMsg(t, database.ChatMessageRoleAssistant, parts)},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
// 2 messages: assistant + synthetic tool result injected
|
||||
@@ -2247,6 +2260,7 @@ func TestConvertMessagesWithFiles_FiltersEmptyTextAndReasoningParts(t *testing.T
|
||||
[]database.ChatMessage{makeMsg(t, database.ChatMessageRoleAssistant, parts)},
|
||||
nil,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, prompt, "all-empty message should be dropped entirely")
|
||||
@@ -2425,6 +2439,7 @@ func TestConvertMessagesWithFiles_AssistantAttachmentIsNotReplayed(t *testing.T)
|
||||
},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resolverCalls, 1)
|
||||
@@ -2478,6 +2493,7 @@ func convertSingleResolvedFileMessage(t *testing.T, fileID uuid.UUID, fileData c
|
||||
}},
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return prompt
|
||||
@@ -2567,6 +2583,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) {
|
||||
require.NoError(t, loadErr)
|
||||
prompt, convErr := chatprompt.ConvertMessagesWithFiles(
|
||||
ctx, dbMsgs, nil, slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, convErr)
|
||||
return prompt
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package chatprompt_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"charm.land/fantasy"
|
||||
"github.com/google/uuid"
|
||||
"github.com/sqlc-dev/pqtype"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"cdr.dev/slog/v3/sloggers/slogtest"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
||||
)
|
||||
|
||||
// userFileMessage builds a single persisted user message holding one
|
||||
// file_id-backed file part, plus a resolver that returns the given
|
||||
// bytes, media type, and filename for that file.
|
||||
func userFileMessage(t *testing.T, name, mediaType string, data []byte) ([]database.ChatMessage, chatprompt.FileResolver) {
|
||||
t.Helper()
|
||||
fileID := uuid.New()
|
||||
rawContent := mustJSON(t, []json.RawMessage{
|
||||
mustJSON(t, map[string]any{
|
||||
"type": "file",
|
||||
"data": map[string]any{
|
||||
"media_type": mediaType,
|
||||
"file_id": fileID.String(),
|
||||
},
|
||||
}),
|
||||
})
|
||||
messages := []database.ChatMessage{{
|
||||
Role: database.ChatMessageRoleUser,
|
||||
Visibility: database.ChatMessageVisibilityBoth,
|
||||
Content: pqtype.NullRawMessage{RawMessage: rawContent, Valid: true},
|
||||
}}
|
||||
resolver := func(_ context.Context, ids []uuid.UUID) (map[uuid.UUID]chatprompt.FileData, error) {
|
||||
result := make(map[uuid.UUID]chatprompt.FileData)
|
||||
for _, id := range ids {
|
||||
if id == fileID {
|
||||
result[id] = chatprompt.FileData{
|
||||
Name: name,
|
||||
Data: data,
|
||||
MediaType: mediaType,
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
return messages, resolver
|
||||
}
|
||||
|
||||
func acceptAll(string) bool { return true }
|
||||
func acceptNone(string) bool { return false }
|
||||
|
||||
func TestConvertMessagesWithFiles_InlinesTextFilePartWhenProviderRejects(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
fileName string
|
||||
mediaType string
|
||||
data []byte
|
||||
accepts func(string) bool
|
||||
wantText bool // true = TextPart, false = FilePart
|
||||
}{
|
||||
{
|
||||
name: "json rejected becomes text",
|
||||
fileName: "data.json",
|
||||
mediaType: "application/json",
|
||||
data: []byte(`{"hello":"world"}`),
|
||||
accepts: acceptNone,
|
||||
wantText: true,
|
||||
},
|
||||
{
|
||||
name: "text accepted stays file",
|
||||
fileName: "notes.txt",
|
||||
mediaType: "text/plain",
|
||||
data: []byte("plain text body"),
|
||||
accepts: acceptAll,
|
||||
wantText: false,
|
||||
},
|
||||
{
|
||||
name: "text rejected becomes text",
|
||||
fileName: "notes.txt",
|
||||
mediaType: "text/plain",
|
||||
data: []byte("plain text body"),
|
||||
accepts: acceptNone,
|
||||
wantText: true,
|
||||
},
|
||||
{
|
||||
name: "markdown rejected becomes text",
|
||||
fileName: "README.md",
|
||||
mediaType: "text/markdown",
|
||||
data: []byte("# Title"),
|
||||
accepts: acceptNone,
|
||||
wantText: true,
|
||||
},
|
||||
{
|
||||
name: "image never decoded even when rejected",
|
||||
fileName: "pic.png",
|
||||
mediaType: "image/png",
|
||||
data: []byte("not really png"),
|
||||
accepts: acceptNone,
|
||||
wantText: false,
|
||||
},
|
||||
{
|
||||
name: "octet-stream never decoded even when rejected",
|
||||
fileName: "blob.bin",
|
||||
mediaType: "application/octet-stream",
|
||||
data: []byte("binary"),
|
||||
accepts: acceptNone,
|
||||
wantText: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
messages, resolver := userFileMessage(t, tc.fileName, tc.mediaType, tc.data)
|
||||
prompt, err := chatprompt.ConvertMessagesWithFiles(
|
||||
context.Background(),
|
||||
messages,
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
tc.accepts,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
require.Len(t, prompt[0].Content, 1)
|
||||
|
||||
if tc.wantText {
|
||||
textPart, ok := fantasy.AsMessagePart[fantasy.TextPart](prompt[0].Content[0])
|
||||
require.True(t, ok, "expected TextPart")
|
||||
require.Contains(t, textPart.Text, tc.fileName)
|
||||
require.Contains(t, textPart.Text, string(tc.data))
|
||||
return
|
||||
}
|
||||
|
||||
filePart, ok := fantasy.AsMessagePart[fantasy.FilePart](prompt[0].Content[0])
|
||||
require.True(t, ok, "expected FilePart")
|
||||
require.Equal(t, tc.data, filePart.Data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertMessagesWithFiles_NilPredicateKeepsFilePart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := []byte(`{"a":1}`)
|
||||
messages, resolver := userFileMessage(t, "data.json", "application/json", data)
|
||||
prompt, err := chatprompt.ConvertMessagesWithFiles(
|
||||
context.Background(),
|
||||
messages,
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
require.Len(t, prompt[0].Content, 1)
|
||||
filePart, ok := fantasy.AsMessagePart[fantasy.FilePart](prompt[0].Content[0])
|
||||
require.True(t, ok, "expected FilePart when predicate is nil")
|
||||
require.Equal(t, data, filePart.Data)
|
||||
}
|
||||
|
||||
func TestConvertMessagesWithFiles_InlinedTextNotTruncated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A large text file is inlined in full, with no silent truncation,
|
||||
// matching how a provider that accepts the media type natively would
|
||||
// receive the whole file.
|
||||
const budget = 128 * 1024
|
||||
data := []byte(strings.Repeat("a", budget+1024))
|
||||
messages, resolver := userFileMessage(t, "big.txt", "text/plain", data)
|
||||
prompt, err := chatprompt.ConvertMessagesWithFiles(
|
||||
context.Background(),
|
||||
messages,
|
||||
resolver,
|
||||
slogtest.Make(t, nil),
|
||||
acceptNone,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompt, 1)
|
||||
require.Len(t, prompt[0].Content, 1)
|
||||
textPart, ok := fantasy.AsMessagePart[fantasy.TextPart](prompt[0].Content[0])
|
||||
require.True(t, ok, "expected TextPart")
|
||||
require.Contains(t, textPart.Text, string(data),
|
||||
"the full file content should be inlined without truncation")
|
||||
require.NotContains(t, textPart.Text, "truncated")
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package chatprovider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mime"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"sort"
|
||||
@@ -76,6 +77,50 @@ func InlineImageCapBytes(provider string) (int, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// AcceptsFilePartMediaType reports whether provider accepts mediaType
|
||||
// as a file content part rather than silently dropping it. modelID
|
||||
// distinguishes API paths within a provider (e.g. OpenAI Responses vs
|
||||
// Chat Completions). Unknown providers return false so callers convert
|
||||
// text-family content to text and guarantee the model still sees it.
|
||||
func AcceptsFilePartMediaType(provider, modelID, mediaType string) bool {
|
||||
baseType := mediaType
|
||||
if parsed, _, err := mime.ParseMediaType(mediaType); err == nil {
|
||||
baseType = parsed
|
||||
}
|
||||
isImage := strings.HasPrefix(baseType, "image/")
|
||||
isText := strings.HasPrefix(baseType, "text/")
|
||||
// Audio types are included for matrix completeness but are not
|
||||
// currently reachable: no audio type is in the storable attachment
|
||||
// allowlist (codersdk.AllChatAttachmentMediaTypes).
|
||||
isAudio := baseType == "audio/wav" || baseType == "audio/mpeg" || baseType == "audio/mp3"
|
||||
isPDF := baseType == "application/pdf"
|
||||
|
||||
switch NormalizeProvider(provider) {
|
||||
case fantasygoogle.Name:
|
||||
// Google passes any file part through unfiltered.
|
||||
return true
|
||||
case fantasyanthropic.Name, fantasybedrock.Name:
|
||||
// Bedrock wraps the anthropic client, so it shares the same
|
||||
// file-part acceptance, including text/* as native documents.
|
||||
return isImage || isText || isPDF
|
||||
case fantasyopenai.Name, fantasyazure.Name:
|
||||
// chatd configures both with WithUseResponsesAPI, but only
|
||||
// Responses-capable models actually use it. Non-Responses models
|
||||
// fall through to the Chat Completions path, which accepts
|
||||
// text/* and audio as native file parts (same as openaicompat).
|
||||
if fantasyopenai.IsResponsesModel(modelID) {
|
||||
return isImage || isPDF
|
||||
}
|
||||
return isImage || isText || isAudio || isPDF
|
||||
case fantasyopenaicompat.Name:
|
||||
return isImage || isText || isAudio || isPDF
|
||||
case fantasyopenrouter.Name, fantasyvercel.Name:
|
||||
return isImage || isAudio || isPDF
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ProviderAPIKeys contains API keys for provider calls.
|
||||
type ProviderAPIKeys struct {
|
||||
OpenAI string
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package chatprovider_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
||||
)
|
||||
|
||||
func TestAcceptsFilePartMediaType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A representative Responses-capable model ID. Non-Responses models
|
||||
// use the Chat Completions path, which accepts text/* natively.
|
||||
const responsesModel = "gpt-4o"
|
||||
const nonResponsesModel = "babbage-002"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
provider string
|
||||
modelID string
|
||||
mediaType string
|
||||
want bool
|
||||
}{
|
||||
// OpenAI Responses accepts only images and PDFs.
|
||||
{"openai-json", "openai", responsesModel, "application/json", false},
|
||||
{"openai-text", "openai", responsesModel, "text/plain", false},
|
||||
{"openai-image", "openai", responsesModel, "image/png", true},
|
||||
{"openai-pdf", "openai", responsesModel, "application/pdf", true},
|
||||
|
||||
// OpenAI Chat Completions (non-Responses models) accepts text/*
|
||||
// and audio as native file parts.
|
||||
{"openai-non-responses-text", "openai", nonResponsesModel, "text/plain", true},
|
||||
{"openai-non-responses-json", "openai", nonResponsesModel, "application/json", false},
|
||||
{"openai-non-responses-image", "openai", nonResponsesModel, "image/png", true},
|
||||
|
||||
// Azure uses the Responses API, same as OpenAI.
|
||||
{"azure-text", "azure", responsesModel, "text/markdown", false},
|
||||
{"azure-image", "azure", responsesModel, "image/jpeg", true},
|
||||
|
||||
// Anthropic accepts text/* as native documents, but not JSON.
|
||||
{"anthropic-text", "anthropic", "", "text/markdown", true},
|
||||
{"anthropic-json", "anthropic", "", "application/json", false},
|
||||
{"anthropic-pdf", "anthropic", "", "application/pdf", true},
|
||||
{"anthropic-image", "anthropic", "", "image/webp", true},
|
||||
|
||||
// Bedrock wraps Anthropic, so it matches Anthropic.
|
||||
{"bedrock-text", "bedrock", "", "text/csv", true},
|
||||
{"bedrock-json", "bedrock", "", "application/json", false},
|
||||
|
||||
// OpenAI-compatible accepts text/*, images, audio, and PDFs.
|
||||
{"openaicompat-text", "openai-compat", "", "text/plain", true},
|
||||
{"openaicompat-json", "openai-compat", "", "application/json", false},
|
||||
{"openaicompat-audio", "openai-compat", "", "audio/mpeg", true},
|
||||
|
||||
// OpenRouter and Vercel do not accept text file parts.
|
||||
{"openrouter-text", "openrouter", "", "text/plain", false},
|
||||
{"openrouter-json", "openrouter", "", "application/json", false},
|
||||
{"openrouter-image", "openrouter", "", "image/png", true},
|
||||
{"vercel-text", "vercel", "", "text/plain", false},
|
||||
{"vercel-json", "vercel", "", "application/json", false},
|
||||
{"vercel-pdf", "vercel", "", "application/pdf", true},
|
||||
|
||||
// Google passes all file parts through unfiltered.
|
||||
{"google-json", "google", "", "application/json", true},
|
||||
{"google-text", "google", "", "text/plain", true},
|
||||
{"google-anything", "google", "", "application/octet-stream", true},
|
||||
|
||||
// Unknown providers reject everything so text-family content is
|
||||
// converted to text and still reaches the model.
|
||||
{"unknown-text", "made-up-provider", "", "text/plain", false},
|
||||
{"empty-text", "", "", "text/plain", false},
|
||||
|
||||
// Base media type handling: parameters are stripped.
|
||||
{"anthropic-text-charset", "anthropic", "", "text/plain; charset=utf-8", true},
|
||||
{"openai-text-charset", "openai", responsesModel, "text/plain; charset=utf-8", false},
|
||||
|
||||
// Provider name normalization is case-insensitive.
|
||||
{"anthropic-uppercase", "Anthropic", "", "text/plain", true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := chatprovider.AcceptsFilePartMediaType(tc.provider, tc.modelID, tc.mediaType)
|
||||
if got != tc.want {
|
||||
t.Fatalf("AcceptsFilePartMediaType(%q, %q, %q) = %v, want %v",
|
||||
tc.provider, tc.modelID, tc.mediaType, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -245,7 +245,15 @@ func (server *Server) prepareGeneration(
|
||||
var g2 errgroup.Group
|
||||
g2.Go(func() error {
|
||||
var err error
|
||||
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, promptRows, server.chatFileResolver(modelConfig.Provider), logger)
|
||||
// Key the file-part acceptance on model.Provider() (the fantasy
|
||||
// transport identity), not the configured provider, because
|
||||
// aibridge routing rewrites the provider (e.g. Bedrock to the
|
||||
// Anthropic transport). The conversion that actually drops or
|
||||
// accepts a file part is the one for model.Provider().
|
||||
acceptsFilePart := func(mediaType string) bool {
|
||||
return chatprovider.AcceptsFilePartMediaType(model.Provider(), model.Model(), mediaType)
|
||||
}
|
||||
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, promptRows, server.chatFileResolver(modelConfig.Provider), logger, acceptsFilePart)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("build chat prompt: %w", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user