From 2890b7daf3cef99f3e6f9d684e4a229b7dc89a8e Mon Sep 17 00:00:00 2001 From: cita Date: Fri, 20 Mar 2026 21:48:05 +0800 Subject: [PATCH] fix cross-protocol conversation file compatibility --- .../conversationFileCapabilities.test.ts | 6 +- .../conversationFileCapabilities.ts | 4 +- src/server/proxy-core/surfaces/chatSurface.ts | 1 + .../proxy-core/surfaces/geminiSurface.ts | 11 ++ .../surfaces/openAiResponsesSurface.ts | 12 +- src/server/routes/proxy/chat.stream.test.ts | 133 +++++++++++++++++- src/server/routes/proxy/gemini.test.ts | 86 +++++++++++ .../routes/proxy/upstreamEndpoint.test.ts | 39 +++++ src/server/routes/proxy/upstreamEndpoint.ts | 5 + .../anthropic/messages/index.test.ts | 40 ++++++ .../gemini/generate-content/compatibility.ts | 61 +++++++- .../gemini/generate-content/index.test.ts | 35 +++++ .../transformers/shared/chatFormatsCore.ts | 111 +++++++++++++-- .../transformers/shared/inputFile.test.ts | 15 ++ src/server/transformers/shared/inputFile.ts | 10 ++ src/web/pages/ModelTester.tsx | 23 +-- .../conversationFileCapabilities.test.ts | 6 +- .../helpers/conversationFileCapabilities.ts | 2 +- .../pages/helpers/modelTesterSession.test.ts | 51 +++++++ src/web/pages/helpers/modelTesterSession.ts | 31 ++++ 20 files changed, 619 insertions(+), 63 deletions(-) diff --git a/src/server/proxy-core/capabilities/conversationFileCapabilities.test.ts b/src/server/proxy-core/capabilities/conversationFileCapabilities.test.ts index 6d00c02..ae8992b 100644 --- a/src/server/proxy-core/capabilities/conversationFileCapabilities.test.ts +++ b/src/server/proxy-core/capabilities/conversationFileCapabilities.test.ts @@ -40,15 +40,15 @@ describe('conversationFileCapabilities', () => { }); }); - it('describes inline-only document support for claude messages and gemini chat paths', () => { + it('describes native remote-document support for claude messages and inline-only support for gemini chat paths', () => { expect(resolveConversationFileEndpointCapability({ sitePlatform: 'claude', endpoint: 'messages', })).toMatchObject({ image: 'native', audio: 'unsupported', - document: 'inline_only', - preservesRemoteDocumentUrl: false, + document: 'native', + preservesRemoteDocumentUrl: true, }); expect(resolveConversationFileEndpointCapability({ diff --git a/src/server/proxy-core/capabilities/conversationFileCapabilities.ts b/src/server/proxy-core/capabilities/conversationFileCapabilities.ts index d2d22fc..38a7cfc 100644 --- a/src/server/proxy-core/capabilities/conversationFileCapabilities.ts +++ b/src/server/proxy-core/capabilities/conversationFileCapabilities.ts @@ -151,8 +151,8 @@ export function resolveConversationFileEndpointCapability(input: { return { image: 'native', audio: 'unsupported', - document: 'inline_only', - preservesRemoteDocumentUrl: false, + document: 'native', + preservesRemoteDocumentUrl: true, }; } return { diff --git a/src/server/proxy-core/surfaces/chatSurface.ts b/src/server/proxy-core/surfaces/chatSurface.ts index d64ecaf..61a09f3 100644 --- a/src/server/proxy-core/surfaces/chatSurface.ts +++ b/src/server/proxy-core/surfaces/chatSurface.ts @@ -164,6 +164,7 @@ export async function handleChatSurfaceRequest( requestedModelHint: requestedModel, requestCapabilities: { hasNonImageFileInput, + conversationFileSummary, }, }; const buildProviderHeaders = () => ( diff --git a/src/server/proxy-core/surfaces/geminiSurface.ts b/src/server/proxy-core/surfaces/geminiSurface.ts index 462b981..bbeb0be 100644 --- a/src/server/proxy-core/surfaces/geminiSurface.ts +++ b/src/server/proxy-core/surfaces/geminiSurface.ts @@ -33,6 +33,7 @@ import { import { dispatchRuntimeRequest } from '../../routes/proxy/runtimeExecutor.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from '../../routes/proxy/downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; +import { summarizeConversationFileInputsInOpenAiBody } from '../capabilities/conversationFileCapabilities.js'; const MAX_RETRIES = 2; const GEMINI_MODEL_PROBES = [ @@ -685,6 +686,8 @@ export async function geminiProxyRoute(app: FastifyInstance) { modelName: actualModel, stream: isStreamAction, }); + const conversationFileSummary = summarizeConversationFileInputsInOpenAiBody(openAiBody); + const hasNonImageFileInput = conversationFileSummary.hasDocument; const endpointCandidates = await resolveUpstreamEndpointCandidates( { site: selected.site, @@ -693,12 +696,20 @@ export async function geminiProxyRoute(app: FastifyInstance) { actualModel, 'openai', requestedModel, + { + hasNonImageFileInput, + conversationFileSummary, + }, ); const endpointRuntimeContext = { siteId: selected.site.id, modelName: actualModel, downstreamFormat: 'openai' as const, requestedModelHint: requestedModel, + requestCapabilities: { + hasNonImageFileInput, + conversationFileSummary, + }, }; const buildEndpointRequest = ( endpoint: 'chat' | 'messages' | 'responses', diff --git a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts index 4ed49ae..0f163d6 100644 --- a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts +++ b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts @@ -237,14 +237,6 @@ export async function handleOpenAiResponsesSurfaceRequest( const responsesConversationFileSummary = summarizeConversationFileInputsInResponsesBody(normalizedResponsesBody); const requiresNativeResponsesFileUrl = responsesConversationFileSummary.hasRemoteDocumentUrl || carriesResponsesFileUrlInput(normalizedResponsesBody.input); - if (requiresNativeResponsesFileUrl && String(selected.site.platform || '').trim().toLowerCase() === 'claude') { - return reply.code(400).send({ - error: { - message: 'Responses input_file.file_url requires an upstream /v1/responses endpoint; current site only supports /v1/messages.', - type: 'invalid_request_error', - }, - }); - } const endpointCandidates = await resolveUpstreamEndpointCandidates( { site: selected.site, @@ -259,9 +251,6 @@ export async function handleOpenAiResponsesSurfaceRequest( wantsNativeResponsesReasoning: prefersNativeResponsesReasoning, }, ); - if (requiresNativeResponsesFileUrl) { - endpointCandidates.splice(0, endpointCandidates.length, 'responses'); - } if (endpointCandidates.length === 0) { endpointCandidates.push('responses', 'chat', 'messages'); } @@ -272,6 +261,7 @@ export async function handleOpenAiResponsesSurfaceRequest( requestedModelHint: requestedModel, requestCapabilities: { hasNonImageFileInput, + conversationFileSummary, wantsNativeResponsesReasoning: prefersNativeResponsesReasoning, }, }; diff --git a/src/server/routes/proxy/chat.stream.test.ts b/src/server/routes/proxy/chat.stream.test.ts index 9d487df..f478029 100644 --- a/src/server/routes/proxy/chat.stream.test.ts +++ b/src/server/routes/proxy/chat.stream.test.ts @@ -2290,7 +2290,7 @@ describe('chat proxy stream behavior', () => { }); }); - it('returns clear 400 when input_file file_url is sent to a claude-only upstream without native responses support', async () => { + it('converts input_file file_url into Claude document url blocks for claude-only upstreams', async () => { selectChannelMock.mockReturnValue({ channel: { id: 11, routeId: 22 }, site: { name: 'claude-site', url: 'https://upstream.example.com', platform: 'claude' }, @@ -2299,6 +2299,18 @@ describe('chat proxy stream behavior', () => { tokenValue: 'sk-demo', actualModel: 'upstream-claude', }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'msg_file_url_1', + type: 'message', + role: 'assistant', + model: 'upstream-claude', + content: [{ type: 'text', text: 'hello from claude messages' }], + stop_reason: 'end_turn', + usage: { input_tokens: 7, output_tokens: 3 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); const response = await app.inject({ method: 'POST', @@ -2322,12 +2334,119 @@ describe('chat proxy stream behavior', () => { }, }); - expect(response.statusCode).toBe(400); - const body = response.json(); - expect(body?.error?.type).toBe('invalid_request_error'); - expect(body?.error?.message).toContain('input_file.file_url'); - expect(body?.error?.message).toContain('/v1/responses'); - expect(fetchMock).not.toHaveBeenCalled(); + expect(response.statusCode).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [targetUrl, options] = fetchMock.mock.calls[0] as [string, any]; + expect(targetUrl).toContain('/v1/messages'); + const forwardedBody = JSON.parse(options.body); + expect(forwardedBody.messages[0].content[1]).toMatchObject({ + type: 'document', + title: 'remote.pdf', + source: { + type: 'url', + url: 'https://example.com/remote.pdf', + }, + }); + }); + + it('does not let remote document url success poison later inline document endpoint preference', async () => { + selectChannelMock.mockReturnValue({ + channel: { id: 11, routeId: 22 }, + site: { name: 'generic-site', url: 'https://upstream.example.com', platform: 'new-api' }, + account: { id: 33, username: 'demo-user' }, + tokenName: 'default', + tokenValue: 'sk-demo', + actualModel: 'upstream-gpt', + }); + fetchMock.mockImplementation(async (target: unknown) => { + const url = String(target); + if (url.includes('/v1/responses')) { + return new Response(JSON.stringify({ + id: 'resp_file_url_runtime_1', + object: 'response', + model: 'upstream-gpt', + output_text: 'hello from responses upstream', + output: [ + { + id: 'msg_file_url_runtime_1', + type: 'message', + role: 'assistant', + content: [{ type: 'output_text', text: 'hello from responses upstream' }], + }, + ], + status: 'completed', + usage: { input_tokens: 7, output_tokens: 3, total_tokens: 10 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/v1/messages')) { + return new Response(JSON.stringify({ + id: 'msg_inline_runtime_1', + type: 'message', + role: 'assistant', + model: 'upstream-gpt', + content: [{ type: 'text', text: 'hello from messages upstream' }], + stop_reason: 'end_turn', + usage: { input_tokens: 7, output_tokens: 3 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + throw new Error(`unexpected target url: ${url}`); + }); + + const remoteResponse = await app.inject({ + method: 'POST', + url: '/v1/responses', + payload: { + model: 'claude-haiku-4-5-20251001', + input: [ + { + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'read this remote file' }, + { + type: 'input_file', + filename: 'remote.pdf', + file_url: 'https://example.com/remote.pdf', + }, + ], + }, + ], + }, + }); + + expect(remoteResponse.statusCode).toBe(200); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/v1/responses'); + + const inlineResponse = await app.inject({ + method: 'POST', + url: '/v1/responses', + payload: { + model: 'claude-haiku-4-5-20251001', + input: [ + { + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'read this inline file' }, + { + type: 'input_file', + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0xLjQK', + }, + ], + }, + ], + }, + }); + + expect(inlineResponse.statusCode).toBe(200); + expect(String(fetchMock.mock.calls[1]?.[0])).toContain('/v1/messages'); }); it('prefers native /v1/responses for claude-family /v1/responses requests that opt into reasoning without injecting a generic default include', async () => { diff --git a/src/server/routes/proxy/gemini.test.ts b/src/server/routes/proxy/gemini.test.ts index 73e89c6..42931da 100644 --- a/src/server/routes/proxy/gemini.test.ts +++ b/src/server/routes/proxy/gemini.test.ts @@ -689,6 +689,92 @@ describe('gemini native proxy routes', () => { }); }); + it('routes Gemini native document requests to responses endpoints on openai-compatible upstreams', async () => { + selectChannelMock.mockReturnValue({ + channel: { id: 42, routeId: 22 }, + site: { id: 78, name: 'openai-site', url: 'https://api.openai.com', platform: 'openai' }, + account: { id: 38, username: 'openai-user@example.com' }, + tokenName: 'default', + tokenValue: 'openai-access-token', + actualModel: 'gpt-4.1', + }); + fetchMock.mockImplementation(async (target: unknown) => { + const url = String(target); + if (url === 'https://api.openai.com/v1/responses') { + return new Response(JSON.stringify({ + id: 'resp-openai-file-1', + object: 'response', + model: 'gpt-4.1', + output: [ + { + type: 'message', + role: 'assistant', + content: [{ type: 'output_text', text: 'document summary from responses' }], + }, + ], + usage: { + input_tokens: 9, + output_tokens: 4, + total_tokens: 13, + }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url === 'https://api.openai.com/v1/chat/completions') { + return new Response(JSON.stringify({ + id: 'chatcmpl-openai-file-1', + object: 'chat.completion', + created: 1_742_160_002, + model: 'gpt-4.1', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'document summary from chat', + }, + finish_reason: 'stop', + }, + ], + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + throw new Error(`unexpected target url: ${url}`); + }); + + const response = await app.inject({ + method: 'POST', + url: '/v1beta/models/gemini-2.5-flash:generateContent', + headers: { + authorization: 'Bearer sk-managed-gemini', + }, + payload: { + contents: [ + { + role: 'user', + parts: [ + { text: 'summarize this pdf' }, + { + fileData: { + fileUri: 'https://example.com/brief.pdf', + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + }, + }); + + expect(response.statusCode).toBe(200); + const [targetUrl] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(targetUrl).toBe('https://api.openai.com/v1/responses'); + }); + it('routes Gemini native generateContent requests to antigravity upstreams through the internal content endpoint', async () => { selectChannelMock.mockReturnValue({ channel: { id: 43, routeId: 22 }, diff --git a/src/server/routes/proxy/upstreamEndpoint.test.ts b/src/server/routes/proxy/upstreamEndpoint.test.ts index d5e7e2c..8492532 100644 --- a/src/server/routes/proxy/upstreamEndpoint.test.ts +++ b/src/server/routes/proxy/upstreamEndpoint.test.ts @@ -216,6 +216,45 @@ describe('resolveUpstreamEndpointCandidates', () => { expect(order).toEqual(['responses', 'chat', 'messages']); }); + it('keeps remote-document-url requests on a separate runtime preference bucket from inline document requests', async () => { + recordUpstreamEndpointSuccess({ + siteId: baseContext.site.id, + endpoint: 'chat', + downstreamFormat: 'openai', + modelName: 'gpt-5.3', + requestCapabilities: { + hasNonImageFileInput: true, + conversationFileSummary: { + hasImage: false, + hasAudio: false, + hasDocument: true, + hasRemoteDocumentUrl: false, + }, + }, + }); + + const order = await resolveUpstreamEndpointCandidates( + { + ...baseContext, + site: { ...baseContext.site, platform: 'new-api' }, + }, + 'gpt-5.3', + 'openai', + undefined, + { + hasNonImageFileInput: true, + conversationFileSummary: { + hasImage: false, + hasAudio: false, + hasDocument: true, + hasRemoteDocumentUrl: true, + }, + }, + ); + + expect(order).toEqual(['responses', 'messages', 'chat']); + }); + it('does not remember messages fallback success for generic /v1/responses requests', async () => { recordUpstreamEndpointSuccess({ siteId: baseContext.site.id, diff --git a/src/server/routes/proxy/upstreamEndpoint.ts b/src/server/routes/proxy/upstreamEndpoint.ts index b97579a..9a7aec9 100644 --- a/src/server/routes/proxy/upstreamEndpoint.ts +++ b/src/server/routes/proxy/upstreamEndpoint.ts @@ -42,6 +42,7 @@ export type EndpointPreference = DownstreamFormat | 'responses'; type EndpointCapabilityProfile = { preferMessagesForClaudeModel: boolean; hasNonImageFileInput: boolean; + hasRemoteDocumentUrl: boolean; wantsNativeResponsesReasoning: boolean; }; @@ -619,6 +620,9 @@ function buildEndpointCapabilityProfile(input?: { input?.requestCapabilities?.conversationFileSummary?.hasDocument === true || input?.requestCapabilities?.hasNonImageFileInput === true ), + hasRemoteDocumentUrl: ( + input?.requestCapabilities?.conversationFileSummary?.hasRemoteDocumentUrl === true + ), wantsNativeResponsesReasoning: input?.requestCapabilities?.wantsNativeResponsesReasoning === true, }; } @@ -634,6 +638,7 @@ function buildEndpointRuntimeStateKey(input: { input.downstreamFormat, capabilityProfile.preferMessagesForClaudeModel ? 'claude' : 'generic', capabilityProfile.hasNonImageFileInput ? 'files' : 'nofiles', + capabilityProfile.hasRemoteDocumentUrl ? 'remoteurl' : 'noremoteurl', capabilityProfile.wantsNativeResponsesReasoning ? 'reasoning' : 'noreasoning', ].join(':'); } diff --git a/src/server/transformers/anthropic/messages/index.test.ts b/src/server/transformers/anthropic/messages/index.test.ts index ab06c9e..4fe1088 100644 --- a/src/server/transformers/anthropic/messages/index.test.ts +++ b/src/server/transformers/anthropic/messages/index.test.ts @@ -34,6 +34,46 @@ describe('anthropicMessagesTransformer protocol contract', () => { }); }); + it('parses native document blocks into canonical file parts', () => { + const result = anthropicMessagesTransformer.parseRequest({ + model: 'claude-sonnet-4-5', + max_tokens: 256, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'summarize this pdf' }, + { + type: 'document', + title: 'brief.pdf', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'JVBERi0xLjQK', + }, + }, + ], + }, + ], + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.messages).toEqual([ + { + role: 'user', + parts: [ + { type: 'text', text: 'summarize this pdf' }, + { + type: 'file', + filename: 'brief.pdf', + mimeType: 'application/pdf', + fileData: 'JVBERi0xLjQK', + }, + ], + }, + ]); + }); + it('builds native messages requests from canonical envelopes', () => { const body = anthropicMessagesTransformer.buildProtocolRequest({ operation: 'count_tokens', diff --git a/src/server/transformers/gemini/generate-content/compatibility.ts b/src/server/transformers/gemini/generate-content/compatibility.ts index 3a946d7..cf93027 100644 --- a/src/server/transformers/gemini/generate-content/compatibility.ts +++ b/src/server/transformers/gemini/generate-content/compatibility.ts @@ -11,6 +11,10 @@ function asTrimmedString(value: unknown): string { return typeof value === 'string' ? value.trim() : ''; } +function isImageMimeType(mimeType: string): boolean { + return mimeType.toLowerCase().startsWith('image/'); +} + function parseJsonIfPossible(value: string): unknown { const trimmed = value.trim(); if (!trimmed) return {}; @@ -30,6 +34,26 @@ function buildDataUrl(part: GeminiRecord): string | null { return `data:${mimeType};base64,${data}`; } +function buildOpenAiFileBlock(input: { + fileData?: string; + fileUrl?: string; + mimeType?: string; +}): Record | null { + const fileData = asTrimmedString(input.fileData); + const fileUrl = asTrimmedString(input.fileUrl); + const mimeType = asTrimmedString(input.mimeType); + if (!fileData && !fileUrl) return null; + + const file: Record = {}; + if (fileData) file.file_data = fileData; + if (fileUrl && !fileData) file.file_url = fileUrl; + if (mimeType) file.mime_type = mimeType; + return { + type: 'file', + file, + }; +} + function toOpenAiContent(contentParts: GeminiRecord[]): string | Array> { const blocks: Array> = []; let textContent = ''; @@ -43,10 +67,39 @@ function toOpenAiContent(contentParts: GeminiRecord[]): string | Array { }); }); + it('parses non-image inlineData parts into canonical file parts', () => { + const result = geminiGenerateContentTransformer.parseRequest({ + model: 'gemini-2.5-pro', + contents: [ + { + role: 'user', + parts: [ + { text: 'summarize this pdf' }, + { + inlineData: { + mimeType: 'application/pdf', + data: 'JVBERi0xLjQK', + }, + }, + ], + }, + ], + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.messages).toEqual([ + { + role: 'user', + parts: [ + { type: 'text', text: 'summarize this pdf' }, + { + type: 'file', + fileData: 'JVBERi0xLjQK', + mimeType: 'application/pdf', + }, + ], + }, + ]); + }); + it('builds native Gemini requests from canonical envelopes', () => { const body = geminiGenerateContentTransformer.buildProtocolRequest({ operation: 'generate', diff --git a/src/server/transformers/shared/chatFormatsCore.ts b/src/server/transformers/shared/chatFormatsCore.ts index 11aa0c9..4eb5710 100644 --- a/src/server/transformers/shared/chatFormatsCore.ts +++ b/src/server/transformers/shared/chatFormatsCore.ts @@ -333,6 +333,90 @@ function parseClaudeMessageContent(content: unknown): string { return extractTextAndReasoning(content).content; } +function buildOpenAiImageUrlBlock(url: string): Record { + return { + type: 'image_url', + image_url: { url }, + }; +} + +function buildOpenAiFileBlock(input: { + fileData?: string; + fileUrl?: string; + filename?: string; + mimeType?: string; +}): Record | null { + const fileData = typeof input.fileData === 'string' ? input.fileData.trim() : ''; + const fileUrl = typeof input.fileUrl === 'string' ? input.fileUrl.trim() : ''; + const filename = typeof input.filename === 'string' ? input.filename.trim() : ''; + const mimeType = typeof input.mimeType === 'string' ? input.mimeType.trim() : ''; + if (!fileData && !fileUrl) return null; + + const file: Record = {}; + if (fileData) file.file_data = fileData; + if (fileUrl && !fileData) file.file_url = fileUrl; + if (filename) file.filename = filename; + if (mimeType) file.mime_type = mimeType; + return { + type: 'file', + file, + }; +} + +function convertClaudeContentBlockToOpenAi(block: Record): Record | null { + const blockType = typeof block.type === 'string' ? block.type : ''; + + if (blockType === 'text') { + const text = parseClaudeMessageContent(block); + return text ? { type: 'text', text } : null; + } + + if (blockType === 'image') { + const source = isRecord(block.source) ? block.source : null; + const sourceType = typeof source?.type === 'string' ? source.type : ''; + if (sourceType === 'url' && typeof source?.url === 'string' && source.url.trim()) { + return buildOpenAiImageUrlBlock(source.url.trim()); + } + if ( + sourceType === 'base64' + && typeof source?.media_type === 'string' + && source.media_type.trim() + && typeof source?.data === 'string' + && source.data.trim() + ) { + return buildOpenAiImageUrlBlock(`data:${source.media_type.trim()};base64,${source.data.trim()}`); + } + return null; + } + + if (blockType === 'document') { + const source = isRecord(block.source) ? block.source : null; + const sourceType = typeof source?.type === 'string' ? source.type : ''; + return buildOpenAiFileBlock({ + fileData: sourceType === 'base64' && typeof source?.data === 'string' ? source.data : undefined, + fileUrl: sourceType === 'url' && typeof source?.url === 'string' ? source.url : undefined, + filename: typeof block.title === 'string' ? block.title : undefined, + mimeType: typeof source?.media_type === 'string' ? source.media_type : undefined, + }); + } + + const text = parseClaudeMessageContent(block); + return text ? { type: 'text', text } : null; +} + +function buildOpenAiMessageContent( + contentBlocks: Array>, +): string | Array> | undefined { + if (contentBlocks.length <= 0) return undefined; + if (contentBlocks.every((block) => block.type === 'text' && typeof block.text === 'string')) { + return contentBlocks + .map((block) => String(block.text).trim()) + .filter(Boolean) + .join('\n\n'); + } + return contentBlocks; +} + function parseResponsesOutputText(payload: Record): string { const direct = typeof payload.output_text === 'string' ? payload.output_text : ''; if (direct) return direct; @@ -521,14 +605,14 @@ function convertClaudeRequestToOpenAiBody(body: Record): { // Claude tool blocks need explicit OpenAI mapping: // - assistant.tool_use -> assistant.tool_calls // - user.tool_result -> tool messages with tool_call_id - const textParts: string[] = []; + const contentBlocks: Array> = []; const toolCalls: Array> = []; - const flushTextAsMessage = () => { - const merged = textParts.map((item) => item.trim()).filter(Boolean).join('\n\n'); - textParts.length = 0; - if (!merged) return; - messages.push({ role: mappedRole, content: merged }); + const flushContentAsMessage = () => { + const contentValue = buildOpenAiMessageContent(contentBlocks); + contentBlocks.length = 0; + if (contentValue === undefined || contentValue === '') return; + messages.push({ role: mappedRole, content: contentValue }); }; for (const block of content) { @@ -536,7 +620,7 @@ function convertClaudeRequestToOpenAiBody(body: Record): { const blockType = typeof block.type === 'string' ? block.type : ''; if (blockType === 'tool_result') { - flushTextAsMessage(); + flushContentAsMessage(); appendToolResultMessage(block.tool_use_id, block.content); continue; } @@ -564,23 +648,22 @@ function convertClaudeRequestToOpenAiBody(body: Record): { continue; } - const text = parseClaudeMessageContent(block); - if (text) textParts.push(text); + const contentBlock = convertClaudeContentBlockToOpenAi(block); + if (contentBlock) contentBlocks.push(contentBlock); } - const merged = textParts.map((item) => item.trim()).filter(Boolean).join('\n\n'); + const contentValue = buildOpenAiMessageContent(contentBlocks); if (toolCalls.length > 0) { const assistantMessage: Record = { role: 'assistant', tool_calls: toolCalls, }; - // Keep textual assistant preface when present. - assistantMessage.content = merged || ''; + assistantMessage.content = contentValue ?? ''; messages.push(assistantMessage); - } else if (merged) { + } else if (contentValue !== undefined && contentValue !== '') { messages.push({ role: mappedRole, - content: merged, + content: contentValue, }); } } diff --git a/src/server/transformers/shared/inputFile.test.ts b/src/server/transformers/shared/inputFile.test.ts index c9ddd53..f1e4d98 100644 --- a/src/server/transformers/shared/inputFile.test.ts +++ b/src/server/transformers/shared/inputFile.test.ts @@ -76,6 +76,21 @@ describe('shared input file helpers', () => { }); }); + it('creates anthropic document blocks from remote file urls', () => { + expect(toAnthropicDocumentBlock({ + fileUrl: 'https://example.com/remote.pdf', + filename: 'remote.pdf', + mimeType: 'application/pdf', + })).toEqual({ + type: 'document', + source: { + type: 'url', + url: 'https://example.com/remote.pdf', + }, + title: 'remote.pdf', + }); + }); + it('infers common mime types from filenames', () => { expect(inferInputFileMimeType({ filename: 'notes.md', mimeType: null })).toBe('text/markdown'); expect(inferInputFileMimeType({ filename: 'photo.jpeg', mimeType: null })).toBe('image/jpeg'); diff --git a/src/server/transformers/shared/inputFile.ts b/src/server/transformers/shared/inputFile.ts index 5ad6bd4..4fa2260 100644 --- a/src/server/transformers/shared/inputFile.ts +++ b/src/server/transformers/shared/inputFile.ts @@ -159,6 +159,16 @@ export function toOpenAiChatFileBlock(file: NormalizedInputFile): Record | null { + if (file.fileUrl) { + return { + type: 'document', + source: { + type: 'url', + url: file.fileUrl, + }, + ...(file.filename ? { title: file.filename } : {}), + }; + } if (!file.fileData) return null; const parsedDataUrl = splitBase64DataUrl(file.fileData); const mimeType = parsedDataUrl?.mimeType || inferInputFileMimeType(file); diff --git a/src/web/pages/ModelTester.tsx b/src/web/pages/ModelTester.tsx index 563e399..086bf15 100644 --- a/src/web/pages/ModelTester.tsx +++ b/src/web/pages/ModelTester.tsx @@ -23,6 +23,7 @@ import { createLoadingAssistantMessage, createMessage, createConversationUserMessage, + extractConversationUploadedFilesFromMessage, filterModelTesterModelNames, finalizeIncompleteMessage, findLastLoadingAssistantIndex, @@ -898,20 +899,6 @@ export default function ModelTester() { setConversationFiles((prev) => prev.filter((item) => item.localId !== localId)); }, [sending]); - const extractUploadedFilesFromMessage = useCallback((message: ChatMessage): ConversationUploadedFile[] => { - const parts = Array.isArray(message.parts) ? message.parts : []; - return parts.flatMap((part) => { - if (part.type !== 'input_file') return []; - const fileId = typeof part.fileId === 'string' ? part.fileId.trim() : ''; - if (!fileId) return []; - return [{ - fileId, - filename: typeof part.filename === 'string' && part.filename.trim() ? part.filename.trim() : null, - mimeType: typeof part.mimeType === 'string' && part.mimeType.trim() ? part.mimeType.trim() : null, - }]; - }); - }, []); - const uploadConversationFiles = useCallback(async (): Promise => { if (conversationFiles.length <= 0) return []; @@ -2047,11 +2034,11 @@ export default function ModelTester() { const base = messages.slice(0, userIndex); const prompt = messages[userIndex].content; - const files = extractUploadedFilesFromMessage(messages[userIndex]); + const files = extractConversationUploadedFilesFromMessage(messages[userIndex]); setEditingMessageId(null); setEditValue(''); void sendWithPrompt(prompt, base, files); - }, [extractUploadedFilesFromMessage, messages, pendingJobId, sendWithPrompt, sending]); + }, [messages, pendingJobId, sendWithPrompt, sending]); const startEditMessage = useCallback((target: ChatMessage) => { if (sending) return; @@ -2085,9 +2072,9 @@ export default function ModelTester() { if (retry && target.role === 'user') { const base = updated.slice(0, targetIndex); - void sendWithPrompt(nextContent, base, extractUploadedFilesFromMessage(target)); + void sendWithPrompt(nextContent, base, extractConversationUploadedFilesFromMessage(target)); } - }, [cancelEditMessage, editValue, editingMessageId, extractUploadedFilesFromMessage, messages, sendWithPrompt]); + }, [cancelEditMessage, editValue, editingMessageId, messages, sendWithPrompt]); const syncMessageToBody = useCallback(() => { const nextBody = syncMessagesToCustomRequestBody(customRequestBody, messages, inputs); diff --git a/src/web/pages/helpers/conversationFileCapabilities.test.ts b/src/web/pages/helpers/conversationFileCapabilities.test.ts index 376d8ac..e5cd273 100644 --- a/src/web/pages/helpers/conversationFileCapabilities.test.ts +++ b/src/web/pages/helpers/conversationFileCapabilities.test.ts @@ -16,16 +16,16 @@ describe('resolveConversationFileCapability', () => { }); }); - it('marks Claude and Gemini as inline-only document transports', () => { + it('marks Claude and Gemini picker flows as inline-only document transports', () => { expect(resolveConversationFileCapability('claude')).toEqual({ supported: true, documentMode: 'inline_only', - reason: '当前协议会以内联文档方式发送会话附件。', + reason: '当前界面的会话附件会以内联文档方式发送。', }); expect(resolveConversationFileCapability('gemini')).toEqual({ supported: true, documentMode: 'inline_only', - reason: '当前协议会以内联文档方式发送会话附件。', + reason: '当前界面的会话附件会以内联文档方式发送。', }); }); }); diff --git a/src/web/pages/helpers/conversationFileCapabilities.ts b/src/web/pages/helpers/conversationFileCapabilities.ts index eca7981..e176de2 100644 --- a/src/web/pages/helpers/conversationFileCapabilities.ts +++ b/src/web/pages/helpers/conversationFileCapabilities.ts @@ -21,7 +21,7 @@ export function resolveConversationFileCapability( return { supported: true, documentMode: 'inline_only', - reason: '当前协议会以内联文档方式发送会话附件。', + reason: '当前界面的会话附件会以内联文档方式发送。', }; } diff --git a/src/web/pages/helpers/modelTesterSession.test.ts b/src/web/pages/helpers/modelTesterSession.test.ts index 173d65f..a706a4e 100644 --- a/src/web/pages/helpers/modelTesterSession.test.ts +++ b/src/web/pages/helpers/modelTesterSession.test.ts @@ -15,6 +15,7 @@ import { collectModelTesterModelNames, countConversationTurns, createConversationUserMessage, + extractConversationUploadedFilesFromMessage, filterModelTesterModelNames, parseCustomRequestBody, parseModelTesterSession, @@ -222,6 +223,56 @@ describe('modelTesterSession', () => { ]); }); + it('extracts inline conversation file data for retry flows', () => { + const files = extractConversationUploadedFilesFromMessage({ + id: 'u1', + role: 'user', + content: '请总结附件', + createAt: 1, + parts: [ + { + type: 'input_file', + filename: 'brief.pdf', + mimeType: 'application/pdf', + data: 'data:application/pdf;base64,JVBERi0xLjQK', + }, + ], + }); + + expect(files).toEqual([ + { + filename: 'brief.pdf', + mimeType: 'application/pdf', + data: 'data:application/pdf;base64,JVBERi0xLjQK', + }, + ]); + }); + + it('extracts uploaded file references for retry flows', () => { + const files = extractConversationUploadedFilesFromMessage({ + id: 'u2', + role: 'user', + content: '请继续分析', + createAt: 2, + parts: [ + { + type: 'input_file', + fileId: 'file-metapi-456', + filename: 'appendix.txt', + mimeType: 'text/plain', + }, + ], + }); + + expect(files).toEqual([ + { + fileId: 'file-metapi-456', + filename: 'appendix.txt', + mimeType: 'text/plain', + }, + ]); + }); + it('preserves file reference parts when building responses conversation payloads', () => { const payload = buildApiPayload( [{ diff --git a/src/web/pages/helpers/modelTesterSession.ts b/src/web/pages/helpers/modelTesterSession.ts index 0336ad7..1f7fe4a 100644 --- a/src/web/pages/helpers/modelTesterSession.ts +++ b/src/web/pages/helpers/modelTesterSession.ts @@ -901,6 +901,37 @@ export const createConversationUserMessage = ( }); }; +export const extractConversationUploadedFilesFromMessage = ( + message: ChatMessage, +): ConversationUploadedFile[] => { + const parts = Array.isArray(message.parts) ? message.parts : []; + return parts.flatMap((part) => { + if (part.type !== 'input_file') return []; + + const fileId = typeof part.fileId === 'string' && part.fileId.trim() + ? part.fileId.trim() + : null; + const filename = typeof part.filename === 'string' && part.filename.trim() + ? part.filename.trim() + : null; + const mimeType = typeof part.mimeType === 'string' && part.mimeType.trim() + ? part.mimeType.trim() + : null; + const data = typeof part.data === 'string' && part.data.trim() + ? part.data.trim() + : null; + + if (!fileId && !data) return []; + + return [{ + ...(fileId ? { fileId } : {}), + ...(filename ? { filename } : {}), + ...(mimeType ? { mimeType } : {}), + ...(data ? { data } : {}), + }]; + }); +}; + export const createLoadingAssistantMessage = (): ChatMessage => createMessage('assistant', '', { status: MESSAGE_STATUS.LOADING,