diff --git a/src/server/routes/proxy/architecture-boundaries.test.ts b/src/server/routes/proxy/architecture-boundaries.test.ts index 9043f63..903109c 100644 --- a/src/server/routes/proxy/architecture-boundaries.test.ts +++ b/src/server/routes/proxy/architecture-boundaries.test.ts @@ -14,16 +14,44 @@ describe('proxy route architecture boundaries', () => { }); it('keeps anthropic-specific stream orchestration out of chat route', () => { const source = readSource('./chat.ts'); - expect(source).not.toContain('function syncClaudeStreamStateFromRawEvent('); - expect(source).not.toContain('function serializeRawSseEvent('); - expect(source).not.toContain('function isClaudeSseEventName('); + expect(source).not.toContain('serializeAnthropicRawSseEvent'); + expect(source).not.toContain('syncAnthropicRawStreamStateFromEvent'); + expect(source).not.toContain('isAnthropicRawSseEventName'); + expect(source).not.toContain('serializeAnthropicFinalAsStream'); + expect(source).not.toContain('function shouldRetryClaudeMessagesWithNormalizedBody('); + expect(source).not.toContain('const emitNormalizedFinalAsStream ='); + expect(source).not.toContain("from './protocolCompat.js'"); + expect(source).not.toContain("from './chatStreamCompat.js'"); + expect(source).not.toContain('const promoteResponsesCandidate ='); + expect(source).not.toContain('shouldRetryClaudeMessagesWithNormalizedBody('); + expect(source).not.toContain('buildOpenAiSyntheticFinalStream('); + expect(source).toContain('anthropicMessagesTransformer.consumeSseEventBlock('); + expect(source).toContain('anthropicMessagesTransformer.serializeUpstreamFinalAsStream('); + expect(source).toContain('openAiChatTransformer.serializeUpstreamFinalAsStream('); }); it('keeps responses protocol assembly out of responses route', () => { const source = readSource('./responses.ts'); expect(source).not.toContain('function toResponsesPayload('); expect(source).not.toContain('function createResponsesStreamState('); - expect(source).not.toContain('function convertResponsesBodyToOpenAiBody('); + expect(source).not.toContain("from '../../transformers/openai/responses/conversion.js'"); + expect(source).not.toContain("from '../../transformers/openai/responses/outbound.js'"); + expect(source).not.toContain("from '../../transformers/openai/responses/aggregator.js'"); + expect(source).not.toContain('function buildResponsesCompatibilityBodies('); + expect(source).not.toContain('function buildResponsesCompatibilityHeaderCandidates('); + expect(source).not.toContain('function shouldRetryResponsesCompatibility('); + expect(source).not.toContain("from './protocolCompat.js'"); + expect(source).not.toContain('function shouldDowngradeFromChatToMessagesForResponses('); + expect(source).not.toContain('function normalizeText('); + expect(source).toContain('openAiResponsesTransformer.inbound.toOpenAiBody('); + expect(source).toContain('openAiResponsesTransformer.compatibility.buildRetryBodies('); + expect(source).toContain('openAiResponsesTransformer.compatibility.buildRetryHeaders('); + expect(source).toContain('openAiResponsesTransformer.compatibility.shouldRetry('); + expect(source).toContain('openAiResponsesTransformer.compatibility.shouldDowngradeChatToMessages('); + expect(source).toContain('openAiResponsesTransformer.aggregator.createState('); + expect(source).toContain('openAiResponsesTransformer.aggregator.serialize('); + expect(source).toContain('openAiResponsesTransformer.aggregator.complete('); + expect(source).toContain('openAiResponsesTransformer.outbound.serializeFinal('); }); it('removes normalizeContentText from upstream endpoint routing', () => { @@ -34,8 +62,14 @@ describe('proxy route architecture boundaries', () => { it('keeps gemini runtime closure in transformer-owned helpers', () => { const source = readSource('./gemini.ts'); - expect(source).toContain('serializeAggregateResponse'); - expect(source).toContain('parseSsePayloads'); - expect(source).toContain('aggregator.apply(aggregateState, parsed)'); + expect(source).not.toContain('outbound.serializeAggregateResponse('); + expect(source).not.toContain('aggregator.apply('); + expect(source).not.toContain('stream.serializeAggregateSsePayload('); + expect(source).not.toContain('stream.serializeAggregateJsonPayload('); + expect(source).not.toContain('stream.applyJsonPayloadToAggregate('); + expect(source).not.toContain('stream.parseSsePayloads('); + expect(source).toContain('stream.consumeUpstreamSseBuffer('); + expect(source).toContain('stream.serializeUpstreamJsonPayload('); }); }); + diff --git a/src/server/routes/proxy/chat.stream.test.ts b/src/server/routes/proxy/chat.stream.test.ts index dc88d0f..e126612 100644 --- a/src/server/routes/proxy/chat.stream.test.ts +++ b/src/server/routes/proxy/chat.stream.test.ts @@ -565,8 +565,15 @@ describe('chat proxy stream behavior', () => { expect(Array.isArray(firstBody.messages)).toBe(true); expect(Array.isArray(secondBody.messages)).toBe(true); - expect(typeof secondBody.messages[0]?.content).toBe('string'); - expect(secondBody.messages[0]?.content).toContain('hello'); + expect(Array.isArray(secondBody.messages[0]?.content)).toBe(true); + expect(secondBody.messages[0]?.content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('hello'), + }), + ]), + ); }); it('downgrades to next endpoint when normalized Claude fallback still returns messages is required', async () => { diff --git a/src/server/routes/proxy/chat.ts b/src/server/routes/proxy/chat.ts index abb3bcd..922934f 100644 --- a/src/server/routes/proxy/chat.ts +++ b/src/server/routes/proxy/chat.ts @@ -16,8 +16,8 @@ import { isEndpointDispatchDeniedError, isEndpointDowngradeError, isUnsupportedMediaTypeError, + promoteResponsesCandidateAfterLegacyChatError, resolveUpstreamEndpointCandidates, - shouldPreferResponsesAfterLegacyChatError, } from './upstreamEndpoint.js'; import { ensureModelAllowedForDownstreamKey, @@ -29,30 +29,9 @@ import { executeEndpointFlow, withUpstreamPath } from './endpointFlow.js'; import { formatUtcSqlDateTime } from '../../services/localTimeService.js'; import { resolveProxyLogBilling } from './proxyBilling.js'; import { openAiChatTransformer } from '../../transformers/openai/chat/index.js'; -import { - anthropicMessagesTransformer, - isAnthropicRawSseEventName, - serializeAnthropicFinalAsStream, - serializeAnthropicRawSseEvent, - syncAnthropicRawStreamStateFromEvent, -} from '../../transformers/anthropic/messages/index.js'; +import { anthropicMessagesTransformer } from '../../transformers/anthropic/messages/index.js'; const MAX_RETRIES = 2; -function shouldRetryClaudeMessagesWithNormalizedBody( - downstreamFormat: DownstreamFormat, - endpointPath: string, - status: number, - upstreamErrorText: string, -): boolean { - if (downstreamFormat !== 'claude') return false; - if (!endpointPath.includes('/v1/messages')) return false; - if (status < 400 || status >= 500) return false; - return /messages\s+is\s+required/i.test(upstreamErrorText); -} - -function isMessagesRequiredError(upstreamErrorText: string): boolean { - return /messages\s+is\s+required/i.test(upstreamErrorText); -} function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object'; @@ -125,27 +104,6 @@ async function handleChatProxyRequest( ]; let startTime = Date.now(); - const promoteResponsesCandidate = (currentEndpoint: string | null | undefined, upstreamErrorText?: string | null, status = 0) => { - if (!shouldPreferResponsesAfterLegacyChatError({ - status, - upstreamErrorText, - downstreamFormat, - sitePlatform: selected.site.platform, - modelName, - requestedModelHint: requestedModel, - currentEndpoint: currentEndpoint as any, - })) { - return; - } - - const currentIndex = endpointCandidates.findIndex((endpoint) => endpoint === currentEndpoint); - const responsesIndex = endpointCandidates.indexOf('responses'); - if (currentIndex < 0 || responsesIndex < 0 || responsesIndex <= currentIndex + 1) return; - - endpointCandidates.splice(responsesIndex, 1); - endpointCandidates.splice(currentIndex + 1, 0, 'responses'); - }; - try { const endpointResult = await executeEndpointFlow({ siteUrl: selected.site.url, @@ -172,12 +130,12 @@ async function handleChatProxyRequest( }; }, tryRecover: async (ctx) => { - if (shouldRetryClaudeMessagesWithNormalizedBody( + if (anthropicMessagesTransformer.compatibility.shouldRetryNormalizedBody({ downstreamFormat, - ctx.request.path, - ctx.response.status, - ctx.rawErrText, - )) { + endpointPath: ctx.request.path, + status: ctx.response.status, + upstreamErrorText: ctx.rawErrText, + })) { const normalizedClaudeRequest = buildUpstreamEndpointRequest({ endpoint: ctx.request.endpoint, modelName, @@ -253,11 +211,19 @@ async function handleChatProxyRequest( }, shouldDowngrade: (ctx) => ( (() => { - promoteResponsesCandidate(ctx.request.endpoint, ctx.rawErrText, ctx.response.status); + promoteResponsesCandidateAfterLegacyChatError(endpointCandidates, { + status: ctx.response.status, + upstreamErrorText: ctx.rawErrText, + downstreamFormat, + sitePlatform: selected.site.platform, + modelName, + requestedModelHint: requestedModel, + currentEndpoint: ctx.request.endpoint, + }); return ( ctx.response.status >= 500 || isEndpointDowngradeError(ctx.response.status, ctx.rawErrText) - || isMessagesRequiredError(ctx.rawErrText) + || anthropicMessagesTransformer.compatibility.isMessagesRequiredError(ctx.rawErrText) || isEndpointDispatchDeniedError(ctx.response.status, ctx.rawErrText) ); })() @@ -338,23 +304,6 @@ async function handleChatProxyRequest( writeLines(downstreamTransformer.serializeDone(streamContext, claudeContext)); }; - const emitNormalizedFinalAsStream = (upstreamData: unknown, fallbackText = '') => { - const normalizedFinal = downstreamTransformer.transformFinalResponse(upstreamData, modelName, fallbackText); - streamContext.id = normalizedFinal.id; - streamContext.model = normalizedFinal.model; - streamContext.created = normalizedFinal.created; - - if (downstreamFormat === 'openai') { - const syntheticChunks = openAiChatTransformer.buildSyntheticChunks(normalizedFinal); - for (const chunk of syntheticChunks) { - reply.raw.write(`data: ${JSON.stringify(chunk)}\n\n`); - } - return; - } - - writeLines(serializeAnthropicFinalAsStream(normalizedFinal, streamContext, claudeContext)); - }; - const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase(); if (!upstreamContentType.includes('text/event-stream')) { const fallbackText = await upstream.text(); @@ -366,7 +315,25 @@ async function handleChatProxyRequest( } parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(fallbackData)); - emitNormalizedFinalAsStream(fallbackData, fallbackText); + if (downstreamFormat === 'openai') { + const syntheticLines = openAiChatTransformer.serializeUpstreamFinalAsStream( + fallbackData, + modelName, + fallbackText, + streamContext, + ); + writeLines(syntheticLines); + } else { + writeLines( + anthropicMessagesTransformer.serializeUpstreamFinalAsStream( + fallbackData, + modelName, + fallbackText, + streamContext, + claudeContext, + ), + ); + } writeDone(); reply.raw.end(); @@ -437,37 +404,37 @@ async function handleChatProxyRequest( } let parsedPayload: unknown = null; - try { - parsedPayload = JSON.parse(eventBlock.data); - } catch { - parsedPayload = null; + if (downstreamFormat === 'claude') { + const consumed = anthropicMessagesTransformer.consumeSseEventBlock( + eventBlock, + streamContext, + claudeContext, + modelName, + ); + parsedPayload = consumed.parsedPayload; + if (parsedPayload && typeof parsedPayload === 'object') { + parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(parsedPayload)); + } + if (consumed.handled) { + writeLines(consumed.lines); + if (consumed.done) { + shouldTerminateEarly = true; + break; + } + continue; + } + } else { + try { + parsedPayload = JSON.parse(eventBlock.data); + } catch { + parsedPayload = null; + } + if (parsedPayload && typeof parsedPayload === 'object') { + parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(parsedPayload)); + } } if (parsedPayload && typeof parsedPayload === 'object') { - parsedUsage = mergeProxyUsage(parsedUsage, parseProxyUsage(parsedPayload)); - if (downstreamFormat === 'claude') { - const payloadType = (isRecord(parsedPayload) && typeof parsedPayload.type === 'string') - ? parsedPayload.type - : ''; - const claudeEventName = isAnthropicRawSseEventName(eventBlock.event) - ? eventBlock.event - : (isAnthropicRawSseEventName(payloadType) ? payloadType : ''); - - if (claudeEventName) { - syncAnthropicRawStreamStateFromEvent( - claudeEventName, - parsedPayload, - streamContext, - claudeContext, - ); - reply.raw.write(serializeAnthropicRawSseEvent(claudeEventName, eventBlock.data)); - if (claudeContext.doneSent) { - shouldTerminateEarly = true; - break; - } - continue; - } - } const normalizedEvent = downstreamTransformer.transformStreamEvent(parsedPayload, streamContext, modelName); writeLines(downstreamTransformer.serializeStreamEvent(normalizedEvent, streamContext, claudeContext)); if (downstreamFormat === 'claude' && claudeContext.doneSent) { diff --git a/src/server/routes/proxy/gemini.test.ts b/src/server/routes/proxy/gemini.test.ts index 8e4a763..9528a76 100644 --- a/src/server/routes/proxy/gemini.test.ts +++ b/src/server/routes/proxy/gemini.test.ts @@ -28,6 +28,21 @@ vi.mock('../../services/downstreamApiKeyService.js', () => ({ consumeManagedKeyRequest: (...args: unknown[]) => consumeManagedKeyRequestMock(...args), })); +function parseSsePayloads(body: string): Array> { + return body + .split('\n\n') + .map((block) => block.trim()) + .filter(Boolean) + .map((block) => block + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trimStart()) + .join('\n'), + ) + .filter((data) => data && data !== '[DONE]') + .map((data) => JSON.parse(data) as Record); +} + describe('gemini native proxy routes', () => { let app: FastifyInstance; @@ -301,7 +316,7 @@ describe('gemini native proxy routes', () => { }); }); - it('aggregates non-sse json-array streaming payloads into a final Gemini response', async () => { + it('keeps non-sse json-array streaming payloads on the wire as chunk responses', async () => { fetchMock.mockResolvedValue(new Response(JSON.stringify([ { candidates: [ @@ -348,32 +363,45 @@ describe('gemini native proxy routes', () => { }); expect(response.statusCode).toBe(200); - expect(response.json()).toEqual({ - responseId: '', - modelVersion: '', - candidates: [ - { - index: 0, - finishReason: 'STOP', - content: { - role: 'model', - parts: [ - { text: 'first' }, - { text: 'second', thoughtSignature: 'sig-1' }, - ], + expect(response.json()).toEqual([ + { + responseId: '', + modelVersion: '', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'first' }], + }, + groundingMetadata: { source: 'web' }, }, - groundingMetadata: { source: 'web' }, - citationMetadata: { citations: [{ startIndex: 0, endIndex: 5 }] }, - }, - ], - usageMetadata: { - promptTokenCount: 11, - candidatesTokenCount: 6, - totalTokenCount: 17, - cachedContentTokenCount: 2, - thoughtsTokenCount: 3, + ], }, - }); + { + responseId: '', + modelVersion: '', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'second', thoughtSignature: 'sig-1' }], + }, + citationMetadata: { citations: [{ startIndex: 0, endIndex: 5 }] }, + }, + ], + usageMetadata: { + promptTokenCount: 11, + candidatesTokenCount: 6, + totalTokenCount: 17, + cachedContentTokenCount: 2, + thoughtsTokenCount: 3, + }, + }, + ]); }); it('derives gemini-3 thinkingLevel from OpenAI-style reasoning inputs in the runtime request path', async () => { @@ -428,12 +456,12 @@ describe('gemini native proxy routes', () => { }, ], generationConfig: { - thinkingConfig: { thinkingLevel: 'high' }, + thinkingConfig: { thinkingLevel: 'high', includeThoughts: true }, }, }); }); - it('aggregates SSE payloads through the transformer instead of raw passthrough', async () => { + it('streams SSE payloads as normalized chunk events instead of cumulative aggregate snapshots', async () => { const encoder = new TextEncoder(); const upstreamBody = new ReadableStream({ start(controller) { @@ -466,12 +494,53 @@ describe('gemini native proxy routes', () => { expect(response.statusCode).toBe(200); expect(response.headers['content-type']).toContain('text/event-stream'); - expect(response.body).toContain('data: {"responseId":"resp-sse"'); - expect(response.body).toContain('"groundingMetadata":{"source":"web"}'); - expect(response.body).toContain('"citationMetadata":{"citations":[{"startIndex":0,"endIndex":5}]}'); - expect(response.body).toContain('"cachedContentTokenCount":2'); - expect(response.body).toContain('"thoughtsTokenCount":3'); - expect(response.body).toContain('"parts":[{"text":"first"},{"text":"second","thoughtSignature":"sig-1"}]'); + const events = parseSsePayloads(response.body); + + expect(events).toHaveLength(2); + expect(events[0]).toMatchObject({ + responseId: 'resp-sse', + modelVersion: 'gemini-2.5-flash', + candidates: [ + { + content: { + parts: [{ text: 'first' }], + }, + groundingMetadata: { source: 'web' }, + }, + ], + }); + expect(events[0]).not.toMatchObject({ + candidates: [ + { + citationMetadata: expect.anything(), + }, + ], + }); + expect(events[1]).toMatchObject({ + usageMetadata: { + promptTokenCount: 11, + candidatesTokenCount: 6, + totalTokenCount: 17, + cachedContentTokenCount: 2, + thoughtsTokenCount: 3, + }, + candidates: [ + { + finishReason: 'STOP', + content: { + parts: [{ text: 'second', thoughtSignature: 'sig-1' }], + }, + citationMetadata: { citations: [{ startIndex: 0, endIndex: 5 }] }, + }, + ], + }); + expect(events[1]).not.toMatchObject({ + candidates: [ + { + groundingMetadata: expect.anything(), + }, + ], + }); expect(response.body).not.toContain('\r\n\r\n'); }); diff --git a/src/server/routes/proxy/gemini.ts b/src/server/routes/proxy/gemini.ts index d8ab432..adfe664 100644 --- a/src/server/routes/proxy/gemini.ts +++ b/src/server/routes/proxy/gemini.ts @@ -119,6 +119,7 @@ export async function geminiProxyRoute(app: FastifyInstance) { const generateContent = async (request: FastifyRequest, reply: FastifyReply) => { const apiVersion = resolveGeminiApiVersion(request); const modelActionPath = extractGeminiModelActionPath(request, apiVersion); + const isStreamAction = modelActionPath.endsWith(':streamGenerateContent'); const requestedModel = modelActionPath.replace(/^models\//, '').split(':')[0].trim(); if (!requestedModel) { return reply.code(400).send({ @@ -206,27 +207,23 @@ export async function geminiProxyRoute(app: FastifyInstance) { if (done) break; if (!value) continue; const chunkText = decoder.decode(value, { stream: true }); - const parsed = geminiGenerateContentTransformer.stream.parseSsePayloads(rest + chunkText); - rest = parsed.rest; - for (const event of parsed.events) { - geminiGenerateContentTransformer.stream.applyAggregate(aggregateState, event); - reply.raw.write( - geminiGenerateContentTransformer.stream.serializeSsePayload( - geminiGenerateContentTransformer.outbound.serializeAggregateResponse(aggregateState), - ), - ); + const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer( + aggregateState, + rest + chunkText, + ); + rest = consumed.rest; + for (const line of consumed.lines) { + reply.raw.write(line); } } const tail = decoder.decode(); if (tail) { - const parsed = geminiGenerateContentTransformer.stream.parseSsePayloads(rest + tail); - for (const event of parsed.events) { - geminiGenerateContentTransformer.stream.applyAggregate(aggregateState, event); - reply.raw.write( - geminiGenerateContentTransformer.stream.serializeSsePayload( - geminiGenerateContentTransformer.outbound.serializeAggregateResponse(aggregateState), - ), - ); + const consumed = geminiGenerateContentTransformer.stream.consumeUpstreamSseBuffer( + aggregateState, + rest + tail, + ); + for (const line of consumed.lines) { + reply.raw.write(line); } } } finally { @@ -239,19 +236,13 @@ export async function geminiProxyRoute(app: FastifyInstance) { const text = await upstream.text(); try { const parsed = JSON.parse(text); - const aggregateState = geminiGenerateContentTransformer.aggregator.createState(); - if (Array.isArray(parsed)) { - for (const chunk of geminiGenerateContentTransformer.stream.parseJsonArrayPayload(parsed)) { - geminiGenerateContentTransformer.aggregator.apply(aggregateState, chunk); - } - return reply.code(upstream.status).send( - geminiGenerateContentTransformer.outbound.serializeAggregateResponse(aggregateState), - ); - } - - geminiGenerateContentTransformer.aggregator.apply(aggregateState, parsed); + const aggregateState = geminiGenerateContentTransformer.stream.createAggregateState(); return reply.code(upstream.status).send( - geminiGenerateContentTransformer.outbound.serializeAggregateResponse(aggregateState), + geminiGenerateContentTransformer.stream.serializeUpstreamJsonPayload( + aggregateState, + parsed, + isStreamAction, + ), ); } catch { return reply.code(upstream.status).type(contentType || 'application/json').send(text); diff --git a/src/server/routes/proxy/responses.ts b/src/server/routes/proxy/responses.ts index 8c02901..de6f556 100644 --- a/src/server/routes/proxy/responses.ts +++ b/src/server/routes/proxy/responses.ts @@ -9,23 +9,13 @@ import { shouldRetryProxyRequest } from '../../services/proxyRetryPolicy.js'; import { resolveProxyUsageWithSelfLogFallback } from '../../services/proxyUsageFallbackService.js'; import { mergeProxyUsage, parseProxyUsage } from '../../services/proxyUsageParser.js'; import { resolveProxyUrlForSite, withSiteRecordProxyRequestInit } from '../../services/siteProxy.js'; -import { type NormalizedStreamEvent } from '../../transformers/shared/normalized.js'; import { openAiResponsesTransformer } from '../../transformers/openai/responses/index.js'; -import { serializeResponsesFinalPayload } from '../../transformers/openai/responses/outbound.js'; -import { convertResponsesBodyToOpenAiBody } from '../../transformers/openai/responses/conversion.js'; -import { - completeResponsesStream, - createOpenAiResponsesAggregateState, - failResponsesStream, - serializeConvertedResponsesEvents, -} from '../../transformers/openai/responses/aggregator.js'; import { buildMinimalJsonHeadersForCompatibility, buildUpstreamEndpointRequest, isEndpointDowngradeError, isUnsupportedMediaTypeError, resolveUpstreamEndpointCandidates, - type UpstreamEndpoint, } from './upstreamEndpoint.js'; import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordDownstreamCostUsage } from './downstreamPolicy.js'; import { composeProxyLogMessage } from './logPathMeta.js'; @@ -35,222 +25,10 @@ import { resolveProxyLogBilling } from './proxyBilling.js'; const MAX_RETRIES = 2; -function shouldDowngradeFromChatToMessagesForResponses( - endpointPath: string, - status: number, - upstreamErrorText: string, -): boolean { - if (!endpointPath.includes('/chat/completions')) return false; - if (status < 400 || status >= 500) return false; - return /messages\s+is\s+required/i.test(upstreamErrorText); -} - -function parseUpstreamErrorShape(rawText: string): { - type: string; - code: string; - message: string; -} { - try { - const parsed = JSON.parse(rawText) as Record; - const error = (parsed.error && typeof parsed.error === 'object') - ? parsed.error as Record - : parsed; - return { - type: typeof error.type === 'string' ? error.type.trim().toLowerCase() : '', - code: typeof error.code === 'string' ? error.code.trim().toLowerCase() : '', - message: typeof error.message === 'string' ? error.message.trim() : '', - }; - } catch { - return { type: '', code: '', message: '' }; - } -} - -function stripResponsesMetadata( - body: Record, -): Record | null { - if (!Object.prototype.hasOwnProperty.call(body, 'metadata')) return null; - const next = { ...body }; - delete next.metadata; - return next; -} - -function toFiniteNumber(value: unknown): number | null { - return typeof value === 'number' && Number.isFinite(value) ? value : null; -} - -function buildCoreResponsesBody( - body: Record, -): Record | null { - const model = typeof body.model === 'string' ? body.model.trim() : ''; - if (!model) return null; - if (body.input === undefined) return null; - - const core: Record = { - model, - input: body.input, - stream: body.stream === true, - }; - - const maxOutputTokens = toFiniteNumber(body.max_output_tokens); - if (maxOutputTokens !== null && maxOutputTokens > 0) { - core.max_output_tokens = Math.trunc(maxOutputTokens); - } - - const temperature = toFiniteNumber(body.temperature); - if (temperature !== null) core.temperature = temperature; - - const topP = toFiniteNumber(body.top_p); - if (topP !== null) core.top_p = topP; - - const instructions = typeof body.instructions === 'string' ? body.instructions.trim() : ''; - if (instructions) core.instructions = instructions; - - return core; -} - -function buildStrictResponsesBody( - body: Record, -): Record | null { - const model = typeof body.model === 'string' ? body.model.trim() : ''; - if (!model) return null; - if (body.input === undefined) return null; - - return { - model, - input: body.input, - stream: body.stream === true, - }; -} - -function buildResponsesCompatibilityBodies( - body: Record, -): Record[] { - const candidates: Record[] = []; - const seen = new Set(); - try { - const originalKey = JSON.stringify(body); - if (originalKey) seen.add(originalKey); - } catch { - // ignore non-serializable bodies - } - const push = (next: Record | null) => { - if (!next) return; - let key = ''; - try { - key = JSON.stringify(next); - } catch { - return; - } - if (!key || seen.has(key)) return; - seen.add(key); - candidates.push(next); - }; - - push(stripResponsesMetadata(body)); - push(buildCoreResponsesBody(body)); - push(buildStrictResponsesBody(body)); - return candidates; -} - -function buildResponsesCompatibilityHeaderCandidates( - headers: Record, - stream: boolean, -): Record[] { - const candidates: Record[] = []; - const seen = new Set(); - const push = (next: Record) => { - const normalizedEntries = Object.entries(next) - .filter(([, value]) => typeof value === 'string' && value.trim().length > 0) - .map(([key, value]) => [key.toLowerCase(), value] as const) - .sort(([a], [b]) => a.localeCompare(b)); - const key = JSON.stringify(normalizedEntries); - if (!key || seen.has(key)) return; - seen.add(key); - candidates.push(Object.fromEntries(normalizedEntries)); - }; - - push(headers); - - const minimal: Record = {}; - for (const [rawKey, rawValue] of Object.entries(headers)) { - const key = rawKey.toLowerCase(); - if ( - key === 'authorization' - || key === 'x-api-key' - || key === 'content-type' - || key === 'accept' - ) { - minimal[key] = rawValue; - } - } - if (!minimal['content-type']) minimal['content-type'] = 'application/json'; - if (stream && !minimal.accept) minimal.accept = 'text/event-stream'; - push(minimal); - - return candidates; -} - -function shouldRetryResponsesCompatibility(input: { - endpoint: UpstreamEndpoint; - status: number; - rawErrText: string; -}): boolean { - if (input.endpoint !== 'responses') return false; - if (input.status !== 400) return false; - const parsedError = parseUpstreamErrorShape(input.rawErrText); - const type = parsedError.type.trim().toLowerCase(); - const code = parsedError.code.trim().toLowerCase(); - const message = parsedError.message.trim().toLowerCase(); - const compact = `${type} ${code} ${message}`.trim(); - const rawCompact = (input.rawErrText || '').toLowerCase(); - - // Authentication/authorization failures should not enter compatibility retries. - if ( - compact.includes('invalid_api_key') - || compact.includes('authentication') - || compact.includes('unauthorized') - || compact.includes('forbidden') - || compact.includes('insufficient_quota') - || compact.includes('rate_limit') - ) { - return false; - } - - if (type === 'upstream_error' || code === 'upstream_error') return true; - if (message === 'upstream_error' || message === 'upstream request failed') return true; - if (rawCompact.includes('upstream_error')) return true; - - // Many sub2api-compatible gateways return generic 400 for field incompatibilities. - // Retry with progressively stricter payload/header candidates to maximize compatibility. - return true; -} - function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object'; } -function asTrimmedString(value: unknown): string { - return typeof value === 'string' ? value.trim() : ''; -} - -function normalizeText(value: unknown): string { - if (typeof value === 'string') return value; - if (Array.isArray(value)) { - return value - .map((item) => normalizeText(item)) - .filter((item) => item.length > 0) - .join('\n'); - } - if (isRecord(value)) { - if (typeof value.text === 'string') return value.text; - if (typeof value.content === 'string') return value.content; - if (typeof value.input_text === 'string') return value.input_text; - if (typeof value.output_text === 'string') return value.output_text; - if (Array.isArray(value.content)) return normalizeText(value.content); - } - return ''; -} - type UsageSummary = ReturnType; export async function responsesProxyRoute(app: FastifyInstance) { @@ -295,7 +73,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { excludeChannelIds.push(selected.channel.id); const modelName = selected.actualModel || requestedModel; - const openAiBody = convertResponsesBodyToOpenAiBody(body, modelName, isStream); + const openAiBody = openAiResponsesTransformer.inbound.toOpenAiBody(body, modelName, isStream); const endpointCandidates = await resolveUpstreamEndpointCandidates( { site: selected.site, @@ -340,15 +118,15 @@ export async function responsesProxyRoute(app: FastifyInstance) { headers: endpointRequest.headers, body: endpointRequest.body as Record, }; - }, - tryRecover: async (ctx) => { - if (shouldRetryResponsesCompatibility({ + }, + tryRecover: async (ctx) => { + if (openAiResponsesTransformer.compatibility.shouldRetry({ endpoint: ctx.request.endpoint, status: ctx.response.status, rawErrText: ctx.rawErrText, })) { - const compatibilityBodies = buildResponsesCompatibilityBodies(ctx.request.body); - const compatibilityHeaders = buildResponsesCompatibilityHeaderCandidates( + const compatibilityBodies = openAiResponsesTransformer.compatibility.buildRetryBodies(ctx.request.body); + const compatibilityHeaders = openAiResponsesTransformer.compatibility.buildRetryHeaders( ctx.request.headers, isStream, ); @@ -416,7 +194,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { shouldDowngrade: (ctx) => ( ctx.response.status >= 500 || isEndpointDowngradeError(ctx.response.status, ctx.rawErrText) - || shouldDowngradeFromChatToMessagesForResponses( + || openAiResponsesTransformer.compatibility.shouldDowngradeChatToMessages( ctx.request.path, ctx.response.status, ctx.rawErrText, @@ -492,7 +270,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { const passthroughResponsesStream = successfulUpstreamPath === '/v1/responses'; const streamContext = openAiResponsesTransformer.createStreamContext(modelName); - const responsesState = createOpenAiResponsesAggregateState(modelName); + const responsesState = openAiResponsesTransformer.aggregator.createState(modelName); const writeLines = (lines: string[]) => { for (const line of lines) reply.raw.write(line); @@ -505,7 +283,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { if (passthroughResponsesStream) { reply.raw.write('data: [DONE]\n\n'); } else if (!responsesState.completed) { - writeLines(completeResponsesStream(responsesState, streamContext, parsedUsage)); + writeLines(openAiResponsesTransformer.aggregator.complete(responsesState, streamContext, parsedUsage)); } continue; } @@ -537,13 +315,13 @@ export async function responsesProxyRoute(app: FastifyInstance) { || payloadType === 'response.failed' ); if (isFailureEvent) { - writeLines(failResponsesStream(responsesState, streamContext, parsedUsage, parsedPayload)); + writeLines(openAiResponsesTransformer.aggregator.fail(responsesState, streamContext, parsedUsage, parsedPayload)); continue; } if (parsedPayload && typeof parsedPayload === 'object') { const normalizedEvent = openAiResponsesTransformer.transformStreamEvent(parsedPayload, streamContext, modelName); - writeLines(serializeConvertedResponsesEvents({ + writeLines(openAiResponsesTransformer.aggregator.serialize({ state: responsesState, streamContext, event: normalizedEvent, @@ -552,7 +330,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { continue; } - writeLines(serializeConvertedResponsesEvents({ + writeLines(openAiResponsesTransformer.aggregator.serialize({ state: responsesState, streamContext, event: { contentDelta: eventBlock.data }, @@ -580,7 +358,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { } finally { reader.releaseLock(); if (!passthroughResponsesStream && !responsesState.completed) { - writeLines(completeResponsesStream(responsesState, streamContext, parsedUsage)); + writeLines(openAiResponsesTransformer.aggregator.complete(responsesState, streamContext, parsedUsage)); } reply.raw.end(); } @@ -632,7 +410,7 @@ export async function responsesProxyRoute(app: FastifyInstance) { modelName, rawText, ); - const downstreamData = serializeResponsesFinalPayload({ + const downstreamData = openAiResponsesTransformer.outbound.serializeFinal({ upstreamPayload: upstreamData, normalized, usage: parsedUsage, diff --git a/src/server/routes/proxy/upstreamEndpoint.test.ts b/src/server/routes/proxy/upstreamEndpoint.test.ts index 045b6f6..88cf6c2 100644 --- a/src/server/routes/proxy/upstreamEndpoint.test.ts +++ b/src/server/routes/proxy/upstreamEndpoint.test.ts @@ -578,7 +578,13 @@ describe('buildUpstreamEndpointRequest', () => { }); expect(request.path).toBe('/v1/messages'); - expect(request.body.system).toBe('be careful'); + expect(request.body.system).toEqual([ + { + type: 'text', + text: 'be careful', + cache_control: { type: 'ephemeral' }, + }, + ]); expect(request.body.messages).toEqual([ { role: 'user', @@ -733,7 +739,13 @@ describe('buildUpstreamEndpointRequest', () => { }); expect(request.path).toBe('/v1/messages'); - expect(request.body.system).toBe('system prompt'); + expect(request.body.system).toEqual([ + { + type: 'text', + text: 'system prompt', + cache_control: { type: 'ephemeral' }, + }, + ]); expect(request.body.messages).toEqual([ { role: 'user', diff --git a/src/server/routes/proxy/upstreamEndpoint.ts b/src/server/routes/proxy/upstreamEndpoint.ts index 62cfd07..4ce7cc6 100644 --- a/src/server/routes/proxy/upstreamEndpoint.ts +++ b/src/server/routes/proxy/upstreamEndpoint.ts @@ -611,6 +611,20 @@ export function shouldPreferResponsesAfterLegacyChatError(input: { ); } +export function promoteResponsesCandidateAfterLegacyChatError( + endpointCandidates: UpstreamEndpoint[], + input: Parameters[0], +): void { + if (!shouldPreferResponsesAfterLegacyChatError(input)) return; + + const currentIndex = endpointCandidates.findIndex((endpoint) => endpoint === input.currentEndpoint); + const responsesIndex = endpointCandidates.indexOf('responses'); + if (currentIndex < 0 || responsesIndex < 0 || responsesIndex <= currentIndex + 1) return; + + endpointCandidates.splice(responsesIndex, 1); + endpointCandidates.splice(currentIndex + 1, 0, 'responses'); +} + export function isEndpointDowngradeError(status: number, upstreamErrorText?: string | null): boolean { if (status < 400) return false; const text = (upstreamErrorText || '').toLowerCase(); diff --git a/src/server/services/modelService.test.ts b/src/server/services/modelService.test.ts index 7436016..61d00c0 100644 --- a/src/server/services/modelService.test.ts +++ b/src/server/services/modelService.test.ts @@ -40,6 +40,51 @@ describe('rebuildTokenRoutesFromAvailability', () => { delete process.env.DATA_DIR; }); + it('creates an exact route with an account-direct channel for apikey model availability', async () => { + const site = await db.insert(schema.sites).values({ + name: 'apikey-site', + url: 'https://apikey-site.example.com', + platform: 'new-api', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'apikey-user', + accessToken: '', + apiToken: 'sk-apikey-route', + status: 'active', + extraConfig: JSON.stringify({ credentialMode: 'apikey' }), + }).returning().get(); + + await db.insert(schema.modelAvailability).values({ + accountId: account.id, + modelName: 'gpt-5.2-codex', + available: true, + latencyMs: 1200, + checkedAt: '2026-03-08T08:00:00.000Z', + }).run(); + + const rebuild = await rebuildTokenRoutesFromAvailability(); + + expect(rebuild.models).toBe(1); + + const route = await db.select().from(schema.tokenRoutes) + .where(eq(schema.tokenRoutes.modelPattern, 'gpt-5.2-codex')) + .get(); + expect(route).toBeDefined(); + + const channels = await db.select().from(schema.routeChannels) + .where(and( + eq(schema.routeChannels.routeId, route!.id), + eq(schema.routeChannels.accountId, account.id), + )) + .all(); + + expect(channels).toHaveLength(1); + expect(channels[0]?.tokenId ?? null).toBeNull(); + expect(channels[0]?.manualOverride).toBe(false); + }); + it('removes stale exact routes and keeps wildcard routes on rebuild', async () => { const site = await db.insert(schema.sites).values({ name: 'site-1', diff --git a/src/server/services/modelService.ts b/src/server/services/modelService.ts index 399108a..1c25190 100644 --- a/src/server/services/modelService.ts +++ b/src/server/services/modelService.ts @@ -250,8 +250,8 @@ async function refreshModelsForAllActiveAccounts() { return results; } -export async function rebuildTokenRoutesFromAvailability() { - const tokenRows = await db.select().from(schema.tokenModelAvailability) +export async function rebuildTokenRoutesFromAvailability() { + const tokenRows = await db.select().from(schema.tokenModelAvailability) .innerJoin(schema.accountTokens, eq(schema.tokenModelAvailability.tokenId, schema.accountTokens.id)) .innerJoin(schema.accounts, eq(schema.accountTokens.accountId, schema.accounts.id)) .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) @@ -262,25 +262,49 @@ export async function rebuildTokenRoutesFromAvailability() { eq(schema.accounts.status, 'active'), eq(schema.sites.status, 'active'), ), - ) - .all(); - - const modelTokens = new Map>(); - for (const row of tokenRows) { - const modelName = row.token_model_availability.modelName; - if (!modelTokens.has(modelName)) modelTokens.set(modelName, new Map()); - modelTokens.get(modelName)!.set(row.account_tokens.id, row.accounts.id); - } - - const routes = await db.select().from(schema.tokenRoutes).all(); - const channels = await db.select().from(schema.routeChannels).all(); + ) + .all(); + + const accountRows = await db.select().from(schema.modelAvailability) + .innerJoin(schema.accounts, eq(schema.modelAvailability.accountId, schema.accounts.id)) + .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) + .where( + and( + eq(schema.modelAvailability.available, true), + eq(schema.accounts.status, 'active'), + eq(schema.sites.status, 'active'), + ), + ) + .all(); + + const modelCandidates = new Map>(); + const addModelCandidate = (modelNameRaw: string | null | undefined, accountId: number, tokenId: number | null) => { + const modelName = (modelNameRaw || '').trim(); + if (!modelName) return; + if (!modelCandidates.has(modelName)) modelCandidates.set(modelName, new Map()); + const candidateKey = `${accountId}:${tokenId ?? 'account'}`; + modelCandidates.get(modelName)!.set(candidateKey, { accountId, tokenId }); + }; + + for (const row of tokenRows) { + addModelCandidate(row.token_model_availability.modelName, row.accounts.id, row.account_tokens.id); + } + + for (const row of accountRows) { + if (!isApiKeyConnection(row.accounts)) continue; + if (!(row.accounts.apiToken || '').trim()) continue; + addModelCandidate(row.model_availability.modelName, row.accounts.id, null); + } + + const routes = await db.select().from(schema.tokenRoutes).all(); + const channels = await db.select().from(schema.routeChannels).all(); let createdRoutes = 0; - let createdChannels = 0; - let removedChannels = 0; - let removedRoutes = 0; - - for (const [modelName, tokenAccountMap] of modelTokens.entries()) { + let createdChannels = 0; + let removedChannels = 0; + let removedRoutes = 0; + + for (const [modelName, candidateMap] of modelCandidates.entries()) { let route = routes.find((r) => r.modelPattern === modelName); if (!route) { const inserted = await db.insert(schema.tokenRoutes).values({ @@ -295,18 +319,21 @@ export async function rebuildTokenRoutesFromAvailability() { routes.push(route); createdRoutes++; } - - const routeChannels = channels.filter((channel) => channel.routeId === route.id); - const desiredTokenIds = new Set(Array.from(tokenAccountMap.keys())); - - for (const [tokenId, accountId] of tokenAccountMap.entries()) { - const exists = routeChannels.some((channel) => channel.tokenId === tokenId); - if (exists) continue; - + + const routeChannels = channels.filter((channel) => channel.routeId === route.id); + const desiredKeys = new Set(Array.from(candidateMap.keys())); + + for (const [candidateKey, candidate] of candidateMap.entries()) { + const exists = routeChannels.some((channel) => ( + channel.accountId === candidate.accountId + && (channel.tokenId ?? null) === candidate.tokenId + )); + if (exists) continue; + const inserted = await db.insert(schema.routeChannels).values({ routeId: route.id, - accountId, - tokenId, + accountId: candidate.accountId, + tokenId: candidate.tokenId, priority: 0, weight: 10, enabled: true, @@ -318,32 +345,34 @@ export async function rebuildTokenRoutesFromAvailability() { if (!created) continue; channels.push(created); createdChannels++; - } - - for (const channel of routeChannels) { - if (channel.tokenId && desiredTokenIds.has(channel.tokenId)) { - continue; - } - - if (!channel.tokenId) { + desiredKeys.add(candidateKey); + } + + for (const channel of routeChannels) { + const channelKey = `${channel.accountId}:${channel.tokenId ?? 'account'}`; + if (desiredKeys.has(channelKey)) { + continue; + } + + if (!channel.tokenId) { const preferred = await getPreferredAccountToken(channel.accountId); - if (preferred && desiredTokenIds.has(preferred.id)) { - await db.update(schema.routeChannels) - .set({ tokenId: preferred.id }) - .where(eq(schema.routeChannels.id, channel.id)) - .run(); - continue; + if (preferred && desiredKeys.has(`${channel.accountId}:${preferred.id}`)) { + await db.update(schema.routeChannels) + .set({ tokenId: preferred.id }) + .where(eq(schema.routeChannels.id, channel.id)) + .run(); + continue; } } if (!channel.manualOverride) { await db.delete(schema.routeChannels).where(eq(schema.routeChannels.id, channel.id)).run(); removedChannels++; - } - } - } - - const latestModelNames = new Set(Array.from(modelTokens.keys())); + } + } + } + + const latestModelNames = new Set(Array.from(modelCandidates.keys())); for (const route of routes) { const modelPattern = (route.modelPattern || '').trim(); if (!modelPattern || !isExactModelPattern(modelPattern) || latestModelNames.has(modelPattern)) { @@ -367,13 +396,13 @@ export async function rebuildTokenRoutesFromAvailability() { invalidateTokenRouterCache(); - return { - models: modelTokens.size, - createdRoutes, - createdChannels, - removedChannels, - removedRoutes, - }; + return { + models: modelCandidates.size, + createdRoutes, + createdChannels, + removedChannels, + removedRoutes, + }; } export async function refreshModelsAndRebuildRoutes() { diff --git a/src/server/transformers/anthropic/messages/aggregator.test.ts b/src/server/transformers/anthropic/messages/aggregator.test.ts new file mode 100644 index 0000000..fcc4c84 --- /dev/null +++ b/src/server/transformers/anthropic/messages/aggregator.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest'; + +import { applyAnthropicMessagesAggregateEvent, createAnthropicMessagesAggregateState } from './aggregator.js'; + +describe('applyAnthropicMessagesAggregateEvent', () => { + it('buffers signature deltas until the thinking block stops', () => { + const state = createAnthropicMessagesAggregateState(); + + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + startBlock: { + kind: 'thinking', + index: 0, + }, + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { reasoningDelta: 'step-1' }); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + signatureDelta: 'sig-buffered', + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { reasoningDelta: 'step-2' }); + + expect(state.pendingSignature).toBe('sig-buffered'); + expect((state as any).contentBlocks).toEqual([ + { + type: 'thinking', + thinking: 'step-1step-2', + signature: '', + }, + ]); + + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + stopBlockIndex: 0, + }, + } as any); + + expect(state.pendingSignature).toBeNull(); + expect((state as any).contentBlocks).toEqual([ + { + type: 'thinking', + thinking: 'step-1step-2', + signature: 'sig-buffered', + }, + ]); + }); + + it('keeps thinking signatures and redacted_thinking blocks in content order', () => { + const state = createAnthropicMessagesAggregateState(); + + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + startBlock: { + kind: 'thinking', + index: 0, + }, + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { reasoningDelta: 'internal-step' }); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + signatureDelta: 'sig-1', + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + stopBlockIndex: 0, + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + startBlock: { + kind: 'redacted_thinking', + index: 1, + }, + redactedThinkingData: 'ciphertext', + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + stopBlockIndex: 1, + }, + } as any); + + expect((state as any).contentBlocks).toEqual([ + { + type: 'thinking', + thinking: 'internal-step', + signature: 'sig-1', + }, + { + type: 'redacted_thinking', + data: 'ciphertext', + }, + ]); + expect(state.blockLifecycle).toEqual([ + { kind: 'thinking', phase: 'start', index: 0 }, + { kind: 'thinking', phase: 'stop', index: 0 }, + { kind: 'redacted_thinking', phase: 'start', index: 1 }, + { kind: 'redacted_thinking', phase: 'stop', index: 1 }, + ]); + }); + + it('accumulates tool_use JSON on one block and preserves block order after thinking', () => { + const state = createAnthropicMessagesAggregateState(); + + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + startBlock: { + kind: 'thinking', + index: 0, + }, + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { reasoningDelta: 'plan' }); + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + stopBlockIndex: 0, + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { + toolCallDeltas: [{ + index: 0, + id: 'call_1', + name: 'lookup_city', + argumentsDelta: '{"city":"par', + }], + }); + applyAnthropicMessagesAggregateEvent(state, { + toolCallDeltas: [{ + index: 0, + argumentsDelta: 'is"}', + }], + }); + + expect((state as any).contentBlocks).toEqual([ + { + type: 'thinking', + thinking: 'plan', + signature: '', + }, + { + type: 'tool_use', + id: 'call_1', + name: 'lookup_city', + inputJson: '{"city":"paris"}', + }, + ]); + expect(state.toolCalls[0]).toEqual({ + id: 'call_1', + name: 'lookup_city', + arguments: '{"city":"paris"}', + }); + }); +}); diff --git a/src/server/transformers/anthropic/messages/aggregator.ts b/src/server/transformers/anthropic/messages/aggregator.ts index 9f50b5a..a32e601 100644 --- a/src/server/transformers/anthropic/messages/aggregator.ts +++ b/src/server/transformers/anthropic/messages/aggregator.ts @@ -1,5 +1,26 @@ import { type NormalizedStreamEvent } from '../../shared/normalized.js'; +export type AnthropicAggregatedContentBlock = + | { + type: 'text'; + text: string; + } + | { + type: 'thinking'; + thinking: string; + signature: string; + } + | { + type: 'redacted_thinking'; + data: string; + } + | { + type: 'tool_use'; + id: string; + name: string; + inputJson: string; + }; + export type AnthropicStreamExtension = { signatureDelta?: string; redactedThinkingData?: string; @@ -31,6 +52,11 @@ export type AnthropicMessagesAggregateState = { thinkingBlockIndex: number | null; redactedBlockIndexes: number[]; blockLifecycle: AnthropicBlockLifecycleEntry[]; + contentBlocks: AnthropicAggregatedContentBlock[]; + textBlockOrder: number | null; + thinkingBlockOrder: number | null; + redactedBlockOrders: Record; + toolBlockOrders: Record; }; export function createAnthropicMessagesAggregateState(): AnthropicMessagesAggregateState { @@ -45,9 +71,75 @@ export function createAnthropicMessagesAggregateState(): AnthropicMessagesAggreg thinkingBlockIndex: null, redactedBlockIndexes: [], blockLifecycle: [], + contentBlocks: [], + textBlockOrder: null, + thinkingBlockOrder: null, + redactedBlockOrders: {}, + toolBlockOrders: {}, }; } +function appendContentBlock( + state: AnthropicMessagesAggregateState, + block: AnthropicAggregatedContentBlock, +): number { + state.contentBlocks.push(block); + return state.contentBlocks.length - 1; +} + +function ensureThinkingContentBlock( + state: AnthropicMessagesAggregateState, +): Extract { + const order = state.thinkingBlockOrder; + if (order !== null && state.contentBlocks[order]?.type === 'thinking') { + return state.contentBlocks[order] as Extract; + } + + const created: Extract = { + type: 'thinking', + thinking: '', + signature: '', + }; + state.thinkingBlockOrder = appendContentBlock(state, created); + return created; +} + +function ensureToolContentBlock( + state: AnthropicMessagesAggregateState, + index: number, +): Extract { + const existingOrder = state.toolBlockOrders[index]; + if (existingOrder !== undefined && state.contentBlocks[existingOrder]?.type === 'tool_use') { + return state.contentBlocks[existingOrder] as Extract; + } + + const created: Extract = { + type: 'tool_use', + id: `toolu_${index}`, + name: '', + inputJson: '', + }; + state.toolBlockOrders[index] = appendContentBlock(state, created); + return created; +} + +function ensureRedactedContentBlock( + state: AnthropicMessagesAggregateState, + index: number, +): Extract { + const existingOrder = state.redactedBlockOrders[index]; + if (existingOrder !== undefined && state.contentBlocks[existingOrder]?.type === 'redacted_thinking') { + return state.contentBlocks[existingOrder] as Extract; + } + + const created: Extract = { + type: 'redacted_thinking', + data: '', + }; + state.redactedBlockOrders[index] = appendContentBlock(state, created); + return created; +} + function ensureToolState( state: AnthropicMessagesAggregateState, index: number, @@ -62,8 +154,24 @@ function ensureToolState( return state.toolCalls[index]; } +function applySignatureToThinkingBlock( + state: AnthropicMessagesAggregateState, + signature: string, +): void { + const thinkingBlock = ensureThinkingContentBlock(state); + thinkingBlock.signature += signature; +} + +function bufferPendingSignature( + state: AnthropicMessagesAggregateState, + signature: string, +): void { + state.pendingSignature = `${state.pendingSignature || ''}${signature}`; +} + function flushPendingSignature(state: AnthropicMessagesAggregateState): void { if (!state.pendingSignature) return; + applySignatureToThinkingBlock(state, state.pendingSignature); state.signatures.push(state.pendingSignature); state.pendingSignature = null; } @@ -76,18 +184,21 @@ function recordBlockStart( state.blockLifecycle.push({ kind, phase: 'start', index }); if (kind === 'thinking') { state.thinkingBlockIndex = typeof index === 'number' ? index : state.thinkingBlockIndex; + ensureThinkingContentBlock(state); return; } if (kind === 'redacted_thinking' && typeof index === 'number') { state.redactedBlockIndexes.push(index); + ensureRedactedContentBlock(state, index); } } function recordBlockStop(state: AnthropicMessagesAggregateState, index: number): void { if (state.thinkingBlockIndex === index) { state.blockLifecycle.push({ kind: 'thinking', phase: 'stop', index }); - state.thinkingBlockIndex = null; flushPendingSignature(state); + state.thinkingBlockIndex = null; + state.thinkingBlockOrder = null; return; } @@ -95,6 +206,7 @@ function recordBlockStop(state: AnthropicMessagesAggregateState, index: number): if (redactedIndex >= 0) { state.blockLifecycle.push({ kind: 'redacted_thinking', phase: 'stop', index }); state.redactedBlockIndexes.splice(redactedIndex, 1); + delete state.redactedBlockOrders[index]; } } @@ -105,7 +217,14 @@ function finalizeOpenBlocks(state: AnthropicMessagesAggregateState): void { } if (state.pendingSignature) { + if (state.thinkingBlockIndex === null && state.thinkingBlockOrder === null) { + state.blockLifecycle.push({ kind: 'thinking', phase: 'start' }); + } flushPendingSignature(state); + if (state.thinkingBlockIndex === null && state.thinkingBlockOrder !== null) { + state.blockLifecycle.push({ kind: 'thinking', phase: 'stop' }); + state.thinkingBlockOrder = null; + } } const openRedactedIndexes = [...state.redactedBlockIndexes]; @@ -130,33 +249,50 @@ export function applyAnthropicMessagesAggregateEvent( if (event.reasoningDelta) { state.reasoning.push(event.reasoningDelta); + ensureThinkingContentBlock(state).thinking += event.reasoningDelta; } if (event.contentDelta) { state.text.push(event.contentDelta); + if (state.textBlockOrder === null || state.contentBlocks[state.textBlockOrder]?.type !== 'text') { + state.textBlockOrder = appendContentBlock(state, { + type: 'text', + text: '', + }); + } + const textBlock = state.contentBlocks[state.textBlockOrder] as Extract; + textBlock.text += event.contentDelta; } if (Array.isArray(event.toolCallDeltas)) { for (const toolDelta of event.toolCallDeltas) { const toolState = ensureToolState(state, toolDelta.index); + const toolBlock = ensureToolContentBlock(state, toolDelta.index); if (toolDelta.id) toolState.id = toolDelta.id; if (toolDelta.name) toolState.name = toolDelta.name; + if (toolDelta.id) toolBlock.id = toolDelta.id; + if (toolDelta.name) toolBlock.name = toolDelta.name; if (toolDelta.argumentsDelta) { toolState.arguments += toolDelta.argumentsDelta; + toolBlock.inputJson += toolDelta.argumentsDelta; } } } if (event.anthropic?.signatureDelta) { - if (state.thinkingBlockIndex !== null) { - state.signatures.push(event.anthropic.signatureDelta); - } else { - state.pendingSignature = event.anthropic.signatureDelta; - } + bufferPendingSignature(state, event.anthropic.signatureDelta); } if (event.anthropic?.redactedThinkingData) { state.redactedReasoning.push(event.anthropic.redactedThinkingData); + const redactedIndex = ( + typeof event.anthropic.startBlock?.index === 'number' + ? event.anthropic.startBlock.index + : state.redactedBlockIndexes[state.redactedBlockIndexes.length - 1] + ); + if (typeof redactedIndex === 'number') { + ensureRedactedContentBlock(state, redactedIndex).data = event.anthropic.redactedThinkingData; + } } if ( diff --git a/src/server/transformers/anthropic/messages/compatibility.ts b/src/server/transformers/anthropic/messages/compatibility.ts new file mode 100644 index 0000000..55a17ac --- /dev/null +++ b/src/server/transformers/anthropic/messages/compatibility.ts @@ -0,0 +1,17 @@ +import { type DownstreamFormat } from '../../shared/normalized.js'; + +export function shouldRetryNormalizedMessagesBody(input: { + downstreamFormat: DownstreamFormat; + endpointPath: string; + status: number; + upstreamErrorText: string; +}): boolean { + if (input.downstreamFormat !== 'claude') return false; + if (!input.endpointPath.includes('/v1/messages')) return false; + if (input.status < 400 || input.status >= 500) return false; + return /messages\s+is\s+required/i.test(input.upstreamErrorText); +} + +export function isMessagesRequiredError(upstreamErrorText: string): boolean { + return /messages\s+is\s+required/i.test(upstreamErrorText); +} diff --git a/src/server/transformers/anthropic/messages/conversion.test.ts b/src/server/transformers/anthropic/messages/conversion.test.ts index 5730780..76d5ab9 100644 --- a/src/server/transformers/anthropic/messages/conversion.test.ts +++ b/src/server/transformers/anthropic/messages/conversion.test.ts @@ -29,6 +29,71 @@ describe('sanitizeAnthropicMessagesBody', () => { }); }); + it('normalizes string system and message content before rebuilding cache anchors', () => { + const result = sanitizeAnthropicMessagesBody({ + model: 'claude-opus-4-6', + system: 'system prompt', + messages: [ + { + role: 'user', + content: 'hello from user', + }, + ], + }); + + expect(result.system).toEqual([ + { + type: 'text', + text: 'system prompt', + cache_control: { type: 'ephemeral' }, + }, + ]); + expect(result.messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'text', + text: 'hello from user', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ]); + }); + + it('drops output_config.effort unless thinking stays adaptive', () => { + const enabled = sanitizeAnthropicMessagesBody({ + model: 'claude-opus-4-6', + thinking: { + type: 'enabled', + budget_tokens: 256, + }, + output_config: { + effort: 'high', + format: 'json', + }, + }); + const adaptive = sanitizeAnthropicMessagesBody({ + model: 'claude-opus-4-6', + thinking: { + type: 'adaptive', + }, + output_config: { + effort: 'high', + format: 'json', + }, + }); + + expect(enabled.output_config).toEqual({ + format: 'json', + }); + expect(adaptive.output_config).toEqual({ + effort: 'high', + format: 'json', + }); + }); + it('drops non-ephemeral cache_control markers while preserving valid tool block markers', () => { const result = sanitizeAnthropicMessagesBody({ model: 'claude-opus-4-6', @@ -344,6 +409,59 @@ describe('convertOpenAiBodyToAnthropicMessagesBody', () => { expect(body.thinking).toEqual({ type: 'adaptive' }); expect(body.output_config).toEqual({ effort: 'high' }); }); + + it('rebuilds cache_control anchors at the anthropic inbound boundary', () => { + const result = anthropicMessagesInbound.parse( + { + model: 'gpt-5', + max_tokens: 512, + tools: [ + { + name: 'lookup', + input_schema: { type: 'object' }, + }, + { + name: 'search', + input_schema: { type: 'object' }, + }, + ], + system: 'system prompt', + messages: [{ role: 'user', content: 'hello' }], + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.value?.claudeOriginalBody?.system).toEqual([ + { + type: 'text', + text: 'system prompt', + cache_control: { type: 'ephemeral' }, + }, + ]); + expect(result.value?.claudeOriginalBody?.tools).toEqual([ + { + name: 'lookup', + input_schema: { type: 'object' }, + }, + { + name: 'search', + input_schema: { type: 'object' }, + cache_control: { type: 'ephemeral' }, + }, + ]); + expect(result.value?.claudeOriginalBody?.messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'text', + text: 'hello', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ]); + }); }); describe('convertOpenAiToolChoiceToAnthropic', () => { @@ -410,6 +528,17 @@ describe('anthropicMessagesInbound', () => { expect(result.error?.statusCode).toBe(400); }); + + it('rejects string tool_choice tool without a tool name', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 512, + messages: [{ role: 'user', content: 'hello' }], + tool_choice: 'tool', + }); + + expect(result.error?.statusCode).toBe(400); + }); }); describe('anthropicMessagesTransformer.stream', () => { @@ -470,13 +599,20 @@ describe('anthropicMessagesTransformer.stream', () => { const streamContext = anthropicMessagesTransformer.createStreamContext('claude-opus-4-6'); const downstreamContext = anthropicMessagesTransformer.createDownstreamContext(); - const redacted = anthropicMessagesTransformer.transformStreamEvent({ + const redactedStart = anthropicMessagesTransformer.transformStreamEvent({ type: 'content_block_start', index: 0, content_block: { type: 'redacted_thinking', data: 'ciphertext' }, }, streamContext, 'claude-opus-4-6'); + const redactedStop = anthropicMessagesTransformer.transformStreamEvent({ + type: 'content_block_stop', + index: 0, + }, streamContext, 'claude-opus-4-6'); - const serialized = anthropicMessagesTransformer.serializeStreamEvent(redacted, streamContext, downstreamContext).join(''); + const serialized = [ + ...anthropicMessagesTransformer.serializeStreamEvent(redactedStart, streamContext, downstreamContext), + ...anthropicMessagesTransformer.serializeStreamEvent(redactedStop, streamContext, downstreamContext), + ].join(''); expect(serialized).toContain('"type":"redacted_thinking"'); expect(serialized).toContain('content_block_start'); expect(serialized).toContain('content_block_stop'); @@ -492,11 +628,14 @@ describe('extractAnthropicUsage', () => { cache_read_input_tokens: 12, cache_creation_input_tokens: 8, }, - })).toEqual({ + })).toMatchObject({ promptTokens: 120, completionTokens: 30, totalTokens: 150, cachedTokens: 12, + cacheReadInputTokens: 12, + cacheCreationInputTokens: 8, + promptTokensIncludingCache: 120, reasoningTokens: 0, audioInputTokens: 0, audioOutputTokens: 0, @@ -517,7 +656,7 @@ describe('extractAnthropicUsage', () => { ephemeral_1h_input_tokens: 3, }, }, - })).toEqual({ + })).toMatchObject({ cacheReadInputTokens: 12, cacheCreationInputTokens: 8, ephemeral5mInputTokens: 5, @@ -537,7 +676,7 @@ describe('extractAnthropicUsage', () => { ephemeral_1h_input_tokens: 3, }, }, - })).toEqual({ + })).toMatchObject({ cacheReadInputTokens: 12, cacheCreationInputTokens: 8, ephemeral5mInputTokens: 5, @@ -663,4 +802,32 @@ describe('applyAnthropicMessagesAggregateEvent', () => { ]); expect(state.finishReason).toBe('stop'); }); + + it('creates a synthetic thinking lifecycle when a buffered signature is flushed on finish', () => { + const state = createAnthropicMessagesAggregateState(); + + applyAnthropicMessagesAggregateEvent(state, { + anthropic: { + signatureDelta: 'sig-only', + }, + } as any); + applyAnthropicMessagesAggregateEvent(state, { + finishReason: 'stop', + done: true, + }); + + expect(state.pendingSignature).toBeNull(); + expect(state.signatures).toEqual(['sig-only']); + expect(state.contentBlocks).toEqual([ + { + type: 'thinking', + thinking: '', + signature: 'sig-only', + }, + ]); + expect(state.blockLifecycle).toEqual([ + { kind: 'thinking', phase: 'start' }, + { kind: 'thinking', phase: 'stop' }, + ]); + }); }); diff --git a/src/server/transformers/anthropic/messages/conversion.ts b/src/server/transformers/anthropic/messages/conversion.ts index 0c7e310..f324b25 100644 --- a/src/server/transformers/anthropic/messages/conversion.ts +++ b/src/server/transformers/anthropic/messages/conversion.ts @@ -5,6 +5,8 @@ function isRecord(value: unknown): value is Record { const VALID_ANTHROPIC_TOOL_CHOICE_TYPES = new Set(['auto', 'none', 'any', 'tool']); const VALID_ANTHROPIC_THINKING_TYPES = new Set(['enabled', 'disabled', 'adaptive']); const VALID_ANTHROPIC_EFFORTS = new Set(['low', 'medium', 'high', 'max']); +const MAX_ANTHROPIC_CACHE_CONTROL_BREAKPOINTS = 4; +const ADAPTIVE_ANTHROPIC_CACHE_CONTROL_BLOCK_WINDOW = 20; function asTrimmedString(value: unknown): string { return typeof value === 'string' ? value.trim() : ''; @@ -47,17 +49,23 @@ function sanitizeCacheControl(value: unknown): Record | undefin function sanitizeAnthropicOutputConfig( value: unknown, + options: { allowEffort?: boolean } = {}, ): { value?: Record; error?: string } { if (!isRecord(value)) return {}; + const allowEffort = options.allowEffort === true; const next: Record = {}; if ('effort' in value) { const effort = asTrimmedString(value.effort).toLowerCase(); - if (!effort) return { value: next }; - if (!VALID_ANTHROPIC_EFFORTS.has(effort)) { + if (!effort) { + // Ignore blank effort and continue preserving other output_config fields. + } else if (!allowEffort) { + // Ignore effort outside adaptive thinking; upstream only applies it there. + } else if (!VALID_ANTHROPIC_EFFORTS.has(effort)) { return { error: 'output_config.effort must be one of: low, medium, high, max' }; + } else { + next.effort = effort; } - next.effort = effort; } for (const [key, entry] of Object.entries(value)) { @@ -77,6 +85,9 @@ function sanitizeAnthropicToolChoiceValue( const normalized = asTrimmedString(value).toLowerCase(); if (!normalized) return {}; if (normalized === 'required') return { value: { type: 'any' } }; + if (normalized === 'tool') { + return { error: 'tool_choice.name is required when type is tool' }; + } if (!VALID_ANTHROPIC_TOOL_CHOICE_TYPES.has(normalized)) { return { error: 'tool_choice.type must be one of: auto, none, any, tool' }; } @@ -359,6 +370,34 @@ function clearAnthropicCacheControls(body: Record) { }); } +function normalizeAnthropicMessageContents(body: Record) { + const rawMessages = Array.isArray(body.messages) ? body.messages : []; + body.messages = rawMessages.map((message) => { + if (!isRecord(message) || Array.isArray(message.content)) return message; + if (typeof message.content !== 'string' || message.content.length === 0) return message; + return { + ...message, + content: [ + { + type: 'text', + text: message.content, + }, + ], + }; + }); +} + +function normalizeAnthropicSystemPrompts(body: Record) { + if (Array.isArray(body.system)) return; + if (typeof body.system !== 'string' || body.system.length === 0) return; + body.system = [ + { + type: 'text', + text: body.system, + }, + ]; +} + function ensureStructuralAnthropicCacheControls(body: Record): number { let count = 0; @@ -454,10 +493,12 @@ function sanitizeUnsupportedAnthropicCacheControls(body: Record } function optimizeAnthropicCacheControls(body: Record) { + normalizeAnthropicMessageContents(body); + normalizeAnthropicSystemPrompts(body); clearAnthropicCacheControls(body); const structuralAnchors = ensureStructuralAnthropicCacheControls(body); - const remaining = Math.max(0, 4 - structuralAnchors); + const remaining = Math.max(0, MAX_ANTHROPIC_CACHE_CONTROL_BREAKPOINTS - structuralAnchors); if (remaining <= 0) { sanitizeUnsupportedAnthropicCacheControls(body); return; @@ -469,7 +510,7 @@ function optimizeAnthropicCacheControls(body: Record) { return; } - const desiredMessageAnchors = refs.length >= 20 ? 2 : 1; + const desiredMessageAnchors = refs.length >= ADAPTIVE_ANTHROPIC_CACHE_CONTROL_BLOCK_WINDOW ? 2 : 1; const targetAnchors = Math.min(desiredMessageAnchors, remaining); const usedIndexes = new Set(); @@ -482,7 +523,7 @@ function optimizeAnthropicCacheControls(body: Record) { applyAnchor(refs.length - 1); if (targetAnchors > 1) { - const targetIndex = Math.max(refs.length - 1 - 20, 0); + const targetIndex = Math.max(refs.length - 1 - ADAPTIVE_ANTHROPIC_CACHE_CONTROL_BLOCK_WINDOW, 0); let chosen = -1; for (let index = targetIndex; index >= 0; index -= 1) { if (!usedIndexes.has(index)) { @@ -546,7 +587,12 @@ export function sanitizeAnthropicMessagesBody( delete sanitized.thinking; } - const outputConfigResult = sanitizeAnthropicOutputConfig(sanitized.output_config ?? sanitized.outputConfig); + const allowOutputConfigEffort = isRecord(sanitized.thinking) + && asTrimmedString(sanitized.thinking.type).toLowerCase() === 'adaptive'; + const outputConfigResult = sanitizeAnthropicOutputConfig( + sanitized.output_config ?? sanitized.outputConfig, + { allowEffort: allowOutputConfigEffort }, + ); if (outputConfigResult.value) { sanitized.output_config = outputConfigResult.value; } else { @@ -653,28 +699,29 @@ export function convertOpenAiToolsToAnthropic(rawTools: unknown): unknown { export function convertOpenAiToolChoiceToAnthropic(rawToolChoice: unknown): unknown { if (rawToolChoice === undefined) return undefined; + let mappedValue: unknown = rawToolChoice; if (typeof rawToolChoice === 'string') { const normalized = rawToolChoice.trim().toLowerCase(); - if (normalized === 'required') return { type: 'any' }; - if (normalized === 'none') return { type: 'none' }; - if (normalized === 'auto') return { type: 'auto' }; - if (normalized === 'any') return { type: 'any' }; - return rawToolChoice; + if (normalized === 'required') mappedValue = { type: 'any' }; + else if (normalized === 'none') mappedValue = { type: 'none' }; + else if (normalized === 'auto') mappedValue = { type: 'auto' }; + else if (normalized === 'any') mappedValue = { type: 'any' }; + else mappedValue = rawToolChoice; } - if (!isRecord(rawToolChoice)) return rawToolChoice; - - const type = asTrimmedString(rawToolChoice.type).toLowerCase(); - if (type === 'function' && isRecord(rawToolChoice.function)) { - const name = asTrimmedString(rawToolChoice.function.name); - return name ? { type: 'tool', name } : { type: 'any' }; + if (isRecord(rawToolChoice)) { + const type = asTrimmedString(rawToolChoice.type).toLowerCase(); + if (type === 'function' && isRecord(rawToolChoice.function)) { + const name = asTrimmedString(rawToolChoice.function.name); + mappedValue = name ? { type: 'tool', name } : undefined; + } else { + mappedValue = rawToolChoice; + } } - if (type === 'tool' || type === 'auto' || type === 'any' || type === 'none') { - return rawToolChoice; - } - - return rawToolChoice; + if (mappedValue === undefined) return undefined; + const toolChoiceResult = sanitizeAnthropicToolChoiceValue(mappedValue); + return toolChoiceResult.value; } export function convertOpenAiBodyToAnthropicMessagesBody( @@ -857,7 +904,11 @@ export function validateAnthropicMessagesBody( }; } - const outputConfigResult = sanitizeAnthropicOutputConfig(body.output_config ?? body.outputConfig); + const allowOutputConfigEffort = !!thinkingResult.value && thinkingResult.value.type === 'adaptive'; + const outputConfigResult = sanitizeAnthropicOutputConfig( + body.output_config ?? body.outputConfig, + { allowEffort: allowOutputConfigEffort }, + ); if (outputConfigResult.error) { return { error: { diff --git a/src/server/transformers/anthropic/messages/inbound.test.ts b/src/server/transformers/anthropic/messages/inbound.test.ts new file mode 100644 index 0000000..13cf261 --- /dev/null +++ b/src/server/transformers/anthropic/messages/inbound.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; + +import { anthropicMessagesInbound } from './inbound.js'; + +describe('anthropicMessagesInbound', () => { + it('rejects requests without a positive max_tokens value', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 0, + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(result.error).toEqual({ + statusCode: 400, + payload: { + error: { + message: 'max_tokens is required and must be positive', + type: 'invalid_request_error', + }, + }, + }); + }); + + it('rejects system prompt blocks that are not text entries', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 256, + system: [ + { type: 'text', text: 'allowed' }, + { type: 'image', source: { type: 'url', url: 'https://example.com/system.png' } }, + ], + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(result.error).toEqual({ + statusCode: 400, + payload: { + error: { + message: 'system prompt must be text', + type: 'invalid_request_error', + }, + }, + }); + }); + + it('normalizes adaptive effort and tool choice at the inbound boundary', () => { + const result = anthropicMessagesInbound.parse({ + model: 'claude-opus-4-6', + max_tokens: 512, + thinking: { type: 'adaptive' }, + output_config: { effort: 'high', preserve: true }, + tool_choice: { type: 'tool', tool: { name: 'lookup' } }, + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.claudeOriginalBody).toMatchObject({ + thinking: { type: 'adaptive' }, + output_config: { effort: 'high', preserve: true }, + tool_choice: { type: 'tool', name: 'lookup' }, + }); + }); +}); diff --git a/src/server/transformers/anthropic/messages/inbound.ts b/src/server/transformers/anthropic/messages/inbound.ts index 28b09f1..d24d0c3 100644 --- a/src/server/transformers/anthropic/messages/inbound.ts +++ b/src/server/transformers/anthropic/messages/inbound.ts @@ -1,20 +1,152 @@ import { parseDownstreamChatRequest, type ParsedDownstreamChatRequest } from '../../shared/normalized.js'; import { validateAnthropicMessagesBody } from './conversion.js'; +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function asTrimmedString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +function toPositiveInteger(value: unknown): number { + const numberValue = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(numberValue) || numberValue <= 0) return 0; + return Math.trunc(numberValue); +} + +function invalidRequest(message: string): { statusCode: number; payload: unknown } { + return { + statusCode: 400, + payload: { + error: { + message, + type: 'invalid_request_error', + }, + }, + }; +} + +function validateMaxTokens(body: Record): { statusCode: number; payload: unknown } | undefined { + const maxTokens = toPositiveInteger(body.max_tokens ?? body.maxTokens); + if (maxTokens > 0) return undefined; + return invalidRequest('max_tokens is required and must be positive'); +} + +function validateSystemPrompts(body: Record): { statusCode: number; payload: unknown } | undefined { + const system = body.system; + if (system === undefined || system === null || typeof system === 'string') return undefined; + if (!Array.isArray(system)) return undefined; + + for (const entry of system) { + if (typeof entry === 'string') continue; + if (!isRecord(entry)) { + return invalidRequest('system prompt must be text'); + } + + const type = asTrimmedString(entry.type).toLowerCase(); + if (type && type !== 'text') { + return invalidRequest('system prompt must be text'); + } + } + + return undefined; +} + +function validateAdaptiveEffort(body: Record): { statusCode: number; payload: unknown } | undefined { + const thinking = isRecord(body.thinking) ? body.thinking : null; + const thinkingType = asTrimmedString(thinking?.type).toLowerCase(); + const outputConfig = isRecord(body.output_config) + ? body.output_config + : (isRecord(body.outputConfig) ? body.outputConfig : null); + + if (!outputConfig || !('effort' in outputConfig)) return undefined; + + const effort = asTrimmedString(outputConfig.effort).toLowerCase(); + if (!effort || thinkingType !== 'adaptive') return undefined; + + if (!['low', 'medium', 'high', 'max'].includes(effort)) { + return invalidRequest('output_config.effort must be one of: low, medium, high, max'); + } + + return undefined; +} + +function validateToolChoice(body: Record): { statusCode: number; payload: unknown } | undefined { + const rawToolChoice = body.tool_choice ?? body.toolChoice; + if (rawToolChoice === undefined) return undefined; + + if (typeof rawToolChoice === 'string') { + const type = asTrimmedString(rawToolChoice).toLowerCase(); + if (!type) return undefined; + if (type === 'required' || type === 'auto' || type === 'none' || type === 'any') return undefined; + if (type === 'tool') { + return invalidRequest('tool_choice.name is required when type is tool'); + } + return invalidRequest('tool_choice.type must be one of: auto, none, any, tool'); + } + + if (!isRecord(rawToolChoice)) { + return invalidRequest('tool_choice must be an object or string'); + } + + const type = asTrimmedString(rawToolChoice.type).toLowerCase(); + if (!['auto', 'none', 'any', 'tool'].includes(type)) { + return invalidRequest('tool_choice.type must be one of: auto, none, any, tool'); + } + + if (type !== 'tool') return undefined; + + const name = asTrimmedString( + rawToolChoice.name + ?? (isRecord(rawToolChoice.tool) ? rawToolChoice.tool.name : undefined), + ); + if (!name) { + return invalidRequest('tool_choice.name is required when type is tool'); + } + + return undefined; +} + +function sanitizeAnthropicInboundBody( + body: Record, +): { sanitizedBody?: Record; error?: { statusCode: number; payload: unknown } } { + const maxTokensError = validateMaxTokens(body); + if (maxTokensError) return { error: maxTokensError }; + + const systemError = validateSystemPrompts(body); + if (systemError) return { error: systemError }; + + const adaptiveEffortError = validateAdaptiveEffort(body); + if (adaptiveEffortError) return { error: adaptiveEffortError }; + + const toolChoiceError = validateToolChoice(body); + if (toolChoiceError) return { error: toolChoiceError }; + + const validation = validateAnthropicMessagesBody(body); + if (validation.error) { + return { error: validation.error }; + } + + return { + sanitizedBody: validation.sanitizedBody ?? body, + }; +} + export const anthropicMessagesInbound = { parse(body: unknown): { value?: ParsedDownstreamChatRequest; error?: { statusCode: number; payload: unknown } } { - const parsed = parseDownstreamChatRequest(body, 'claude'); + const rawBody = isRecord(body) ? body : null; + const inboundValidation = rawBody ? sanitizeAnthropicInboundBody(rawBody) : null; + if (inboundValidation?.error) { + return { error: inboundValidation.error }; + } + + const effectiveBody = inboundValidation?.sanitizedBody ?? body; + const parsed = parseDownstreamChatRequest(effectiveBody, 'claude'); if (parsed.error || !parsed.value) return parsed; - const originalBody = parsed.value.claudeOriginalBody; - if (originalBody) { - const validation = validateAnthropicMessagesBody(originalBody); - if (validation.error) { - return { error: validation.error }; - } - if (validation.sanitizedBody) { - parsed.value.claudeOriginalBody = validation.sanitizedBody; - } + if (inboundValidation?.sanitizedBody) { + parsed.value.claudeOriginalBody = inboundValidation.sanitizedBody; } return parsed; diff --git a/src/server/transformers/anthropic/messages/index.ts b/src/server/transformers/anthropic/messages/index.ts index c1bd871..01eafac 100644 --- a/src/server/transformers/anthropic/messages/index.ts +++ b/src/server/transformers/anthropic/messages/index.ts @@ -4,6 +4,7 @@ import { anthropicMessagesOutbound } from './outbound.js'; import { anthropicMessagesStream, consumeAnthropicSseEvent } from './stream.js'; import { anthropicMessagesUsage } from './usage.js'; import { createAnthropicMessagesAggregateState } from './aggregator.js'; +import { isMessagesRequiredError, shouldRetryNormalizedMessagesBody } from './compatibility.js'; export { ANTHROPIC_RAW_SSE_EVENT_NAMES, consumeAnthropicSseEvent, @@ -20,6 +21,10 @@ export const anthropicMessagesTransformer = { outbound: anthropicMessagesOutbound, stream: anthropicMessagesStream, usage: anthropicMessagesUsage, + compatibility: { + shouldRetryNormalizedBody: shouldRetryNormalizedMessagesBody, + isMessagesRequiredError, + }, aggregator: { createState: createAnthropicMessagesAggregateState, }, diff --git a/src/server/transformers/anthropic/messages/outbound.test.ts b/src/server/transformers/anthropic/messages/outbound.test.ts new file mode 100644 index 0000000..436dcd4 --- /dev/null +++ b/src/server/transformers/anthropic/messages/outbound.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from 'vitest'; + +import type { NormalizedFinalResponse } from '../../shared/normalized.js'; +import { anthropicMessagesOutbound } from './outbound.js'; +import { anthropicMessagesUsage } from './usage.js'; + +const normalized: NormalizedFinalResponse = { + id: 'chatcmpl-1', + model: 'claude-test', + created: 456, + content: 'done', + reasoningContent: '', + finishReason: 'stop', + toolCalls: [], +}; + +describe('anthropicMessagesOutbound.serializeFinal', () => { + it('round-trips native anthropic thinking/tool/text block order and signature', () => { + const payload = { + id: 'msg_native-1', + type: 'message', + role: 'assistant', + model: 'claude-test', + content: [ + { + type: 'thinking', + thinking: 'plan first', + signature: 'sig-native', + }, + { + type: 'tool_use', + id: 'toolu_lookup', + name: 'lookup_weather', + input: { city: 'Paris' }, + }, + { + type: 'text', + text: 'done', + }, + ], + stop_reason: 'tool_use', + stop_sequence: null, + usage: { + input_tokens: 100, + output_tokens: 30, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + cache_creation: { + ephemeral_5m_input_tokens: 5, + ephemeral_1h_input_tokens: 3, + }, + }, + }; + + const normalized = anthropicMessagesOutbound.normalizeFinal(payload, 'claude-test'); + + expect(anthropicMessagesOutbound.serializeFinal( + normalized, + anthropicMessagesUsage.fromPayload(payload), + )).toEqual({ + id: 'msg_native-1', + type: 'message', + role: 'assistant', + model: 'claude-test', + content: [ + { + type: 'thinking', + thinking: 'plan first', + signature: 'sig-native', + }, + { + type: 'tool_use', + id: 'toolu_lookup', + name: 'lookup_weather', + input: { city: 'Paris' }, + }, + { + type: 'text', + text: 'done', + }, + ], + stop_reason: 'tool_use', + stop_sequence: null, + usage: { + input_tokens: 100, + output_tokens: 30, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + cache_creation: { + ephemeral_5m_input_tokens: 5, + ephemeral_1h_input_tokens: 3, + }, + }, + }); + }); + + it('preserves anthropic cache read and cache creation semantics from parsed usage', () => { + expect(anthropicMessagesOutbound.serializeFinal(normalized, { + promptTokens: 100, + completionTokens: 30, + totalTokens: 150, + cacheReadTokens: 12, + cacheCreationTokens: 8, + promptTokensIncludeCache: false, + })).toEqual({ + id: 'msg_chatcmpl-1', + type: 'message', + role: 'assistant', + model: 'claude-test', + content: [{ type: 'text', text: 'done' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 100, + output_tokens: 30, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + }, + }); + }); + + it('emits cache_creation breakdown when detailed anthropic usage fields are available', () => { + expect(anthropicMessagesOutbound.serializeFinal(normalized, { + promptTokens: 100, + completionTokens: 30, + totalTokens: 150, + cacheReadInputTokens: 12, + cacheCreationInputTokens: 8, + ephemeral5mInputTokens: 5, + ephemeral1hInputTokens: 3, + promptTokensIncludingCache: 120, + })).toEqual({ + id: 'msg_chatcmpl-1', + type: 'message', + role: 'assistant', + model: 'claude-test', + content: [{ type: 'text', text: 'done' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 100, + output_tokens: 30, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + cache_creation: { + ephemeral_5m_input_tokens: 5, + ephemeral_1h_input_tokens: 3, + }, + }, + }); + }); +}); diff --git a/src/server/transformers/anthropic/messages/outbound.ts b/src/server/transformers/anthropic/messages/outbound.ts index 571fc02..df71412 100644 --- a/src/server/transformers/anthropic/messages/outbound.ts +++ b/src/server/transformers/anthropic/messages/outbound.ts @@ -1,10 +1,147 @@ -import { normalizeUpstreamFinalResponse, serializeFinalResponse, type NormalizedFinalResponse } from '../../shared/normalized.js'; +import { normalizeUpstreamFinalResponse, toClaudeStopReason, type NormalizedFinalResponse } from '../../shared/normalized.js'; +import { toAnthropicUsagePayload } from './usage.js'; + +type AnthropicRecord = Record; + +type AnthropicMessagesNormalizedFinalResponse = NormalizedFinalResponse & { + nativeContent?: AnthropicRecord[]; + stopSequence?: string | null; + usagePayload?: AnthropicRecord; +}; + +function isRecord(value: unknown): value is AnthropicRecord { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function cloneJsonValue(value: T): T { + if (Array.isArray(value)) { + return value.map((item) => cloneJsonValue(item)) as T; + } + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, cloneJsonValue(item)]), + ) as T; + } + return value; +} + +function parseJsonLike(raw: string): unknown { + const trimmed = raw.trim(); + if (!trimmed) return {}; + try { + return JSON.parse(trimmed); + } catch { + return { value: raw }; + } +} + +function buildClaudeMessageId(sourceId: string): string { + if (sourceId.startsWith('msg_')) return sourceId; + const sanitized = sourceId.replace(/[^A-Za-z0-9_-]/g, '_'); + return `msg_${sanitized || Date.now()}`; +} + +function buildAnthropicContent(normalized: NormalizedFinalResponse): Array> { + const anthropicNormalized = normalized as AnthropicMessagesNormalizedFinalResponse; + if (Array.isArray(anthropicNormalized.nativeContent) && anthropicNormalized.nativeContent.length > 0) { + return anthropicNormalized.nativeContent.map((block) => cloneJsonValue(block)); + } + + const contentBlocks: Array> = []; + if (normalized.reasoningContent) { + contentBlocks.push({ + type: 'thinking', + thinking: normalized.reasoningContent, + }); + } + if (normalized.content) { + contentBlocks.push({ + type: 'text', + text: normalized.content, + }); + } + const toolCalls = Array.isArray(normalized.toolCalls) ? normalized.toolCalls : []; + for (let index = 0; index < toolCalls.length; index += 1) { + const toolCall = toolCalls[index]; + contentBlocks.push({ + type: 'tool_use', + id: toolCall.id || `toolu_${index}`, + name: toolCall.name || `tool_${index}`, + input: parseJsonLike(toolCall.arguments || ''), + }); + } + return contentBlocks.length > 0 + ? contentBlocks + : [{ type: 'text', text: '' }]; +} + +function extractNativeAnthropicContent(payload: unknown): AnthropicRecord[] | undefined { + if (!isRecord(payload) || !Array.isArray(payload.content)) return undefined; + const content = payload.content + .filter((block): block is AnthropicRecord => isRecord(block)) + .map((block) => cloneJsonValue(block)); + return content.length > 0 ? content : undefined; +} + +function extractStopSequence(payload: unknown): string | null { + if (!isRecord(payload) || typeof payload.stop_sequence !== 'string') return null; + return payload.stop_sequence; +} + +function extractUsagePayload(payload: unknown): AnthropicRecord | undefined { + if (!isRecord(payload) || !isRecord(payload.usage)) return undefined; + return cloneJsonValue(payload.usage); +} + +function mergeUsagePayload( + usage: unknown, + normalized: AnthropicMessagesNormalizedFinalResponse, +): AnthropicRecord { + const merged = toAnthropicUsagePayload(usage ?? normalized.usagePayload); + const usagePayload = normalized.usagePayload; + if (!usagePayload) return merged; + + for (const [key, value] of Object.entries(usagePayload)) { + if (key === 'input_tokens' || key === 'output_tokens') continue; + if (key === 'cache_creation' && isRecord(value) && isRecord(merged.cache_creation)) { + merged.cache_creation = { + ...cloneJsonValue(value), + ...merged.cache_creation, + }; + continue; + } + if (merged[key] === undefined) { + merged[key] = cloneJsonValue(value); + } + } + + return merged; +} export const anthropicMessagesOutbound = { - normalizeFinal(payload: unknown, modelName: string, fallbackText = ''): NormalizedFinalResponse { - return normalizeUpstreamFinalResponse(payload, modelName, fallbackText); + normalizeFinal(payload: unknown, modelName: string, fallbackText = ''): AnthropicMessagesNormalizedFinalResponse { + const nativeContent = extractNativeAnthropicContent(payload); + const stopSequence = extractStopSequence(payload); + const usagePayload = extractUsagePayload(payload); + + return { + ...normalizeUpstreamFinalResponse(payload, modelName, fallbackText), + ...(nativeContent ? { nativeContent } : {}), + ...(stopSequence !== null ? { stopSequence } : {}), + ...(usagePayload ? { usagePayload } : {}), + }; }, serializeFinal(normalized: NormalizedFinalResponse, usage?: unknown) { - return serializeFinalResponse('claude', normalized, usage as any); + const anthropicNormalized = normalized as AnthropicMessagesNormalizedFinalResponse; + return { + id: buildClaudeMessageId(normalized.id), + type: 'message', + role: 'assistant', + model: normalized.model, + content: buildAnthropicContent(normalized), + stop_reason: toClaudeStopReason(normalized.finishReason), + stop_sequence: anthropicNormalized.stopSequence ?? null, + usage: mergeUsagePayload(usage, anthropicNormalized), + }; }, }; diff --git a/src/server/transformers/anthropic/messages/stream.test.ts b/src/server/transformers/anthropic/messages/stream.test.ts new file mode 100644 index 0000000..80ffe04 --- /dev/null +++ b/src/server/transformers/anthropic/messages/stream.test.ts @@ -0,0 +1,306 @@ +import { describe, expect, it } from 'vitest'; + +import { anthropicMessagesTransformer } from './index.js'; + +type SerializedAnthropicEvent = { + event: string; + payload: Record; +}; + +function collectSerializedEvents(events: Array>): SerializedAnthropicEvent[] { + const streamContext = anthropicMessagesTransformer.createStreamContext('claude-opus-4-6'); + const downstreamContext = anthropicMessagesTransformer.createDownstreamContext(); + const serialized = events + .flatMap((event) => anthropicMessagesTransformer.serializeStreamEvent(event as any, streamContext, downstreamContext)) + .join(''); + + return anthropicMessagesTransformer.pullSseEvents(serialized).events.map((item) => ({ + event: item.event, + payload: JSON.parse(item.data), + })); +} + +describe('anthropicMessagesStream.serializeEvent', () => { + it('buffers signature deltas until the active thinking block closes', () => { + const events = collectSerializedEvents([ + { + anthropic: { + startBlock: { + kind: 'thinking', + index: 0, + }, + }, + }, + { + reasoningDelta: 'step-1', + }, + { + anthropic: { + signatureDelta: 'sig-buffered-until-stop', + }, + }, + { + reasoningDelta: 'step-2', + }, + { + anthropic: { + stopBlockIndex: 0, + }, + }, + ]); + + expect(events.map((item) => ({ + type: item.payload.type, + deltaType: item.payload.delta?.type, + thinking: item.payload.delta?.thinking, + signature: item.payload.delta?.signature, + }))).toEqual([ + { + type: 'message_start', + deltaType: undefined, + thinking: undefined, + signature: undefined, + }, + { + type: 'content_block_start', + deltaType: undefined, + thinking: undefined, + signature: undefined, + }, + { + type: 'content_block_delta', + deltaType: 'thinking_delta', + thinking: 'step-1', + signature: undefined, + }, + { + type: 'content_block_delta', + deltaType: 'thinking_delta', + thinking: 'step-2', + signature: undefined, + }, + { + type: 'content_block_delta', + deltaType: 'signature_delta', + thinking: undefined, + signature: 'sig-buffered-until-stop', + }, + { + type: 'content_block_stop', + deltaType: undefined, + thinking: undefined, + signature: undefined, + }, + ]); + }); + + it('flushes a buffered signature when an explicit thinking block closes before redacted thinking', () => { + const events = collectSerializedEvents([ + { + anthropic: { + startBlock: { + kind: 'thinking', + index: 0, + }, + }, + }, + { + anthropic: { + signatureDelta: 'sig-buffered', + }, + }, + { + anthropic: { + startBlock: { + kind: 'redacted_thinking', + index: 1, + }, + redactedThinkingData: 'ciphertext', + }, + }, + ]); + + expect(events.map((item) => item.payload.type)).toEqual([ + 'message_start', + 'content_block_start', + 'content_block_delta', + 'content_block_stop', + 'content_block_start', + ]); + expect(events[1]?.payload.content_block).toEqual({ + type: 'thinking', + thinking: '', + }); + expect(events[2]?.payload.delta).toEqual({ + type: 'signature_delta', + signature: 'sig-buffered', + }); + expect(events[3]?.payload.index).toBe(0); + expect(events[4]?.payload.content_block).toEqual({ + type: 'redacted_thinking', + data: 'ciphertext', + }); + }); + + it('closes the previous tool_use block before starting the next one and keeps JSON increments on the same block', () => { + const events = collectSerializedEvents([ + { + toolCallDeltas: [{ + index: 0, + id: 'call_1', + name: 'lookup_city', + argumentsDelta: '{"city":"par', + }], + }, + { + toolCallDeltas: [{ + index: 0, + argumentsDelta: 'is"}', + }], + }, + { + toolCallDeltas: [{ + index: 1, + id: 'call_2', + name: 'lookup_weather', + argumentsDelta: '{"days":2}', + }], + }, + ]); + + expect(events.map((item) => ({ + type: item.payload.type, + index: item.payload.index, + blockType: item.payload.content_block?.type, + deltaType: item.payload.delta?.type, + partialJson: item.payload.delta?.partial_json, + }))).toEqual([ + { + type: 'message_start', + index: undefined, + blockType: undefined, + deltaType: undefined, + partialJson: undefined, + }, + { + type: 'content_block_start', + index: 0, + blockType: 'tool_use', + deltaType: undefined, + partialJson: undefined, + }, + { + type: 'content_block_delta', + index: 0, + blockType: undefined, + deltaType: 'input_json_delta', + partialJson: '{"city":"par', + }, + { + type: 'content_block_delta', + index: 0, + blockType: undefined, + deltaType: 'input_json_delta', + partialJson: 'is"}', + }, + { + type: 'content_block_stop', + index: 0, + blockType: undefined, + deltaType: undefined, + partialJson: undefined, + }, + { + type: 'content_block_start', + index: 1, + blockType: 'tool_use', + deltaType: undefined, + partialJson: undefined, + }, + { + type: 'content_block_delta', + index: 1, + blockType: undefined, + deltaType: 'input_json_delta', + partialJson: '{"days":2}', + }, + ]); + }); + + it('flushes a buffered signature on finish even when no thinking delta ever arrived', () => { + const events = collectSerializedEvents([ + { + anthropic: { + signatureDelta: 'sig-finish-only', + }, + }, + { + finishReason: 'stop', + done: true, + }, + ]); + + expect(events.map((item) => item.payload.type)).toEqual([ + 'message_start', + 'content_block_start', + 'content_block_delta', + 'content_block_stop', + 'message_delta', + 'message_stop', + ]); + expect(events[1]?.payload.content_block).toEqual({ + type: 'thinking', + thinking: '', + }); + expect(events[2]?.payload.delta).toEqual({ + type: 'signature_delta', + signature: 'sig-finish-only', + }); + expect(events[4]?.payload.delta).toEqual({ + stop_reason: 'end_turn', + stop_sequence: null, + }); + }); + + it('keeps redacted_thinking open until an explicit stop arrives', () => { + const streamContext = anthropicMessagesTransformer.createStreamContext('claude-opus-4-6'); + const downstreamContext = anthropicMessagesTransformer.createDownstreamContext(); + + const startSerialized = anthropicMessagesTransformer.serializeStreamEvent({ + anthropic: { + startBlock: { + kind: 'redacted_thinking', + index: 1, + }, + redactedThinkingData: 'ciphertext', + }, + } as any, streamContext, downstreamContext).join(''); + const stopSerialized = anthropicMessagesTransformer.serializeStreamEvent({ + anthropic: { + stopBlockIndex: 1, + }, + } as any, streamContext, downstreamContext).join(''); + + const startEvents = anthropicMessagesTransformer.pullSseEvents(startSerialized).events.map((item) => ({ + event: item.event, + payload: JSON.parse(item.data), + })); + const stopEvents = anthropicMessagesTransformer.pullSseEvents(stopSerialized).events.map((item) => ({ + event: item.event, + payload: JSON.parse(item.data), + })); + + expect(startEvents.map((item) => item.payload.type)).toEqual([ + 'message_start', + 'content_block_start', + ]); + expect(startEvents[1]?.payload.content_block).toEqual({ + type: 'redacted_thinking', + data: 'ciphertext', + }); + expect(stopEvents.map((item) => item.payload.type)).toEqual([ + 'content_block_stop', + ]); + expect(stopEvents[0]?.payload.index).toBe(1); + }); +}); diff --git a/src/server/transformers/anthropic/messages/stream.ts b/src/server/transformers/anthropic/messages/stream.ts index f76657f..700e9e3 100644 --- a/src/server/transformers/anthropic/messages/stream.ts +++ b/src/server/transformers/anthropic/messages/stream.ts @@ -13,9 +13,25 @@ import { type AnthropicExtendedStreamEvent } from './aggregator.js'; type AnthropicStreamPayload = Record; +type AnthropicBlockKind = 'thinking' | 'text' | 'tool_use' | 'redacted_thinking'; + +type ExtendedToolBlockState = { + contentIndex: number; + id: string; + name: string; + open: boolean; + sourceIndex: number | null; +}; + type ExtendedClaudeDownstreamContext = ClaudeDownstreamContext & { + toolBlocks: Record; thinkingBlockIndex?: number | null; + thinkingSourceIndex?: number | null; + redactedBlockIndex?: number | null; + redactedSourceIndex?: number | null; + textSourceIndex?: number | null; pendingSignature?: string | null; + activeToolSlot?: number | null; }; export const ANTHROPIC_RAW_SSE_EVENT_NAMES = new Set([ @@ -56,7 +72,12 @@ export function serializeAnthropicRawSseEvent(event: string, data: string): stri function ensureContext(context: ClaudeDownstreamContext): ExtendedClaudeDownstreamContext { const extended = context as ExtendedClaudeDownstreamContext; if (extended.thinkingBlockIndex === undefined) extended.thinkingBlockIndex = null; + if (extended.thinkingSourceIndex === undefined) extended.thinkingSourceIndex = null; + if (extended.redactedBlockIndex === undefined) extended.redactedBlockIndex = null; + if (extended.redactedSourceIndex === undefined) extended.redactedSourceIndex = null; + if (extended.textSourceIndex === undefined) extended.textSourceIndex = null; if (extended.pendingSignature === undefined) extended.pendingSignature = null; + if (extended.activeToolSlot === undefined) extended.activeToolSlot = null; return extended; } @@ -132,6 +153,7 @@ function closeTextBlock(context: ExtendedClaudeDownstreamContext): string[] { if (context.textBlockIndex === null || context.textBlockIndex === undefined) return []; const index = context.textBlockIndex; context.textBlockIndex = null; + context.textSourceIndex = null; context.contentBlockStarted = false; return [ serializeSse('content_block_stop', { @@ -159,6 +181,13 @@ function emitPendingSignature(context: ExtendedClaudeDownstreamContext): string[ ]; } +function bufferPendingSignature( + context: ExtendedClaudeDownstreamContext, + signature: string, +): void { + context.pendingSignature = `${context.pendingSignature || ''}${signature}`; +} + function closeThinkingBlock(context: ExtendedClaudeDownstreamContext): string[] { if (context.thinkingBlockIndex === null || context.thinkingBlockIndex === undefined) return []; const index = context.thinkingBlockIndex; @@ -170,9 +199,23 @@ function closeThinkingBlock(context: ExtendedClaudeDownstreamContext): string[] }), ]; context.thinkingBlockIndex = null; + context.thinkingSourceIndex = null; return events; } +function closeRedactedBlock(context: ExtendedClaudeDownstreamContext): string[] { + if (context.redactedBlockIndex === null || context.redactedBlockIndex === undefined) return []; + const index = context.redactedBlockIndex; + context.redactedBlockIndex = null; + context.redactedSourceIndex = null; + return [ + serializeSse('content_block_stop', { + type: 'content_block_stop', + index, + }), + ]; +} + function closeToolBlocks(context: ExtendedClaudeDownstreamContext): string[] { const openBlocks = Object.values(context.toolBlocks) .filter((item) => item.open) @@ -188,6 +231,21 @@ function closeToolBlocks(context: ExtendedClaudeDownstreamContext): string[] { index: block.contentIndex, })); } + context.activeToolSlot = null; + return events; +} + +function emitPendingSignatureAsThinkingBlock( + context: ExtendedClaudeDownstreamContext, +): string[] { + if (!context.pendingSignature) return []; + const signature = context.pendingSignature; + const events = [ + ...ensureThinkingBlockStart(context), + ]; + context.pendingSignature = signature; + events.push(...emitPendingSignature(context)); + events.push(...closeThinkingBlock(context)); return events; } @@ -195,15 +253,41 @@ function closeAllBlocks(context: ExtendedClaudeDownstreamContext): string[] { return [ ...closeTextBlock(context), ...closeThinkingBlock(context), + ...closeRedactedBlock(context), ...closeToolBlocks(context), ]; } -function ensureTextBlockStart(context: ExtendedClaudeDownstreamContext): string[] { - if (context.textBlockIndex !== null && context.textBlockIndex !== undefined) return []; +function normalizeBlockIndex(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) return null; + return Math.max(0, Math.trunc(value)); +} + +function allocateContentIndex( + context: ExtendedClaudeDownstreamContext, + preferredIndex?: number | null, +): number { + const normalizedPreferred = normalizeBlockIndex(preferredIndex); + if (normalizedPreferred !== null) { + if (context.nextContentBlockIndex <= normalizedPreferred) { + context.nextContentBlockIndex = normalizedPreferred + 1; + } + return normalizedPreferred; + } + const index = context.nextContentBlockIndex; context.nextContentBlockIndex += 1; + return index; +} + +function ensureTextBlockStart( + context: ExtendedClaudeDownstreamContext, + preferredIndex?: number | null, +): string[] { + if (context.textBlockIndex !== null && context.textBlockIndex !== undefined) return []; + const index = allocateContentIndex(context, preferredIndex); context.textBlockIndex = index; + context.textSourceIndex = normalizeBlockIndex(preferredIndex) ?? index; context.contentBlockStarted = true; return [ serializeSse('content_block_start', { @@ -217,11 +301,14 @@ function ensureTextBlockStart(context: ExtendedClaudeDownstreamContext): string[ ]; } -function ensureThinkingBlockStart(context: ExtendedClaudeDownstreamContext): string[] { +function ensureThinkingBlockStart( + context: ExtendedClaudeDownstreamContext, + preferredIndex?: number | null, +): string[] { if (context.thinkingBlockIndex !== null && context.thinkingBlockIndex !== undefined) return []; - const index = context.nextContentBlockIndex; - context.nextContentBlockIndex += 1; + const index = allocateContentIndex(context, preferredIndex); context.thinkingBlockIndex = index; + context.thinkingSourceIndex = normalizeBlockIndex(preferredIndex) ?? index; return [ serializeSse('content_block_start', { type: 'content_block_start', @@ -234,6 +321,27 @@ function ensureThinkingBlockStart(context: ExtendedClaudeDownstreamContext): str ]; } +function ensureRedactedBlockStart( + context: ExtendedClaudeDownstreamContext, + data: string, + preferredIndex?: number | null, +): string[] { + if (context.redactedBlockIndex !== null && context.redactedBlockIndex !== undefined) return []; + const index = allocateContentIndex(context, preferredIndex); + context.redactedBlockIndex = index; + context.redactedSourceIndex = normalizeBlockIndex(preferredIndex) ?? index; + return [ + serializeSse('content_block_start', { + type: 'content_block_start', + index, + content_block: { + type: 'redacted_thinking', + data, + }, + }), + ]; +} + function ensureToolBlockStart( context: ExtendedClaudeDownstreamContext, toolDelta: NonNullable[number], @@ -242,12 +350,12 @@ function ensureToolBlockStart( let state = context.toolBlocks[toolSlot]; if (!state) { state = { - contentIndex: context.nextContentBlockIndex, + contentIndex: allocateContentIndex(context), id: toolDelta.id || `toolu_${toolSlot}`, name: toolDelta.name || `tool_${toolSlot}`, open: false, + sourceIndex: toolSlot, }; - context.nextContentBlockIndex += 1; context.toolBlocks[toolSlot] = state; } else { if (toolDelta.id) state.id = toolDelta.id; @@ -269,9 +377,81 @@ function ensureToolBlockStart( })); } + context.activeToolSlot = toolSlot; return { events, contentIndex: state.contentIndex }; } +function ensureExplicitToolBlockStart( + context: ExtendedClaudeDownstreamContext, + sourceIndex?: number | null, +): string[] { + const toolSlot = normalizeBlockIndex(sourceIndex) ?? 0; + const state = context.toolBlocks[toolSlot]; + const toolDelta: NonNullable[number] = { + index: toolSlot, + id: state?.id, + name: state?.name, + }; + return ensureToolBlockStart(context, toolDelta).events; +} + +function closePreviousToolBlockIfNeeded( + context: ExtendedClaudeDownstreamContext, + toolSlot: number, +): string[] { + const activeToolSlot = context.activeToolSlot; + if (activeToolSlot === null || activeToolSlot === undefined || activeToolSlot === toolSlot) { + return []; + } + return closeToolBlocks(context); +} + +function isMatchingBlockIndex( + targetIndex: number, + contentIndex: number | null | undefined, + sourceIndex: number | null | undefined, +): boolean { + if (sourceIndex !== null && sourceIndex !== undefined) return sourceIndex === targetIndex; + if (contentIndex !== null && contentIndex !== undefined) return contentIndex === targetIndex; + return false; +} + +function handleExplicitBlockStart( + kind: AnthropicBlockKind, + sourceIndex: number | null, + context: ExtendedClaudeDownstreamContext, +): string[] { + if (kind === 'thinking') { + return [ + ...closeRedactedBlock(context), + ...closeToolBlocks(context), + ...closeTextBlock(context), + ...ensureThinkingBlockStart(context, sourceIndex), + ]; + } + + if (kind === 'text') { + return [ + ...closeRedactedBlock(context), + ...closeToolBlocks(context), + ...closeThinkingBlock(context), + ...ensureTextBlockStart(context, sourceIndex), + ]; + } + + if (kind === 'tool_use') { + return [ + ...closeRedactedBlock(context), + ...closeTextBlock(context), + ...closeThinkingBlock(context), + ...closePreviousToolBlockIfNeeded(context, normalizeBlockIndex(sourceIndex) ?? 0), + ...ensureExplicitToolBlockStart(context, sourceIndex), + ]; + } + + return []; +} + function buildDoneEvents( streamContext: StreamTransformContext, context: ExtendedClaudeDownstreamContext, @@ -282,6 +462,7 @@ function buildDoneEvents( const events = [ ...ensureClaudeStartEvents(streamContext, context), ...closeAllBlocks(context), + ...emitPendingSignatureAsThinkingBlock(context), serializeSse('message_delta', { type: 'message_delta', delta: { @@ -384,6 +565,17 @@ function normalizeAnthropicRawEvent( const blockType = asTrimmedString(contentBlock.type); const index = typeof payload.index === 'number' ? payload.index : undefined; + if (blockType === 'text') { + return { + anthropic: { + startBlock: { + kind: 'text', + index, + }, + }, + }; + } + if (blockType === 'thinking') { return { anthropic: { @@ -409,6 +601,12 @@ function normalizeAnthropicRawEvent( if (blockType === 'tool_use') { return { + anthropic: { + startBlock: { + kind: 'tool_use', + index, + }, + }, toolCallDeltas: [{ index: typeof index === 'number' ? index : 0, id: asTrimmedString(contentBlock.id) || undefined, @@ -548,6 +746,8 @@ export const anthropicMessagesStream = { const context = ensureContext(downstreamContext); const anthropicEvent = event as AnthropicExtendedStreamEvent; const events: string[] = []; + const startBlock = anthropicEvent.anthropic?.startBlock; + const normalizedStartIndex = normalizeBlockIndex(startBlock?.index); const needsStart = ( event.role === 'assistant' @@ -562,43 +762,30 @@ export const anthropicMessagesStream = { events.push(...ensureClaudeStartEvents(streamContext, context)); } + if (startBlock && startBlock.kind !== 'redacted_thinking') { + events.push(...handleExplicitBlockStart(startBlock.kind, normalizedStartIndex, context)); + } + if (anthropicEvent.anthropic?.signatureDelta) { - if (context.thinkingBlockIndex !== null && context.thinkingBlockIndex !== undefined) { - events.push(serializeSse('content_block_delta', { - type: 'content_block_delta', - index: context.thinkingBlockIndex, - delta: { - type: 'signature_delta', - signature: anthropicEvent.anthropic.signatureDelta, - }, - })); - } else { - context.pendingSignature = anthropicEvent.anthropic.signatureDelta; - } + bufferPendingSignature(context, anthropicEvent.anthropic.signatureDelta); } if (anthropicEvent.anthropic?.redactedThinkingData) { - events.push(...closeAllBlocks(context)); - const index = context.nextContentBlockIndex; - context.nextContentBlockIndex += 1; - events.push(serializeSse('content_block_start', { - type: 'content_block_start', - index, - content_block: { - type: 'redacted_thinking', - data: anthropicEvent.anthropic.redactedThinkingData, - }, - })); - events.push(serializeSse('content_block_stop', { - type: 'content_block_stop', - index, - })); + events.push(...closeToolBlocks(context)); + events.push(...closeTextBlock(context)); + events.push(...closeThinkingBlock(context)); + events.push(...ensureRedactedBlockStart( + context, + anthropicEvent.anthropic.redactedThinkingData, + normalizedStartIndex, + )); } if (event.reasoningDelta) { + events.push(...closeRedactedBlock(context)); events.push(...closeToolBlocks(context)); events.push(...closeTextBlock(context)); - events.push(...ensureThinkingBlockStart(context)); + events.push(...ensureThinkingBlockStart(context, normalizedStartIndex)); events.push(serializeSse('content_block_delta', { type: 'content_block_delta', index: context.thinkingBlockIndex ?? 0, @@ -610,9 +797,14 @@ export const anthropicMessagesStream = { } if (Array.isArray(event.toolCallDeltas) && event.toolCallDeltas.length > 0) { + events.push(...closeRedactedBlock(context)); events.push(...closeTextBlock(context)); events.push(...closeThinkingBlock(context)); for (const toolDelta of event.toolCallDeltas) { + events.push(...closePreviousToolBlockIfNeeded( + context, + Number.isFinite(toolDelta.index) ? Math.max(0, Math.trunc(toolDelta.index)) : 0, + )); const toolBlock = ensureToolBlockStart(context, toolDelta); events.push(...toolBlock.events); if (toolDelta.argumentsDelta) { @@ -629,9 +821,10 @@ export const anthropicMessagesStream = { } if (event.contentDelta) { + events.push(...closeRedactedBlock(context)); events.push(...closeToolBlocks(context)); events.push(...closeThinkingBlock(context)); - events.push(...ensureTextBlockStart(context)); + events.push(...ensureTextBlockStart(context, normalizedStartIndex)); events.push(serializeSse('content_block_delta', { type: 'content_block_delta', index: context.textBlockIndex ?? 0, @@ -647,15 +840,27 @@ export const anthropicMessagesStream = { && anthropicEvent.anthropic.stopBlockIndex !== null ) { const targetIndex = anthropicEvent.anthropic.stopBlockIndex; - if (context.thinkingBlockIndex === targetIndex) { + if (isMatchingBlockIndex(targetIndex, context.thinkingBlockIndex, context.thinkingSourceIndex)) { events.push(...closeThinkingBlock(context)); } - if (context.textBlockIndex === targetIndex) { + if (isMatchingBlockIndex(targetIndex, context.textBlockIndex, context.textSourceIndex)) { events.push(...closeTextBlock(context)); } - const matchingToolBlock = Object.values(context.toolBlocks).find((item) => item.open && item.contentIndex === targetIndex); + if (isMatchingBlockIndex(targetIndex, context.redactedBlockIndex, context.redactedSourceIndex)) { + events.push(...closeRedactedBlock(context)); + } + const matchingToolBlock = Object.values(context.toolBlocks).find((item) => ( + item.open + && isMatchingBlockIndex(targetIndex, item.contentIndex, item.sourceIndex) + )); if (matchingToolBlock) { matchingToolBlock.open = false; + if (typeof context.activeToolSlot === 'number') { + const activeState = context.toolBlocks[context.activeToolSlot]; + if (activeState?.contentIndex === matchingToolBlock.contentIndex) { + context.activeToolSlot = null; + } + } events.push(serializeSse('content_block_stop', { type: 'content_block_stop', index: matchingToolBlock.contentIndex, diff --git a/src/server/transformers/anthropic/messages/usage.test.ts b/src/server/transformers/anthropic/messages/usage.test.ts new file mode 100644 index 0000000..0624280 --- /dev/null +++ b/src/server/transformers/anthropic/messages/usage.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; + +import { + anthropicMessagesUsage, + extractAnthropicUsage, + extractAnthropicUsageMetadata, +} from './usage.js'; + +describe('extractAnthropicUsage', () => { + it('treats cached_tokens as already included in input_tokens for moonshot-style payloads', () => { + expect(extractAnthropicUsage({ + usage: { + input_tokens: 100, + output_tokens: 30, + cached_tokens: 12, + }, + })).toEqual({ + promptTokens: 100, + completionTokens: 30, + totalTokens: 130, + cachedTokens: 12, + reasoningTokens: 0, + audioInputTokens: 0, + audioOutputTokens: 0, + acceptedPredictionTokens: 0, + rejectedPredictionTokens: 0, + cacheReadInputTokens: 12, + cacheCreationInputTokens: 0, + ephemeral5mInputTokens: 0, + ephemeral1hInputTokens: 0, + promptTokensIncludingCache: 100, + serviceTier: null, + promptTokensIncludeCache: true, + }); + }); + + it('reports prompt tokens including cache without double-counting cached_tokens aliases', () => { + expect(extractAnthropicUsageMetadata({ + usage: { + input_tokens: 100, + output_tokens: 30, + cached_tokens: 12, + }, + })).toEqual({ + cacheReadInputTokens: 12, + cacheCreationInputTokens: 0, + ephemeral5mInputTokens: 0, + ephemeral1hInputTokens: 0, + promptTokensIncludingCache: 100, + serviceTier: null, + cachedTokens: 12, + promptTokensIncludeCache: true, + }); + }); +}); + +describe('anthropicMessagesUsage.toPayload', () => { + it('reconstructs anthropic cache breakdown for downstream responses', () => { + const toPayload = ( + anthropicMessagesUsage as typeof anthropicMessagesUsage & { + toPayload?: (usage: unknown) => unknown; + } + ).toPayload; + + expect(typeof toPayload).toBe('function'); + expect(toPayload?.({ + promptTokens: 100, + completionTokens: 30, + totalTokens: 150, + cacheReadInputTokens: 12, + cacheCreationInputTokens: 8, + ephemeral5mInputTokens: 5, + ephemeral1hInputTokens: 3, + promptTokensIncludingCache: 120, + })).toEqual({ + input_tokens: 100, + output_tokens: 30, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + cache_creation: { + ephemeral_5m_input_tokens: 5, + ephemeral_1h_input_tokens: 3, + }, + }); + }); +}); diff --git a/src/server/transformers/anthropic/messages/usage.ts b/src/server/transformers/anthropic/messages/usage.ts index 3ecaed0..e304563 100644 --- a/src/server/transformers/anthropic/messages/usage.ts +++ b/src/server/transformers/anthropic/messages/usage.ts @@ -6,6 +6,10 @@ function isRecord(value: unknown): value is AnthropicRecord { return !!value && typeof value === 'object' && !Array.isArray(value); } +function hasOwn(record: AnthropicRecord, key: string): boolean { + return Object.prototype.hasOwnProperty.call(record, key); +} + function toPositiveInt(value: unknown): number { const numberValue = typeof value === 'number' ? value : Number(value); if (!Number.isFinite(numberValue)) return 0; @@ -18,90 +22,330 @@ function resolveAnthropicUsageRecord(payload: unknown): AnthropicRecord | null { return null; } +function readEphemeralBreakdown(record: AnthropicRecord): { + ephemeral5mInputTokens: number; + ephemeral1hInputTokens: number; +} { + const snake = isRecord(record.cache_creation) ? record.cache_creation : null; + const camel = isRecord(record.cacheCreation) ? record.cacheCreation : null; + + return { + ephemeral5mInputTokens: Math.max( + toPositiveInt(snake?.ephemeral_5m_input_tokens), + toPositiveInt(camel?.ephemeral5mInputTokens), + toPositiveInt(record.ephemeral_5m_input_tokens), + toPositiveInt(record.ephemeral5mInputTokens), + ), + ephemeral1hInputTokens: Math.max( + toPositiveInt(snake?.ephemeral_1h_input_tokens), + toPositiveInt(camel?.ephemeral1hInputTokens), + toPositiveInt(record.ephemeral_1h_input_tokens), + toPositiveInt(record.ephemeral1hInputTokens), + ), + }; +} + +function readServiceTier(record: AnthropicRecord): string | null { + if (typeof record.service_tier === 'string' && record.service_tier.trim().length > 0) { + return record.service_tier.trim(); + } + if (typeof record.serviceTier === 'string' && record.serviceTier.trim().length > 0) { + return record.serviceTier.trim(); + } + return null; +} + +function readInputTokens(record: AnthropicRecord): number { + return Math.max( + toPositiveInt(record.input_tokens), + toPositiveInt(record.inputTokens), + toPositiveInt(record.prompt_tokens), + toPositiveInt(record.promptTokens), + ); +} + +function readCompletionTokens(record: AnthropicRecord): number { + return Math.max( + toPositiveInt(record.output_tokens), + toPositiveInt(record.outputTokens), + toPositiveInt(record.completion_tokens), + toPositiveInt(record.completionTokens), + ); +} + +function readTotalTokens(record: AnthropicRecord): number { + return Math.max( + toPositiveInt(record.total_tokens), + toPositiveInt(record.totalTokens), + ); +} + +function readCacheReadTokens(record: AnthropicRecord): { + cacheReadInputTokens: number; + cachedTokens: number; + hasCachedTokensAlias: boolean; + hasExplicitCacheReadField: boolean; +} { + const cachedTokens = Math.max( + toPositiveInt(record.cached_tokens), + toPositiveInt(record.cachedTokens), + ); + const cacheReadInputTokens = Math.max( + toPositiveInt(record.cache_read_input_tokens), + toPositiveInt(record.cacheReadInputTokens), + toPositiveInt(record.cacheReadTokens), + cachedTokens, + ); + + return { + cacheReadInputTokens, + cachedTokens: Math.max(cachedTokens, cacheReadInputTokens), + hasCachedTokensAlias: hasOwn(record, 'cached_tokens') || hasOwn(record, 'cachedTokens'), + hasExplicitCacheReadField: ( + hasOwn(record, 'cache_read_input_tokens') + || hasOwn(record, 'cacheReadInputTokens') + || hasOwn(record, 'cacheReadTokens') + ), + }; +} + +function readCacheCreationInputTokens( + record: AnthropicRecord, + breakdown: { ephemeral5mInputTokens: number; ephemeral1hInputTokens: number }, +): number { + return Math.max( + toPositiveInt(record.cache_creation_input_tokens), + toPositiveInt(record.cacheCreationInputTokens), + toPositiveInt(record.cacheCreationTokens), + breakdown.ephemeral5mInputTokens + breakdown.ephemeral1hInputTokens, + ); +} + +function readPromptTokensIncludeCache(record: AnthropicRecord, cacheRead: { + hasCachedTokensAlias: boolean; + hasExplicitCacheReadField: boolean; +}): boolean | null { + if (typeof record.promptTokensIncludeCache === 'boolean') { + return record.promptTokensIncludeCache; + } + if (typeof record.prompt_tokens_include_cache === 'boolean') { + return record.prompt_tokens_include_cache; + } + + if (cacheRead.hasCachedTokensAlias && !cacheRead.hasExplicitCacheReadField) { + return true; + } + + if ( + cacheRead.hasExplicitCacheReadField + || hasOwn(record, 'cache_creation_input_tokens') + || hasOwn(record, 'cacheCreationInputTokens') + || hasOwn(record, 'cacheCreationTokens') + || hasOwn(record, 'cache_creation') + || hasOwn(record, 'cacheCreation') + ) { + return false; + } + + return null; +} + export type AnthropicUsageMetadata = { cacheReadInputTokens: number; cacheCreationInputTokens: number; ephemeral5mInputTokens: number; ephemeral1hInputTokens: number; promptTokensIncludingCache: number; + promptTokensIncludeCache?: boolean | null; + serviceTier?: string | null; + cachedTokens?: number; }; -export function extractAnthropicUsage(payload: unknown): NormalizedUsage { - const usageRecord = resolveAnthropicUsageRecord(payload); - if (!usageRecord) { - return createEmptyNormalizedUsage(); - } +export type AnthropicNormalizedUsage = NormalizedUsage & AnthropicUsageMetadata & { + promptTokensIncludeCache?: boolean | null; +}; - const inputTokens = toPositiveInt(usageRecord.input_tokens ?? usageRecord.inputTokens); - const outputTokens = toPositiveInt(usageRecord.output_tokens ?? usageRecord.outputTokens); - const cacheReadTokens = Math.max( - toPositiveInt(usageRecord.cache_read_input_tokens ?? usageRecord.cacheReadInputTokens), - toPositiveInt(usageRecord.cached_tokens ?? usageRecord.cachedTokens), - ); - const cacheCreationTokens = Math.max( - toPositiveInt(usageRecord.cache_creation_input_tokens ?? usageRecord.cacheCreationInputTokens), - toPositiveInt((usageRecord.cache_creation as AnthropicRecord | undefined)?.ephemeral_5m_input_tokens) - + toPositiveInt((usageRecord.cache_creation as AnthropicRecord | undefined)?.ephemeral_1h_input_tokens), - toPositiveInt((usageRecord.cacheCreation as AnthropicRecord | undefined)?.ephemeral5mInputTokens) - + toPositiveInt((usageRecord.cacheCreation as AnthropicRecord | undefined)?.ephemeral1hInputTokens), - ); +type AnthropicUsageSummary = { + inputTokens: number; + completionTokens: number; + totalTokens: number; + cacheReadInputTokens: number; + cacheCreationInputTokens: number; + ephemeral5mInputTokens: number; + ephemeral1hInputTokens: number; + promptTokensIncludingCache: number; + promptTokensIncludeCache: boolean | null; + serviceTier: string | null; + cachedTokens: number; +}; - const promptTokens = inputTokens + cacheReadTokens + cacheCreationTokens; - const completionTokens = outputTokens; - const totalTokens = Math.max( - toPositiveInt(usageRecord.total_tokens ?? usageRecord.totalTokens), - promptTokens + completionTokens, - ); - - return { - promptTokens, - completionTokens, - totalTokens, - cachedTokens: cacheReadTokens, - reasoningTokens: 0, - audioInputTokens: 0, - audioOutputTokens: 0, - acceptedPredictionTokens: 0, - rejectedPredictionTokens: 0, - }; -} - -export function extractAnthropicUsageMetadata(payload: unknown): AnthropicUsageMetadata { - const usageRecord = resolveAnthropicUsageRecord(payload); - if (!usageRecord) { +function summarizeAnthropicUsage(payload: unknown): AnthropicUsageSummary { + const record = resolveAnthropicUsageRecord(payload); + if (!record) { return { + inputTokens: 0, + completionTokens: 0, + totalTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, ephemeral5mInputTokens: 0, ephemeral1hInputTokens: 0, promptTokensIncludingCache: 0, + promptTokensIncludeCache: null, + serviceTier: null, + cachedTokens: 0, }; } - const inputTokens = toPositiveInt(usageRecord.input_tokens ?? usageRecord.inputTokens); - const cacheReadInputTokens = Math.max( - toPositiveInt(usageRecord.cache_read_input_tokens ?? usageRecord.cacheReadInputTokens), - toPositiveInt(usageRecord.cached_tokens ?? usageRecord.cachedTokens), - ); - const ephemeral5mInputTokens = Math.max( - toPositiveInt((usageRecord.cache_creation as AnthropicRecord | undefined)?.ephemeral_5m_input_tokens), - toPositiveInt((usageRecord.cacheCreation as AnthropicRecord | undefined)?.ephemeral5mInputTokens), - ); - const ephemeral1hInputTokens = Math.max( - toPositiveInt((usageRecord.cache_creation as AnthropicRecord | undefined)?.ephemeral_1h_input_tokens), - toPositiveInt((usageRecord.cacheCreation as AnthropicRecord | undefined)?.ephemeral1hInputTokens), - ); - const cacheCreationInputTokens = Math.max( - toPositiveInt(usageRecord.cache_creation_input_tokens ?? usageRecord.cacheCreationInputTokens), - ephemeral5mInputTokens + ephemeral1hInputTokens, + const breakdown = readEphemeralBreakdown(record); + const cacheRead = readCacheReadTokens(record); + const cacheCreationInputTokens = readCacheCreationInputTokens(record, breakdown); + const promptTokensIncludeCache = readPromptTokensIncludeCache(record, cacheRead); + + let inputTokens = readInputTokens(record); + const completionTokens = readCompletionTokens(record); + + const explicitPromptTokensIncludingCache = Math.max( + toPositiveInt(record.promptTokensIncludingCache), + toPositiveInt(record.prompt_tokens_including_cache), ); + let promptTokensIncludingCache = explicitPromptTokensIncludingCache; + if (promptTokensIncludingCache <= 0) { + if (promptTokensIncludeCache === true) { + promptTokensIncludingCache = inputTokens; + } else if (promptTokensIncludeCache === false) { + promptTokensIncludingCache = inputTokens + cacheRead.cacheReadInputTokens + cacheCreationInputTokens; + } else { + promptTokensIncludingCache = Math.max( + inputTokens, + inputTokens + cacheRead.cacheReadInputTokens + cacheCreationInputTokens, + ); + } + } + + if (inputTokens <= 0 && promptTokensIncludingCache > 0) { + inputTokens = Math.max( + promptTokensIncludingCache - cacheRead.cacheReadInputTokens - cacheCreationInputTokens, + 0, + ); + } + return { - cacheReadInputTokens, + inputTokens, + completionTokens, + totalTokens: Math.max( + readTotalTokens(record), + promptTokensIncludingCache + completionTokens, + ), + cacheReadInputTokens: cacheRead.cacheReadInputTokens, cacheCreationInputTokens, - ephemeral5mInputTokens, - ephemeral1hInputTokens, - promptTokensIncludingCache: inputTokens + cacheReadInputTokens + cacheCreationInputTokens, + ephemeral5mInputTokens: breakdown.ephemeral5mInputTokens, + ephemeral1hInputTokens: breakdown.ephemeral1hInputTokens, + promptTokensIncludingCache, + promptTokensIncludeCache, + serviceTier: readServiceTier(record), + cachedTokens: cacheRead.cachedTokens, + }; +} + +export function extractAnthropicUsage(payload: unknown): NormalizedUsage { + const summary = summarizeAnthropicUsage(payload); + return { + promptTokens: summary.promptTokensIncludingCache, + completionTokens: summary.completionTokens, + totalTokens: summary.totalTokens, + cachedTokens: summary.cacheReadInputTokens, + reasoningTokens: 0, + audioInputTokens: 0, + audioOutputTokens: 0, + acceptedPredictionTokens: 0, + rejectedPredictionTokens: 0, + cacheReadInputTokens: summary.cacheReadInputTokens, + cacheCreationInputTokens: summary.cacheCreationInputTokens, + ephemeral5mInputTokens: summary.ephemeral5mInputTokens, + ephemeral1hInputTokens: summary.ephemeral1hInputTokens, + promptTokensIncludingCache: summary.promptTokensIncludingCache, + promptTokensIncludeCache: summary.promptTokensIncludeCache, + serviceTier: summary.serviceTier, + } as AnthropicNormalizedUsage; +} + +export function extractAnthropicUsageMetadata(payload: unknown): AnthropicUsageMetadata { + const summary = summarizeAnthropicUsage(payload); + return { + cacheReadInputTokens: summary.cacheReadInputTokens, + cacheCreationInputTokens: summary.cacheCreationInputTokens, + ephemeral5mInputTokens: summary.ephemeral5mInputTokens, + ephemeral1hInputTokens: summary.ephemeral1hInputTokens, + promptTokensIncludingCache: summary.promptTokensIncludingCache, + promptTokensIncludeCache: summary.promptTokensIncludeCache, + serviceTier: summary.serviceTier, + cachedTokens: summary.cachedTokens, + }; +} + +export function toAnthropicUsagePayload(usage: unknown): Record { + const summary = summarizeAnthropicUsage(usage); + const shouldDeriveInputTokens = ( + summary.cacheReadInputTokens > 0 + || summary.cacheCreationInputTokens > 0 + || summary.promptTokensIncludeCache === true + || summary.promptTokensIncludingCache !== summary.inputTokens + ); + const inputTokens = shouldDeriveInputTokens + ? Math.max( + summary.promptTokensIncludingCache - summary.cacheReadInputTokens - summary.cacheCreationInputTokens, + 0, + ) + : summary.inputTokens; + const payload: Record = { + input_tokens: inputTokens, + output_tokens: summary.completionTokens, + }; + + if (summary.cacheReadInputTokens > 0) { + payload.cache_read_input_tokens = summary.cacheReadInputTokens; + } + if (summary.cacheCreationInputTokens > 0) { + payload.cache_creation_input_tokens = summary.cacheCreationInputTokens; + } + if (summary.ephemeral5mInputTokens > 0 || summary.ephemeral1hInputTokens > 0) { + payload.cache_creation = { + ephemeral_5m_input_tokens: summary.ephemeral5mInputTokens, + ephemeral_1h_input_tokens: summary.ephemeral1hInputTokens, + }; + } + if (summary.serviceTier) { + payload.service_tier = summary.serviceTier; + } + + return payload; +} + +function mergeAnthropicUsage( + base: NormalizedUsage | undefined, + next: NormalizedUsage | undefined, +): AnthropicNormalizedUsage { + const mergedBase = mergeNormalizedUsage(base, next) as AnthropicNormalizedUsage; + const baseUsage = (base ?? {}) as AnthropicNormalizedUsage; + const nextUsage = (next ?? {}) as AnthropicNormalizedUsage; + + const basePromptTokensIncludingCache = toPositiveInt(baseUsage.promptTokensIncludingCache ?? baseUsage.promptTokens); + const nextPromptTokensIncludingCache = toPositiveInt(nextUsage.promptTokensIncludingCache ?? nextUsage.promptTokens); + + return { + ...mergedBase, + cacheReadInputTokens: toPositiveInt(baseUsage.cacheReadInputTokens) + toPositiveInt(nextUsage.cacheReadInputTokens), + cacheCreationInputTokens: toPositiveInt(baseUsage.cacheCreationInputTokens) + toPositiveInt(nextUsage.cacheCreationInputTokens), + ephemeral5mInputTokens: toPositiveInt(baseUsage.ephemeral5mInputTokens) + toPositiveInt(nextUsage.ephemeral5mInputTokens), + ephemeral1hInputTokens: toPositiveInt(baseUsage.ephemeral1hInputTokens) + toPositiveInt(nextUsage.ephemeral1hInputTokens), + promptTokensIncludingCache: basePromptTokensIncludingCache + nextPromptTokensIncludingCache, + promptTokensIncludeCache: nextUsage.promptTokensIncludeCache ?? baseUsage.promptTokensIncludeCache ?? null, + serviceTier: typeof nextUsage.serviceTier === 'string' + ? nextUsage.serviceTier + : (typeof baseUsage.serviceTier === 'string' ? baseUsage.serviceTier : null), + cachedTokens: toPositiveInt(baseUsage.cachedTokens) + toPositiveInt(nextUsage.cachedTokens), }; } @@ -109,5 +353,6 @@ export const anthropicMessagesUsage = { empty: createEmptyNormalizedUsage, fromPayload: extractAnthropicUsage, metadataFromPayload: extractAnthropicUsageMetadata, - merge: mergeNormalizedUsage, + toPayload: toAnthropicUsagePayload, + merge: mergeAnthropicUsage, }; diff --git a/src/server/transformers/gemini/generate-content/aggregator-outbound-usage.test.ts b/src/server/transformers/gemini/generate-content/aggregator-outbound-usage.test.ts new file mode 100644 index 0000000..8880bf9 --- /dev/null +++ b/src/server/transformers/gemini/generate-content/aggregator-outbound-usage.test.ts @@ -0,0 +1,603 @@ +import { describe, expect, it } from 'vitest'; + +import { + applyGeminiGenerateContentAggregate, + createGeminiGenerateContentAggregateState, +} from './aggregator.js'; +import { + extractResponseMetadata, + serializeGeminiAggregateResponse, +} from './outbound.js'; +import { extractGeminiUsage } from './usage.js'; + +describe('Gemini aggregate/outbound/usage', () => { + it('coalesces chunked text and thought parts per candidate while preserving tool thought signatures', () => { + const state = createGeminiGenerateContentAggregateState(); + + applyGeminiGenerateContentAggregate(state, [ + { + responseId: 'resp-coalesce', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + content: { + parts: [ + { text: 'Think ', thought: true, thoughtSignature: 'sig-think-1' }, + { text: 'Hello ' }, + ], + }, + }, + { + index: 1, + content: { + parts: [ + { text: 'Alt ', thought: true, thoughtSignature: 'sig-think-2' }, + ], + }, + }, + ], + }, + { + responseId: 'resp-coalesce', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + groundingMetadata: { webSearchQueries: ['cats'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/cats' }] }, + content: { + parts: [ + { text: 'deeply', thought: true, thoughtSignature: 'sig-think-1' }, + { text: 'world' }, + { + functionCall: { + id: 'tool-call-1', + name: 'lookupDocs', + args: { query: 'cats' }, + }, + thoughtSignature: 'sig-tool-1', + }, + ], + }, + }, + { + index: 1, + finishReason: 'MAX_TOKENS', + groundingMetadata: { webSearchQueries: ['dogs'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/dogs' }] }, + content: { + parts: [ + { text: 'answer', thought: true, thoughtSignature: 'sig-think-2' }, + { text: 'candidate' }, + ], + }, + }, + ], + }, + ]); + + expect(serializeGeminiAggregateResponse(state)).toEqual({ + responseId: 'resp-coalesce', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [ + { text: 'Think deeply', thought: true, thoughtSignature: 'sig-think-1' }, + { text: 'Hello world' }, + { + functionCall: { + id: 'tool-call-1', + name: 'lookupDocs', + args: { query: 'cats' }, + }, + thoughtSignature: 'sig-tool-1', + }, + ], + }, + groundingMetadata: { webSearchQueries: ['cats'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/cats' }] }, + }, + { + index: 1, + finishReason: 'MAX_TOKENS', + content: { + role: 'model', + parts: [ + { text: 'Alt answer', thought: true, thoughtSignature: 'sig-think-2' }, + { text: 'candidate' }, + ], + }, + groundingMetadata: { webSearchQueries: ['dogs'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/dogs' }] }, + }, + ], + }); + }); + + it('reflows preserved request metadata alongside aggregated grounding, citation, thought, and usage data', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, { + responseId: 'resp-meta', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + groundingMetadata: { webSearchQueries: ['cats'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/cats' }] }, + content: { + parts: [ + { text: 'reasoning', thought: true, thoughtSignature: 'sig-think-1' }, + { + functionCall: { + id: 'call_1', + name: 'lookupWeather', + args: { city: 'Paris' }, + }, + thoughtSignature: 'sig-tool-1', + }, + { text: 'final answer' }, + ], + }, + }, + ], + usageMetadata: { + promptTokenCount: 15, + candidatesTokenCount: 5, + totalTokenCount: 24, + cachedContentTokenCount: 2, + thoughtsTokenCount: 4, + }, + }); + + const metadata = extractResponseMetadata(state, { + systemInstruction: { role: 'system', parts: [{ text: 'stay concise' }] }, + cachedContent: 'cached/item-1', + safetySettings: [{ category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_LOW_AND_ABOVE' }], + toolConfig: { functionCallingConfig: { mode: 'ANY' } }, + generationConfig: { + stopSequences: ['END'], + responseModalities: ['TEXT', 'AUDIO'], + responseMimeType: 'application/json', + responseSchema: { type: 'object', properties: { answer: { type: 'string' } } }, + candidateCount: 2, + temperature: 0.3, + topP: 0.8, + topK: 20, + presencePenalty: 0.1, + frequencyPenalty: 0.2, + seed: 42, + responseLogprobs: true, + logprobs: 5, + thinkingConfig: { thinkingLevel: 'high', includeThoughts: true }, + imageConfig: { aspectRatio: '16:9' }, + }, + tools: [ + { + functionDeclarations: [ + { + name: 'lookupWeather', + description: 'Lookup weather', + }, + ], + }, + { googleSearch: {} }, + { urlContext: {} }, + { codeExecution: {} }, + ], + }); + + expect(metadata).toEqual({ + systemInstruction: { role: 'system', parts: [{ text: 'stay concise' }] }, + cachedContent: 'cached/item-1', + safetySettings: [{ category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_LOW_AND_ABOVE' }], + toolConfig: { functionCallingConfig: { mode: 'ANY' } }, + stopSequences: ['END'], + responseModalities: ['TEXT', 'AUDIO'], + responseMimeType: 'application/json', + responseSchema: { type: 'object', properties: { answer: { type: 'string' } } }, + candidateCount: 2, + temperature: 0.3, + topP: 0.8, + topK: 20, + presencePenalty: 0.1, + frequencyPenalty: 0.2, + seed: 42, + responseLogprobs: true, + logprobs: 5, + thinkingConfig: { thinkingLevel: 'high', includeThoughts: true }, + imageConfig: { aspectRatio: '16:9' }, + tools: [ + { + functionDeclarations: [ + { + name: 'lookupWeather', + description: 'Lookup weather', + }, + ], + }, + { googleSearch: {} }, + { urlContext: {} }, + { codeExecution: {} }, + ], + citations: [{ citations: [{ uri: 'https://example.com/cats' }] }], + groundingMetadata: [{ webSearchQueries: ['cats'] }], + thoughtSignature: 'sig-think-1', + thoughtSignatures: ['sig-think-1', 'sig-tool-1'], + usageMetadata: { + promptTokenCount: 15, + candidatesTokenCount: 5, + totalTokenCount: 24, + cachedContentTokenCount: 2, + thoughtsTokenCount: 4, + }, + }); + }); + + it('merges split candidate metadata and emits top-level metadata in candidate order', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, [ + { + responseId: 'resp-metadata-merge', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 1, + groundingMetadata: { + webSearchQueries: ['dogs'], + }, + citationMetadata: { + citations: [{ uri: 'https://example.com/dogs-1' }], + }, + content: { + parts: [{ text: 'candidate-1' }], + }, + }, + { + index: 0, + groundingMetadata: { + webSearchQueries: ['cats'], + }, + citationMetadata: { + citations: [{ uri: 'https://example.com/cats-1' }], + }, + content: { + parts: [{ text: 'candidate-0' }], + }, + }, + ], + }, + { + responseId: 'resp-metadata-merge', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 1, + finishReason: 'STOP', + groundingMetadata: { + groundingChunks: [{ id: 'dogs-chunk' }], + }, + citationMetadata: { + citations: [{ uri: 'https://example.com/dogs-2' }], + }, + content: { + parts: [{ text: '-done' }], + }, + }, + { + index: 0, + finishReason: 'STOP', + groundingMetadata: { + groundingChunks: [{ id: 'cats-chunk' }], + }, + citationMetadata: { + citations: [{ uri: 'https://example.com/cats-2' }], + }, + content: { + parts: [{ text: '-done' }], + }, + }, + ], + }, + ]); + + const response = serializeGeminiAggregateResponse(state); + const metadata = extractResponseMetadata(state); + + expect(response.candidates).toEqual([ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'candidate-0-done' }], + }, + groundingMetadata: { + webSearchQueries: ['cats'], + groundingChunks: [{ id: 'cats-chunk' }], + }, + citationMetadata: { + citations: [ + { uri: 'https://example.com/cats-1' }, + { uri: 'https://example.com/cats-2' }, + ], + }, + }, + { + index: 1, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'candidate-1-done' }], + }, + groundingMetadata: { + webSearchQueries: ['dogs'], + groundingChunks: [{ id: 'dogs-chunk' }], + }, + citationMetadata: { + citations: [ + { uri: 'https://example.com/dogs-1' }, + { uri: 'https://example.com/dogs-2' }, + ], + }, + }, + ]); + expect(metadata.groundingMetadata).toEqual([ + { + webSearchQueries: ['cats'], + groundingChunks: [{ id: 'cats-chunk' }], + }, + { + webSearchQueries: ['dogs'], + groundingChunks: [{ id: 'dogs-chunk' }], + }, + ]); + expect(metadata.citations).toEqual([ + { + citations: [ + { uri: 'https://example.com/cats-1' }, + { uri: 'https://example.com/cats-2' }, + ], + }, + { + citations: [ + { uri: 'https://example.com/dogs-1' }, + { uri: 'https://example.com/dogs-2' }, + ], + }, + ]); + }); + + it('preserves multi-candidate tool thought signatures in serialized Gemini responses', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, [ + { + responseId: 'resp-multi-tools', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 1, + finishReason: 'STOP', + content: { + parts: [ + { + functionCall: { + id: 'call_2', + name: 'lookupDocs', + args: { query: 'transformers' }, + }, + thoughtSignature: 'sig-candidate-2', + }, + ], + }, + groundingMetadata: { webSearchQueries: ['transformers'] }, + }, + ], + }, + { + responseId: 'resp-multi-tools', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [ + { text: 'candidate zero answer', thoughtSignature: 'sig-candidate-1' }, + ], + }, + citationMetadata: { citations: [{ uri: 'https://example.com/zero' }] }, + }, + ], + }, + ]); + + expect(serializeGeminiAggregateResponse(state)).toEqual({ + responseId: 'resp-multi-tools', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'candidate zero answer', thoughtSignature: 'sig-candidate-1' }], + }, + citationMetadata: { citations: [{ uri: 'https://example.com/zero' }] }, + }, + { + index: 1, + finishReason: 'STOP', + content: { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_2', + name: 'lookupDocs', + args: { query: 'transformers' }, + }, + thoughtSignature: 'sig-candidate-2', + }, + ], + }, + groundingMetadata: { webSearchQueries: ['transformers'] }, + }, + ], + }); + }); + + it('prefers final answer thought signature for scalar metadata while retaining tool signatures in the array', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, [ + { + responseId: 'resp-thought-order', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + content: { + parts: [ + { + functionCall: { + id: 'call_1', + name: 'lookupWeather', + args: { city: 'Paris' }, + }, + thoughtSignature: 'sig-tool-first', + }, + ], + }, + }, + ], + }, + { + responseId: 'resp-thought-order', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [ + { text: 'reasoning', thought: true, thoughtSignature: 'sig-answer-final' }, + { text: 'answer' }, + ], + }, + }, + ], + }, + ]); + + const metadata = extractResponseMetadata(state); + + expect(metadata.thoughtSignature).toBe('sig-answer-final'); + expect(metadata.thoughtSignatures).toEqual([ + 'sig-tool-first', + 'sig-answer-final', + ]); + }); + + it('extracts Gemini usage from aggregate state and serialized response consistently', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, { + usageMetadata: { + promptTokenCount: 20, + candidatesTokenCount: 7, + totalTokenCount: 31, + cachedContentTokenCount: 4, + thoughtsTokenCount: 4, + }, + }); + + const serialized = serializeGeminiAggregateResponse(state); + + expect(extractGeminiUsage(state)).toEqual({ + promptTokens: 20, + completionTokens: 11, + totalTokens: 31, + cachedTokens: 4, + reasoningTokens: 4, + }); + expect(extractGeminiUsage(serialized)).toEqual({ + promptTokens: 20, + completionTokens: 11, + totalTokens: 31, + cachedTokens: 4, + reasoningTokens: 4, + }); + }); + + it('drops non-finite Gemini usage values instead of leaking NaN or Infinity into normalized usage', () => { + const usage = extractGeminiUsage({ + usageMetadata: { + promptTokenCount: Number.NaN, + candidatesTokenCount: 6, + totalTokenCount: Number.POSITIVE_INFINITY, + cachedContentTokenCount: Number.NaN, + thoughtsTokenCount: Number.NEGATIVE_INFINITY, + }, + }); + + expect(usage).toEqual({ + promptTokens: 0, + completionTokens: 6, + totalTokens: 6, + cachedTokens: 0, + reasoningTokens: 0, + }); + }); + + it('keeps request-preserved metadata stable after serialize -> metadata extraction round-trip', () => { + const state = createGeminiGenerateContentAggregateState(); + applyGeminiGenerateContentAggregate(state, { + responseId: 'resp-roundtrip', + modelVersion: 'gemini-3.1-pro-preview', + candidates: [ + { + index: 0, + finishReason: 'STOP', + groundingMetadata: { webSearchQueries: ['roundtrip'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/roundtrip' }] }, + content: { + parts: [ + { text: 'thinking', thought: true, thoughtSignature: 'sig-roundtrip' }, + { text: 'final answer' }, + ], + }, + }, + ], + usageMetadata: { + promptTokenCount: 8, + candidatesTokenCount: 3, + totalTokenCount: 13, + cachedContentTokenCount: 1, + thoughtsTokenCount: 2, + }, + }); + + const requestPayload = { + systemInstruction: { role: 'system', parts: [{ text: 'preserve me' }] }, + cachedContent: 'cached/roundtrip', + generationConfig: { + responseModalities: ['TEXT', 'AUDIO'], + responseMimeType: 'application/json', + responseSchema: { type: 'object' }, + thinkingConfig: { thinkingLevel: 'high', includeThoughts: true }, + }, + tools: [{ googleSearch: {} }, { codeExecution: {} }], + }; + + const fromState = extractResponseMetadata(state, requestPayload); + const serialized = serializeGeminiAggregateResponse(state); + const fromSerialized = extractResponseMetadata(serialized, requestPayload); + + expect(fromSerialized).toEqual(fromState); + }); +}); diff --git a/src/server/transformers/gemini/generate-content/aggregator.ts b/src/server/transformers/gemini/generate-content/aggregator.ts index e91fb61..7d5b196 100644 --- a/src/server/transformers/gemini/generate-content/aggregator.ts +++ b/src/server/transformers/gemini/generate-content/aggregator.ts @@ -16,13 +16,78 @@ function cloneJsonValue(value: T): T { return value; } +function stableSerialize(value: unknown): string { + return JSON.stringify(value); +} + +function mergeJsonArrays(existing: unknown[], incoming: unknown[]): unknown[] { + const merged = existing.map((item) => cloneJsonValue(item)); + const seen = new Set(merged.map((item) => stableSerialize(item))); + for (const item of incoming) { + const cloned = cloneJsonValue(item); + const serialized = stableSerialize(cloned); + if (seen.has(serialized)) continue; + seen.add(serialized); + merged.push(cloned); + } + return merged; +} + +function mergeJsonValues(existing: unknown, incoming: unknown): unknown { + if (incoming === undefined) return cloneJsonValue(existing); + if (existing === undefined) return cloneJsonValue(incoming); + if (Array.isArray(existing) && Array.isArray(incoming)) { + return mergeJsonArrays(existing, incoming); + } + if (isRecord(existing) && isRecord(incoming)) { + return mergeJsonRecords(existing, incoming); + } + return cloneJsonValue(incoming); +} + +function mergeJsonRecords(existing: GeminiRecord, incoming: GeminiRecord): GeminiRecord { + const merged: GeminiRecord = Object.fromEntries( + Object.entries(existing).map(([key, value]) => [key, cloneJsonValue(value)]), + ); + + for (const [key, value] of Object.entries(incoming)) { + merged[key] = mergeJsonValues(merged[key], value); + } + + return merged; +} + function pushUniqueJson(target: GeminiRecord[], incoming: unknown): void { if (!isRecord(incoming)) return; - const serialized = JSON.stringify(incoming); - if (target.some((item) => JSON.stringify(item) === serialized)) return; + const serialized = stableSerialize(incoming); + if (target.some((item) => stableSerialize(item) === serialized)) return; target.push(cloneJsonValue(incoming)); } +function isTextPart(value: unknown): value is GeminiRecord & { text: string } { + return isRecord(value) && typeof value.text === 'string'; +} + +function partComparableShape(value: GeminiRecord & { text: string }): string { + const { text: _text, ...rest } = value; + return stableSerialize(rest); +} + +function appendPart(target: GeminiRecord[], incoming: unknown): void { + if (!isRecord(incoming)) return; + const next = cloneJsonValue(incoming); + if (isTextPart(next)) { + for (let index = target.length - 1; index >= 0; index -= 1) { + const existing = target[index]; + if (!isTextPart(existing)) continue; + if (partComparableShape(existing) !== partComparableShape(next)) continue; + existing.text = `${existing.text}${next.text}`; + return; + } + } + target.push(next); +} + function collectPartsFromPayload(payload: unknown): GeminiRecord[] { if (Array.isArray(payload)) { return payload.flatMap((item) => collectPartsFromPayload(item)); @@ -119,7 +184,7 @@ export function applyGeminiGenerateContentAggregate( payload: unknown, ): GeminiGenerateContentAggregateState { for (const part of collectPartsFromPayload(payload)) { - state.parts.push(part); + appendPart(state.parts, part); if (typeof part.thoughtSignature === 'string' && part.thoughtSignature.trim()) { if (!state.thoughtSignatures.includes(part.thoughtSignature)) { state.thoughtSignatures.push(part.thoughtSignature); @@ -145,13 +210,17 @@ export function applyGeminiGenerateContentAggregate( if (candidate.groundingMetadata !== undefined) { pushUniqueJson(state.groundingMetadata, candidate.groundingMetadata); if (isRecord(candidate.groundingMetadata)) { - candidateAggregate.groundingMetadata = cloneJsonValue(candidate.groundingMetadata); + candidateAggregate.groundingMetadata = candidateAggregate.groundingMetadata + ? mergeJsonRecords(candidateAggregate.groundingMetadata, candidate.groundingMetadata) + : cloneJsonValue(candidate.groundingMetadata); } } if (candidate.citationMetadata !== undefined) { pushUniqueJson(state.citations, candidate.citationMetadata); if (isRecord(candidate.citationMetadata)) { - candidateAggregate.citationMetadata = cloneJsonValue(candidate.citationMetadata); + candidateAggregate.citationMetadata = candidateAggregate.citationMetadata + ? mergeJsonRecords(candidateAggregate.citationMetadata, candidate.citationMetadata) + : cloneJsonValue(candidate.citationMetadata); } } if (typeof candidate.finishReason === 'string' && candidate.finishReason.trim()) { @@ -163,7 +232,7 @@ export function applyGeminiGenerateContentAggregate( if (content && Array.isArray(content.parts)) { for (const part of content.parts) { if (!isRecord(part)) continue; - candidateAggregate.parts.push(cloneJsonValue(part)); + appendPart(candidateAggregate.parts, part); } } } diff --git a/src/server/transformers/gemini/generate-content/convert.ts b/src/server/transformers/gemini/generate-content/convert.ts index df18f0e..16a90b2 100644 --- a/src/server/transformers/gemini/generate-content/convert.ts +++ b/src/server/transformers/gemini/generate-content/convert.ts @@ -3,8 +3,20 @@ export type GeminiReasoningEffort = 'none' | 'low' | 'medium' | 'high' | 'max'; type GeminiThinkingConfig = { thinkingBudget?: number; thinkingLevel?: string; + includeThoughts?: boolean; }; +const GEMINI_3_STANDARD_BUDGETS: ReadonlyMap = new Map([ + [0, 'none'], + [1024, 'low'], + [8192, 'medium'], + [32768, 'high'], +]); + +function isGemini3Model(model: string): boolean { + return /gemini-3(?:[.-]|$)/i.test(model); +} + function normalizeReasoningEffort(value: unknown): GeminiReasoningEffort { const normalized = typeof value === 'string' ? value.trim().toLowerCase() : ''; if ( @@ -43,10 +55,14 @@ function thinkingBudgetToReasoningEffort(budget: number): GeminiReasoningEffort } function shouldUseThinkingLevel(model: string, effort: GeminiReasoningEffort): boolean { - if (!/gemini-3-/i.test(model)) return false; + if (!isGemini3Model(model)) return false; return effort === 'none' || effort === 'low' || effort === 'medium' || effort === 'high'; } +function shouldUseThinkingLevelForBudget(model: string, budget: number): boolean { + return isGemini3Model(model) && GEMINI_3_STANDARD_BUDGETS.has(budget); +} + function reasoningEffortToThinkingBudget(effort: GeminiReasoningEffort): number { switch (effort) { case 'none': @@ -74,6 +90,23 @@ export function reasoningEffortToGeminiThinkingConfig( return { thinkingBudget: reasoningEffortToThinkingBudget(effort) }; } +export function thinkingBudgetToGeminiThinkingConfig( + model: string, + thinkingBudget: unknown, +): GeminiThinkingConfig | null { + const budget = toFiniteInt(thinkingBudget); + if (budget === null) return null; + if (shouldUseThinkingLevelForBudget(model, budget)) { + const standardEffort = GEMINI_3_STANDARD_BUDGETS.get(budget); + return { + thinkingLevel: standardEffort ?? thinkingBudgetToReasoningEffort(budget), + }; + } + return { + thinkingBudget: budget, + }; +} + export function geminiThinkingConfigToReasoning( thinkingConfig: Record | null | undefined, ): { reasoningEffort: GeminiReasoningEffort; reasoningBudget: number } | null { @@ -108,6 +141,16 @@ function toFiniteInt(value: unknown): number | null { return Math.max(0, Math.trunc(numeric)); } +function withSynthesizedThoughts( + thinkingConfig: GeminiThinkingConfig | null, +): GeminiThinkingConfig | null { + if (!thinkingConfig) return null; + return { + ...thinkingConfig, + includeThoughts: true, + }; +} + export function resolveGeminiThinkingConfigFromRequest( model: string, body: Record, @@ -124,7 +167,7 @@ export function resolveGeminiThinkingConfigFromRequest( return { thinkingLevel: nativeLevel }; } if (nativeBudget !== null) { - return { thinkingBudget: nativeBudget }; + return thinkingBudgetToGeminiThinkingConfig(model, nativeBudget); } } @@ -142,11 +185,11 @@ export function resolveGeminiThinkingConfigFromRequest( const parsedBudget = toFiniteInt(reasoningBudget); if (parsedBudget !== null) { - return reasoningEffortToGeminiThinkingConfig(model, thinkingBudgetToReasoningEffort(parsedBudget)); + return withSynthesizedThoughts(thinkingBudgetToGeminiThinkingConfig(model, parsedBudget)); } if (reasoningEffort !== undefined) { - return reasoningEffortToGeminiThinkingConfig(model, reasoningEffort); + return withSynthesizedThoughts(reasoningEffortToGeminiThinkingConfig(model, reasoningEffort)); } return null; @@ -161,5 +204,13 @@ export function extractReasoningMetadataFromGeminiRequest( const thinkingConfig = isRecord(generationConfig?.thinkingConfig) ? generationConfig!.thinkingConfig as Record : null; - return geminiThinkingConfigToReasoning(thinkingConfig); + if (thinkingConfig) { + return geminiThinkingConfigToReasoning(thinkingConfig); + } + + const synthesizedThinkingConfig = resolveGeminiThinkingConfigFromRequest( + typeof body.model === 'string' ? body.model : '', + body, + ); + return geminiThinkingConfigToReasoning(synthesizedThinkingConfig ?? null); } diff --git a/src/server/transformers/gemini/generate-content/inbound.ts b/src/server/transformers/gemini/generate-content/inbound.ts index 0dcb041..6a90633 100644 --- a/src/server/transformers/gemini/generate-content/inbound.ts +++ b/src/server/transformers/gemini/generate-content/inbound.ts @@ -1,5 +1,8 @@ type GeminiRecord = Record; -import { resolveGeminiThinkingConfigFromRequest } from './convert.js'; +import { + geminiThinkingConfigToReasoning, + resolveGeminiThinkingConfigFromRequest, +} from './convert.js'; function isRecord(value: unknown): value is GeminiRecord { return !!value && typeof value === 'object' && !Array.isArray(value); @@ -30,6 +33,37 @@ function cloneContents(value: unknown): unknown[] | undefined { }); } +function cloneThinkingConfig(value: unknown): GeminiRecord | undefined { + if (!isRecord(value)) return undefined; + + const next: GeminiRecord = {}; + for (const [key, item] of Object.entries(value)) { + if (key === 'thinkingLevel') { + const normalizedLevel = typeof item === 'string' ? item.trim().toLowerCase() : ''; + if (normalizedLevel === 'minimal') { + next.thinkingLevel = 'low'; + continue; + } + if (normalizedLevel) { + next.thinkingLevel = normalizedLevel; + continue; + } + } + + if (key === 'thinkingBudget') { + const numeric = typeof item === 'number' ? item : Number(item); + if (Number.isFinite(numeric)) { + next.thinkingBudget = Math.max(0, Math.trunc(numeric)); + continue; + } + } + + next[key] = cloneJsonValue(item); + } + + return Object.keys(next).length > 0 ? next : undefined; +} + function cloneGenerationConfig(value: unknown): GeminiRecord | undefined { if (!isRecord(value)) return undefined; const allowedKeys = [ @@ -52,7 +86,12 @@ function cloneGenerationConfig(value: unknown): GeminiRecord | undefined { ]; const next: GeminiRecord = {}; for (const key of allowedKeys) { - if (value[key] !== undefined) next[key] = cloneJsonValue(value[key]); + if (value[key] === undefined) continue; + if (key === 'thinkingConfig') { + next[key] = cloneThinkingConfig(value[key]) ?? cloneJsonValue(value[key]); + continue; + } + next[key] = cloneJsonValue(value[key]); } return Object.keys(next).length > 0 ? next : undefined; } @@ -71,6 +110,55 @@ function cloneTools(value: unknown): unknown[] | undefined { }); } +function hasMeaningfulThinkingConfig(value: unknown): boolean { + return isRecord(value) && geminiThinkingConfigToReasoning(value) !== null; +} + +function sanitizeThinkingConfig(value: unknown): GeminiRecord | undefined { + if (!isRecord(value)) return undefined; + + const next = cloneJsonValue(value); + const normalizedReasoning = geminiThinkingConfigToReasoning(next); + if (normalizedReasoning) { + return next; + } + + if ('thinkingLevel' in next) { + delete next.thinkingLevel; + } + + if ('thinkingBudget' in next) { + const numeric = typeof next.thinkingBudget === 'number' ? next.thinkingBudget : Number(next.thinkingBudget); + if (Number.isFinite(numeric)) { + next.thinkingBudget = Math.max(0, Math.trunc(numeric)); + } else { + delete next.thinkingBudget; + } + } + + return Object.keys(next).length > 0 ? next : undefined; +} + +function mergeThinkingConfig( + currentValue: unknown, + derivedThinkingConfig: GeminiRecord | null, +): GeminiRecord | undefined { + const currentThinkingConfig = sanitizeThinkingConfig(currentValue); + + if (!derivedThinkingConfig) { + return currentThinkingConfig; + } + + if (!hasMeaningfulThinkingConfig(currentThinkingConfig)) { + return { + ...(currentThinkingConfig ?? {}), + ...cloneJsonValue(derivedThinkingConfig), + }; + } + + return currentThinkingConfig; +} + export type GeminiGenerateContentRequest = GeminiRecord; export const geminiGenerateContentInbound = { @@ -96,8 +184,12 @@ export const geminiGenerateContentInbound = { const generationConfig = isRecord(next.generationConfig) ? { ...next.generationConfig } : {}; - if (!isRecord(generationConfig.thinkingConfig)) { - generationConfig.thinkingConfig = derivedThinkingConfig; + const thinkingConfig = mergeThinkingConfig( + generationConfig.thinkingConfig, + derivedThinkingConfig, + ); + if (thinkingConfig) { + generationConfig.thinkingConfig = thinkingConfig; } next.generationConfig = generationConfig; } diff --git a/src/server/transformers/gemini/generate-content/index.test.ts b/src/server/transformers/gemini/generate-content/index.test.ts index 8e1f078..a12338e 100644 --- a/src/server/transformers/gemini/generate-content/index.test.ts +++ b/src/server/transformers/gemini/generate-content/index.test.ts @@ -7,6 +7,12 @@ import { } from './index.js'; import { extractGeminiUsage } from './usage.js'; import { serializeGeminiAggregateResponse, extractResponseMetadata } from './outbound.js'; +import { resolveGeminiThinkingConfigFromRequest } from './convert.js'; +import { + parseGeminiStreamPayload, + serializeAggregateJsonPayload, + serializeAggregatePayload, +} from './stream.js'; describe('geminiGenerateContentTransformer.inbound', () => { it('preserves native Gemini request fields through normalization', () => { @@ -96,6 +102,130 @@ describe('geminiGenerateContentTransformer.inbound', () => { ], }); }); + + it('uses thinkingLevel for Gemini 3 reasoning budgets within standard ranges and preserves very high budgets', () => { + const standardBudget = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + generationConfig: {}, + reasoning_budget: 1024, + }, + 'gemini-3-pro', + ); + + expect(standardBudget.generationConfig).toEqual({ + thinkingConfig: { + thinkingLevel: 'low', + includeThoughts: true, + }, + }); + + const higherBudget = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + generationConfig: {}, + reasoning: { + budget_tokens: 12000, + }, + }, + 'gemini-3-pro', + ); + + expect(higherBudget.generationConfig).toEqual({ + thinkingConfig: { + thinkingBudget: 12000, + includeThoughts: true, + }, + }); + }); + + it('prefers thinkingLevel for Gemini 3.1 standard reasoning budgets and preserves non-standard budgets', () => { + const standardBudgetBody = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + generationConfig: {}, + reasoning: { + budget_tokens: 1024, + }, + }, + 'gemini-3.1-pro-preview', + ); + + expect(standardBudgetBody.generationConfig).toEqual({ + thinkingConfig: { + thinkingLevel: 'low', + includeThoughts: true, + }, + }); + + const nonStandardBudgetBody = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + generationConfig: {}, + reasoning: { + budget_tokens: 50000, + }, + }, + 'gemini-3-flash', + ); + + expect(nonStandardBudgetBody.generationConfig).toEqual({ + thinkingConfig: { + thinkingBudget: 50000, + includeThoughts: true, + }, + }); + }); + + it('normalizes native thinking config and keeps request metadata stable when injecting derived thinking config', () => { + const body = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + model: 'gemini-3-pro', + contents: [ + { + role: 'user', + parts: [{ text: 'hello' }], + }, + ], + systemInstruction: { + role: 'system', + parts: [{ text: 'system prompt' }], + }, + cachedContent: 'cached/abc', + generationConfig: { + responseModalities: ['TEXT', 'AUDIO'], + responseMimeType: 'application/json', + thinkingConfig: { + thinkingLevel: 'minimal', + }, + }, + tools: [ + { googleSearch: {} }, + { urlContext: {} }, + { codeExecution: {} }, + ], + reasoning_effort: 'high', + reasoning_budget: 4096, + }, + 'gemini-3-pro', + ); + + expect(body.systemInstruction).toEqual({ + role: 'system', + parts: [{ text: 'system prompt' }], + }); + expect(body.cachedContent).toBe('cached/abc'); + expect(body.tools).toEqual([ + { googleSearch: {} }, + { urlContext: {} }, + { codeExecution: {} }, + ]); + expect(body.generationConfig).toEqual({ + responseModalities: ['TEXT', 'AUDIO'], + responseMimeType: 'application/json', + thinkingConfig: { + thinkingLevel: 'low', + }, + }); + expect(body.reasoning_effort).toBeUndefined(); + expect(body.reasoning_budget).toBeUndefined(); + }); }); describe('geminiGenerateContentTransformer.aggregator', () => { @@ -302,6 +432,127 @@ describe('geminiGenerateContentTransformer.aggregator', () => { { codeExecution: {} }, ]); }); + + it('preserves tool thought signatures and request-side Gemini metadata in outbound helpers', () => { + const state = geminiGenerateContentTransformer.aggregator.createState(); + geminiGenerateContentTransformer.aggregator.apply(state, { + responseId: 'resp-tools', + modelVersion: 'gemini-3-pro', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [ + { + functionCall: { + id: 'call-1', + name: 'lookup', + args: { query: 'cat' }, + }, + thoughtSignature: 'tool-sig-1', + }, + ], + }, + groundingMetadata: { webSearchQueries: ['cat'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/cat' }] }, + }, + ], + usageMetadata: { + promptTokenCount: 12, + candidatesTokenCount: 4, + totalTokenCount: 18, + cachedContentTokenCount: 6, + thoughtsTokenCount: 2, + }, + }); + + expect(serializeGeminiAggregateResponse(state)).toEqual({ + responseId: 'resp-tools', + modelVersion: 'gemini-3-pro', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [ + { + functionCall: { + id: 'call-1', + name: 'lookup', + args: { query: 'cat' }, + }, + thoughtSignature: 'tool-sig-1', + }, + ], + }, + groundingMetadata: { webSearchQueries: ['cat'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/cat' }] }, + }, + ], + usageMetadata: { + promptTokenCount: 12, + candidatesTokenCount: 4, + totalTokenCount: 18, + cachedContentTokenCount: 6, + thoughtsTokenCount: 2, + }, + }); + + expect(extractResponseMetadata(state, { + systemInstruction: { role: 'system', parts: [{ text: 'system prompt' }] }, + cachedContent: 'cached/tooling', + safetySettings: [{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH' }], + toolConfig: { + functionCallingConfig: { + mode: 'ANY', + allowedFunctionNames: ['lookup'], + }, + }, + generationConfig: { + responseModalities: ['TEXT'], + responseSchema: { type: 'object' }, + responseMimeType: 'application/json', + thinkingConfig: { thinkingLevel: 'high' }, + imageConfig: { aspectRatio: '1:1' }, + }, + tools: [ + { functionDeclarations: [{ name: 'lookup', description: 'find facts' }] }, + { googleSearch: {} }, + ], + })).toEqual({ + systemInstruction: { role: 'system', parts: [{ text: 'system prompt' }] }, + cachedContent: 'cached/tooling', + safetySettings: [{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH' }], + toolConfig: { + functionCallingConfig: { + mode: 'ANY', + allowedFunctionNames: ['lookup'], + }, + }, + responseModalities: ['TEXT'], + responseSchema: { type: 'object' }, + responseMimeType: 'application/json', + thinkingConfig: { thinkingLevel: 'high' }, + imageConfig: { aspectRatio: '1:1' }, + tools: [ + { functionDeclarations: [{ name: 'lookup', description: 'find facts' }] }, + { googleSearch: {} }, + ], + citations: [{ citations: [{ uri: 'https://example.com/cat' }] }], + groundingMetadata: [{ webSearchQueries: ['cat'] }], + thoughtSignature: 'tool-sig-1', + thoughtSignatures: ['tool-sig-1'], + usageMetadata: { + promptTokenCount: 12, + candidatesTokenCount: 4, + totalTokenCount: 18, + cachedContentTokenCount: 6, + thoughtsTokenCount: 2, + }, + }); + }); }); describe('extractGeminiUsage', () => { @@ -350,6 +601,44 @@ describe('Gemini reasoning mapping', () => { }); }); + it('uses thinkingLevel for Gemini 3 budget values up to the high tier and preserves very high budgets', () => { + expect(resolveGeminiThinkingConfigFromRequest('gemini-3-pro', { + reasoning_budget: 8192, + })).toEqual({ + thinkingLevel: 'medium', + includeThoughts: true, + }); + + expect(resolveGeminiThinkingConfigFromRequest('gemini-3-pro', { + reasoning_budget: 12000, + })).toEqual({ + thinkingBudget: 12000, + includeThoughts: true, + }); + }); + + it('injects derived thinking config into normalized requests when runtime model name is provided', () => { + const body = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + contents: [{ role: 'user', parts: [{ text: 'hello' }] }], + generationConfig: { + responseModalities: ['TEXT'], + thinkingConfig: {}, + }, + reasoning_effort: 'high', + }, + 'gemini-3-pro', + ); + + expect(body.generationConfig).toEqual({ + responseModalities: ['TEXT'], + thinkingConfig: { + thinkingLevel: 'high', + includeThoughts: true, + }, + }); + }); + it('maps Gemini thinking config back to normalized reasoning hints', () => { expect(geminiThinkingConfigToReasoning({ thinkingLevel: 'medium', @@ -365,6 +654,132 @@ describe('Gemini reasoning mapping', () => { reasoningBudget: 512, }); }); + + it('uses thinkingLevel for Gemini 3 models when reasoning_budget is a standard tier', () => { + const normalized = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + contents: [{ role: 'user', parts: [{ text: 'hello' }] }], + reasoning_budget: 8192, + }, + 'gemini-3-flash', + ); + + expect(normalized.generationConfig).toEqual({ + thinkingConfig: { + thinkingLevel: 'medium', + includeThoughts: true, + }, + }); + }); + + it('preserves non-standard Gemini 3 reasoning_budget values instead of coercing them to a tier', () => { + const normalized = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + contents: [{ role: 'user', parts: [{ text: 'hello' }] }], + reasoning_budget: 5000, + }, + 'gemini-3-flash', + ); + + expect(normalized.generationConfig).toEqual({ + thinkingConfig: { + thinkingBudget: 5000, + includeThoughts: true, + }, + }); + }); + + it('fills missing native thinkingConfig from reasoning_effort while preserving request metadata', () => { + const normalized = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + cachedContent: 'cached/abc', + generationConfig: { + responseModalities: ['TEXT'], + thinkingConfig: { + includeThoughts: true, + }, + }, + reasoning_effort: 'high', + }, + 'gemini-3-flash', + ); + + expect(normalized.cachedContent).toBe('cached/abc'); + expect(normalized.generationConfig).toEqual({ + responseModalities: ['TEXT'], + thinkingConfig: { + includeThoughts: true, + thinkingLevel: 'high', + }, + }); + }); + + it('preserves non-standard explicit Gemini 3 budgets instead of coercing them to a standard effort level', () => { + expect( + geminiGenerateContentTransformer.inbound.normalizeRequest( + { + reasoning_budget: 50000, + }, + 'gemini-3-pro', + ), + ).toEqual({ + generationConfig: { + thinkingConfig: { + thinkingBudget: 50000, + includeThoughts: true, + }, + }, + }); + }); +}); + +describe('Gemini convert/inbound runtime strategy', () => { + it('uses thinkingLevel only for standard Gemini 3 budget tiers and preserves non-standard budgets', () => { + expect(resolveGeminiThinkingConfigFromRequest('gemini-3-pro', { + reasoning_budget: 1024, + })).toEqual({ + thinkingLevel: 'low', + includeThoughts: true, + }); + + expect(resolveGeminiThinkingConfigFromRequest('gemini-3-pro', { + reasoning_budget: 512, + })).toEqual({ + thinkingBudget: 512, + includeThoughts: true, + }); + }); + + it('prefers the runtime model name when injecting thinkingConfig and keeps request metadata stable', () => { + const normalized = geminiGenerateContentTransformer.inbound.normalizeRequest( + { + model: 'gemini-2.5-flash', + cachedContent: 'cached/runtime', + systemInstruction: { + role: 'system', + parts: [{ text: 'system prompt' }], + }, + generationConfig: { + responseModalities: ['TEXT'], + }, + reasoning_budget: 512, + }, + 'gemini-3-pro', + ); + + expect(normalized.cachedContent).toBe('cached/runtime'); + expect(normalized.systemInstruction).toEqual({ + role: 'system', + parts: [{ text: 'system prompt' }], + }); + expect(normalized.generationConfig).toEqual({ + responseModalities: ['TEXT'], + thinkingConfig: { + thinkingBudget: 512, + includeThoughts: true, + }, + }); + }); }); describe('geminiGenerateContentTransformer.stream', () => { @@ -436,4 +851,91 @@ describe('geminiGenerateContentTransformer.stream', () => { extractGeminiUsage(jsonState), ); }); + + it('normalizes SSE and JSON-array payloads through the same parse/apply boundary', () => { + const chunks = [ + { + responseId: 'resp-boundary', + modelVersion: 'gemini-2.5-pro', + candidates: [ + { + index: 0, + content: { + parts: [{ text: 'hello', thought: true, thoughtSignature: 'sig-a' }], + }, + groundingMetadata: { webSearchQueries: ['cat'] }, + citationMetadata: { citations: [{ uri: 'https://example.com/a' }] }, + }, + ], + }, + { + responseId: 'resp-boundary', + modelVersion: 'gemini-2.5-pro', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'world' }], + }, + }, + ], + usageMetadata: { + promptTokenCount: 10, + candidatesTokenCount: 4, + totalTokenCount: 14, + cachedContentTokenCount: 3, + thoughtsTokenCount: 2, + }, + }, + ]; + + const sseInput = chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join(''); + const parsedSse = parseGeminiStreamPayload(sseInput, 'text/event-stream'); + const parsedJson = parseGeminiStreamPayload(chunks, 'application/json'); + + expect(parsedSse.events).toEqual(parsedJson.events); + expect(parsedSse.rest).toBe(''); + expect(parsedJson.rest).toBe(''); + + const sseState = geminiGenerateContentTransformer.stream.createAggregateState(); + const jsonState = geminiGenerateContentTransformer.stream.createAggregateState(); + geminiGenerateContentTransformer.stream.applyParsedPayloadToAggregate(sseState, parsedSse); + geminiGenerateContentTransformer.stream.applyParsedPayloadToAggregate(jsonState, parsedJson); + + expect(serializeGeminiAggregateResponse(sseState)).toEqual( + serializeGeminiAggregateResponse(jsonState), + ); + }); + + it('serializes aggregate payloads with identical final semantics for SSE and JSON modes', () => { + const state = geminiGenerateContentTransformer.aggregator.createState(); + geminiGenerateContentTransformer.aggregator.apply(state, { + responseId: 'resp-serialize', + modelVersion: 'gemini-2.5-pro', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'answer', thoughtSignature: 'sig-final' }], + }, + }, + ], + usageMetadata: { + promptTokenCount: 9, + candidatesTokenCount: 5, + totalTokenCount: 14, + cachedContentTokenCount: 4, + thoughtsTokenCount: 1, + }, + }); + + const jsonPayload = serializeAggregateJsonPayload(state); + const ssePayload = serializeAggregatePayload(state, 'sse'); + const reparsedSse = parseGeminiStreamPayload(ssePayload, 'text/event-stream'); + + expect(jsonPayload).toEqual(serializeGeminiAggregateResponse(state)); + expect(reparsedSse.events).toEqual([jsonPayload]); + }); }); diff --git a/src/server/transformers/gemini/generate-content/outbound.ts b/src/server/transformers/gemini/generate-content/outbound.ts index 168222d..6d40db0 100644 --- a/src/server/transformers/gemini/generate-content/outbound.ts +++ b/src/server/transformers/gemini/generate-content/outbound.ts @@ -49,6 +49,18 @@ function isRecord(value: unknown): value is GeminiRecord { return !!value && typeof value === 'object' && !Array.isArray(value); } +function cloneJsonValue(value: T): T { + if (Array.isArray(value)) { + return value.map((item) => cloneJsonValue(item)) as T; + } + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, cloneJsonValue(item)]), + ) as T; + } + return value; +} + function isAggregateState(value: unknown): value is GeminiGenerateContentAggregateState { return isRecord(value) && Array.isArray(value.parts) && Array.isArray(value.groundingMetadata) && Array.isArray(value.citations); } @@ -112,18 +124,92 @@ function buildCandidates(state: GeminiGenerateContentAggregateState): GeminiReco return [fallback]; } +function extractOrderedCandidateMetadata( + state: GeminiGenerateContentAggregateState, + key: 'groundingMetadata' | 'citationMetadata', +): GeminiRecord[] { + if (state.candidates.length > 0) { + return state.candidates + .slice() + .sort((left, right) => left.index - right.index) + .map((candidate) => candidate[key]) + .filter((item): item is GeminiRecord => isRecord(item)) + .map((item) => cloneJsonValue(item)); + } + + const fallback = key === 'groundingMetadata' ? state.groundingMetadata : state.citations; + return fallback.map((item) => cloneJsonValue(item)); +} + +function extractOrderedThoughtSignatures( + state: GeminiGenerateContentAggregateState, +): { + all: string[]; + preferred: string | undefined; +} { + const ordered = new Set(); + const preferredThoughts = new Set(); + + const candidates = state.candidates.length > 0 + ? state.candidates.slice().sort((left, right) => left.index - right.index) + : [{ index: 0, finishReason: state.finishReason, parts: state.parts }]; + + for (const candidate of candidates) { + for (const part of candidate.parts) { + if (!isRecord(part) || typeof part.thoughtSignature !== 'string' || !part.thoughtSignature.trim()) { + continue; + } + ordered.add(part.thoughtSignature); + if (part.thought === true) { + preferredThoughts.add(part.thoughtSignature); + } + } + } + + const orderedList = [...ordered]; + const preferred = [...preferredThoughts][0] + ?? (orderedList.length === 1 ? orderedList[0] : undefined); + + return { + all: orderedList.length > 0 ? orderedList : [...state.thoughtSignatures], + preferred, + }; +} + function extractRequestSemantics(requestPayload: unknown): GeminiRecord { if (!isRecord(requestPayload)) return {}; const metadata: GeminiRecord = {}; - if (requestPayload.systemInstruction !== undefined) metadata.systemInstruction = requestPayload.systemInstruction; - if (requestPayload.cachedContent !== undefined) metadata.cachedContent = requestPayload.cachedContent; + if (requestPayload.systemInstruction !== undefined) metadata.systemInstruction = cloneJsonValue(requestPayload.systemInstruction); + if (requestPayload.cachedContent !== undefined) metadata.cachedContent = cloneJsonValue(requestPayload.cachedContent); + if (requestPayload.safetySettings !== undefined) metadata.safetySettings = cloneJsonValue(requestPayload.safetySettings); + if (requestPayload.toolConfig !== undefined) metadata.toolConfig = cloneJsonValue(requestPayload.toolConfig); const generationConfig = isRecord(requestPayload.generationConfig) ? requestPayload.generationConfig : null; if (generationConfig) { - if (generationConfig.responseModalities !== undefined) metadata.responseModalities = generationConfig.responseModalities; - if (generationConfig.responseSchema !== undefined) metadata.responseSchema = generationConfig.responseSchema; - if (generationConfig.responseMimeType !== undefined) metadata.responseMimeType = generationConfig.responseMimeType; + const preservedKeys = [ + 'stopSequences', + 'responseModalities', + 'responseMimeType', + 'responseSchema', + 'candidateCount', + 'maxOutputTokens', + 'temperature', + 'topP', + 'topK', + 'presencePenalty', + 'frequencyPenalty', + 'seed', + 'responseLogprobs', + 'logprobs', + 'thinkingConfig', + 'imageConfig', + ]; + for (const key of preservedKeys) { + if (generationConfig[key] !== undefined) { + metadata[key] = cloneJsonValue(generationConfig[key]); + } + } } if (Array.isArray(requestPayload.tools)) { @@ -131,10 +217,10 @@ function extractRequestSemantics(requestPayload: unknown): GeminiRecord { .filter((item) => isRecord(item)) .map((item) => { const next: GeminiRecord = {}; - if (item.googleSearch !== undefined) next.googleSearch = item.googleSearch; - if (item.urlContext !== undefined) next.urlContext = item.urlContext; - if (item.codeExecution !== undefined) next.codeExecution = item.codeExecution; - if (item.functionDeclarations !== undefined) next.functionDeclarations = item.functionDeclarations; + if (item.googleSearch !== undefined) next.googleSearch = cloneJsonValue(item.googleSearch); + if (item.urlContext !== undefined) next.urlContext = cloneJsonValue(item.urlContext); + if (item.codeExecution !== undefined) next.codeExecution = cloneJsonValue(item.codeExecution); + if (item.functionDeclarations !== undefined) next.functionDeclarations = cloneJsonValue(item.functionDeclarations); return next; }) .filter((item) => Object.keys(item).length > 0); @@ -169,11 +255,17 @@ export function extractResponseMetadata(payload: unknown, requestPayload?: unkno const state = ensureAggregateState(payload); const metadata: GeminiRecord = extractRequestSemantics(requestPayload); - if (state.citations.length > 0) metadata.citations = state.citations; - if (state.groundingMetadata.length > 0) metadata.groundingMetadata = state.groundingMetadata; - if (state.thoughtSignatures.length > 0) { - metadata.thoughtSignature = state.thoughtSignatures[0]; - metadata.thoughtSignatures = state.thoughtSignatures; + const citations = extractOrderedCandidateMetadata(state, 'citationMetadata'); + const groundingMetadata = extractOrderedCandidateMetadata(state, 'groundingMetadata'); + const thoughtSignatures = extractOrderedThoughtSignatures(state); + + if (citations.length > 0) metadata.citations = citations; + if (groundingMetadata.length > 0) metadata.groundingMetadata = groundingMetadata; + if (thoughtSignatures.all.length > 0) { + if (thoughtSignatures.preferred) { + metadata.thoughtSignature = thoughtSignatures.preferred; + } + metadata.thoughtSignatures = thoughtSignatures.all; } const usageMetadata = buildUsageMetadata(state); if (usageMetadata) metadata.usageMetadata = usageMetadata; diff --git a/src/server/transformers/gemini/generate-content/stream.test.ts b/src/server/transformers/gemini/generate-content/stream.test.ts new file mode 100644 index 0000000..f171c02 --- /dev/null +++ b/src/server/transformers/gemini/generate-content/stream.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from 'vitest'; + +import { createGeminiGenerateContentAggregateState } from './aggregator.js'; +import { extractResponseMetadata, serializeGeminiAggregateResponse } from './outbound.js'; +import { + applyJsonPayloadToAggregate, + applySsePayloadsToAggregate, + geminiGenerateContentStream, + parseGeminiStreamPayload, + serializeAggregateSsePayload, +} from './stream.js'; +import { extractGeminiUsage } from './usage.js'; + +describe('geminiGenerateContentStream', () => { + it('treats stringified json payloads as json when content type is not sse', () => { + const payload = JSON.stringify([ + { + responseId: 'resp-json-string', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'json-string-body' }], + }, + }, + ], + }, + ]); + + expect(parseGeminiStreamPayload(payload, 'application/json')).toEqual({ + format: 'json', + events: [ + { + responseId: 'resp-json-string', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'json-string-body' }], + }, + }, + ], + }, + ], + rest: '', + }); + }); + + it('aggregates SSE and JSON-array payloads into the same final semantics', () => { + const requestPayload = { + systemInstruction: { role: 'system', parts: [{ text: 'system prompt' }] }, + cachedContent: 'cached/abc', + generationConfig: { + responseModalities: ['TEXT'], + responseSchema: { type: 'object' }, + responseMimeType: 'application/json', + }, + tools: [ + { googleSearch: {} }, + { codeExecution: {} }, + ], + }; + + const chunks = [ + { + responseId: 'resp-1', + modelVersion: 'gemini-2.5-pro', + candidates: [ + { + index: 0, + content: { + parts: [{ text: 'thinking', thought: true, thoughtSignature: 'sig-1' }], + }, + groundingMetadata: { webSearchQueries: ['cat'] }, + citationMetadata: { citations: [{ uri: 'https://a.example.com' }] }, + }, + { + index: 1, + content: { + parts: [{ text: 'alt-answer' }], + }, + groundingMetadata: { webSearchQueries: ['dog'] }, + citationMetadata: { citations: [{ uri: 'https://b.example.com' }] }, + }, + ], + }, + { + responseId: 'resp-1', + modelVersion: 'gemini-2.5-pro', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'answer' }], + }, + }, + { + index: 1, + finishReason: 'MAX_TOKENS', + content: { + parts: [{ functionCall: { id: 'tool-1', name: 'lookup', args: { q: 'cat' } }, thoughtSignature: 'sig-tool-1' }], + }, + }, + ], + usageMetadata: { + promptTokenCount: 10, + candidatesTokenCount: 4, + totalTokenCount: 16, + cachedContentTokenCount: 3, + thoughtsTokenCount: 2, + }, + }, + ]; + + const ssePayload = `${chunks + .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`) + .join('')}data: [DONE]\n\n`; + + const sseState = createGeminiGenerateContentAggregateState(); + const sseResult = applySsePayloadsToAggregate(sseState, ssePayload); + + const jsonState = createGeminiGenerateContentAggregateState(); + applyJsonPayloadToAggregate(jsonState, chunks); + + expect(sseResult.rest).toBe(''); + expect(serializeGeminiAggregateResponse(sseResult.state)).toEqual( + serializeGeminiAggregateResponse(jsonState), + ); + expect(extractResponseMetadata(sseResult.state, requestPayload)).toEqual( + extractResponseMetadata(jsonState, requestPayload), + ); + expect(extractGeminiUsage(sseResult.state)).toEqual( + extractGeminiUsage(jsonState), + ); + expect(serializeAggregateSsePayload(sseResult.state)).toBe( + `data: ${JSON.stringify(serializeGeminiAggregateResponse(jsonState))}\n\n`, + ); + }); + + it('keeps trailing rest for partial SSE blocks and ignores done markers', () => { + const state = createGeminiGenerateContentAggregateState(); + const payload = [ + 'data: {"responseId":"resp-1","candidates":[{"content":{"parts":[{"text":"hello"}]}}]}', + '', + 'data: [DONE]', + '', + 'data: {"responseId":"resp-1"', + ].join('\n'); + + const result = applySsePayloadsToAggregate(state, payload); + + expect(result.rest).toBe('data: {"responseId":"resp-1"'); + expect(serializeGeminiAggregateResponse(result.state)).toEqual({ + responseId: 'resp-1', + modelVersion: '', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'hello' }], + }, + }, + ], + }); + }); + + it('normalizes object payloads through the JSON-array path', () => { + const state = createGeminiGenerateContentAggregateState(); + + applyJsonPayloadToAggregate(state, { + responseId: 'resp-2', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + parts: [{ text: 'single-object' }], + }, + }, + ], + }); + + expect(geminiGenerateContentStream.parseJsonArrayPayload({ + responseId: 'resp-2', + })).toEqual([{ responseId: 'resp-2' }]); + expect(serializeGeminiAggregateResponse(state)).toEqual({ + responseId: 'resp-2', + modelVersion: '', + candidates: [ + { + index: 0, + finishReason: 'STOP', + content: { + role: 'model', + parts: [{ text: 'single-object' }], + }, + }, + ], + }); + }); +}); diff --git a/src/server/transformers/gemini/generate-content/stream.ts b/src/server/transformers/gemini/generate-content/stream.ts index 03f7185..1393597 100644 --- a/src/server/transformers/gemini/generate-content/stream.ts +++ b/src/server/transformers/gemini/generate-content/stream.ts @@ -3,12 +3,25 @@ import { createGeminiGenerateContentAggregateState, type GeminiGenerateContentAggregateState, } from './aggregator.js'; +import { serializeGeminiAggregateResponse } from './outbound.js'; type ParsedSsePayloads = { events: unknown[]; rest: string; }; +type GeminiGenerateContentStreamFormat = 'sse' | 'json'; + +type ParsedGeminiStreamPayload = { + format: GeminiGenerateContentStreamFormat; + events: unknown[]; + rest: string; +}; + +type AppliedGeminiStreamPayloads = ParsedSsePayloads & { + state: GeminiGenerateContentAggregateState; +}; + function serializeSsePayload(payload: unknown): string { return `data: ${JSON.stringify(payload)}\n\n`; } @@ -45,17 +58,157 @@ function parseSsePayloads(buffer: string): ParsedSsePayloads { return { events, rest }; } +function isRecord(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function parseJsonArrayPayload(payload: unknown): unknown[] { + if (Array.isArray(payload)) return payload; + if (isRecord(payload)) return [payload]; + return []; +} + +function parseGeminiStreamPayload( + payload: unknown, + contentType?: string | null, +): ParsedGeminiStreamPayload { + if (geminiGenerateContentStream.isSseContentType(contentType)) { + const parsed = parseSsePayloads(String(payload ?? '')); + return { + format: 'sse', + events: parsed.events, + rest: parsed.rest, + }; + } + + if (typeof payload === 'string') { + const trimmed = payload.trim(); + if (trimmed) { + try { + const parsedJson = JSON.parse(trimmed); + return { + format: 'json', + events: parseJsonArrayPayload(parsedJson), + rest: '', + }; + } catch { + const parsed = parseSsePayloads(payload); + return { + format: 'sse', + events: parsed.events, + rest: parsed.rest, + }; + } + } + return { + format: 'json', + events: [], + rest: '', + }; + } + + return { + format: 'json', + events: parseJsonArrayPayload(payload), + rest: '', + }; +} + +function applyParsedPayloadToAggregate( + state: GeminiGenerateContentAggregateState, + parsed: ParsedGeminiStreamPayload, +): AppliedGeminiStreamPayloads { + for (const event of parsed.events) { + applyGeminiGenerateContentAggregate(state, event); + } + + return { + events: parsed.events, + rest: parsed.rest, + state, + }; +} + +function applyJsonPayloadToAggregate( + state: GeminiGenerateContentAggregateState, + payload: unknown, +): GeminiGenerateContentAggregateState { + applyParsedPayloadToAggregate(state, parseGeminiStreamPayload(payload, 'application/json')); + return state; +} + +function applySsePayloadsToAggregate( + state: GeminiGenerateContentAggregateState, + buffer: string, +): AppliedGeminiStreamPayloads { + return applyParsedPayloadToAggregate(state, parseGeminiStreamPayload(buffer, 'text/event-stream')); +} + +function consumeUpstreamSseBuffer( + state: GeminiGenerateContentAggregateState, + buffer: string, +): AppliedGeminiStreamPayloads & { lines: string[] } { + const applied = applySsePayloadsToAggregate(state, buffer); + return { + ...applied, + lines: applied.events.map((event) => serializeAggregateSsePayload(event)), + }; +} + +function serializeAggregateJsonPayload( + payload: GeminiGenerateContentAggregateState | unknown, +): unknown { + return serializeGeminiAggregateResponse(payload); +} + +function serializeAggregateSsePayload( + payload: GeminiGenerateContentAggregateState | unknown, +): string { + return serializeSsePayload(serializeAggregateJsonPayload(payload)); +} + +function serializeAggregatePayload( + payload: GeminiGenerateContentAggregateState | unknown, + format: GeminiGenerateContentStreamFormat = 'json', +): unknown { + return format === 'sse' + ? serializeAggregateSsePayload(payload) + : serializeAggregateJsonPayload(payload); +} + +function serializeUpstreamJsonPayload( + state: GeminiGenerateContentAggregateState, + payload: unknown, + streamAction = false, +): unknown { + if (streamAction) { + return parseJsonArrayPayload(payload).map((event) => { + applyGeminiGenerateContentAggregate(state, event); + return serializeAggregateJsonPayload(event); + }); + } + + applyJsonPayloadToAggregate(state, payload); + return serializeAggregateJsonPayload(state); +} + export const geminiGenerateContentStream = { isSseContentType(contentType: string | null | undefined): boolean { return (contentType || '').toLowerCase().includes('text/event-stream'); }, - parseJsonArrayPayload(payload: unknown): unknown[] { - return Array.isArray(payload) ? payload : []; - }, - + parseJsonArrayPayload, + parseGeminiStreamPayload, parseSsePayloads, serializeSsePayload, + serializeAggregateJsonPayload, + serializeAggregatePayload, + serializeAggregateSsePayload, + serializeUpstreamJsonPayload, + applyParsedPayloadToAggregate, + applyJsonPayloadToAggregate, + applySsePayloadsToAggregate, + consumeUpstreamSseBuffer, createAggregateState(): GeminiGenerateContentAggregateState { return createGeminiGenerateContentAggregateState(); @@ -65,3 +218,24 @@ export const geminiGenerateContentStream = { return applyGeminiGenerateContentAggregate(state, payload); }, }; + +export { + applyParsedPayloadToAggregate, + applyJsonPayloadToAggregate, + applySsePayloadsToAggregate, + parseGeminiStreamPayload, + parseJsonArrayPayload, + parseSsePayloads, + serializeAggregateJsonPayload, + serializeAggregatePayload, + serializeAggregateSsePayload, + serializeUpstreamJsonPayload, + serializeSsePayload, + consumeUpstreamSseBuffer, +}; +export type { + AppliedGeminiStreamPayloads, + GeminiGenerateContentStreamFormat, + ParsedGeminiStreamPayload, + ParsedSsePayloads, +}; diff --git a/src/server/transformers/gemini/generate-content/usage.ts b/src/server/transformers/gemini/generate-content/usage.ts index d02138b..0de1611 100644 --- a/src/server/transformers/gemini/generate-content/usage.ts +++ b/src/server/transformers/gemini/generate-content/usage.ts @@ -11,6 +11,10 @@ function isAggregateState(value: unknown): value is GeminiGenerateContentAggrega return isRecord(value) && Array.isArray(value.parts) && isRecord(value.usage); } +function finiteNumber(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + export function extractGeminiUsage(payload: unknown): NormalizedUsage { const usageMetadata = isAggregateState(payload) ? payload.usage @@ -20,22 +24,12 @@ export function extractGeminiUsage(payload: unknown): NormalizedUsage { return createEmptyNormalizedUsage(); } - const promptTokens = typeof usageMetadata.promptTokenCount === 'number' - ? usageMetadata.promptTokenCount - : 0; - const candidatesTokenCount = typeof usageMetadata.candidatesTokenCount === 'number' - ? usageMetadata.candidatesTokenCount - : 0; - const thoughtsTokenCount = typeof usageMetadata.thoughtsTokenCount === 'number' - ? usageMetadata.thoughtsTokenCount - : 0; + const promptTokens = finiteNumber(usageMetadata.promptTokenCount) ?? 0; + const candidatesTokenCount = finiteNumber(usageMetadata.candidatesTokenCount) ?? 0; + const thoughtsTokenCount = finiteNumber(usageMetadata.thoughtsTokenCount) ?? 0; const completionTokens = candidatesTokenCount + thoughtsTokenCount; - const totalTokens = typeof usageMetadata.totalTokenCount === 'number' - ? usageMetadata.totalTokenCount - : promptTokens + completionTokens; - const cachedTokens = typeof usageMetadata.cachedContentTokenCount === 'number' - ? usageMetadata.cachedContentTokenCount - : 0; + const totalTokens = finiteNumber(usageMetadata.totalTokenCount) ?? (promptTokens + completionTokens); + const cachedTokens = finiteNumber(usageMetadata.cachedContentTokenCount) ?? 0; const reasoningTokens = thoughtsTokenCount; return { diff --git a/src/server/transformers/openai/chat/aggregator.ts b/src/server/transformers/openai/chat/aggregator.ts index 25d0a90..257411b 100644 --- a/src/server/transformers/openai/chat/aggregator.ts +++ b/src/server/transformers/openai/chat/aggregator.ts @@ -1,9 +1,12 @@ import { type NormalizedFinalResponse, type NormalizedStreamEvent } from '../../shared/normalized.js'; import type { + OpenAiChatChoice, + OpenAiChatChoiceDelta, OpenAiChatNormalizedFinalResponse, OpenAiChatNormalizedStreamEvent, OpenAiChatUsageDetails, } from './model.js'; +import { mergeChatUsageDetails } from './helpers.js'; function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object'; @@ -22,18 +25,9 @@ function pushUniqueAnnotation( annotations.push(candidate); } -function mergeUsageDetails( - target: OpenAiChatUsageDetails | undefined, - next: OpenAiChatUsageDetails | undefined, -): OpenAiChatUsageDetails | undefined { - if (!target && !next) return undefined; - return { - ...(target || {}), - ...(next || {}), - }; -} - -export type OpenAiChatAggregateState = { +type ChoiceAggregate = { + index: number; + role?: 'assistant'; content: string[]; reasoning: string[]; toolCalls: Array<{ id: string; name: string; arguments: string }>; @@ -41,11 +35,18 @@ export type OpenAiChatAggregateState = { annotations: Array>; annotationUrls: Set; citations: Set; +}; + +export type OpenAiChatAggregateState = { + choices: Map; + citations: Set; usageDetails?: OpenAiChatUsageDetails; }; -export function createOpenAiChatAggregateState(): OpenAiChatAggregateState { +function createChoiceAggregate(index: number): ChoiceAggregate { return { + index, + role: undefined, content: [], reasoning: [], toolCalls: [], @@ -53,6 +54,53 @@ export function createOpenAiChatAggregateState(): OpenAiChatAggregateState { annotations: [], annotationUrls: new Set(), citations: new Set(), + }; +} + +function getChoiceAggregate(state: OpenAiChatAggregateState, index: number): ChoiceAggregate { + const existing = state.choices.get(index); + if (existing) return existing; + const created = createChoiceAggregate(index); + state.choices.set(index, created); + return created; +} + +function applyChoiceDelta(choice: ChoiceAggregate, event: OpenAiChatChoiceDelta) { + if (event.role) choice.role = event.role; + if (event.contentDelta) choice.content.push(event.contentDelta); + if (event.reasoningDelta) choice.reasoning.push(event.reasoningDelta); + if (event.finishReason !== undefined) choice.finishReason = event.finishReason ?? null; + if (Array.isArray(event.toolCallDeltas)) { + for (const delta of event.toolCallDeltas) { + if (!delta.id && !delta.name && !delta.argumentsDelta) continue; + const index = Number.isFinite(delta.index) ? Math.max(0, Math.trunc(delta.index)) : choice.toolCalls.length; + while (choice.toolCalls.length <= index) { + choice.toolCalls.push({ id: '', name: '', arguments: '' }); + } + const existing = choice.toolCalls[index]; + if (delta.id) existing.id = delta.id; + if (delta.name) existing.name = delta.name; + if (delta.argumentsDelta) existing.arguments += delta.argumentsDelta; + } + } + if (Array.isArray(event.annotations)) { + for (const annotation of event.annotations) { + pushUniqueAnnotation(choice.annotations, choice.annotationUrls, annotation); + } + } + if (Array.isArray(event.citations)) { + for (const citation of event.citations) { + if (typeof citation === 'string' && citation.trim()) { + choice.citations.add(citation.trim()); + } + } + } +} + +export function createOpenAiChatAggregateState(): OpenAiChatAggregateState { + return { + choices: new Map(), + citations: new Set(), usageDetails: undefined, }; } @@ -61,35 +109,34 @@ export function applyOpenAiChatStreamEvent( state: OpenAiChatAggregateState, event: OpenAiChatNormalizedStreamEvent, ): OpenAiChatAggregateState { - if (event.contentDelta) state.content.push(event.contentDelta); - if (event.reasoningDelta) state.reasoning.push(event.reasoningDelta); - if (event.finishReason !== undefined) state.finishReason = event.finishReason ?? null; - if (Array.isArray(event.toolCallDeltas)) { - for (const delta of event.toolCallDeltas) { - if (!delta.id && !delta.name && !delta.argumentsDelta) continue; - const index = Number.isFinite(delta.index) ? Math.max(0, Math.trunc(delta.index)) : state.toolCalls.length; - while (state.toolCalls.length <= index) { - state.toolCalls.push({ id: '', name: '', arguments: '' }); - } - const existing = state.toolCalls[index]; - if (delta.id) existing.id = delta.id; - if (delta.name) existing.name = delta.name; - if (delta.argumentsDelta) existing.arguments += delta.argumentsDelta; - } - } - if (Array.isArray(event.annotations)) { - for (const annotation of event.annotations) { - pushUniqueAnnotation(state.annotations, state.annotationUrls, annotation); - } + const choiceEvents = Array.isArray(event.choiceEvents) && event.choiceEvents.length > 0 + ? event.choiceEvents + : [{ + index: Number.isFinite(event.choiceIndex) ? Math.max(0, Math.trunc(event.choiceIndex!)) : 0, + role: event.role, + contentDelta: event.contentDelta, + reasoningDelta: event.reasoningDelta, + toolCallDeltas: event.toolCallDeltas, + finishReason: event.finishReason, + annotations: event.annotations, + citations: event.citations, + } satisfies OpenAiChatChoiceDelta]; + + for (const choiceEvent of choiceEvents) { + const choice = getChoiceAggregate(state, choiceEvent.index); + applyChoiceDelta(choice, choiceEvent); } + if (Array.isArray(event.citations)) { for (const citation of event.citations) { - if (typeof citation === 'string' && citation.trim()) { - state.citations.add(citation.trim()); - } + if (typeof citation === 'string' && citation.trim()) state.citations.add(citation.trim()); } } - state.usageDetails = mergeUsageDetails(state.usageDetails, event.usageDetails); + for (const choice of state.choices.values()) { + for (const citation of choice.citations) state.citations.add(citation); + } + + state.usageDetails = mergeChatUsageDetails(state.usageDetails, event.usageDetails); return state; } @@ -97,25 +144,36 @@ export function finalizeOpenAiChatAggregate( state: OpenAiChatAggregateState, normalized: OpenAiChatNormalizedFinalResponse, ): OpenAiChatNormalizedFinalResponse { - const mergedToolCalls = state.toolCalls.filter((item) => item.id || item.name || item.arguments); - if ( - state.content.length === 0 - && state.reasoning.length === 0 - && mergedToolCalls.length === 0 - && state.annotations.length === 0 - && state.citations.size === 0 - && !state.usageDetails - ) { + if (state.choices.size === 0 && state.citations.size === 0 && !state.usageDetails) { return normalized; } + + const finalizedChoices: OpenAiChatChoice[] = Array.from(state.choices.values()) + .sort((left, right) => left.index - right.index) + .map((choice) => ({ + index: choice.index, + ...(choice.role ? { role: choice.role } : {}), + content: choice.content.join(''), + reasoningContent: choice.reasoning.join(''), + toolCalls: choice.toolCalls.filter((item) => item.id || item.name || item.arguments), + finishReason: choice.finishReason || (choice.toolCalls.length > 0 ? 'tool_calls' : 'stop'), + ...(choice.annotations.length > 0 ? { annotations: choice.annotations } : {}), + ...(choice.citations.size > 0 ? { citations: Array.from(choice.citations).sort() } : {}), + })); + + const primaryChoice = finalizedChoices[0]; + const normalizedChoices = Array.isArray(normalized.choices) ? normalized.choices : []; + const mergedChoices = finalizedChoices.length > 0 ? finalizedChoices : normalizedChoices; + return { ...normalized, - content: state.content.join('') || normalized.content, - reasoningContent: state.reasoning.join('') || normalized.reasoningContent, - finishReason: state.finishReason ?? normalized.finishReason, - toolCalls: mergedToolCalls.length > 0 ? mergedToolCalls : normalized.toolCalls, - annotations: state.annotations.length > 0 ? state.annotations : normalized.annotations, + content: primaryChoice?.content || normalized.content, + reasoningContent: primaryChoice?.reasoningContent || normalized.reasoningContent, + finishReason: primaryChoice?.finishReason || normalized.finishReason, + toolCalls: primaryChoice?.toolCalls?.length ? primaryChoice.toolCalls : normalized.toolCalls, + annotations: primaryChoice?.annotations?.length ? primaryChoice.annotations : normalized.annotations, citations: state.citations.size > 0 ? Array.from(state.citations).sort() : normalized.citations, - usageDetails: mergeUsageDetails(normalized.usageDetails, state.usageDetails), + usageDetails: mergeChatUsageDetails(normalized.usageDetails, state.usageDetails), + ...(mergedChoices.length > 0 ? { choices: mergedChoices } : {}), }; } diff --git a/src/server/transformers/openai/chat/helpers.ts b/src/server/transformers/openai/chat/helpers.ts index d932c57..10ba632 100644 --- a/src/server/transformers/openai/chat/helpers.ts +++ b/src/server/transformers/openai/chat/helpers.ts @@ -1,7 +1,10 @@ import type { + OpenAiChatChoice, + OpenAiChatChoiceDelta, OpenAiChatNormalizedFinalResponse, OpenAiChatNormalizedStreamEvent, OpenAiChatRequestMetadata, + OpenAiChatToolCall, OpenAiChatUsageDetails, } from './model.js'; @@ -31,11 +34,20 @@ function toStringArray(value: unknown): string[] { .filter((item) => item.length > 0); } +function toFiniteNumber(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim().length > 0) { + const numeric = Number(value); + if (Number.isFinite(numeric)) return numeric; + } + return undefined; +} + function toNumericRecord(value: unknown): Record | undefined { if (!isRecord(value)) return undefined; const entries = Object.entries(value) .map(([key, raw]) => { - const numeric = typeof raw === 'number' ? raw : Number(raw); + const numeric = toFiniteNumber(raw); if (!Number.isFinite(numeric)) return null; return [key, numeric] as const; }) @@ -64,6 +76,112 @@ function annotationUrl(annotation: Record): string { return (direct || nested).trim(); } +function extractTextPart(value: unknown): string { + if (typeof value === 'string') return value; + if (!isRecord(value)) return ''; + + if (typeof value.text === 'string') return value.text; + if (typeof value.output_text === 'string') return value.output_text; + if (typeof value.content === 'string') return value.content; + if (typeof value.reasoning_content === 'string') return value.reasoning_content; + if (typeof value.reasoning === 'string') return value.reasoning; + + if (Array.isArray(value.content)) { + return value.content.map((item) => extractTextPart(item)).join(''); + } + + if (Array.isArray(value.parts)) { + return value.parts.map((item) => extractTextPart(item)).join(''); + } + + return ''; +} + +function extractTextAndReasoning(value: unknown): { content: string; reasoning: string } { + if (typeof value === 'string') return { content: value, reasoning: '' }; + if (Array.isArray(value)) { + const contentParts: string[] = []; + const reasoningParts: string[] = []; + + for (const item of value) { + if (!isRecord(item)) { + if (typeof item === 'string') contentParts.push(item); + continue; + } + + const type = typeof item.type === 'string' ? item.type : ''; + if ((type === 'reasoning' || type === 'thinking') && typeof item.text === 'string') { + reasoningParts.push(item.text); + continue; + } + if ((type === 'reasoning' || type === 'thinking') && typeof item.reasoning === 'string') { + reasoningParts.push(item.reasoning); + continue; + } + const text = extractTextPart(item); + if (text) contentParts.push(text); + } + + return { + content: contentParts.join(''), + reasoning: reasoningParts.join(''), + }; + } + + if (!isRecord(value)) return { content: '', reasoning: '' }; + return { + content: extractTextPart(value.content ?? value), + reasoning: typeof value.reasoning_content === 'string' + ? value.reasoning_content + : typeof value.reasoning === 'string' + ? value.reasoning + : '', + }; +} + +function collectToolCalls(value: unknown): OpenAiChatToolCall[] { + if (!Array.isArray(value)) return []; + const toolCalls: OpenAiChatToolCall[] = []; + + for (const item of value) { + if (!isRecord(item)) continue; + const functionPart = isRecord(item.function) ? item.function : null; + if (!functionPart) continue; + const name = typeof functionPart.name === 'string' ? functionPart.name : ''; + const id = typeof item.id === 'string' ? item.id : ''; + const args = typeof functionPart.arguments === 'string' ? functionPart.arguments : ''; + if (!id && !name && !args) continue; + toolCalls.push({ + id, + name, + arguments: args, + }); + } + + return toolCalls; +} + +function collectToolCallDeltas(value: unknown): OpenAiChatChoiceDelta['toolCallDeltas'] { + if (!Array.isArray(value)) return undefined; + const deltas = value + .map((item, itemIndex) => { + if (!isRecord(item)) return null; + const functionPart = isRecord(item.function) ? item.function : {}; + const index = toFiniteNumber(item.index); + const delta = { + index: index !== undefined ? Math.max(0, Math.trunc(index)) : itemIndex, + id: typeof item.id === 'string' && item.id.trim() ? item.id : undefined, + name: typeof functionPart.name === 'string' && functionPart.name.trim() ? functionPart.name : undefined, + argumentsDelta: typeof functionPart.arguments === 'string' ? functionPart.arguments : undefined, + }; + if (!delta.id && !delta.name && !delta.argumentsDelta) return null; + return delta; + }) + .filter((item): item is NonNullable => !!item); + + return deltas.length > 0 ? deltas : undefined; +} + export function dedupeAnnotations(input: unknown): Array> { if (!Array.isArray(input)) return []; const seen = new Set(); @@ -98,6 +216,104 @@ export function dedupeCitations(input: unknown): string[] { return Array.from(citations); } +export function extractChoiceAnnotations(choice: unknown): Array> { + if (!isRecord(choice)) return []; + const choiceMessage = asRecord(choice.message); + const choiceDelta = asRecord(choice.delta); + return dedupeAnnotations([ + ...(Array.isArray(choice.annotations) ? choice.annotations : []), + ...(Array.isArray(choiceMessage?.annotations) ? choiceMessage.annotations : []), + ...(Array.isArray(choiceDelta?.annotations) ? choiceDelta.annotations : []), + ]); +} + +export function extractChoiceCitations( + choice: unknown, + sharedCitations: Iterable = [], +): string[] { + const citations = new Set(sharedCitations); + for (const annotation of extractChoiceAnnotations(choice)) { + const url = annotationUrl(annotation); + if (url) citations.add(url); + } + if (isRecord(choice)) { + for (const citation of dedupeCitations(choice.citations)) citations.add(citation); + } + return Array.from(citations); +} + +export function extractChatChoices(payload: unknown): OpenAiChatChoice[] { + const record = asRecord(payload); + if (!record || !Array.isArray(record.choices)) return []; + + const sharedCitations = dedupeCitations(record.citations); + const choices: OpenAiChatChoice[] = []; + + for (const [index, choice] of record.choices.entries()) { + const choiceRecord = asRecord(choice); + if (!choiceRecord) continue; + const message = asRecord(choiceRecord.message) ?? asRecord(choiceRecord.delta) ?? {}; + const parsed = extractTextAndReasoning(message.content ?? message); + const toolCalls = collectToolCalls(message.tool_calls); + const annotations = extractChoiceAnnotations(choiceRecord); + const citations = extractChoiceCitations(choiceRecord, sharedCitations); + const finishReason = typeof choiceRecord.finish_reason === 'string' && choiceRecord.finish_reason.length > 0 + ? choiceRecord.finish_reason + : (toolCalls.length > 0 ? 'tool_calls' : 'stop'); + + choices.push({ + index: toFiniteNumber(choiceRecord.index) !== undefined ? Math.max(0, Math.trunc(toFiniteNumber(choiceRecord.index)!)) : index, + ...(message.role === 'assistant' ? { role: 'assistant' as const } : {}), + content: parsed.content || (toolCalls.length > 0 ? '' : ''), + reasoningContent: typeof message.reasoning_content === 'string' + ? message.reasoning_content + : parsed.reasoning, + toolCalls, + finishReason, + ...(annotations.length > 0 ? { annotations } : {}), + ...(citations.length > 0 ? { citations } : {}), + }); + } + + return choices.sort((left, right) => left.index - right.index); +} + +export function extractChatChoiceEvents(payload: unknown): OpenAiChatChoiceDelta[] { + const record = asRecord(payload); + if (!record || !Array.isArray(record.choices)) return []; + + const sharedCitations = dedupeCitations(record.citations); + const choiceEvents: OpenAiChatChoiceDelta[] = []; + + for (const [index, choice] of record.choices.entries()) { + const choiceRecord = asRecord(choice); + if (!choiceRecord) continue; + const delta = asRecord(choiceRecord.delta) ?? {}; + const parsed = extractTextAndReasoning(delta.content ?? delta); + const toolCallDeltas = collectToolCallDeltas(delta.tool_calls); + const annotations = extractChoiceAnnotations(choiceRecord); + const citations = extractChoiceCitations(choiceRecord, sharedCitations); + const choiceIndex = toFiniteNumber(choiceRecord.index) !== undefined + ? Math.max(0, Math.trunc(toFiniteNumber(choiceRecord.index)!)) + : index; + + choiceEvents.push({ + index: choiceIndex, + ...(delta.role === 'assistant' ? { role: 'assistant' as const } : {}), + ...(parsed.content ? { contentDelta: parsed.content } : {}), + ...((typeof delta.reasoning_content === 'string' ? delta.reasoning_content : parsed.reasoning) + ? { reasoningDelta: (typeof delta.reasoning_content === 'string' ? delta.reasoning_content : parsed.reasoning) } + : {}), + ...(toolCallDeltas ? { toolCallDeltas } : {}), + finishReason: typeof choiceRecord.finish_reason === 'string' ? choiceRecord.finish_reason : null, + ...(annotations.length > 0 ? { annotations } : {}), + ...(citations.length > 0 ? { citations } : {}), + }); + } + + return choiceEvents; +} + function extractAnnotationsFromPayload(payload: Record): Array> { const collected: Array> = []; const append = (value: unknown) => { @@ -216,18 +432,25 @@ export function extractChatRequestMetadata(body: unknown): OpenAiChatRequestMeta if (!raw) return undefined; const streamOptions = asRecord(raw.stream_options) ?? asRecord(raw.streamOptions); + const modalities = Array.isArray(raw.modalities) + ? raw.modalities.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) + : undefined; + const audio = asRecord(raw.audio) as OpenAiChatRequestMetadata['audio'] | null; + const reasoningBudget = toFiniteNumber(raw.reasoning_budget); + const topLogprobs = toFiniteNumber(raw.top_logprobs); + const logitBias = toNumericRecord(raw.logit_bias); const metadata: OpenAiChatRequestMetadata = { - ...(raw.modalities !== undefined ? { modalities: raw.modalities } : {}), - ...(raw.audio !== undefined ? { audio: raw.audio } : {}), - ...(raw.reasoning_effort !== undefined ? { reasoningEffort: raw.reasoning_effort } : {}), - ...(raw.reasoning_budget !== undefined ? { reasoningBudget: raw.reasoning_budget } : {}), - ...(raw.reasoning_summary !== undefined ? { reasoningSummary: raw.reasoning_summary } : {}), - ...(raw.service_tier !== undefined ? { serviceTier: raw.service_tier } : {}), - ...(raw.top_logprobs !== undefined ? { topLogprobs: raw.top_logprobs } : {}), - ...(raw.logit_bias !== undefined ? { logitBias: raw.logit_bias } : {}), - ...(raw.prompt_cache_key !== undefined ? { promptCacheKey: raw.prompt_cache_key } : {}), - ...(raw.safety_identifier !== undefined ? { safetyIdentifier: raw.safety_identifier } : {}), - ...(raw.verbosity !== undefined ? { verbosity: raw.verbosity } : {}), + ...(modalities && modalities.length > 0 ? { modalities } : {}), + ...(audio ? { audio } : {}), + ...(typeof raw.reasoning_effort === 'string' ? { reasoningEffort: raw.reasoning_effort } : {}), + ...(reasoningBudget !== undefined ? { reasoningBudget } : {}), + ...(typeof raw.reasoning_summary === 'string' ? { reasoningSummary: raw.reasoning_summary } : {}), + ...(typeof raw.service_tier === 'string' ? { serviceTier: raw.service_tier } : {}), + ...(topLogprobs !== undefined ? { topLogprobs } : {}), + ...(logitBias ? { logitBias } : {}), + ...(typeof raw.prompt_cache_key === 'string' ? { promptCacheKey: raw.prompt_cache_key } : {}), + ...(typeof raw.safety_identifier === 'string' ? { safetyIdentifier: raw.safety_identifier } : {}), + ...(typeof raw.verbosity === 'string' ? { verbosity: raw.verbosity } : {}), ...(raw.response_format !== undefined ? { responseFormat: raw.response_format } : {}), ...(streamOptions && streamOptions.include_usage !== undefined ? { streamOptionsIncludeUsage: Boolean(streamOptions.include_usage) } diff --git a/src/server/transformers/openai/chat/index.test.ts b/src/server/transformers/openai/chat/index.test.ts index 2ceba9c..642db76 100644 --- a/src/server/transformers/openai/chat/index.test.ts +++ b/src/server/transformers/openai/chat/index.test.ts @@ -61,6 +61,48 @@ describe('openAiChatTransformer.inbound', () => { streamOptionsIncludeUsage: true, }); }); + + it('normalizes typed inbound metadata without mutating upstream body', () => { + const result = openAiChatTransformer.transformRequest({ + model: 'gpt-5', + messages: [{ role: 'user', content: 'hello' }], + modalities: ['text', 42, 'audio', '', null], + audio: { voice: 'alloy', format: 'wav' }, + reasoning_effort: 'medium', + reasoning_budget: '2048', + reasoning_summary: 'concise', + service_tier: 'priority', + top_logprobs: '5', + logit_bias: { '42': '7', invalid: 'oops' }, + prompt_cache_key: 'cache-key', + safety_identifier: 'safety-id', + verbosity: 'high', + stream_options: { include_usage: 1 }, + }); + + expect(result.error).toBeUndefined(); + expect(result.value?.upstreamBody).toMatchObject({ + modalities: ['text', 42, 'audio', '', null], + reasoning_budget: '2048', + top_logprobs: '5', + logit_bias: { '42': '7', invalid: 'oops' }, + stream_options: { include_usage: 1 }, + }); + expect((result.value as any)?.requestMetadata).toEqual({ + modalities: ['text', 'audio'], + audio: { voice: 'alloy', format: 'wav' }, + reasoningEffort: 'medium', + reasoningBudget: 2048, + reasoningSummary: 'concise', + serviceTier: 'priority', + topLogprobs: 5, + logitBias: { '42': 7 }, + promptCacheKey: 'cache-key', + safetyIdentifier: 'safety-id', + verbosity: 'high', + streamOptionsIncludeUsage: true, + }); + }); }); describe('openAiChatTransformer.outbound', () => { @@ -111,6 +153,99 @@ describe('openAiChatTransformer.outbound', () => { completion_tokens_details: { reasoning_tokens: 2 }, }); }); + + it('serializes multi-choice chat responses with per-choice annotations, reasoning, tool calls, and shared citations', () => { + const normalized = openAiChatTransformer.transformFinalResponse({ + id: 'chatcmpl-multi', + model: 'gpt-5', + created: 123, + citations: ['https://shared.example', 'https://shared.example'], + usage: { + prompt_tokens: 21, + completion_tokens: 9, + total_tokens: 30, + prompt_tokens_details: { cached_tokens: 4, audio_tokens: 1 }, + completion_tokens_details: { reasoning_tokens: 3, audio_tokens: 2 }, + }, + choices: [ + { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'choice-0', + reasoning_content: 'reason-0', + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + }, + }, + { + index: 1, + finish_reason: 'tool_calls', + message: { + role: 'assistant', + content: '', + reasoning_content: 'reason-1', + tool_calls: [{ + id: 'call_1', + type: 'function', + function: { name: 'search', arguments: '{"q":"cat"}' }, + }], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + }, + }, + ], + }, 'gpt-5'); + + const payload = openAiChatTransformer.serializeFinalResponse(normalized, { + promptTokens: 21, + completionTokens: 9, + totalTokens: 30, + }) as any; + + expect(payload.choices).toHaveLength(2); + expect(payload.choices[0]).toMatchObject({ + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'choice-0', + reasoning_content: 'reason-0', + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + }, + }); + expect(payload.choices[1]).toMatchObject({ + index: 1, + finish_reason: 'tool_calls', + message: { + role: 'assistant', + content: '', + reasoning_content: 'reason-1', + tool_calls: [{ + id: 'call_1', + type: 'function', + function: { name: 'search', arguments: '{"q":"cat"}' }, + }], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + }, + }); + expect(payload.citations).toEqual([ + 'https://shared.example', + 'https://a.example', + 'https://b.example', + ]); + expect(payload.usage).toMatchObject({ + prompt_tokens_details: { cached_tokens: 4, audio_tokens: 1 }, + completion_tokens_details: { reasoning_tokens: 3, audio_tokens: 2 }, + }); + }); }); describe('openAiChatTransformer.stream', () => { @@ -157,6 +292,89 @@ describe('openAiChatTransformer.stream', () => { { type: 'url_citation', url_citation: { url: 'https://a.example' } }, ]); }); + + it('serializes multi-choice stream chunks without collapsing choice-specific deltas', () => { + const context = openAiChatTransformer.createStreamContext('gpt-5'); + const event = openAiChatTransformer.transformStreamEvent({ + id: 'chatcmpl-stream-multi', + model: 'gpt-5', + choices: [ + { + index: 0, + finish_reason: null, + delta: { + role: 'assistant', + content: 'choice-0', + reasoning_content: 'reason-0', + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + }, + }, + { + index: 1, + finish_reason: null, + delta: { + role: 'assistant', + content: 'choice-1', + tool_calls: [{ + index: 0, + id: 'call_1', + type: 'function', + function: { name: 'search', arguments: '{"q":"dog"}' }, + }], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + }, + }, + ], + citations: ['https://shared.example'], + usage: { + prompt_tokens: 11, + completion_tokens: 7, + total_tokens: 18, + prompt_tokens_details: { cached_tokens: 3 }, + }, + }, context, 'gpt-5'); + + const payloads = parseSsePayloads( + openAiChatTransformer.serializeStreamEvent(event, context, createClaudeDownstreamContext()), + ); + + expect(payloads[0].choices).toHaveLength(2); + expect((payloads[0] as any).choices[0]).toMatchObject({ + index: 0, + delta: { + role: 'assistant', + content: 'choice-0', + reasoning_content: 'reason-0', + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + }, + }); + expect((payloads[0] as any).choices[1]).toMatchObject({ + index: 1, + delta: { + role: 'assistant', + content: 'choice-1', + tool_calls: [{ + index: 0, + id: 'call_1', + function: { name: 'search', arguments: '{"q":"dog"}' }, + }], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + }, + }); + expect((payloads[0] as any).citations).toEqual([ + 'https://shared.example', + 'https://a.example', + 'https://b.example', + ]); + }); }); describe('openAiChatTransformer.aggregator', () => { @@ -231,4 +449,85 @@ describe('openAiChatTransformer.aggregator', () => { { type: 'url_citation', url_citation: { url: 'https://b.example' } }, ]); }); + + it('aggregates stream events by choice index and preserves per-choice annotations, citations, tools, reasoning, and usage details', () => { + const state = openAiChatTransformer.aggregator.createState(); + + openAiChatTransformer.aggregator.applyEvent(state, { + choiceIndex: 0, + role: 'assistant', + contentDelta: 'hello', + reasoningDelta: 'why-0', + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + citations: ['https://shared.example'], + usageDetails: { + prompt_tokens_details: { cached_tokens: 2 }, + }, + } as any); + + openAiChatTransformer.aggregator.applyEvent(state, { + choiceIndex: 1, + role: 'assistant', + contentDelta: 'tool-choice', + reasoningDelta: 'why-1', + toolCallDeltas: [{ + index: 0, + id: 'call_1', + name: 'search', + argumentsDelta: '{"q":"bird"}', + }], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + citations: ['https://other.example'], + usageDetails: { + completion_tokens_details: { reasoning_tokens: 4 }, + }, + finishReason: 'tool_calls', + } as any); + + const normalized = openAiChatTransformer.aggregator.finalize(state, { + id: 'chatcmpl-multi', + model: 'gpt-5', + created: 123, + content: '', + reasoningContent: '', + finishReason: 'stop', + toolCalls: [], + }) as any; + + expect(normalized.choices).toHaveLength(2); + expect(normalized.choices[0]).toMatchObject({ + index: 0, + content: 'hello', + reasoningContent: 'why-0', + finishReason: 'stop', + citations: ['https://shared.example'], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://a.example' } }, + ], + }); + expect(normalized.choices[1]).toMatchObject({ + index: 1, + content: 'tool-choice', + reasoningContent: 'why-1', + finishReason: 'tool_calls', + toolCalls: [{ + id: 'call_1', + name: 'search', + arguments: '{"q":"bird"}', + }], + citations: ['https://other.example'], + annotations: [ + { type: 'url_citation', url_citation: { url: 'https://b.example' } }, + ], + }); + expect(normalized.citations).toEqual(['https://other.example', 'https://shared.example']); + expect(normalized.usageDetails).toEqual({ + prompt_tokens_details: { cached_tokens: 2 }, + completion_tokens_details: { reasoning_tokens: 4 }, + }); + }); }); diff --git a/src/server/transformers/openai/chat/index.ts b/src/server/transformers/openai/chat/index.ts index ec45858..ee08686 100644 --- a/src/server/transformers/openai/chat/index.ts +++ b/src/server/transformers/openai/chat/index.ts @@ -1,9 +1,10 @@ -import { type NormalizedFinalResponse, type NormalizedStreamEvent, type ParsedDownstreamChatRequest, type StreamTransformContext } from '../../shared/normalized.js'; +import { type NormalizedFinalResponse, type NormalizedStreamEvent, type StreamTransformContext } from '../../shared/normalized.js'; import { openAiChatInbound } from './inbound.js'; import { openAiChatOutbound } from './outbound.js'; import { openAiChatStream } from './stream.js'; import { openAiChatUsage } from './usage.js'; import { createOpenAiChatAggregateState, applyOpenAiChatStreamEvent, finalizeOpenAiChatAggregate } from './aggregator.js'; +import type { OpenAiChatParsedRequest as OpenAiChatParsedRequestModel } from './model.js'; export const openAiChatTransformer = { protocol: 'openai/chat' as const, @@ -47,6 +48,20 @@ export const openAiChatTransformer = { ) { return openAiChatOutbound.serializeFinal(normalized, usage); }, + serializeUpstreamFinalAsStream( + payload: unknown, + modelName: string, + fallbackText: string, + streamContext: StreamTransformContext, + ) { + const normalizedFinal = openAiChatOutbound.normalizeFinal(payload, modelName, fallbackText); + streamContext.id = normalizedFinal.id; + streamContext.model = normalizedFinal.model; + streamContext.created = normalizedFinal.created; + return openAiChatOutbound + .buildSyntheticChunks(normalizedFinal) + .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`); + }, buildSyntheticChunks(normalized: NormalizedFinalResponse) { return openAiChatOutbound.buildSyntheticChunks(normalized); }, @@ -56,4 +71,4 @@ export const openAiChatTransformer = { }; export type OpenAiChatTransformer = typeof openAiChatTransformer; -export type OpenAiChatParsedRequest = ParsedDownstreamChatRequest; +export type OpenAiChatParsedRequest = OpenAiChatParsedRequestModel; diff --git a/src/server/transformers/openai/chat/model.ts b/src/server/transformers/openai/chat/model.ts index 218fcb8..074295d 100644 --- a/src/server/transformers/openai/chat/model.ts +++ b/src/server/transformers/openai/chat/model.ts @@ -4,18 +4,24 @@ import type { ParsedDownstreamChatRequest, } from '../../shared/normalized.js'; +export type OpenAiChatAudioRequest = { + format?: string; + voice?: string; + [key: string]: unknown; +}; + export type OpenAiChatRequestMetadata = { - modalities?: unknown; - audio?: unknown; - reasoningEffort?: unknown; - reasoningBudget?: unknown; - reasoningSummary?: unknown; - serviceTier?: unknown; - topLogprobs?: unknown; - logitBias?: unknown; - promptCacheKey?: unknown; - safetyIdentifier?: unknown; - verbosity?: unknown; + modalities?: string[]; + audio?: OpenAiChatAudioRequest; + reasoningEffort?: string; + reasoningBudget?: number; + reasoningSummary?: string; + serviceTier?: string; + topLogprobs?: number; + logitBias?: Record; + promptCacheKey?: string; + safetyIdentifier?: string; + verbosity?: string; responseFormat?: unknown; streamOptionsIncludeUsage?: boolean | null; }; @@ -29,14 +35,50 @@ export type OpenAiChatParsedRequest = ParsedDownstreamChatRequest & { requestMetadata?: OpenAiChatRequestMetadata; }; +export type OpenAiChatToolCall = { + id: string; + name: string; + arguments: string; +}; + +export type OpenAiChatChoice = { + index: number; + role?: 'assistant'; + content: string; + reasoningContent: string; + toolCalls: OpenAiChatToolCall[]; + finishReason: string; + annotations?: Array>; + citations?: string[]; +}; + export type OpenAiChatNormalizedFinalResponse = NormalizedFinalResponse & { + choices?: OpenAiChatChoice[]; annotations?: Array>; citations?: string[]; usageDetails?: OpenAiChatUsageDetails; usagePayload?: Record; }; +export type OpenAiChatChoiceDelta = { + index: number; + role?: 'assistant'; + contentDelta?: string; + reasoningDelta?: string; + toolCallDeltas?: Array<{ + index: number; + id?: string; + name?: string; + argumentsDelta?: string; + }>; + finishReason?: string | null; + annotations?: Array>; + citations?: string[]; +}; + export type OpenAiChatNormalizedStreamEvent = NormalizedStreamEvent & { + choiceIndex?: number; + choiceEvents?: OpenAiChatChoiceDelta[]; annotations?: Array>; citations?: string[]; usageDetails?: OpenAiChatUsageDetails; diff --git a/src/server/transformers/openai/chat/outbound.ts b/src/server/transformers/openai/chat/outbound.ts index a3f9c44..c844057 100644 --- a/src/server/transformers/openai/chat/outbound.ts +++ b/src/server/transformers/openai/chat/outbound.ts @@ -4,7 +4,7 @@ import { serializeFinalResponse, type NormalizedFinalResponse, } from '../../shared/normalized.js'; -import { extractChatResponseExtras } from './helpers.js'; +import { extractChatChoices, extractChatResponseExtras } from './helpers.js'; import type { OpenAiChatNormalizedFinalResponse } from './model.js'; function isRecord(value: unknown): value is Record { @@ -39,8 +39,13 @@ function mergeUsagePayload( export const openAiChatOutbound = { normalizeFinal(payload: unknown, modelName: string, fallbackText = ''): OpenAiChatNormalizedFinalResponse { + const choices = extractChatChoices(payload); + const primaryChoice = choices[0]; return { ...normalizeUpstreamFinalResponse(payload, modelName, fallbackText), + ...(choices.length > 0 ? { choices } : {}), + ...(primaryChoice?.annotations?.length ? { annotations: primaryChoice.annotations } : {}), + ...(primaryChoice?.citations?.length ? { citations: primaryChoice.citations } : {}), ...extractChatResponseExtras(payload), }; }, @@ -52,18 +57,54 @@ export const openAiChatOutbound = { usage as { promptTokens: number; completionTokens: number; totalTokens: number }, ) as Record; + const choices = Array.isArray(chatNormalized.choices) && chatNormalized.choices.length > 0 + ? chatNormalized.choices + : [{ + index: 0, + content: normalized.content, + reasoningContent: normalized.reasoningContent, + toolCalls: normalized.toolCalls, + finishReason: normalized.toolCalls.length > 0 ? 'tool_calls' : normalized.finishReason, + annotations: chatNormalized.annotations, + citations: chatNormalized.citations, + }]; + + payload.choices = choices.map((choice) => { + const message: Record = { + role: choice.role || 'assistant', + content: choice.content, + }; + if (choice.reasoningContent) { + message.reasoning_content = choice.reasoningContent; + } + if (Array.isArray(choice.toolCalls) && choice.toolCalls.length > 0) { + message.tool_calls = choice.toolCalls.map((toolCall) => ({ + id: toolCall.id, + type: 'function', + function: { + name: toolCall.name, + arguments: toolCall.arguments, + }, + })); + if (!choice.content) { + message.content = ''; + } + } + if (Array.isArray(choice.annotations) && choice.annotations.length > 0) { + message.annotations = choice.annotations; + } + + return { + index: choice.index, + message, + finish_reason: choice.finishReason || (choice.toolCalls?.length ? 'tool_calls' : 'stop'), + }; + }).sort((left, right) => left.index - right.index); + if (Array.isArray(chatNormalized.citations) && chatNormalized.citations.length > 0) { payload.citations = chatNormalized.citations; } - if (Array.isArray(chatNormalized.annotations) && chatNormalized.annotations.length > 0) { - const choices = Array.isArray(payload.choices) ? payload.choices : []; - const firstChoice = choices[0]; - if (isRecord(firstChoice) && isRecord(firstChoice.message)) { - firstChoice.message.annotations = chatNormalized.annotations; - } - } - const mergedUsage = mergeUsagePayload(payload.usage, chatNormalized); if (mergedUsage) { payload.usage = { @@ -75,8 +116,57 @@ export const openAiChatOutbound = { return payload; }, buildSyntheticChunks(normalized: NormalizedFinalResponse) { - const chunks = buildSyntheticOpenAiChunks(normalized); const chatNormalized = normalized as OpenAiChatNormalizedFinalResponse; + const choices = Array.isArray(chatNormalized.choices) && chatNormalized.choices.length > 0 + ? chatNormalized.choices + : undefined; + const chunks = choices && choices.length > 1 + ? [ + { + id: normalized.id, + object: 'chat.completion.chunk', + created: normalized.created, + model: normalized.model, + choices: choices.map((choice) => { + const delta: Record = { + role: choice.role || 'assistant', + content: choice.content || '', + }; + if (choice.reasoningContent) delta.reasoning_content = choice.reasoningContent; + if (choice.toolCalls.length > 0) { + delta.tool_calls = choice.toolCalls.map((toolCall, toolIndex) => ({ + index: toolIndex, + id: toolCall.id, + type: 'function', + function: { + name: toolCall.name, + arguments: toolCall.arguments, + }, + })); + } + if (Array.isArray(choice.annotations) && choice.annotations.length > 0) { + delta.annotations = choice.annotations; + } + return { + index: choice.index, + delta, + finish_reason: null, + }; + }).sort((left, right) => left.index - right.index), + }, + { + id: normalized.id, + object: 'chat.completion.chunk', + created: normalized.created, + model: normalized.model, + choices: choices.map((choice) => ({ + index: choice.index, + delta: {}, + finish_reason: choice.finishReason || (choice.toolCalls.length > 0 ? 'tool_calls' : 'stop'), + })).sort((left, right) => left.index - right.index), + }, + ] + : buildSyntheticOpenAiChunks(normalized); if (chunks.length <= 0) return chunks; if (Array.isArray(chatNormalized.citations) && chatNormalized.citations.length > 0) { @@ -86,7 +176,7 @@ export const openAiChatOutbound = { }; } - if (Array.isArray(chatNormalized.annotations) && chatNormalized.annotations.length > 0) { + if (!choices && Array.isArray(chatNormalized.annotations) && chatNormalized.annotations.length > 0) { const firstChunk = isRecord(chunks[0]) ? chunks[0] : null; const firstChoice = firstChunk && Array.isArray(firstChunk.choices) ? firstChunk.choices[0] : null; if (isRecord(firstChoice) && isRecord(firstChoice.delta)) { diff --git a/src/server/transformers/openai/chat/stream.ts b/src/server/transformers/openai/chat/stream.ts index 8418285..3d34bb9 100644 --- a/src/server/transformers/openai/chat/stream.ts +++ b/src/server/transformers/openai/chat/stream.ts @@ -8,7 +8,7 @@ import { type ClaudeDownstreamContext, type StreamTransformContext, } from '../../shared/normalized.js'; -import { extractChatResponseExtras } from './helpers.js'; +import { extractChatChoiceEvents, extractChatResponseExtras } from './helpers.js'; import type { OpenAiChatNormalizedStreamEvent } from './model.js'; function isRecord(value: unknown): value is Record { @@ -36,8 +36,23 @@ export const openAiChatStream = { return createStreamTransformContext(modelName); }, normalizeEvent(payload: unknown, context: StreamTransformContext, modelName: string): OpenAiChatNormalizedStreamEvent { + const choiceEvents = extractChatChoiceEvents(payload); + const primaryChoice = choiceEvents[0]; return { ...normalizeUpstreamStreamEvent(payload, context, modelName), + ...(primaryChoice + ? { + choiceIndex: primaryChoice.index, + role: primaryChoice.role, + contentDelta: primaryChoice.contentDelta, + reasoningDelta: primaryChoice.reasoningDelta, + toolCallDeltas: primaryChoice.toolCallDeltas, + finishReason: primaryChoice.finishReason, + annotations: primaryChoice.annotations, + citations: primaryChoice.citations, + } + : {}), + ...(choiceEvents.length > 0 ? { choiceEvents } : {}), ...extractChatResponseExtras(payload), }; }, @@ -46,6 +61,60 @@ export const openAiChatStream = { context: StreamTransformContext, downstreamContext?: ClaudeDownstreamContext, ): string[] { + if ( + Array.isArray(event.choiceEvents) + && event.choiceEvents.length > 0 + && (event.choiceEvents.length > 1 || event.choiceEvents[0]?.index !== 0) + ) { + const usagePayload = event.usagePayload + ? { + ...(isRecord(event.usagePayload) ? event.usagePayload : {}), + ...(event.usageDetails?.prompt_tokens_details + ? { prompt_tokens_details: event.usageDetails.prompt_tokens_details } + : {}), + ...(event.usageDetails?.completion_tokens_details + ? { completion_tokens_details: event.usageDetails.completion_tokens_details } + : {}), + } + : undefined; + return [`data: ${JSON.stringify({ + id: context.id || `chatcmpl-${Date.now()}`, + object: 'chat.completion.chunk', + created: context.created, + model: context.model, + choices: event.choiceEvents + .map((choiceEvent) => { + const delta: Record = {}; + if (choiceEvent.role) delta.role = choiceEvent.role; + if (choiceEvent.contentDelta !== undefined) delta.content = choiceEvent.contentDelta; + if (choiceEvent.reasoningDelta) delta.reasoning_content = choiceEvent.reasoningDelta; + if (Array.isArray(choiceEvent.toolCallDeltas) && choiceEvent.toolCallDeltas.length > 0) { + delta.tool_calls = choiceEvent.toolCallDeltas.map((toolCall) => ({ + index: toolCall.index, + ...(toolCall.id ? { id: toolCall.id } : {}), + type: 'function', + function: { + ...(toolCall.name ? { name: toolCall.name } : {}), + ...(toolCall.argumentsDelta !== undefined ? { arguments: toolCall.argumentsDelta } : {}), + }, + })); + } + if (Array.isArray(choiceEvent.annotations) && choiceEvent.annotations.length > 0) { + delta.annotations = choiceEvent.annotations; + } + + return { + index: choiceEvent.index, + delta, + finish_reason: choiceEvent.finishReason ?? null, + }; + }) + .sort((left, right) => left.index - right.index), + ...(Array.isArray(event.citations) && event.citations.length > 0 ? { citations: event.citations } : {}), + ...(usagePayload && Object.keys(usagePayload).length > 0 ? { usage: usagePayload } : {}), + })}\n\n`]; + } + const lines = serializeNormalizedStreamEvent( 'openai', event, diff --git a/src/server/transformers/openai/responses/aggregator.test.ts b/src/server/transformers/openai/responses/aggregator.test.ts index c7f20bf..cf9079c 100644 --- a/src/server/transformers/openai/responses/aggregator.test.ts +++ b/src/server/transformers/openai/responses/aggregator.test.ts @@ -1,336 +1,180 @@ import { describe, expect, it } from 'vitest'; +import { createStreamTransformContext } from '../../shared/normalized.js'; import { createOpenAiResponsesAggregateState, serializeConvertedResponsesEvents, - type OpenAiResponsesAggregateState, - completeResponsesStream, } from './aggregator.js'; -import { openAiResponsesStream } from './stream.js'; -function collectSerializedEvents(input: string[]): Array<{ event: string; data: any }> { - const events: Array<{ event: string; data: any }> = []; - for (const chunk of input) { - const blocks = chunk.split('\n\n').filter((item) => item.trim().length > 0); - for (const block of blocks) { - const lines = block.split('\n'); - const eventLine = lines.find((line) => line.startsWith('event: ')); - const dataLine = lines.find((line) => line.startsWith('data: ')); - if (!dataLine) continue; - const rawData = dataLine.slice('data: '.length); - if (rawData === '[DONE]') { - events.push({ event: eventLine?.slice('event: '.length) || 'done', data: '[DONE]' }); - continue; +function parseSsePayloads(lines: string[]): Array> { + return lines + .flatMap((line) => line.split('\n\n').filter((block) => block.trim().length > 0)) + .map((block) => { + const dataLine = block + .split('\n') + .find((line) => line.startsWith('data: ')); + if (!dataLine) return null; + try { + return JSON.parse(dataLine.slice('data: '.length)) as Record; + } catch { + return null; } - events.push({ - event: eventLine?.slice('event: '.length) || '', - data: JSON.parse(rawData), - }); - } - } - return events; + }) + .filter((item): item is Record => !!item); } -function feedResponsesEvent( - state: OpenAiResponsesAggregateState, - payload: Record, -): Array<{ event: string; data: any }> { - const context = openAiResponsesStream.createContext('gpt-5'); - const normalizedEvent = openAiResponsesStream.normalizeEvent(payload, context, 'gpt-5'); - return collectSerializedEvents( +describe('serializeConvertedResponsesEvents', () => { + it('aggregates reasoning summary events into the completed response payload', () => { + const state = createOpenAiResponsesAggregateState('gpt-5'); + const streamContext = createStreamTransformContext('gpt-5'); + const usage = { + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }; + serializeConvertedResponsesEvents({ state, - streamContext: context, - event: normalizedEvent, - usage: { - promptTokens: 0, - completionTokens: 0, - totalTokens: 0, - }, - }), - ); -} - -describe('openAiResponsesStream / aggregator', () => { - it('keeps reasoning separate from assistant text for normalized events and merges usage details', () => { - const state = createOpenAiResponsesAggregateState('gpt-5'); - const context = openAiResponsesStream.createContext('gpt-5'); - - const contentEvent = { - role: 'assistant', - contentDelta: 'hello', - }; - const reasoningEvent = { - reasoningDelta: 'plan first', - responsesPayload: { - usage: { - input_tokens: 5, - output_tokens: 3, - total_tokens: 8, - output_tokens_details: { reasoning_tokens: 2 }, + streamContext, + usage, + event: { + responsesEventType: 'response.reasoning_summary_part.added', + responsesPayload: { + type: 'response.reasoning_summary_part.added', + item_id: 'rs_1', + output_index: 0, + summary_index: 0, + part: { + type: 'summary_text', + text: '', + }, }, }, - }; + }); - collectSerializedEvents( - serializeConvertedResponsesEvents({ - state, - streamContext: context, - event: contentEvent as any, - usage: { promptTokens: 5, completionTokens: 3, totalTokens: 8 }, - }), - ); - - const reasoningEmitted = collectSerializedEvents( - serializeConvertedResponsesEvents({ - state, - streamContext: context, - event: reasoningEvent as any, - usage: { promptTokens: 5, completionTokens: 3, totalTokens: 8 }, - }), - ); - - expect(reasoningEmitted.some((item) => item.event === 'response.reasoning_summary_part.added')).toBe(true); - expect(reasoningEmitted.some((item) => item.event === 'response.reasoning_summary_text.delta')).toBe(true); - - const completed = collectSerializedEvents( - completeResponsesStream(state, context, { promptTokens: 5, completionTokens: 3, totalTokens: 8 }), - ).find((item) => item.event === 'response.completed'); - - expect(completed?.data.response.output).toEqual([ - { - id: expect.any(String), - type: 'message', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: 'hello', - }, - ], + serializeConvertedResponsesEvents({ + state, + streamContext, + usage, + event: { + responsesEventType: 'response.reasoning_summary_text.delta', + responsesPayload: { + type: 'response.reasoning_summary_text.delta', + item_id: 'rs_1', + output_index: 0, + summary_index: 0, + delta: 'Think ', + }, }, - { - id: expect.any(String), - type: 'reasoning', - status: 'completed', - summary: [ - { - type: 'summary_text', - text: 'plan first', + }); + + const completedLines = serializeConvertedResponsesEvents({ + state, + streamContext, + usage, + event: { + responsesEventType: 'response.completed', + responsesPayload: { + type: 'response.completed', + response: { + id: 'resp_1', + model: 'gpt-5', + usage: { + input_tokens: 1, + output_tokens: 2, + total_tokens: 3, + }, }, - ], + }, }, - ]); - expect(completed?.data.response.usage).toMatchObject({ - input_tokens: 5, - output_tokens: 3, - total_tokens: 8, - output_tokens_details: { reasoning_tokens: 2 }, + }); + + const payloads = parseSsePayloads(completedLines); + const completed = payloads.find((item) => item.type === 'response.completed'); + expect(completed).toBeTruthy(); + expect(completed?.response).toMatchObject({ + id: 'resp_1', + output: [ + { + id: 'rs_1', + type: 'reasoning', + summary: [ + { + type: 'summary_text', + text: 'Think ', + }, + ], + }, + ], }); }); - it('tracks reasoning summary events into completed response output', () => { + it('preserves richer image_generation_call fields while aggregating progress events', () => { const state = createOpenAiResponsesAggregateState('gpt-5'); + const streamContext = createStreamTransformContext('gpt-5'); + const usage = { + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }; - const emitted = [ - ...feedResponsesEvent(state, { - type: 'response.output_item.added', - output_index: 0, - item: { - id: 'rs_1', - type: 'reasoning', - status: 'in_progress', - summary: [], + serializeConvertedResponsesEvents({ + state, + streamContext, + usage, + event: { + responsesEventType: 'response.image_generation_call.partial_image', + responsesPayload: { + type: 'response.image_generation_call.partial_image', + item_id: 'img_1', + output_index: 0, + partial_image_index: 0, + partial_image_b64: 'partial', + background: 'transparent', + output_format: 'png', + quality: 'high', + size: '1024x1024', }, - }), - ...feedResponsesEvent(state, { - type: 'response.reasoning_summary_part.added', - item_id: 'rs_1', - output_index: 0, - summary_index: 0, - part: { - type: 'summary_text', - text: '', - }, - }), - ...feedResponsesEvent(state, { - type: 'response.reasoning_summary_text.delta', - item_id: 'rs_1', - output_index: 0, - summary_index: 0, - delta: 'Plan', - }), - ...feedResponsesEvent(state, { - type: 'response.reasoning_summary_text.done', - item_id: 'rs_1', - output_index: 0, - summary_index: 0, - text: 'Plan it out', - }), - ...feedResponsesEvent(state, { - type: 'response.reasoning_summary_part.done', - item_id: 'rs_1', - output_index: 0, - summary_index: 0, - part: { - type: 'summary_text', - text: 'Plan it out', - }, - }), - ...feedResponsesEvent(state, { - type: 'response.output_item.done', - output_index: 0, - item: { - id: 'rs_1', - type: 'reasoning', - status: 'completed', - summary: [], - }, - }), - ...feedResponsesEvent(state, { - type: 'response.completed', - response: { - id: 'resp_reasoning', - object: 'response', - created_at: 1700000000, - status: 'completed', - model: 'gpt-5', - output: [], - }, - }), - ]; - - expect(emitted.some((item) => item.event === 'response.reasoning_summary_part.added')).toBe(true); - expect(emitted.some((item) => item.event === 'response.reasoning_summary_text.done')).toBe(true); - - const completed = emitted.find((item) => item.event === 'response.completed'); - expect(completed?.data.response.output).toEqual([ - { - id: 'rs_1', - type: 'reasoning', - status: 'completed', - summary: [ - { - type: 'summary_text', - text: 'Plan it out', - }, - ], }, - ]); - }); + }); - it('tracks custom tool input and image generation results', () => { - const state = createOpenAiResponsesAggregateState('gpt-5'); - - const emitted = [ - ...feedResponsesEvent(state, { - type: 'response.output_item.added', - output_index: 0, - item: { - id: 'ct_1', - type: 'custom_tool_call', - status: 'in_progress', - call_id: 'ct_1', - name: 'browser', - input: '', + const completedLines = serializeConvertedResponsesEvents({ + state, + streamContext, + usage, + event: { + responsesEventType: 'response.image_generation_call.completed', + responsesPayload: { + type: 'response.image_generation_call.completed', + item_id: 'img_1', + output_index: 0, + result: 'final-image', + background: 'transparent', + output_format: 'png', + quality: 'high', + size: '1024x1024', }, - }), - ...feedResponsesEvent(state, { - type: 'response.custom_tool_call_input.delta', - item_id: 'ct_1', - output_index: 0, - delta: 'open ', - }), - ...feedResponsesEvent(state, { - type: 'response.custom_tool_call_input.done', - item_id: 'ct_1', - output_index: 0, - input: 'open example.com', - }), - ...feedResponsesEvent(state, { - type: 'response.output_item.done', - output_index: 0, - item: { - id: 'ct_1', - type: 'custom_tool_call', - status: 'completed', - call_id: 'ct_1', - name: 'browser', - input: 'open example.com', - }, - }), - ...feedResponsesEvent(state, { - type: 'response.output_item.added', - output_index: 1, - item: { - id: 'img_1', - type: 'image_generation_call', - status: 'in_progress', - result: null, - }, - }), - ...feedResponsesEvent(state, { - type: 'response.image_generation_call.partial_image', - item_id: 'img_1', - output_index: 1, - partial_image_index: 0, - partial_image_b64: 'abc123', - }), - ...feedResponsesEvent(state, { - type: 'response.image_generation_call.completed', - item_id: 'img_1', - output_index: 1, - result: 'data:image/png;base64,final', - }), - ...feedResponsesEvent(state, { - type: 'response.output_item.done', - output_index: 1, - item: { - id: 'img_1', - type: 'image_generation_call', - status: 'completed', - result: 'data:image/png;base64,final', - }, - }), - ...feedResponsesEvent(state, { - type: 'response.completed', - response: { - id: 'resp_tools', - object: 'response', - created_at: 1700000001, - status: 'completed', - model: 'gpt-5', - output: [], - }, - }), - ]; - - expect(emitted.some((item) => item.event === 'response.custom_tool_call_input.done')).toBe(true); - expect(emitted.some((item) => item.event === 'response.image_generation_call.completed')).toBe(true); - - const completed = emitted.find((item) => item.event === 'response.completed'); - expect(completed?.data.response.output).toEqual([ - { - id: 'ct_1', - type: 'custom_tool_call', - status: 'completed', - call_id: 'ct_1', - name: 'browser', - input: 'open example.com', }, - { - id: 'img_1', - type: 'image_generation_call', - status: 'completed', - result: 'data:image/png;base64,final', - partial_images: [ - { - partial_image_index: 0, - partial_image_b64: 'abc123', - }, - ], - }, - ]); + }); + + const payloads = parseSsePayloads(completedLines); + const imageEvent = payloads.find((item) => item.type === 'response.image_generation_call.completed'); + expect(imageEvent).toBeTruthy(); + expect(state.outputItems[0]).toMatchObject({ + id: 'img_1', + type: 'image_generation_call', + status: 'completed', + result: 'final-image', + background: 'transparent', + output_format: 'png', + quality: 'high', + size: '1024x1024', + partial_images: [ + { + partial_image_index: 0, + partial_image_b64: 'partial', + }, + ], + }); }); }); diff --git a/src/server/transformers/openai/responses/aggregator.ts b/src/server/transformers/openai/responses/aggregator.ts index 3a07a22..64e4e3b 100644 --- a/src/server/transformers/openai/responses/aggregator.ts +++ b/src/server/transformers/openai/responses/aggregator.ts @@ -407,6 +407,26 @@ function serializeOriginalResponsesEvent(eventType: string, payload: Record, +): void { + const passthroughKeys = [ + 'background', + 'output_format', + 'quality', + 'size', + 'revised_prompt', + 'mime_type', + ] as const; + + for (const key of passthroughKeys) { + if (payload[key] !== undefined) { + item[key] = cloneJson(payload[key]); + } + } +} + function computeNovelDelta(existingText: string, incomingDelta: string): string { if (!incomingDelta) return ''; if (!existingText) return incomingDelta; @@ -555,6 +575,7 @@ function applyOriginalResponsesPayload( }); entry.item.partial_images = partialImages; } + mergeImageGenerationFields(entry.item, payload); if (payload.result !== undefined) { entry.item.result = payload.result; } diff --git a/src/server/transformers/openai/responses/compatibility.ts b/src/server/transformers/openai/responses/compatibility.ts index 8dc51fa..6a596d5 100644 --- a/src/server/transformers/openai/responses/compatibility.ts +++ b/src/server/transformers/openai/responses/compatibility.ts @@ -165,6 +165,117 @@ export function normalizeResponsesInputForCompatibility(input: unknown): unknown return input; } +export function buildResponsesCompatibilityBodies( + body: Record, +): Record[] { + const candidates: Record[] = []; + const seen = new Set(); + try { + const originalKey = JSON.stringify(body); + if (originalKey) seen.add(originalKey); + } catch { + // ignore non-serializable bodies + } + + const push = (next: Record | null) => { + if (!next) return; + let key = ''; + try { + key = JSON.stringify(next); + } catch { + return; + } + if (!key || seen.has(key)) return; + seen.add(key); + candidates.push(next); + }; + + push(stripResponsesMetadata(body)); + push(buildCoreResponsesBody(body)); + push(buildStrictResponsesBody(body)); + return candidates; +} + +export function buildResponsesCompatibilityHeaderCandidates( + headers: Record, + stream: boolean, +): Record[] { + const candidates: Record[] = []; + const seen = new Set(); + const push = (next: Record) => { + const normalizedEntries = Object.entries(next) + .filter(([, value]) => typeof value === 'string' && value.trim().length > 0) + .map(([key, value]) => [key.toLowerCase(), value] as const) + .sort(([a], [b]) => a.localeCompare(b)); + const key = JSON.stringify(normalizedEntries); + if (!key || seen.has(key)) return; + seen.add(key); + candidates.push(Object.fromEntries(normalizedEntries)); + }; + + push(headers); + + const minimal: Record = {}; + for (const [rawKey, rawValue] of Object.entries(headers)) { + const key = rawKey.toLowerCase(); + if ( + key === 'authorization' + || key === 'x-api-key' + || key === 'content-type' + || key === 'accept' + ) { + minimal[key] = rawValue; + } + } + if (!minimal['content-type']) minimal['content-type'] = 'application/json'; + if (stream && !minimal.accept) minimal.accept = 'text/event-stream'; + push(minimal); + + return candidates; +} + +export function shouldRetryResponsesCompatibility(input: { + endpoint: string; + status: number; + rawErrText: string; +}): boolean { + if (input.endpoint !== 'responses') return false; + if (input.status !== 400) return false; + const parsedError = parseUpstreamErrorShape(input.rawErrText); + const type = parsedError.type.trim().toLowerCase(); + const code = parsedError.code.trim().toLowerCase(); + const message = parsedError.message.trim().toLowerCase(); + const compact = `${type} ${code} ${message}`.trim(); + const rawCompact = (input.rawErrText || '').toLowerCase(); + + if ( + compact.includes('invalid_api_key') + || compact.includes('authentication') + || compact.includes('unauthorized') + || compact.includes('forbidden') + || compact.includes('insufficient_quota') + || compact.includes('rate_limit') + ) { + return false; + } + + if (type === 'upstream_error' || code === 'upstream_error') return true; + if (message === 'upstream_error' || message === 'upstream request failed') return true; + if (rawCompact.includes('upstream_error')) return true; + + return true; +} + +export function shouldDowngradeResponsesChatToMessages( + endpointPath: string, + status: number, + upstreamErrorText: string, +): boolean { + if (!endpointPath.includes('/chat/completions')) return false; + if (status < 400 || status >= 500) return false; + return /messages\s+is\s+required/i.test(upstreamErrorText); +} + function extractTextFromResponsesContent(content: unknown): string { if (typeof content === 'string') return content.trim(); if (!Array.isArray(content)) return ''; @@ -216,6 +327,79 @@ function toFiniteNumber(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null; } +function parseUpstreamErrorShape(rawText: string): { + type: string; + code: string; + message: string; +} { + try { + const parsed = JSON.parse(rawText) as Record; + const error = (parsed.error && typeof parsed.error === 'object') + ? parsed.error as Record + : parsed; + return { + type: typeof error.type === 'string' ? error.type.trim().toLowerCase() : '', + code: typeof error.code === 'string' ? error.code.trim().toLowerCase() : '', + message: typeof error.message === 'string' ? error.message.trim() : '', + }; + } catch { + return { type: '', code: '', message: '' }; + } +} + +function stripResponsesMetadata( + body: Record, +): Record | null { + if (!Object.prototype.hasOwnProperty.call(body, 'metadata')) return null; + const next = { ...body }; + delete next.metadata; + return next; +} + +function buildCoreResponsesBody( + body: Record, +): Record | null { + const model = typeof body.model === 'string' ? body.model.trim() : ''; + if (!model) return null; + if (body.input === undefined) return null; + + const core: Record = { + model, + input: body.input, + stream: body.stream === true, + }; + + const maxOutputTokens = toFiniteNumber(body.max_output_tokens); + if (maxOutputTokens !== null && maxOutputTokens > 0) { + core.max_output_tokens = Math.trunc(maxOutputTokens); + } + + const temperature = toFiniteNumber(body.temperature); + if (temperature !== null) core.temperature = temperature; + + const topP = toFiniteNumber(body.top_p); + if (topP !== null) core.top_p = topP; + + const instructions = typeof body.instructions === 'string' ? body.instructions.trim() : ''; + if (instructions) core.instructions = instructions; + + return core; +} + +function buildStrictResponsesBody( + body: Record, +): Record | null { + const model = typeof body.model === 'string' ? body.model.trim() : ''; + if (!model) return null; + if (body.input === undefined) return null; + + return { + model, + input: body.input, + stream: body.stream === true, + }; +} + const ALLOWED_RESPONSES_FIELDS = new Set([ 'model', 'input', diff --git a/src/server/transformers/openai/responses/conversion.test.ts b/src/server/transformers/openai/responses/conversion.test.ts index a359f25..0945d46 100644 --- a/src/server/transformers/openai/responses/conversion.test.ts +++ b/src/server/transformers/openai/responses/conversion.test.ts @@ -19,6 +19,7 @@ describe('sanitizeResponsesBodyForProxy', () => { stream_options: { include_obfuscation: true }, background: true, text: { format: { type: 'text' }, verbosity: 'high' }, + top_logprobs: 2, }, 'gpt-5', true, @@ -34,6 +35,48 @@ describe('sanitizeResponsesBodyForProxy', () => { stream_options: { include_obfuscation: true }, background: true, text: { format: { type: 'text' }, verbosity: 'high' }, + top_logprobs: 2, + }); + }); + + it('normalizes current Responses inbound parity fields', () => { + const result = sanitizeResponsesBodyForProxy( + { + input: 'hello', + safety_identifier: ' safe-user-4 ', + max_tool_calls: '5', + prompt_cache_key: ' cache-key-2 ', + prompt_cache_retention: ' 24h ', + stream_options: { include_obfuscation: 'true', extra: 'keep-me' }, + background: 'false', + text: { format: { type: 'text' }, verbosity: ' high ' }, + truncation: ' auto ', + previous_response_id: ' resp_prev_2 ', + include: [' reasoning.encrypted_content ', '', 123, 'message.input_image.image_url'], + top_logprobs: '7', + user: ' user-456 ', + service_tier: ' priority ', + }, + 'gpt-5', + false, + ); + + expect(result).toMatchObject({ + model: 'gpt-5', + stream: false, + safety_identifier: 'safe-user-4', + max_tool_calls: 5, + prompt_cache_key: 'cache-key-2', + prompt_cache_retention: '24h', + stream_options: { include_obfuscation: true, extra: 'keep-me' }, + background: false, + text: { format: { type: 'text' }, verbosity: 'high' }, + truncation: 'auto', + previous_response_id: 'resp_prev_2', + include: ['reasoning.encrypted_content', 'message.input_image.image_url'], + top_logprobs: 7, + user: 'user-456', + service_tier: 'priority', }); }); }); @@ -94,6 +137,92 @@ describe('convertOpenAiBodyToResponsesBody', () => { ], }); }); + + it('maps OpenAI response_format into Responses text.format while preserving verbosity', () => { + const result = convertOpenAiBodyToResponsesBody( + { + model: 'gpt-5', + messages: [{ role: 'user', content: 'return structured data' }], + response_format: { + type: 'json_schema', + json_schema: { + name: 'payload', + schema: { + type: 'object', + properties: { + name: { type: 'string' }, + }, + required: ['name'], + }, + }, + }, + verbosity: 'high', + }, + 'gpt-5', + false, + ); + + expect(result).toMatchObject({ + text: { + format: { + type: 'json_schema', + json_schema: { + name: 'payload', + schema: { + type: 'object', + properties: { + name: { type: 'string' }, + }, + required: ['name'], + }, + }, + }, + verbosity: 'high', + }, + }); + }); + + it('normalizes and preserves field parity when converting from OpenAI-compatible input', () => { + const result = convertOpenAiBodyToResponsesBody( + { + model: 'gpt-5', + messages: [{ role: 'user', content: 'hello' }], + safety_identifier: ' safe-user-5 ', + max_tool_calls: '6', + prompt_cache_key: ' cache-key-3 ', + prompt_cache_retention: ' in-memory ', + stream_options: { include_obfuscation: 'false' }, + background: 'true', + verbosity: ' medium ', + truncation: ' disabled ', + previous_response_id: ' resp_prev_3 ', + include: 'reasoning.encrypted_content', + top_logprobs: '4', + user: ' user-789 ', + service_tier: ' flex ', + }, + 'gpt-5', + true, + ); + + expect(result).toMatchObject({ + model: 'gpt-5', + stream: true, + safety_identifier: 'safe-user-5', + max_tool_calls: 6, + prompt_cache_key: 'cache-key-3', + prompt_cache_retention: 'in-memory', + stream_options: { include_obfuscation: false }, + background: true, + text: { verbosity: 'medium' }, + truncation: 'disabled', + previous_response_id: 'resp_prev_3', + include: ['reasoning.encrypted_content'], + top_logprobs: 4, + user: 'user-789', + service_tier: 'flex', + }); + }); }); describe('convertResponsesBodyToOpenAiBody', () => { @@ -258,4 +387,88 @@ describe('convertResponsesBodyToOpenAiBody', () => { top_logprobs: 4, }); }); + + it('maps Responses text.format back into OpenAI response_format', () => { + const result = convertResponsesBodyToOpenAiBody( + { + model: 'gpt-5', + input: 'hello', + text: { + format: { + type: 'json_schema', + json_schema: { + name: 'payload', + schema: { + type: 'object', + properties: { + value: { type: 'string' }, + }, + }, + }, + }, + verbosity: 'medium', + }, + }, + 'gpt-5', + false, + ); + + expect(result).toMatchObject({ + response_format: { + type: 'json_schema', + json_schema: { + name: 'payload', + schema: { + type: 'object', + properties: { + value: { type: 'string' }, + }, + }, + }, + }, + verbosity: 'medium', + }); + }); + + it('normalizes field parity when converting Responses input back to OpenAI-compatible input', () => { + const result = convertResponsesBodyToOpenAiBody( + { + model: 'gpt-5', + input: 'hello', + safety_identifier: ' safe-user-6 ', + max_tool_calls: '8', + prompt_cache_key: ' cache-key-4 ', + prompt_cache_retention: ' 24h ', + stream_options: { include_obfuscation: 'true' }, + background: 'false', + text: { verbosity: ' low ' }, + truncation: ' auto ', + previous_response_id: ' resp_prev_4 ', + include: [' reasoning.encrypted_content ', ''], + top_logprobs: '9', + user: ' user-999 ', + service_tier: ' default ', + }, + 'gpt-5', + false, + ); + + expect(result).toMatchObject({ + model: 'gpt-5', + stream: false, + safety_identifier: 'safe-user-6', + max_tool_calls: 8, + prompt_cache_key: 'cache-key-4', + prompt_cache_retention: '24h', + stream_options: { include_obfuscation: true }, + background: false, + verbosity: 'low', + truncation: 'auto', + previous_response_id: 'resp_prev_4', + include: ['reasoning.encrypted_content'], + top_logprobs: 9, + user: 'user-999', + service_tier: 'default', + }); + }); }); diff --git a/src/server/transformers/openai/responses/conversion.ts b/src/server/transformers/openai/responses/conversion.ts index 2ce4582..f8b181d 100644 --- a/src/server/transformers/openai/responses/conversion.ts +++ b/src/server/transformers/openai/responses/conversion.ts @@ -15,6 +15,29 @@ function toFiniteNumber(value: unknown): number | null { return typeof value === 'number' && Number.isFinite(value) ? value : null; } +function toFiniteIntegerLike(value: unknown): number | null { + if (typeof value === 'number' && Number.isFinite(value)) { + return Math.trunc(value); + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed) return null; + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? Math.trunc(parsed) : null; + } + return null; +} + +function toBooleanLike(value: unknown): boolean | undefined { + if (typeof value === 'boolean') return value; + if (typeof value === 'string') { + const trimmed = value.trim().toLowerCase(); + if (trimmed === 'true') return true; + if (trimmed === 'false') return false; + } + return undefined; +} + function safeJsonStringify(value: unknown): string { try { return JSON.stringify(value); @@ -40,6 +63,103 @@ function cloneJsonValue(value: T): T { return value; } +function normalizeOptionalTrimmedString(value: unknown): string | undefined { + const trimmed = asTrimmedString(value); + return trimmed || undefined; +} + +function normalizeIncludeList(value: unknown): unknown { + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed ? [trimmed] : []; + } + + if (!Array.isArray(value)) return value; + + return value + .map((item) => (typeof item === 'string' ? item.trim() : '')) + .filter((item) => item.length > 0); +} + +function normalizeTextConfig( + rawText: unknown, + fallbackVerbosity?: unknown, +): Record | undefined { + const textConfig = cloneRecord(rawText) || {}; + const verbosity = ( + normalizeOptionalTrimmedString(textConfig.verbosity) + ?? normalizeOptionalTrimmedString(fallbackVerbosity) + ); + if (verbosity) { + textConfig.verbosity = verbosity; + } + return Object.keys(textConfig).length > 0 ? textConfig : undefined; +} + +function normalizeStreamOptions(rawStreamOptions: unknown): unknown { + const streamOptions = cloneRecord(rawStreamOptions); + if (!streamOptions) return rawStreamOptions; + + const includeObfuscation = toBooleanLike(streamOptions.include_obfuscation); + if (includeObfuscation !== undefined) { + streamOptions.include_obfuscation = includeObfuscation; + } + + return streamOptions; +} + +function normalizeResponsesRequestFieldParity( + body: Record, + options?: { verbositySource?: unknown }, +): Record { + const normalized: Record = { ...body }; + + const safetyIdentifier = normalizeOptionalTrimmedString(normalized.safety_identifier); + if (safetyIdentifier) normalized.safety_identifier = safetyIdentifier; + + const maxToolCalls = toFiniteIntegerLike(normalized.max_tool_calls); + if (maxToolCalls !== null) normalized.max_tool_calls = maxToolCalls; + + const promptCacheKey = normalizeOptionalTrimmedString(normalized.prompt_cache_key); + if (promptCacheKey) normalized.prompt_cache_key = promptCacheKey; + + const promptCacheRetention = normalizeOptionalTrimmedString(normalized.prompt_cache_retention); + if (promptCacheRetention) normalized.prompt_cache_retention = promptCacheRetention; + + const background = toBooleanLike(normalized.background); + if (background !== undefined) normalized.background = background; + + const user = normalizeOptionalTrimmedString(normalized.user); + if (user) normalized.user = user; + + const previousResponseId = normalizeOptionalTrimmedString(normalized.previous_response_id); + if (previousResponseId) normalized.previous_response_id = previousResponseId; + + const truncation = normalizeOptionalTrimmedString(normalized.truncation); + if (truncation) normalized.truncation = truncation; + + const serviceTier = normalizeOptionalTrimmedString(normalized.service_tier); + if (serviceTier) normalized.service_tier = serviceTier; + + const topLogprobs = toFiniteIntegerLike(normalized.top_logprobs); + if (topLogprobs !== null) normalized.top_logprobs = topLogprobs; + + if (normalized.include !== undefined) { + normalized.include = normalizeIncludeList(normalized.include); + } + + if (normalized.stream_options !== undefined) { + normalized.stream_options = normalizeStreamOptions(normalized.stream_options); + } + + const textConfig = normalizeTextConfig(normalized.text, options?.verbositySource); + if (textConfig) { + normalized.text = textConfig; + } + + return normalized; +} + function parseJsonString(raw: string): unknown { const trimmed = raw.trim(); if (!trimmed) return {}; @@ -219,6 +339,7 @@ const ALLOWED_RESPONSES_FIELDS = new Set([ 'include', 'response_format', 'service_tier', + 'top_logprobs', 'stop', 'n', ]); @@ -250,6 +371,10 @@ export function sanitizeResponsesBodyForProxy( } } + normalized = normalizeResponsesRequestFieldParity(normalized, { + verbositySource: body.verbosity, + }); + const sanitized: Record = {}; for (const [key, value] of Object.entries(normalized)) { if (!ALLOWED_RESPONSES_FIELDS.has(key)) continue; @@ -388,23 +513,23 @@ export function convertOpenAiBodyToResponsesBody( if (openaiBody.service_tier !== undefined) body.service_tier = openaiBody.service_tier; if (openaiBody.top_logprobs !== undefined) body.top_logprobs = openaiBody.top_logprobs; if (openaiBody.stream_options !== undefined) body.stream_options = openaiBody.stream_options; - - const textConfig = cloneRecord(openaiBody.text) || {}; - const verbosity = ( - asTrimmedString(openaiBody.verbosity) - || asTrimmedString(isRecord(openaiBody.text) ? openaiBody.text.verbosity : undefined) - ); - if (verbosity) { - textConfig.verbosity = verbosity; + if (openaiBody.response_format !== undefined) { + const existingTextConfig = cloneRecord(body.text) || {}; + existingTextConfig.format = cloneJsonValue(openaiBody.response_format); + body.text = existingTextConfig; } - if (Object.keys(textConfig).length > 0) { + + const textConfig = normalizeTextConfig(body.text, openaiBody.verbosity); + if (textConfig) { body.text = textConfig; } const responsesToolChoice = convertOpenAiToolChoiceToResponses(openaiBody.tool_choice); if (responsesToolChoice !== undefined) body.tool_choice = responsesToolChoice; - return normalizeResponsesBodyForCompatibility(body); + return normalizeResponsesBodyForCompatibility( + normalizeResponsesRequestFieldParity(body, { verbositySource: openaiBody.verbosity }), + ); } type OpenAiToolCall = { @@ -560,8 +685,9 @@ export function convertResponsesBodyToOpenAiBody( modelName: string, stream: boolean, ): Record { + const normalizedBody = normalizeResponsesRequestFieldParity(body); const messages: Array> = []; - const input = body.input; + const input = normalizedBody.input; let functionCallIndex = 0; let pendingToolCalls: OpenAiToolCall[] = []; @@ -655,7 +781,7 @@ export function convertResponsesBodyToOpenAiBody( } flushPendingToolCalls(); - const instructions = asTrimmedString(body.instructions); + const instructions = asTrimmedString(normalizedBody.instructions); if (instructions) { messages.unshift({ role: 'system', content: instructions }); } @@ -666,32 +792,35 @@ export function convertResponsesBodyToOpenAiBody( messages, }; - if (typeof body.temperature === 'number' && Number.isFinite(body.temperature)) { - payload.temperature = body.temperature; + if (typeof normalizedBody.temperature === 'number' && Number.isFinite(normalizedBody.temperature)) { + payload.temperature = normalizedBody.temperature; } - if (typeof body.top_p === 'number' && Number.isFinite(body.top_p)) { - payload.top_p = body.top_p; + if (typeof normalizedBody.top_p === 'number' && Number.isFinite(normalizedBody.top_p)) { + payload.top_p = normalizedBody.top_p; } - if (typeof body.max_output_tokens === 'number' && Number.isFinite(body.max_output_tokens)) { - payload.max_tokens = body.max_output_tokens; + if (typeof normalizedBody.max_output_tokens === 'number' && Number.isFinite(normalizedBody.max_output_tokens)) { + payload.max_tokens = normalizedBody.max_output_tokens; } - if (body.parallel_tool_calls !== undefined) payload.parallel_tool_calls = body.parallel_tool_calls; - if (body.tools !== undefined) payload.tools = convertResponsesToolsToOpenAi(body.tools); - if (body.tool_choice !== undefined) payload.tool_choice = convertResponsesToolChoiceToOpenAi(body.tool_choice); - if (body.safety_identifier !== undefined) payload.safety_identifier = body.safety_identifier; - if (body.max_tool_calls !== undefined) payload.max_tool_calls = body.max_tool_calls; - if (body.prompt_cache_key !== undefined) payload.prompt_cache_key = body.prompt_cache_key; - if (body.prompt_cache_retention !== undefined) payload.prompt_cache_retention = body.prompt_cache_retention; - if (body.background !== undefined) payload.background = body.background; - if (body.user !== undefined) payload.user = body.user; - if (body.include !== undefined) payload.include = cloneJsonValue(body.include); - if (body.previous_response_id !== undefined) payload.previous_response_id = body.previous_response_id; - if (body.truncation !== undefined) payload.truncation = body.truncation; - if (body.service_tier !== undefined) payload.service_tier = body.service_tier; - if (body.top_logprobs !== undefined) payload.top_logprobs = body.top_logprobs; - if (body.stream_options !== undefined) payload.stream_options = body.stream_options; - if (isRecord(body.text) && asTrimmedString(body.text.verbosity)) { - payload.verbosity = asTrimmedString(body.text.verbosity); + if (normalizedBody.parallel_tool_calls !== undefined) payload.parallel_tool_calls = normalizedBody.parallel_tool_calls; + if (normalizedBody.tools !== undefined) payload.tools = convertResponsesToolsToOpenAi(normalizedBody.tools); + if (normalizedBody.tool_choice !== undefined) payload.tool_choice = convertResponsesToolChoiceToOpenAi(normalizedBody.tool_choice); + if (normalizedBody.safety_identifier !== undefined) payload.safety_identifier = normalizedBody.safety_identifier; + if (normalizedBody.max_tool_calls !== undefined) payload.max_tool_calls = normalizedBody.max_tool_calls; + if (normalizedBody.prompt_cache_key !== undefined) payload.prompt_cache_key = normalizedBody.prompt_cache_key; + if (normalizedBody.prompt_cache_retention !== undefined) payload.prompt_cache_retention = normalizedBody.prompt_cache_retention; + if (normalizedBody.background !== undefined) payload.background = normalizedBody.background; + if (normalizedBody.user !== undefined) payload.user = normalizedBody.user; + if (normalizedBody.include !== undefined) payload.include = cloneJsonValue(normalizedBody.include); + if (normalizedBody.previous_response_id !== undefined) payload.previous_response_id = normalizedBody.previous_response_id; + if (normalizedBody.truncation !== undefined) payload.truncation = normalizedBody.truncation; + if (normalizedBody.service_tier !== undefined) payload.service_tier = normalizedBody.service_tier; + if (normalizedBody.top_logprobs !== undefined) payload.top_logprobs = normalizedBody.top_logprobs; + if (normalizedBody.stream_options !== undefined) payload.stream_options = normalizedBody.stream_options; + if (isRecord(normalizedBody.text) && normalizedBody.text.format !== undefined) { + payload.response_format = cloneJsonValue(normalizedBody.text.format); + } + if (isRecord(normalizedBody.text) && asTrimmedString(normalizedBody.text.verbosity)) { + payload.verbosity = asTrimmedString(normalizedBody.text.verbosity); } return payload; diff --git a/src/server/transformers/openai/responses/index.ts b/src/server/transformers/openai/responses/index.ts index 7c3209f..c98bcf5 100644 --- a/src/server/transformers/openai/responses/index.ts +++ b/src/server/transformers/openai/responses/index.ts @@ -2,11 +2,17 @@ import { type StreamTransformContext } from '../../shared/normalized.js'; import { convertOpenAiBodyToResponsesBody, convertResponsesBodyToOpenAiBody, - normalizeResponsesInputForCompatibility, - normalizeResponsesMessageContent, sanitizeResponsesBodyForProxy, } from './conversion.js'; -import { normalizeResponsesMessageItem } from './compatibility.js'; +import { + buildResponsesCompatibilityBodies as buildRetryBodies, + buildResponsesCompatibilityHeaderCandidates as buildRetryHeaders, + normalizeResponsesInputForCompatibility, + normalizeResponsesMessageContent, + normalizeResponsesMessageItem, + shouldDowngradeResponsesChatToMessages as shouldDowngradeChatToMessages, + shouldRetryResponsesCompatibility as shouldRetry, +} from './compatibility.js'; import { type OpenAiResponsesAggregateState, completeResponsesStream, @@ -31,6 +37,12 @@ export const openAiResponsesTransformer = { outbound: openAiResponsesOutbound, stream: openAiResponsesStream, usage: openAiResponsesUsage, + compatibility: { + buildRetryBodies, + buildRetryHeaders, + shouldRetry, + shouldDowngradeChatToMessages, + }, aggregator: { createState: createOpenAiResponsesAggregateState, serialize: serializeConvertedResponsesEvents, diff --git a/src/server/transformers/openai/responses/outbound.test.ts b/src/server/transformers/openai/responses/outbound.test.ts new file mode 100644 index 0000000..a05378e --- /dev/null +++ b/src/server/transformers/openai/responses/outbound.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, it } from 'vitest'; + +import { serializeResponsesFinalPayload } from './outbound.js'; + +describe('serializeResponsesFinalPayload', () => { + it('preserves top-level chat annotations on synthetic assistant messages', () => { + const payload = serializeResponsesFinalPayload({ + upstreamPayload: { + id: 'chatcmpl_1', + object: 'chat.completion', + created: 1700000000, + model: 'gpt-5', + choices: [ + { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'hello', + annotations: [ + { + type: 'url_citation', + url: 'https://example.com', + title: 'Example', + }, + ], + }, + }, + ], + }, + normalized: { + id: 'chatcmpl_1', + model: 'gpt-5', + created: 1700000000, + content: 'hello', + reasoningContent: '', + finishReason: 'stop', + toolCalls: [], + }, + usage: { + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }, + }); + + expect(payload.output).toEqual([ + { + id: 'msg_chatcmpl_1', + type: 'message', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: 'hello', + annotations: [ + { + type: 'url_citation', + url: 'https://example.com', + title: 'Example', + }, + ], + }, + ], + }, + ]); + }); + + it('falls back to normalized tool calls when upstream payload no longer exposes them directly', () => { + const payload = serializeResponsesFinalPayload({ + upstreamPayload: { + id: 'opaque_1', + model: 'gpt-5', + }, + normalized: { + id: 'opaque_1', + model: 'gpt-5', + created: 1700000000, + content: '', + reasoningContent: '', + finishReason: 'tool_calls', + toolCalls: [ + { + id: 'call_1', + name: 'browser', + arguments: '{"url":"https://example.com"}', + }, + ], + }, + usage: { + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }, + }); + + expect(payload.output).toEqual([ + { + id: 'call_1', + type: 'function_call', + status: 'completed', + call_id: 'call_1', + name: 'browser', + arguments: '{"url":"https://example.com"}', + }, + ]); + }); + + it('preserves response-like custom tool and image generation items when synthesizing object=response payloads', () => { + const payload = serializeResponsesFinalPayload({ + upstreamPayload: { + id: 'resp_like_1', + model: 'gpt-5', + output: [ + { + id: 'ct_1', + type: 'custom_tool_call', + status: 'completed', + call_id: 'ct_1', + name: 'browser', + input: 'open example.com', + }, + { + id: 'img_1', + type: 'image_generation_call', + status: 'completed', + result: 'data:image/png;base64,final', + background: 'transparent', + output_format: 'png', + quality: 'high', + size: '1024x1024', + partial_images: [ + { + partial_image_index: 0, + partial_image_b64: 'partial', + }, + ], + }, + ], + }, + normalized: { + id: 'resp_like_1', + model: 'gpt-5', + created: 1700000000, + content: '', + reasoningContent: '', + finishReason: 'stop', + toolCalls: [], + }, + usage: { + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }, + }); + + expect(payload.output).toEqual([ + { + id: 'ct_1', + type: 'custom_tool_call', + status: 'completed', + call_id: 'ct_1', + name: 'browser', + input: 'open example.com', + }, + { + id: 'img_1', + type: 'image_generation_call', + status: 'completed', + result: 'data:image/png;base64,final', + background: 'transparent', + output_format: 'png', + quality: 'high', + size: '1024x1024', + partial_images: [ + { + partial_image_index: 0, + partial_image_b64: 'partial', + }, + ], + }, + ]); + }); + + it('maps upstream chat-completion usage details into Responses usage details', () => { + const payload = serializeResponsesFinalPayload({ + upstreamPayload: { + id: 'chatcmpl_usage', + object: 'chat.completion', + created: 1700000000, + model: 'gpt-5', + choices: [ + { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'hello', + }, + }, + ], + usage: { + prompt_tokens: 11, + completion_tokens: 7, + total_tokens: 18, + prompt_tokens_details: { + cached_tokens: 5, + audio_tokens: 2, + }, + completion_tokens_details: { + reasoning_tokens: 3, + audio_tokens: 1, + accepted_prediction_tokens: 4, + rejected_prediction_tokens: 6, + }, + }, + }, + normalized: { + id: 'chatcmpl_usage', + model: 'gpt-5', + created: 1700000000, + content: 'hello', + reasoningContent: '', + finishReason: 'stop', + toolCalls: [], + }, + usage: { + promptTokens: 11, + completionTokens: 7, + totalTokens: 18, + }, + }); + + expect(payload.usage).toMatchObject({ + input_tokens: 11, + output_tokens: 7, + total_tokens: 18, + input_tokens_details: { + cached_tokens: 5, + audio_tokens: 2, + }, + output_tokens_details: { + reasoning_tokens: 3, + audio_tokens: 1, + accepted_prediction_tokens: 4, + rejected_prediction_tokens: 6, + }, + }); + }); +}); diff --git a/src/server/transformers/openai/responses/outbound.ts b/src/server/transformers/openai/responses/outbound.ts index f690dec..79c694d 100644 --- a/src/server/transformers/openai/responses/outbound.ts +++ b/src/server/transformers/openai/responses/outbound.ts @@ -11,6 +11,18 @@ function asTrimmedString(value: unknown): string { return typeof value === 'string' ? value.trim() : ''; } +function cloneJson(value: T): T { + if (Array.isArray(value)) { + return value.map((item) => cloneJson(item)) as T; + } + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, cloneJson(item)]), + ) as T; + } + return value; +} + function ensureResponseId(rawId: string): string { const trimmed = rawId.trim() || `resp_${Date.now()}`; return trimmed.startsWith('resp_') ? trimmed : `resp_${trimmed}`; @@ -41,6 +53,8 @@ export type ResponsesUsageSummary = { promptTokens: number; completionTokens: number; totalTokens: number; + inputTokensDetails?: Record; + outputTokensDetails?: Record; }; export type ResponsesToolCall = { @@ -49,6 +63,8 @@ export type ResponsesToolCall = { arguments: string; }; +type ResponsesOutputItem = Record; + function extractToolCallsFromUpstream(payload: unknown): ResponsesToolCall[] { if (!isRecord(payload)) return []; @@ -95,6 +111,166 @@ function extractToolCallsFromUpstream(payload: unknown): ResponsesToolCall[] { return []; } +function extractToolCallsFromNormalized(normalized: NormalizedFinalResponse): ResponsesToolCall[] { + return Array.isArray(normalized.toolCalls) + ? normalized.toolCalls + .map((item) => { + const id = ensureFunctionCallId(asTrimmedString(item.id)); + const name = asTrimmedString(item.name); + if (!name) return null; + return { + id, + name, + arguments: typeof item.arguments === 'string' ? item.arguments : '', + }; + }) + .filter((item): item is ResponsesToolCall => !!item) + : []; +} + +function extractAnnotationsFromUpstream(payload: unknown): unknown[] { + if (!isRecord(payload)) return []; + + if (Array.isArray(payload.choices)) { + const choice = payload.choices[0]; + const message = isRecord((choice as any)?.message) ? (choice as any).message : {}; + const annotations = Array.isArray((message as any).annotations) ? (message as any).annotations : []; + return annotations.map((item) => cloneJson(item)); + } + + if (payload.type === 'message' && Array.isArray(payload.content)) { + const annotations = payload.content + .filter((item): item is Record => isRecord(item)) + .flatMap((item) => (Array.isArray(item.annotations) ? item.annotations : [])); + return annotations.map((item) => cloneJson(item)); + } + + return []; +} + +function extractSyntheticOutputItemsFromUpstream(payload: unknown): ResponsesOutputItem[] { + if (!isRecord(payload) || !Array.isArray(payload.output)) return []; + + return payload.output + .map((rawItem): ResponsesOutputItem | null => { + if (!isRecord(rawItem)) return null; + const itemType = asTrimmedString(rawItem.type).toLowerCase(); + if (!itemType) return null; + + if (itemType === 'message') { + return { + id: asTrimmedString(rawItem.id) || ensureMessageId(''), + type: 'message', + role: asTrimmedString(rawItem.role) || 'assistant', + status: asTrimmedString(rawItem.status) || 'completed', + content: Array.isArray(rawItem.content) ? cloneJson(rawItem.content) : [], + }; + } + + if (itemType === 'reasoning') { + return { + id: asTrimmedString(rawItem.id) || ensureMessageId(''), + type: 'reasoning', + status: asTrimmedString(rawItem.status) || 'completed', + encrypted_content: rawItem.encrypted_content, + summary: Array.isArray(rawItem.summary) ? cloneJson(rawItem.summary) : [], + }; + } + + if (itemType === 'function_call') { + return { + id: asTrimmedString(rawItem.id) || ensureFunctionCallId(''), + type: 'function_call', + status: asTrimmedString(rawItem.status) || 'completed', + call_id: asTrimmedString(rawItem.call_id) || ensureFunctionCallId(asTrimmedString(rawItem.id)), + name: asTrimmedString(rawItem.name), + arguments: typeof rawItem.arguments === 'string' ? rawItem.arguments : '', + }; + } + + if (itemType === 'custom_tool_call') { + return { + id: asTrimmedString(rawItem.id) || ensureFunctionCallId(''), + type: 'custom_tool_call', + status: asTrimmedString(rawItem.status) || 'completed', + call_id: asTrimmedString(rawItem.call_id) || ensureFunctionCallId(asTrimmedString(rawItem.id)), + name: asTrimmedString(rawItem.name), + input: typeof rawItem.input === 'string' ? rawItem.input : '', + }; + } + + if (itemType === 'function_call_output' || itemType === 'custom_tool_call_output') { + return { + id: asTrimmedString(rawItem.id) || ensureMessageId(''), + type: itemType, + status: asTrimmedString(rawItem.status) || 'completed', + call_id: asTrimmedString(rawItem.call_id), + output: cloneJson(rawItem.output), + }; + } + + if (itemType === 'image_generation_call') { + return { + id: asTrimmedString(rawItem.id) || ensureMessageId(''), + type: 'image_generation_call', + status: asTrimmedString(rawItem.status) || 'completed', + result: cloneJson(rawItem.result), + partial_images: Array.isArray(rawItem.partial_images) ? cloneJson(rawItem.partial_images) : [], + ...(rawItem.background !== undefined ? { background: cloneJson(rawItem.background) } : {}), + ...(rawItem.output_format !== undefined ? { output_format: cloneJson(rawItem.output_format) } : {}), + ...(rawItem.quality !== undefined ? { quality: cloneJson(rawItem.quality) } : {}), + ...(rawItem.size !== undefined ? { size: cloneJson(rawItem.size) } : {}), + ...(rawItem.revised_prompt !== undefined ? { revised_prompt: cloneJson(rawItem.revised_prompt) } : {}), + }; + } + + return null; + }) + .filter((item): item is ResponsesOutputItem => item !== null); +} + +function collectOutputTextFromItems(items: ResponsesOutputItem[]): string { + const textParts: string[] = []; + + for (const item of items) { + if (asTrimmedString(item.type).toLowerCase() !== 'message') continue; + const content = Array.isArray(item.content) ? item.content : []; + for (const part of content) { + if (!isRecord(part)) continue; + const partType = asTrimmedString(part.type).toLowerCase(); + if ((partType === 'output_text' || partType === 'text') && typeof part.text === 'string') { + textParts.push(part.text); + } + } + } + + return textParts.join(''); +} + +function extractSyntheticUsageDetails(upstreamPayload: unknown): { + inputTokensDetails?: Record; + outputTokensDetails?: Record; +} { + if (!isRecord(upstreamPayload) || !isRecord(upstreamPayload.usage)) { + return {}; + } + + const usage = upstreamPayload.usage; + const promptDetails = ( + (isRecord(usage.input_tokens_details) ? usage.input_tokens_details : null) + ?? (isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : null) + ); + const completionDetails = ( + (isRecord(usage.output_tokens_details) ? usage.output_tokens_details : null) + ?? (isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : null) + ); + + return { + ...(promptDetails ? { inputTokensDetails: cloneJson(promptDetails) } : {}), + ...(completionDetails ? { outputTokensDetails: cloneJson(completionDetails) } : {}), + }; +} + export function normalizeResponsesFinalPayload( payload: unknown, modelName: string, @@ -118,10 +294,30 @@ export function serializeResponsesFinalPayload(input: { : `resp_${Date.now()}`; const responseId = ensureResponseId(normalizedId); const messageId = ensureMessageId(normalizedId); - const toolCalls = extractToolCallsFromUpstream(upstreamPayload); + const syntheticOutput = extractSyntheticOutputItemsFromUpstream(upstreamPayload); + const syntheticUsageDetails = extractSyntheticUsageDetails(upstreamPayload); + const toolCalls = (() => { + const extracted = extractToolCallsFromUpstream(upstreamPayload); + if (extracted.length > 0) return extracted; + return extractToolCallsFromNormalized(normalized); + })(); + const annotations = extractAnnotationsFromUpstream(upstreamPayload); - const output: Array> = []; - if (normalized.reasoningContent) { + const output: Array> = syntheticOutput.map((item) => cloneJson(item)); + const hasReasoningItem = output.some((item) => asTrimmedString(item.type).toLowerCase() === 'reasoning'); + const hasMessageItem = output.some((item) => asTrimmedString(item.type).toLowerCase() === 'message'); + const hasToolLikeItem = output.some((item) => { + const itemType = asTrimmedString(item.type).toLowerCase(); + return ( + itemType === 'function_call' + || itemType === 'custom_tool_call' + || itemType === 'function_call_output' + || itemType === 'custom_tool_call_output' + || itemType === 'image_generation_call' + ); + }); + + if (normalized.reasoningContent && !hasReasoningItem) { output.push({ id: ensureMessageId(`${normalizedId}_reasoning`), type: 'reasoning', @@ -132,28 +328,35 @@ export function serializeResponsesFinalPayload(input: { }], }); } - if (normalized.content || toolCalls.length === 0) { + + if ((normalized.content || (!hasToolLikeItem && output.length === 0 && toolCalls.length === 0)) && !hasMessageItem) { + const textPart: Record = { + type: 'output_text', + text: normalized.content || '', + }; + if (annotations.length > 0) { + textPart.annotations = annotations; + } output.push({ id: messageId, type: 'message', role: 'assistant', status: 'completed', - content: [{ - type: 'output_text', - text: normalized.content || '', - }], + content: [textPart], }); } - for (const toolCall of toolCalls) { - output.push({ - id: toolCall.id, - type: 'function_call', - status: 'completed', - call_id: toolCall.id, - name: toolCall.name, - arguments: toolCall.arguments, - }); + if (toolCalls.length > 0 && !hasToolLikeItem) { + for (const toolCall of toolCalls) { + output.push({ + id: toolCall.id, + type: 'function_call', + status: 'completed', + call_id: toolCall.id, + name: toolCall.name, + arguments: toolCall.arguments, + }); + } } return { @@ -163,11 +366,17 @@ export function serializeResponsesFinalPayload(input: { status: 'completed', model: normalized.model, output, - output_text: normalized.content || '', + output_text: normalized.content || collectOutputTextFromItems(output), usage: { input_tokens: usage.promptTokens, output_tokens: usage.completionTokens, total_tokens: usage.totalTokens, + ...(usage.inputTokensDetails ?? syntheticUsageDetails.inputTokensDetails + ? { input_tokens_details: cloneJson(usage.inputTokensDetails ?? syntheticUsageDetails.inputTokensDetails) } + : {}), + ...(usage.outputTokensDetails ?? syntheticUsageDetails.outputTokensDetails + ? { output_tokens_details: cloneJson(usage.outputTokensDetails ?? syntheticUsageDetails.outputTokensDetails) } + : {}), }, }; }