mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
refactor(providers,executor): deepen three shallow modules (#5052)
* refactor(executor): collapse subflow node-ID logic behind a codec Extract the ~20 scattered subflow node-ID parsing/building helpers and their regex patterns into a single SubflowNodeIdCodec. All patterns now live in one place; subflow-utils and execution state delegate to it. Pure refactor — byte-identical output, ordering, and error modes. * refactor(providers): extract StreamingExecution assembly into a factory The near-identical streaming-response assembly (timing segments, cost, token counts, onComplete wiring, success/logs/metadata envelope) that was copy-pasted across ~16 providers now goes through createStreamingExecution. Each provider injects only its stream iterable and delta extractors. Gemini is intentionally left as-is (its assembly is structurally divergent). Pure refactor — identical StreamingExecution shape, cost, tokens, timing, and callback order per provider. * refactor(providers): dedupe tool-schema wrapping into adapters Replace the identical inline tool-schema literals repeated across ~16 providers with shared adaptOpenAIChatToolSchema / adaptAnthropicToolSchema helpers. Content building reverts to calling the attachments builders directly (the per-provider content seam was pure pass-through indirection). Pure refactor — byte-identical tool-schema and content output. * chore(skills): point add-model provider edits at shared response/tool-schema helpers When editing provider code, reuse createStreamingExecution and the tool-schema-adapter helpers instead of hand-rolling. * refactor(providers): drop placeholder stream double-cast in streaming factory Build the output object before creating the stream so the factory returns a fully-typed StreamingExecution without 'undefined as unknown as ReadableStream'. Keeps the strict API-validation boundary ratchet green (double-cast count back to baseline). No behavior change: the same output reference is mutated on drain.
This commit is contained in:
Waleed
@@ -143,12 +143,12 @@ If anything matches, run the affected provider tests and update assertions as ne
|
||||
|
||||
### New API behavior is NOT data-driven
|
||||
|
||||
The Consumption Matrix (Step 2) tells you which capability *flags* are honored by existing provider code. But if the new model needs **net-new** request handling that the provider doesn't implement yet — a new beta header (e.g. Anthropic's `anthropic-beta` structured-outputs header in `anthropic/index.ts`), a new thinking/reasoning encoding, a Responses-API quirk — you must edit `apps/sim/providers/<provider>/core.ts` / `index.ts`. Setting a flag whose behavior isn't implemented is a silent no-op.
|
||||
The Consumption Matrix (Step 2) tells you which capability *flags* are honored by existing provider code. But if the new model needs **net-new** request handling that the provider doesn't implement yet — a new beta header (e.g. Anthropic's `anthropic-beta` structured-outputs header in `anthropic/index.ts`), a new thinking/reasoning encoding, a Responses-API quirk — you must edit `apps/sim/providers/<provider>/core.ts` / `index.ts`. Setting a flag whose behavior isn't implemented is a silent no-op. When you do edit provider code, reuse the shared helpers rather than hand-rolling: streaming responses are assembled via `createStreamingExecution` (`@/providers/streaming-execution`) and tool schemas via `adaptOpenAIChatToolSchema` / `adaptAnthropicToolSchema` (`@/providers/tool-schema-adapter`).
|
||||
|
||||
### Wrong family entirely?
|
||||
|
||||
- **Embedding or rerank model** → it does NOT go in the `models[]` array. Use `EMBEDDING_MODEL_PRICING` / `RERANK_MODEL_PRICING` in `models.ts` instead.
|
||||
- **Brand-new provider** (not just a new model under an existing one) → much larger surface: add the id to `ProviderId` in `providers/types.ts`, a registry entry in `providers/registry.ts`, a provider implementation under `providers/<id>/`, an icon in `components/icons.tsx`, and the `PROVIDER_DEFINITIONS` block. That is beyond this skill — tell the user.
|
||||
- **Brand-new provider** (not just a new model under an existing one) → much larger surface: add the id to `ProviderId` in `providers/types.ts`, a registry entry in `providers/registry.ts`, a provider implementation under `providers/<id>/` (assemble streaming responses with `createStreamingExecution` and wrap tool schemas with the `@/providers/tool-schema-adapter` helpers), an icon in `components/icons.tsx`, and the `PROVIDER_DEFINITIONS` block. That is beyond this skill — tell the user.
|
||||
|
||||
## Step 5: Write, lint
|
||||
|
||||
|
||||
@@ -1,24 +1,22 @@
|
||||
import type { BlockStateController } from '@/executor/execution/types'
|
||||
import type { BlockState, NormalizedBlockOutput } from '@/executor/types'
|
||||
import { SubflowNodeIdCodec } from '@/executor/utils/subflow-node-id-codec'
|
||||
import {
|
||||
buildOuterBranchScopedId,
|
||||
extractOuterBranchIndex,
|
||||
stripCloneSuffixes,
|
||||
} from '@/executor/utils/subflow-utils'
|
||||
|
||||
const BRANCH_SUFFIX_PATTERN = /₍\d+₎/u
|
||||
const LOOP_SUFFIX_PATTERN = /_loop\d+/
|
||||
|
||||
function normalizeLookupId(id: string): string {
|
||||
return id.replace(/₍\d+₎/gu, '').replace(/_loop\d+/g, '')
|
||||
return SubflowNodeIdCodec.normalizeLookupId(id)
|
||||
}
|
||||
|
||||
function extractBranchSuffix(id: string): string {
|
||||
return id.match(BRANCH_SUFFIX_PATTERN)?.[0] ?? ''
|
||||
return SubflowNodeIdCodec.extractBranchSuffix(id)
|
||||
}
|
||||
|
||||
function extractLoopSuffix(id: string): string {
|
||||
return id.match(LOOP_SUFFIX_PATTERN)?.[0] ?? ''
|
||||
return SubflowNodeIdCodec.extractLoopSuffix(id)
|
||||
}
|
||||
export interface LoopScope {
|
||||
iteration: number
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SubflowNodeIdCodec } from '@/executor/utils/subflow-node-id-codec'
|
||||
|
||||
describe('SubflowNodeIdCodec', () => {
|
||||
describe('branch subscripts', () => {
|
||||
it('builds and round-trips branch node IDs', () => {
|
||||
const id = SubflowNodeIdCodec.buildBranchNodeId('block-1', 2)
|
||||
expect(id).toBe('block-1₍2₎')
|
||||
expect(SubflowNodeIdCodec.isBranchNodeId(id)).toBe(true)
|
||||
expect(SubflowNodeIdCodec.extractBaseBlockId(id)).toBe('block-1')
|
||||
expect(SubflowNodeIdCodec.extractBranchIndex(id)).toBe(2)
|
||||
})
|
||||
|
||||
it('returns null index and false predicate for non-branch IDs', () => {
|
||||
expect(SubflowNodeIdCodec.isBranchNodeId('block-1')).toBe(false)
|
||||
expect(SubflowNodeIdCodec.extractBranchIndex('block-1')).toBeNull()
|
||||
expect(SubflowNodeIdCodec.extractBaseBlockId('block-1')).toBe('block-1')
|
||||
})
|
||||
|
||||
it('only strips a trailing branch subscript', () => {
|
||||
expect(SubflowNodeIdCodec.extractBaseBlockId('a₍1₎b')).toBe('a₍1₎b')
|
||||
expect(SubflowNodeIdCodec.extractBaseBlockId('a₍1₎b₍2₎')).toBe('a₍1₎b')
|
||||
})
|
||||
})
|
||||
|
||||
describe('loop sentinels', () => {
|
||||
it('builds and parses loop sentinel IDs', () => {
|
||||
const start = SubflowNodeIdCodec.buildLoopSentinelStartId('loop-1')
|
||||
const end = SubflowNodeIdCodec.buildLoopSentinelEndId('loop-1')
|
||||
expect(start).toBe('loop-loop-1-sentinel-start')
|
||||
expect(end).toBe('loop-loop-1-sentinel-end')
|
||||
expect(SubflowNodeIdCodec.isLoopSentinelNodeId(start)).toBe(true)
|
||||
expect(SubflowNodeIdCodec.isLoopSentinelNodeId(end)).toBe(true)
|
||||
expect(SubflowNodeIdCodec.extractLoopIdFromSentinel(start)).toBe('loop-1')
|
||||
expect(SubflowNodeIdCodec.extractLoopIdFromSentinel(end)).toBe('loop-1')
|
||||
})
|
||||
|
||||
it('returns null when not a loop sentinel', () => {
|
||||
expect(SubflowNodeIdCodec.isLoopSentinelNodeId('block-1')).toBe(false)
|
||||
expect(SubflowNodeIdCodec.extractLoopIdFromSentinel('block-1')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('parallel sentinels', () => {
|
||||
it('builds and parses parallel sentinel IDs', () => {
|
||||
const start = SubflowNodeIdCodec.buildParallelSentinelStartId('p-1')
|
||||
const end = SubflowNodeIdCodec.buildParallelSentinelEndId('p-1')
|
||||
expect(start).toBe('parallel-p-1-sentinel-start')
|
||||
expect(end).toBe('parallel-p-1-sentinel-end')
|
||||
expect(SubflowNodeIdCodec.isParallelSentinelNodeId(start)).toBe(true)
|
||||
expect(SubflowNodeIdCodec.isParallelSentinelNodeId(end)).toBe(true)
|
||||
expect(SubflowNodeIdCodec.extractParallelIdFromSentinel(start)).toBe('p-1')
|
||||
expect(SubflowNodeIdCodec.extractParallelIdFromSentinel(end)).toBe('p-1')
|
||||
})
|
||||
|
||||
it('returns null when not a parallel sentinel', () => {
|
||||
expect(SubflowNodeIdCodec.isParallelSentinelNodeId('block-1')).toBe(false)
|
||||
expect(SubflowNodeIdCodec.extractParallelIdFromSentinel('block-1')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('outer-branch clone scoping', () => {
|
||||
it('builds and extracts outer branch index', () => {
|
||||
const id = SubflowNodeIdCodec.buildOuterBranchScopedId('loop-1', 3)
|
||||
expect(id).toBe('loop-1__obranch-3')
|
||||
expect(SubflowNodeIdCodec.extractOuterBranchIndex(id)).toBe(3)
|
||||
})
|
||||
|
||||
it('extracts the innermost outer branch index for nested clones', () => {
|
||||
const id = 'loop-1__obranch-2__obranch-5'
|
||||
expect(SubflowNodeIdCodec.extractOuterBranchIndex(id)).toBe(2)
|
||||
expect(SubflowNodeIdCodec.extractInnermostOuterBranchIndex(id)).toBe(5)
|
||||
})
|
||||
|
||||
it('returns undefined when no outer branch suffix is present', () => {
|
||||
expect(SubflowNodeIdCodec.extractOuterBranchIndex('loop-1')).toBeUndefined()
|
||||
expect(SubflowNodeIdCodec.extractInnermostOuterBranchIndex('loop-1')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('strips outer-branch and clone-digest suffixes', () => {
|
||||
expect(SubflowNodeIdCodec.stripOuterBranchSuffix('loop-1__obranch-2')).toBe('loop-1')
|
||||
expect(SubflowNodeIdCodec.stripOuterBranchSuffix('loop-1__cloneABCDEF__obranch-2')).toBe(
|
||||
'loop-1'
|
||||
)
|
||||
})
|
||||
|
||||
it('strips all clone suffixes and branch subscripts to the base block ID', () => {
|
||||
expect(SubflowNodeIdCodec.stripCloneSuffixes('block-1__obranch-2₍3₎')).toBe('block-1')
|
||||
expect(SubflowNodeIdCodec.stripCloneSuffixes('block-1__clone0a1f__obranch-2₍0₎')).toBe(
|
||||
'block-1'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizeNodeId', () => {
|
||||
it('normalizes branch, loop sentinel, and parallel sentinel IDs', () => {
|
||||
expect(SubflowNodeIdCodec.normalizeNodeId('block-1₍2₎')).toBe('block-1')
|
||||
expect(SubflowNodeIdCodec.normalizeNodeId('loop-loop-1-sentinel-start')).toBe('loop-1')
|
||||
expect(SubflowNodeIdCodec.normalizeNodeId('parallel-p-1-sentinel-end')).toBe('p-1')
|
||||
expect(SubflowNodeIdCodec.normalizeNodeId('block-1')).toBe('block-1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('loop digest lookup helpers', () => {
|
||||
it('strips branch subscripts and loop digests for lookup keys', () => {
|
||||
expect(SubflowNodeIdCodec.normalizeLookupId('block-1₍2₎_loop3')).toBe('block-1')
|
||||
expect(SubflowNodeIdCodec.normalizeLookupId('block-1')).toBe('block-1')
|
||||
})
|
||||
|
||||
it('extracts the leading branch suffix and loop digest segments', () => {
|
||||
expect(SubflowNodeIdCodec.extractBranchSuffix('block-1₍2₎_loop3')).toBe('₍2₎')
|
||||
expect(SubflowNodeIdCodec.extractBranchSuffix('block-1')).toBe('')
|
||||
expect(SubflowNodeIdCodec.extractLoopSuffix('block-1₍2₎_loop3')).toBe('_loop3')
|
||||
expect(SubflowNodeIdCodec.extractLoopSuffix('block-1')).toBe('')
|
||||
})
|
||||
})
|
||||
|
||||
describe('findEffectiveContainerId', () => {
|
||||
it('returns the original ID for branch 0 / missing scope', () => {
|
||||
const map = new Map<string, unknown>([['loop-1', {}]])
|
||||
expect(SubflowNodeIdCodec.findEffectiveContainerId('loop-1', 'block-1', map)).toBe('loop-1')
|
||||
})
|
||||
|
||||
it('prefers the mapped cloned scope when present', () => {
|
||||
const map = new Map<string, unknown>([
|
||||
['loop-1', {}],
|
||||
['loop-1__obranch-2', {}],
|
||||
])
|
||||
expect(SubflowNodeIdCodec.findEffectiveContainerId('loop-1', 'block-1', map, 2)).toBe(
|
||||
'loop-1__obranch-2'
|
||||
)
|
||||
})
|
||||
|
||||
it('resolves the cloned scope from the current node ID suffix', () => {
|
||||
const map = new Map<string, unknown>([
|
||||
['loop-1', {}],
|
||||
['loop-1__obranch-3', {}],
|
||||
])
|
||||
expect(SubflowNodeIdCodec.findEffectiveContainerId('loop-1', 'block-1__obranch-3', map)).toBe(
|
||||
'loop-1__obranch-3'
|
||||
)
|
||||
})
|
||||
|
||||
it('prefers __clone scopes when the current node carries a clone marker', () => {
|
||||
const map = new Map<string, unknown>([
|
||||
['loop-1__obranch-2', {}],
|
||||
['loop-1__cloneabc__obranch-2', {}],
|
||||
])
|
||||
expect(
|
||||
SubflowNodeIdCodec.findEffectiveContainerId('loop-1', 'block-1__cloneabc__obranch-2', map)
|
||||
).toBe('loop-1__cloneabc__obranch-2')
|
||||
})
|
||||
})
|
||||
|
||||
describe('round-trip parse ∘ build', () => {
|
||||
it('builds then parses branch IDs symmetrically', () => {
|
||||
for (const index of [0, 1, 7, 20]) {
|
||||
const id = SubflowNodeIdCodec.buildBranchNodeId('base-id', index)
|
||||
expect(SubflowNodeIdCodec.extractBranchIndex(id)).toBe(index)
|
||||
expect(SubflowNodeIdCodec.extractBaseBlockId(id)).toBe('base-id')
|
||||
}
|
||||
})
|
||||
|
||||
it('builds then parses outer-branch scoped IDs symmetrically', () => {
|
||||
for (const index of [1, 4, 19]) {
|
||||
const id = SubflowNodeIdCodec.buildOuterBranchScopedId('base-id', index)
|
||||
expect(SubflowNodeIdCodec.extractOuterBranchIndex(id)).toBe(index)
|
||||
expect(SubflowNodeIdCodec.stripOuterBranchSuffix(id)).toBe('base-id')
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,263 @@
|
||||
import { LOOP, PARALLEL } from '@/executor/constants'
|
||||
|
||||
/**
|
||||
* Single source of truth for parsing and building subflow node IDs.
|
||||
*
|
||||
* Runtime node IDs layer several encodings onto a workflow-level block ID:
|
||||
* - Branch subscripts `₍N₎` mark a parallel branch instance.
|
||||
* - Loop/parallel sentinels wrap a container ID (`loop-{id}-sentinel-start`).
|
||||
* - Outer-branch clone suffixes `__obranch-N` and clone digests `__clone{hex}`
|
||||
* scope a cloned subflow to a global outer parallel branch.
|
||||
* - Loop digests `_loopN` scope a block output to a loop iteration.
|
||||
*
|
||||
* All regexes and string templates for these encodings live here so callers
|
||||
* never reconstruct them inline.
|
||||
*/
|
||||
|
||||
const SENTINEL = {
|
||||
LOOP_START: new RegExp(`${LOOP.SENTINEL.PREFIX}(.+)${LOOP.SENTINEL.START_SUFFIX}`),
|
||||
LOOP_END: new RegExp(`${LOOP.SENTINEL.PREFIX}(.+)${LOOP.SENTINEL.END_SUFFIX}`),
|
||||
PARALLEL_START: new RegExp(`${PARALLEL.SENTINEL.PREFIX}(.+)${PARALLEL.SENTINEL.START_SUFFIX}`),
|
||||
PARALLEL_END: new RegExp(`${PARALLEL.SENTINEL.PREFIX}(.+)${PARALLEL.SENTINEL.END_SUFFIX}`),
|
||||
} as const
|
||||
|
||||
const BRANCH = {
|
||||
MATCH: new RegExp(`${PARALLEL.BRANCH.PREFIX}\\d+${PARALLEL.BRANCH.SUFFIX}$`),
|
||||
INDEX: new RegExp(`${PARALLEL.BRANCH.PREFIX}(\\d+)${PARALLEL.BRANCH.SUFFIX}$`),
|
||||
SUFFIX: /₍\d+₎/u,
|
||||
SUFFIX_GLOBAL: /₍\d+₎/gu,
|
||||
} as const
|
||||
|
||||
const OUTER_BRANCH = {
|
||||
MATCH: /__obranch-(\d+)/,
|
||||
MATCH_GLOBAL: /__obranch-(\d+)/g,
|
||||
STRIP: /__obranch-\d+/g,
|
||||
} as const
|
||||
|
||||
const CLONE = {
|
||||
DIGEST_STRIP: /__clone[0-9a-f]+/gi,
|
||||
MARKER: '__clone',
|
||||
} as const
|
||||
|
||||
const LOOP_DIGEST = {
|
||||
MATCH: /_loop\d+/,
|
||||
STRIP: /_loop\d+/g,
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Builds the loop sentinel-start node ID for a container.
|
||||
*/
|
||||
function buildLoopSentinelStartId(loopId: string): string {
|
||||
return `${LOOP.SENTINEL.PREFIX}${loopId}${LOOP.SENTINEL.START_SUFFIX}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the loop sentinel-end node ID for a container.
|
||||
*/
|
||||
function buildLoopSentinelEndId(loopId: string): string {
|
||||
return `${LOOP.SENTINEL.PREFIX}${loopId}${LOOP.SENTINEL.END_SUFFIX}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the parallel sentinel-start node ID for a container.
|
||||
*/
|
||||
function buildParallelSentinelStartId(parallelId: string): string {
|
||||
return `${PARALLEL.SENTINEL.PREFIX}${parallelId}${PARALLEL.SENTINEL.START_SUFFIX}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the parallel sentinel-end node ID for a container.
|
||||
*/
|
||||
function buildParallelSentinelEndId(parallelId: string): string {
|
||||
return `${PARALLEL.SENTINEL.PREFIX}${parallelId}${PARALLEL.SENTINEL.END_SUFFIX}`
|
||||
}
|
||||
|
||||
function isLoopSentinelNodeId(nodeId: string): boolean {
|
||||
return (
|
||||
nodeId.startsWith(LOOP.SENTINEL.PREFIX) &&
|
||||
(nodeId.endsWith(LOOP.SENTINEL.START_SUFFIX) || nodeId.endsWith(LOOP.SENTINEL.END_SUFFIX))
|
||||
)
|
||||
}
|
||||
|
||||
function isParallelSentinelNodeId(nodeId: string): boolean {
|
||||
return (
|
||||
nodeId.startsWith(PARALLEL.SENTINEL.PREFIX) &&
|
||||
(nodeId.endsWith(PARALLEL.SENTINEL.START_SUFFIX) ||
|
||||
nodeId.endsWith(PARALLEL.SENTINEL.END_SUFFIX))
|
||||
)
|
||||
}
|
||||
|
||||
function extractLoopIdFromSentinel(sentinelId: string): string | null {
|
||||
const startMatch = sentinelId.match(SENTINEL.LOOP_START)
|
||||
if (startMatch) return startMatch[1]
|
||||
const endMatch = sentinelId.match(SENTINEL.LOOP_END)
|
||||
if (endMatch) return endMatch[1]
|
||||
return null
|
||||
}
|
||||
|
||||
function extractParallelIdFromSentinel(sentinelId: string): string | null {
|
||||
const startMatch = sentinelId.match(SENTINEL.PARALLEL_START)
|
||||
if (startMatch) return startMatch[1]
|
||||
const endMatch = sentinelId.match(SENTINEL.PARALLEL_END)
|
||||
if (endMatch) return endMatch[1]
|
||||
return null
|
||||
}
|
||||
|
||||
function buildBranchNodeId(baseId: string, branchIndex: number): string {
|
||||
return `${baseId}${PARALLEL.BRANCH.PREFIX}${branchIndex}${PARALLEL.BRANCH.SUFFIX}`
|
||||
}
|
||||
|
||||
function extractBaseBlockId(branchNodeId: string): string {
|
||||
return branchNodeId.replace(BRANCH.MATCH, '')
|
||||
}
|
||||
|
||||
function extractBranchIndex(branchNodeId: string): number | null {
|
||||
const match = branchNodeId.match(BRANCH.INDEX)
|
||||
return match ? Number.parseInt(match[1], 10) : null
|
||||
}
|
||||
|
||||
function isBranchNodeId(nodeId: string): boolean {
|
||||
return BRANCH.MATCH.test(nodeId)
|
||||
}
|
||||
|
||||
function extractOuterBranchIndex(clonedId: string): number | undefined {
|
||||
const match = clonedId.match(OUTER_BRANCH.MATCH)
|
||||
return match ? Number.parseInt(match[1], 10) : undefined
|
||||
}
|
||||
|
||||
function extractInnermostOuterBranchIndex(clonedId: string): number | undefined {
|
||||
const matches = Array.from(clonedId.matchAll(OUTER_BRANCH.MATCH_GLOBAL))
|
||||
const lastMatch = matches.at(-1)
|
||||
return lastMatch ? Number.parseInt(lastMatch[1], 10) : undefined
|
||||
}
|
||||
|
||||
function stripCloneSuffixes(nodeId: string): string {
|
||||
return extractBaseBlockId(nodeId.replace(OUTER_BRANCH.STRIP, '').replace(CLONE.DIGEST_STRIP, ''))
|
||||
}
|
||||
|
||||
function buildOuterBranchScopedId(originalId: string, branchIndex: number): string {
|
||||
return `${originalId}__obranch-${branchIndex}`
|
||||
}
|
||||
|
||||
function stripOuterBranchSuffix(id: string): string {
|
||||
return id.replace(OUTER_BRANCH.STRIP, '').replace(CLONE.DIGEST_STRIP, '')
|
||||
}
|
||||
|
||||
function hasCloneMarker(id: string): boolean {
|
||||
return id.includes(CLONE.MARKER)
|
||||
}
|
||||
|
||||
function normalizeNodeId(nodeId: string): string {
|
||||
if (isBranchNodeId(nodeId)) {
|
||||
return extractBaseBlockId(nodeId)
|
||||
}
|
||||
if (isLoopSentinelNodeId(nodeId)) {
|
||||
return extractLoopIdFromSentinel(nodeId) || nodeId
|
||||
}
|
||||
if (isParallelSentinelNodeId(nodeId)) {
|
||||
return extractParallelIdFromSentinel(nodeId) || nodeId
|
||||
}
|
||||
return nodeId
|
||||
}
|
||||
|
||||
function findEffectiveContainerId(
|
||||
originalId: string,
|
||||
currentNodeId: string,
|
||||
executionMap: Map<string, unknown>,
|
||||
mappedBranchIndex?: number
|
||||
): string {
|
||||
if (mappedBranchIndex !== undefined && mappedBranchIndex > 0) {
|
||||
const cloneSuffix = `__obranch-${mappedBranchIndex}`
|
||||
const candidateId = buildOuterBranchScopedId(originalId, mappedBranchIndex)
|
||||
if (executionMap.has(candidateId)) {
|
||||
return candidateId
|
||||
}
|
||||
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const match = currentNodeId.match(OUTER_BRANCH.MATCH)
|
||||
if (match) {
|
||||
const branchIndex = Number.parseInt(match[1], 10)
|
||||
const cloneSuffix = `__obranch-${branchIndex}`
|
||||
if (hasCloneMarker(currentNodeId)) {
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (
|
||||
hasCloneMarker(scopeId) &&
|
||||
scopeId.endsWith(cloneSuffix) &&
|
||||
stripOuterBranchSuffix(scopeId) === originalId
|
||||
) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidateId = buildOuterBranchScopedId(originalId, branchIndex)
|
||||
if (executionMap.has(candidateId)) {
|
||||
return candidateId
|
||||
}
|
||||
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return originalId
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips branch subscripts (`₍N₎`) and loop digests (`_loopN`) from a node ID,
|
||||
* yielding the lookup key used by execution-state block-output resolution.
|
||||
*/
|
||||
function normalizeLookupId(id: string): string {
|
||||
return id.replace(BRANCH.SUFFIX_GLOBAL, '').replace(LOOP_DIGEST.STRIP, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the leading branch subscript (`₍N₎`) of a node ID, or '' when absent.
|
||||
*/
|
||||
function extractBranchSuffix(id: string): string {
|
||||
return id.match(BRANCH.SUFFIX)?.[0] ?? ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the loop digest segment (`_loopN`) of a node ID, or '' when absent.
|
||||
*/
|
||||
function extractLoopSuffix(id: string): string {
|
||||
return id.match(LOOP_DIGEST.MATCH)?.[0] ?? ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Codec exposing all subflow node-ID parsing/building operations as a single,
|
||||
* pattern-free interface. Implementation owns every regex and string template.
|
||||
*/
|
||||
export const SubflowNodeIdCodec = {
|
||||
buildLoopSentinelStartId,
|
||||
buildLoopSentinelEndId,
|
||||
buildParallelSentinelStartId,
|
||||
buildParallelSentinelEndId,
|
||||
isLoopSentinelNodeId,
|
||||
isParallelSentinelNodeId,
|
||||
extractLoopIdFromSentinel,
|
||||
extractParallelIdFromSentinel,
|
||||
buildBranchNodeId,
|
||||
extractBaseBlockId,
|
||||
extractBranchIndex,
|
||||
isBranchNodeId,
|
||||
extractOuterBranchIndex,
|
||||
extractInnermostOuterBranchIndex,
|
||||
stripCloneSuffixes,
|
||||
buildOuterBranchScopedId,
|
||||
stripOuterBranchSuffix,
|
||||
normalizeNodeId,
|
||||
findEffectiveContainerId,
|
||||
normalizeLookupId,
|
||||
extractBranchSuffix,
|
||||
extractLoopSuffix,
|
||||
} as const
|
||||
@@ -1,57 +1,42 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { DEFAULTS, LOOP, PARALLEL } from '@/executor/constants'
|
||||
import { DEFAULTS } from '@/executor/constants'
|
||||
import type { ContextExtensions } from '@/executor/execution/types'
|
||||
import { type BlockLog, type ExecutionContext, getNextExecutionOrder } from '@/executor/types'
|
||||
import { buildContainerIterationContext } from '@/executor/utils/iteration-context'
|
||||
import { SubflowNodeIdCodec } from '@/executor/utils/subflow-node-id-codec'
|
||||
import type { SerializedWorkflow } from '@/serializer/types'
|
||||
|
||||
const logger = createLogger('SubflowUtils')
|
||||
|
||||
const BRANCH_PATTERN = new RegExp(`${PARALLEL.BRANCH.PREFIX}\\d+${PARALLEL.BRANCH.SUFFIX}$`)
|
||||
const BRANCH_INDEX_PATTERN = new RegExp(`${PARALLEL.BRANCH.PREFIX}(\\d+)${PARALLEL.BRANCH.SUFFIX}$`)
|
||||
const LOOP_SENTINEL_START_PATTERN = new RegExp(
|
||||
`${LOOP.SENTINEL.PREFIX}(.+)${LOOP.SENTINEL.START_SUFFIX}`
|
||||
)
|
||||
const LOOP_SENTINEL_END_PATTERN = new RegExp(
|
||||
`${LOOP.SENTINEL.PREFIX}(.+)${LOOP.SENTINEL.END_SUFFIX}`
|
||||
)
|
||||
const PARALLEL_SENTINEL_START_PATTERN = new RegExp(
|
||||
`${PARALLEL.SENTINEL.PREFIX}(.+)${PARALLEL.SENTINEL.START_SUFFIX}`
|
||||
)
|
||||
const PARALLEL_SENTINEL_END_PATTERN = new RegExp(
|
||||
`${PARALLEL.SENTINEL.PREFIX}(.+)${PARALLEL.SENTINEL.END_SUFFIX}`
|
||||
)
|
||||
|
||||
/**
|
||||
* Builds the loop sentinel-start node ID for a container.
|
||||
*/
|
||||
export function buildSentinelStartId(loopId: string): string {
|
||||
return `${LOOP.SENTINEL.PREFIX}${loopId}${LOOP.SENTINEL.START_SUFFIX}`
|
||||
return SubflowNodeIdCodec.buildLoopSentinelStartId(loopId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the loop sentinel-end node ID for a container.
|
||||
*/
|
||||
export function buildSentinelEndId(loopId: string): string {
|
||||
return `${LOOP.SENTINEL.PREFIX}${loopId}${LOOP.SENTINEL.END_SUFFIX}`
|
||||
return SubflowNodeIdCodec.buildLoopSentinelEndId(loopId)
|
||||
}
|
||||
|
||||
export function buildParallelSentinelStartId(parallelId: string): string {
|
||||
return `${PARALLEL.SENTINEL.PREFIX}${parallelId}${PARALLEL.SENTINEL.START_SUFFIX}`
|
||||
return SubflowNodeIdCodec.buildParallelSentinelStartId(parallelId)
|
||||
}
|
||||
|
||||
export function buildParallelSentinelEndId(parallelId: string): string {
|
||||
return `${PARALLEL.SENTINEL.PREFIX}${parallelId}${PARALLEL.SENTINEL.END_SUFFIX}`
|
||||
return SubflowNodeIdCodec.buildParallelSentinelEndId(parallelId)
|
||||
}
|
||||
|
||||
export function isLoopSentinelNodeId(nodeId: string): boolean {
|
||||
return (
|
||||
nodeId.startsWith(LOOP.SENTINEL.PREFIX) &&
|
||||
(nodeId.endsWith(LOOP.SENTINEL.START_SUFFIX) || nodeId.endsWith(LOOP.SENTINEL.END_SUFFIX))
|
||||
)
|
||||
return SubflowNodeIdCodec.isLoopSentinelNodeId(nodeId)
|
||||
}
|
||||
|
||||
export function isParallelSentinelNodeId(nodeId: string): boolean {
|
||||
return (
|
||||
nodeId.startsWith(PARALLEL.SENTINEL.PREFIX) &&
|
||||
(nodeId.endsWith(PARALLEL.SENTINEL.START_SUFFIX) ||
|
||||
nodeId.endsWith(PARALLEL.SENTINEL.END_SUFFIX))
|
||||
)
|
||||
return SubflowNodeIdCodec.isParallelSentinelNodeId(nodeId)
|
||||
}
|
||||
|
||||
export function isSentinelNodeId(nodeId: string): boolean {
|
||||
@@ -59,19 +44,11 @@ export function isSentinelNodeId(nodeId: string): boolean {
|
||||
}
|
||||
|
||||
export function extractLoopIdFromSentinel(sentinelId: string): string | null {
|
||||
const startMatch = sentinelId.match(LOOP_SENTINEL_START_PATTERN)
|
||||
if (startMatch) return startMatch[1]
|
||||
const endMatch = sentinelId.match(LOOP_SENTINEL_END_PATTERN)
|
||||
if (endMatch) return endMatch[1]
|
||||
return null
|
||||
return SubflowNodeIdCodec.extractLoopIdFromSentinel(sentinelId)
|
||||
}
|
||||
|
||||
export function extractParallelIdFromSentinel(sentinelId: string): string | null {
|
||||
const startMatch = sentinelId.match(PARALLEL_SENTINEL_START_PATTERN)
|
||||
if (startMatch) return startMatch[1]
|
||||
const endMatch = sentinelId.match(PARALLEL_SENTINEL_END_PATTERN)
|
||||
if (endMatch) return endMatch[1]
|
||||
return null
|
||||
return SubflowNodeIdCodec.extractParallelIdFromSentinel(sentinelId)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -79,39 +56,32 @@ export function extractParallelIdFromSentinel(sentinelId: string): string | null
|
||||
* Example: ("blockId", 2) → "blockId₍2₎"
|
||||
*/
|
||||
export function buildBranchNodeId(baseId: string, branchIndex: number): string {
|
||||
return `${baseId}${PARALLEL.BRANCH.PREFIX}${branchIndex}${PARALLEL.BRANCH.SUFFIX}`
|
||||
return SubflowNodeIdCodec.buildBranchNodeId(baseId, branchIndex)
|
||||
}
|
||||
|
||||
export function extractBaseBlockId(branchNodeId: string): string {
|
||||
return branchNodeId.replace(BRANCH_PATTERN, '')
|
||||
return SubflowNodeIdCodec.extractBaseBlockId(branchNodeId)
|
||||
}
|
||||
|
||||
export function extractBranchIndex(branchNodeId: string): number | null {
|
||||
const match = branchNodeId.match(BRANCH_INDEX_PATTERN)
|
||||
return match ? Number.parseInt(match[1], 10) : null
|
||||
return SubflowNodeIdCodec.extractBranchIndex(branchNodeId)
|
||||
}
|
||||
|
||||
export function isBranchNodeId(nodeId: string): boolean {
|
||||
return BRANCH_PATTERN.test(nodeId)
|
||||
return SubflowNodeIdCodec.isBranchNodeId(nodeId)
|
||||
}
|
||||
|
||||
const OUTER_BRANCH_PATTERN = /__obranch-(\d+)/
|
||||
const OUTER_BRANCH_STRIP_PATTERN = /__obranch-\d+/g
|
||||
const CLONE_DIGEST_STRIP_PATTERN = /__clone[0-9a-f]+/gi
|
||||
|
||||
/**
|
||||
* Extracts the outer branch index from a cloned subflow ID.
|
||||
* Cloned IDs follow the pattern `{originalId}__obranch-{index}`.
|
||||
* Returns undefined if the ID is not a clone.
|
||||
*/
|
||||
export function extractOuterBranchIndex(clonedId: string): number | undefined {
|
||||
const match = clonedId.match(OUTER_BRANCH_PATTERN)
|
||||
return match ? Number.parseInt(match[1], 10) : undefined
|
||||
return SubflowNodeIdCodec.extractOuterBranchIndex(clonedId)
|
||||
}
|
||||
|
||||
export function extractInnermostOuterBranchIndex(clonedId: string): number | undefined {
|
||||
const matches = Array.from(clonedId.matchAll(/__obranch-(\d+)/g))
|
||||
const lastMatch = matches.at(-1)
|
||||
return lastMatch ? Number.parseInt(lastMatch[1], 10) : undefined
|
||||
return SubflowNodeIdCodec.extractInnermostOuterBranchIndex(clonedId)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -119,23 +89,21 @@ export function extractInnermostOuterBranchIndex(clonedId: string): number | und
|
||||
* from a node ID, returning the original workflow-level block ID.
|
||||
*/
|
||||
export function stripCloneSuffixes(nodeId: string): string {
|
||||
return extractBaseBlockId(
|
||||
nodeId.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_DIGEST_STRIP_PATTERN, '')
|
||||
)
|
||||
return SubflowNodeIdCodec.stripCloneSuffixes(nodeId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a stable ID for an output scoped to a global outer parallel branch.
|
||||
*/
|
||||
export function buildOuterBranchScopedId(originalId: string, branchIndex: number): string {
|
||||
return `${originalId}__obranch-${branchIndex}`
|
||||
return SubflowNodeIdCodec.buildOuterBranchScopedId(originalId, branchIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a cloned subflow ID from an original ID and outer branch index.
|
||||
*/
|
||||
export function buildClonedSubflowId(originalId: string, branchIndex: number): string {
|
||||
return buildOuterBranchScopedId(originalId, branchIndex)
|
||||
return SubflowNodeIdCodec.buildOuterBranchScopedId(originalId, branchIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -143,7 +111,7 @@ export function buildClonedSubflowId(originalId: string, branchIndex: number): s
|
||||
* returning the original workflow-level subflow ID.
|
||||
*/
|
||||
export function stripOuterBranchSuffix(id: string): string {
|
||||
return id.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_DIGEST_STRIP_PATTERN, '')
|
||||
return SubflowNodeIdCodec.stripOuterBranchSuffix(id)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -163,67 +131,16 @@ export function findEffectiveContainerId(
|
||||
executionMap: Map<string, unknown>,
|
||||
mappedBranchIndex?: number
|
||||
): string {
|
||||
if (mappedBranchIndex !== undefined && mappedBranchIndex > 0) {
|
||||
const cloneSuffix = `__obranch-${mappedBranchIndex}`
|
||||
const candidateId = buildClonedSubflowId(originalId, mappedBranchIndex)
|
||||
if (executionMap.has(candidateId)) {
|
||||
return candidateId
|
||||
}
|
||||
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer the cloned variant when currentNodeId carries an __obranch-N suffix.
|
||||
// During concurrent parallel-in-loop execution both the original (branch 0)
|
||||
// and cloned variants coexist in the map; the clone is the correct scope.
|
||||
const match = currentNodeId.match(OUTER_BRANCH_PATTERN)
|
||||
if (match) {
|
||||
const branchIndex = Number.parseInt(match[1], 10)
|
||||
const cloneSuffix = `__obranch-${branchIndex}`
|
||||
if (currentNodeId.includes('__clone')) {
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (
|
||||
scopeId.includes('__clone') &&
|
||||
scopeId.endsWith(cloneSuffix) &&
|
||||
stripOuterBranchSuffix(scopeId) === originalId
|
||||
) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidateId = buildClonedSubflowId(originalId, branchIndex)
|
||||
if (executionMap.has(candidateId)) {
|
||||
return candidateId
|
||||
}
|
||||
|
||||
for (const scopeId of executionMap.keys()) {
|
||||
if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) {
|
||||
return scopeId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return original ID — for branch-0 (non-cloned) or when scope is missing.
|
||||
// Callers handle the missing-scope case gracefully.
|
||||
return originalId
|
||||
return SubflowNodeIdCodec.findEffectiveContainerId(
|
||||
originalId,
|
||||
currentNodeId,
|
||||
executionMap,
|
||||
mappedBranchIndex
|
||||
)
|
||||
}
|
||||
|
||||
export function normalizeNodeId(nodeId: string): string {
|
||||
if (isBranchNodeId(nodeId)) {
|
||||
return extractBaseBlockId(nodeId)
|
||||
}
|
||||
if (isLoopSentinelNodeId(nodeId)) {
|
||||
return extractLoopIdFromSentinel(nodeId) || nodeId
|
||||
}
|
||||
if (isParallelSentinelNodeId(nodeId)) {
|
||||
return extractParallelIdFromSentinel(nodeId) || nodeId
|
||||
}
|
||||
return nodeId
|
||||
return SubflowNodeIdCodec.normalizeNodeId(nodeId)
|
||||
}
|
||||
|
||||
type SubflowContainerType = 'loop' | 'parallel'
|
||||
|
||||
@@ -16,6 +16,8 @@ import {
|
||||
supportsNativeStructuredOutputs,
|
||||
supportsTemperature,
|
||||
} from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptAnthropicToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegment } from '@/providers/trace-enrichment'
|
||||
import type { ProviderRequest, ProviderResponse, TimeSegment } from '@/providers/types'
|
||||
import { ProviderError } from '@/providers/types'
|
||||
@@ -251,15 +253,7 @@ export async function executeAnthropicProviderRequest(
|
||||
}
|
||||
|
||||
let anthropicTools: Anthropic.Messages.Tool[] | undefined = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
input_schema: {
|
||||
type: 'object' as const,
|
||||
properties: tool.parameters.properties,
|
||||
required: tool.parameters.required,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptAnthropicToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
let toolChoice: 'none' | 'auto' | { type: 'tool'; name: string } = 'auto'
|
||||
@@ -403,79 +397,38 @@ export async function executeAnthropicProviderRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.input_tokens,
|
||||
output: usage.output_tokens,
|
||||
total: usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(request.model, usage.input_tokens, usage.output_tokens)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { total: 0.0, input: 0.0, output: 0.0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.input_tokens,
|
||||
output: usage.output_tokens,
|
||||
total: usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: {
|
||||
total: 0.0,
|
||||
input: 0.0,
|
||||
output: 0.0,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(request.model, usage.input_tokens, usage.output_tokens)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}
|
||||
),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (request.stream && !shouldStreamToolCalls) {
|
||||
@@ -811,81 +764,57 @@ export async function executeAnthropicProviderRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(streamContent, usage) => {
|
||||
streamingResult.execution.output.content = streamContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.input_tokens,
|
||||
output: tokens.output + usage.output_tokens,
|
||||
total: tokens.total + usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(request.model, usage.input_tokens, usage.output_tokens)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
}
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls: toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(streamContent, usage) => {
|
||||
output.content = streamContent
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.input_tokens,
|
||||
output: tokens.output + usage.output_tokens,
|
||||
total: tokens.total + usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.input_tokens,
|
||||
usage.output_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}
|
||||
),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
} catch (error) {
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
@@ -1258,81 +1187,57 @@ export async function executeAnthropicProviderRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(streamContent, usage) => {
|
||||
streamingResult.execution.output.content = streamContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.input_tokens,
|
||||
output: tokens.output + usage.output_tokens,
|
||||
total: tokens.total + usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(request.model, usage.input_tokens, usage.output_tokens)
|
||||
const tc2 = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: cost.input + streamCost.input,
|
||||
output: cost.output + streamCost.output,
|
||||
toolCost: tc2 || undefined,
|
||||
total: cost.total + streamCost.total + tc2,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: cost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
}
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: cost.total,
|
||||
},
|
||||
toolCalls: toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromAnthropicStream(
|
||||
streamResponse as AsyncIterable<RawMessageStreamEvent>,
|
||||
(streamContent, usage) => {
|
||||
output.content = streamContent
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.input_tokens,
|
||||
output: tokens.output + usage.output_tokens,
|
||||
total: tokens.total + usage.input_tokens + usage.output_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.input_tokens,
|
||||
usage.output_tokens
|
||||
)
|
||||
const tc2 = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: cost.input + streamCost.input,
|
||||
output: cost.output + streamCost.output,
|
||||
toolCost: tc2 || undefined,
|
||||
total: cost.total + streamCost.total + tc2,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}
|
||||
),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -27,6 +27,8 @@ import {
|
||||
} from '@/providers/azure-openai/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { executeResponsesProviderRequest } from '@/providers/openai/core'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -117,14 +119,7 @@ async function executeChatCompletionsRequest(
|
||||
}
|
||||
|
||||
const tools: ChatCompletionTool[] | undefined = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function' as const,
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: ChatCompletionCreateParamsBase & { verbosity?: string } = {
|
||||
@@ -197,75 +192,38 @@ async function executeChatCompletionsRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromAzureOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromAzureOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -528,80 +486,62 @@ async function executeChatCompletionsRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromAzureOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromAzureOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
supportsNativeStructuredOutputs,
|
||||
} from '@/providers/baseten/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -111,14 +113,7 @@ export const basetenProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'baseten') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -165,71 +160,38 @@ export const basetenProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const end = Date.now()
|
||||
const endISO = new Date(end).toISOString()
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = endISO
|
||||
streamingResult.execution.output.providerTiming.duration = end - providerStartTime
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime = end
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
end - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -487,67 +449,51 @@ export const basetenProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (request.responseFormat && hasActiveTools) {
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
getBedrockInferenceProfileId,
|
||||
} from '@/providers/bedrock/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { enrichLastModelSegment } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -360,76 +361,36 @@ export const bedrockProvider: ProviderConfig = {
|
||||
throw new Error('No stream returned from Bedrock')
|
||||
}
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromBedrockStream(streamResponse.stream, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.inputTokens,
|
||||
output: usage.outputTokens,
|
||||
total: usage.inputTokens + usage.outputTokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(request.model, usage.inputTokens, usage.outputTokens)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const bedrockStream = streamResponse.stream
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { total: 0.0, input: 0.0, output: 0.0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromBedrockStream(bedrockStream, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.inputTokens,
|
||||
output: usage.outputTokens,
|
||||
total: usage.inputTokens + usage.outputTokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: {
|
||||
total: 0.0,
|
||||
input: 0.0,
|
||||
output: 0.0,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(request.model, usage.inputTokens, usage.outputTokens)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerStartTime = Date.now()
|
||||
@@ -884,12 +845,33 @@ export const bedrockProvider: ProviderConfig = {
|
||||
throw new Error('No stream returned from Bedrock')
|
||||
}
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromBedrockStream(
|
||||
streamResponse.stream,
|
||||
(streamContent, usage) => {
|
||||
streamingResult.execution.output.content = streamContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
const bedrockStream = streamResponse.stream
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: cost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromBedrockStream(bedrockStream, (streamContent, usage) => {
|
||||
output.content = streamContent
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.inputTokens,
|
||||
output: tokens.output + usage.outputTokens,
|
||||
total: tokens.total + usage.inputTokens + usage.outputTokens,
|
||||
@@ -897,68 +879,18 @@ export const bedrockProvider: ProviderConfig = {
|
||||
|
||||
const streamCost = calculateCost(request.model, usage.inputTokens, usage.outputTokens)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
output.cost = {
|
||||
input: cost.input + streamCost.input,
|
||||
output: cost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: cost.total + streamCost.total + tc,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: cost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -7,6 +7,8 @@ import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import type { CerebrasResponse } from '@/providers/cerebras/types'
|
||||
import { createReadableStreamFromCerebrasStream } from '@/providers/cerebras/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
@@ -68,14 +70,7 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'cerebras')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -130,60 +125,37 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromCerebrasStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromCerebrasStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
const initialCallTime = Date.now()
|
||||
|
||||
@@ -489,73 +461,62 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromCerebrasStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
}
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromCerebrasStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -6,6 +6,8 @@ import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromDeepseekStream } from '@/providers/deepseek/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
@@ -71,14 +73,7 @@ export const deepseekProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'deepseek')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -127,12 +122,18 @@ export const deepseekProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromDeepseekStream(
|
||||
streamResponse as any,
|
||||
(content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromDeepseekStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
@@ -143,47 +144,15 @@ export const deepseekProvider: ProviderConfig = {
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -480,12 +449,41 @@ export const deepseekProvider: ProviderConfig = {
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromDeepseekStream(
|
||||
streamResponse as any,
|
||||
(content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromDeepseekStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
@@ -497,59 +495,16 @@ export const deepseekProvider: ProviderConfig = {
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}
|
||||
),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
supportsNativeStructuredOutputs,
|
||||
} from '@/providers/fireworks/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -111,14 +113,7 @@ export const fireworksProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'fireworks') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -165,71 +160,38 @@ export const fireworksProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const end = Date.now()
|
||||
const endISO = new Date(end).toISOString()
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = endISO
|
||||
streamingResult.execution.output.providerTiming.duration = end - providerStartTime
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime = end
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
end - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -487,67 +449,51 @@ export const fireworksProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (request.responseFormat && hasActiveTools) {
|
||||
|
||||
@@ -6,6 +6,8 @@ import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromGroqStream } from '@/providers/groq/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
@@ -64,14 +66,7 @@ export const groqProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'groq')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -131,60 +126,37 @@ export const groqProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromGroqStream(streamResponse as any, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromGroqStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerStartTime = Date.now()
|
||||
@@ -437,73 +409,62 @@ export const groqProvider: ProviderConfig = {
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromGroqStream(streamResponse as any, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
}
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromGroqStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
|
||||
@@ -8,6 +8,8 @@ import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromLiteLLMStream } from '@/providers/litellm/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
Message,
|
||||
@@ -129,14 +131,7 @@ export const litellmProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'litellm') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -216,81 +211,44 @@ export const litellmProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.content = cleanContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
output.content = cleanContent
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -600,77 +558,66 @@ export const litellmProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.content = cleanContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
output.content = cleanContent
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (deferResponseFormat && responseFormatPayload) {
|
||||
|
||||
@@ -7,6 +7,8 @@ import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromMistralStream } from '@/providers/mistral/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
@@ -81,14 +83,7 @@ export const mistralProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'mistral')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -153,75 +148,38 @@ export const mistralProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromMistralStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromMistralStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -493,71 +451,60 @@ export const mistralProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromMistralStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromMistralStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
|
||||
@@ -9,6 +9,8 @@ import type { CompletionUsage } from 'openai/resources/completions'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type { Message, ProviderRequest, ProviderResponse, TimeSegment } from '@/providers/types'
|
||||
import { ProviderError } from '@/providers/types'
|
||||
@@ -103,14 +105,7 @@ export async function executeOllamaProviderRequest(
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, providerId) as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -179,82 +174,43 @@ export async function executeOllamaProviderRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: config.createStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
config.createStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
|
||||
if (content && request.responseFormat) {
|
||||
streamingResult.execution.output.content = content
|
||||
.replace(/```json\n?|\n?```/g, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
if (content && request.responseFormat) {
|
||||
output.content = content.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -504,78 +460,65 @@ export async function executeOllamaProviderRequest(
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: config.createStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
|
||||
if (content && request.responseFormat) {
|
||||
streamingResult.execution.output.content = content
|
||||
.replace(/```json\n?|\n?```/g, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
createStream: ({ output }) =>
|
||||
config.createStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
if (content && request.responseFormat) {
|
||||
output.content = content.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
// Deferred structured output: one final JSON-mode call now that tools have run.
|
||||
|
||||
@@ -3,6 +3,8 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import type OpenAI from 'openai'
|
||||
import type { IterationToolCall, StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegment, parseToolCallArguments } from '@/providers/trace-enrichment'
|
||||
import type { Message, ProviderRequest, ProviderResponse, TimeSegment } from '@/providers/types'
|
||||
import { ProviderError } from '@/providers/types'
|
||||
@@ -138,14 +140,7 @@ export async function executeResponsesProviderRequest(
|
||||
}
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
let preparedTools: PreparedTools | null = null
|
||||
@@ -246,75 +241,38 @@ export async function executeResponsesProviderRequest(
|
||||
throw new Error(`${config.providerLabel} API error (${streamResponse.status}): ${message}`)
|
||||
}
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromResponses(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage?.promptTokens || 0,
|
||||
output: usage?.completionTokens || 0,
|
||||
total: usage?.totalTokens || 0,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage?.promptTokens || 0,
|
||||
usage?.completionTokens || 0
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromResponses(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage?.promptTokens || 0,
|
||||
output: usage?.completionTokens || 0,
|
||||
total: usage?.totalTokens || 0,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage?.promptTokens || 0,
|
||||
usage?.completionTokens || 0
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -697,71 +655,50 @@ export async function executeResponsesProviderRequest(
|
||||
throw new Error(`${config.providerLabel} API error (${streamResponse.status}): ${message}`)
|
||||
}
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromResponses(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + (usage?.promptTokens || 0),
|
||||
output: tokens.output + (usage?.completionTokens || 0),
|
||||
total: tokens.total + (usage?.totalTokens || 0),
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage?.promptTokens || 0,
|
||||
usage?.completionTokens || 0
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls: toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromResponses(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + (usage?.promptTokens || 0),
|
||||
output: tokens.output + (usage?.completionTokens || 0),
|
||||
total: tokens.total + (usage?.totalTokens || 0),
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage?.promptTokens || 0,
|
||||
usage?.completionTokens || 0
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
createReadableStreamFromOpenAIStream,
|
||||
supportsNativeStructuredOutputs,
|
||||
} from '@/providers/openrouter/utils'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -113,14 +115,7 @@ export const openRouterProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'openrouter') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -167,71 +162,38 @@ export const openRouterProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const end = Date.now()
|
||||
const endISO = new Date(end).toISOString()
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = endISO
|
||||
streamingResult.execution.output.providerTiming.duration = end - providerStartTime
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime = end
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
end - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -489,67 +451,51 @@ export const openRouterProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (request.responseFormat && hasActiveTools) {
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { NormalizedBlockOutput } from '@/executor/types'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
|
||||
/**
|
||||
* Builds a fake stream factory mirroring the providers' `createReadableStreamFrom*`
|
||||
* helpers: it returns a sentinel stream and synchronously invokes the drain
|
||||
* callback so the test can assert the populated output without a real stream.
|
||||
*/
|
||||
function fakeStreamFactory(
|
||||
drain: (handles: { output: NormalizedBlockOutput; finalizeTiming: () => void }) => void
|
||||
) {
|
||||
const stream = new ReadableStream()
|
||||
return {
|
||||
stream,
|
||||
createStream: (handles: { output: NormalizedBlockOutput; finalizeTiming: () => void }) => {
|
||||
drain(handles)
|
||||
return stream
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe('createStreamingExecution', () => {
|
||||
const providerStartTime = 1_000
|
||||
const providerStartTimeISO = new Date(providerStartTime).toISOString()
|
||||
|
||||
it('assembles the simple (no-tools) shape and finalizes timing on drain', () => {
|
||||
const drainTime = 5_000
|
||||
vi.spyOn(Date, 'now').mockReturnValue(drainTime)
|
||||
|
||||
const { stream, createStream } = fakeStreamFactory(({ output, finalizeTiming }) => {
|
||||
output.content = 'hello'
|
||||
output.tokens = { input: 10, output: 20, total: 30 }
|
||||
output.cost = { input: 0.1, output: 0.2, total: 0.3 }
|
||||
finalizeTiming()
|
||||
})
|
||||
|
||||
const result = createStreamingExecution({
|
||||
model: 'test-model',
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: 'test-model' },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream,
|
||||
})
|
||||
|
||||
expect(result.stream).toBe(stream)
|
||||
|
||||
const output = result.execution.output
|
||||
expect(output.content).toBe('hello')
|
||||
expect(output.model).toBe('test-model')
|
||||
expect(output.tokens).toEqual({ input: 10, output: 20, total: 30 })
|
||||
expect(output.cost).toEqual({ input: 0.1, output: 0.2, total: 0.3 })
|
||||
expect(output.toolCalls).toBeUndefined()
|
||||
|
||||
const timing = output.providerTiming
|
||||
expect(timing?.startTime).toBe(providerStartTimeISO)
|
||||
expect(timing?.endTime).toBe(new Date(drainTime).toISOString())
|
||||
expect(timing?.duration).toBe(drainTime - providerStartTime)
|
||||
expect(timing?.modelTime).toBeUndefined()
|
||||
|
||||
const segment = timing?.timeSegments?.[0]
|
||||
expect(segment).toMatchObject({
|
||||
type: 'model',
|
||||
name: 'test-model',
|
||||
startTime: providerStartTime,
|
||||
})
|
||||
expect(segment?.endTime).toBe(drainTime)
|
||||
expect(segment?.duration).toBe(drainTime - providerStartTime)
|
||||
|
||||
expect(result.execution.success).toBe(true)
|
||||
expect(result.execution.logs).toEqual([])
|
||||
expect(result.execution.isStreaming).toBe(true)
|
||||
expect(result.execution.metadata?.startTime).toBe(providerStartTimeISO)
|
||||
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('assembles the accumulated (post-tools) shape with pre-built segments', () => {
|
||||
const drainTime = 7_000
|
||||
vi.spyOn(Date, 'now').mockReturnValue(drainTime)
|
||||
|
||||
const timeSegments = [
|
||||
{ type: 'model' as const, name: 'iter 1', startTime: 1_000, endTime: 2_000, duration: 1_000 },
|
||||
{ type: 'tool' as const, name: 'lookup', startTime: 2_000, endTime: 2_500, duration: 500 },
|
||||
]
|
||||
|
||||
const { createStream } = fakeStreamFactory(({ output }) => {
|
||||
output.content = 'final'
|
||||
output.tokens = { input: 110, output: 220, total: 330 }
|
||||
output.cost = { input: 1.1, output: 2.2, toolCost: 0.5, total: 3.8 }
|
||||
})
|
||||
|
||||
const result = createStreamingExecution({
|
||||
model: 'tool-model',
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime: 1_500,
|
||||
toolsTime: 500,
|
||||
firstResponseTime: 800,
|
||||
iterations: 2,
|
||||
timeSegments,
|
||||
},
|
||||
initialTokens: { input: 100, output: 200, total: 300 },
|
||||
initialCost: { input: 1, output: 2, toolCost: undefined, total: 3 },
|
||||
toolCalls: { list: [{ name: 'lookup' }], count: 1 },
|
||||
isStreaming: true,
|
||||
createStream,
|
||||
})
|
||||
|
||||
const output = result.execution.output
|
||||
expect(output.content).toBe('final')
|
||||
expect(output.tokens).toEqual({ input: 110, output: 220, total: 330 })
|
||||
expect(output.cost).toEqual({ input: 1.1, output: 2.2, toolCost: 0.5, total: 3.8 })
|
||||
expect(output.toolCalls).toEqual({ list: [{ name: 'lookup' }], count: 1 })
|
||||
|
||||
const timing = output.providerTiming
|
||||
expect(timing?.modelTime).toBe(1_500)
|
||||
expect(timing?.toolsTime).toBe(500)
|
||||
expect(timing?.firstResponseTime).toBe(800)
|
||||
expect(timing?.iterations).toBe(2)
|
||||
expect(timing?.timeSegments).toBe(timeSegments)
|
||||
expect(timing?.startTime).toBe(providerStartTimeISO)
|
||||
expect(timing?.endTime).toBe(new Date(drainTime).toISOString())
|
||||
expect(timing?.duration).toBe(drainTime - providerStartTime)
|
||||
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('only finalizes timing when the provider calls finalizeTiming', () => {
|
||||
const constructTime = 1_200
|
||||
vi.spyOn(Date, 'now').mockReturnValue(constructTime)
|
||||
|
||||
const result = createStreamingExecution({
|
||||
model: 'no-finalize',
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime: 0,
|
||||
toolsTime: 0,
|
||||
firstResponseTime: 0,
|
||||
iterations: 1,
|
||||
timeSegments: [],
|
||||
},
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output }) => {
|
||||
output.content = 'no-timing-mutation'
|
||||
return new ReadableStream()
|
||||
},
|
||||
})
|
||||
|
||||
const timing = result.execution.output.providerTiming
|
||||
expect(timing?.endTime).toBe(new Date(constructTime).toISOString())
|
||||
expect(timing?.duration).toBe(constructTime - providerStartTime)
|
||||
expect(result.execution.isStreaming).toBeUndefined()
|
||||
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('finalizeTiming touches only top-level aggregate for accumulated timing', () => {
|
||||
const constructTime = 1_000
|
||||
const drainTime = 9_000
|
||||
const nowMock = vi.spyOn(Date, 'now').mockReturnValue(constructTime)
|
||||
|
||||
const segment = { type: 'model' as const, name: 's', startTime: 1, endTime: 2, duration: 1 }
|
||||
|
||||
const result = createStreamingExecution({
|
||||
model: 'm',
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime: 0,
|
||||
toolsTime: 0,
|
||||
firstResponseTime: 0,
|
||||
iterations: 1,
|
||||
timeSegments: [segment],
|
||||
},
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ finalizeTiming }) => {
|
||||
nowMock.mockReturnValue(drainTime)
|
||||
finalizeTiming()
|
||||
return new ReadableStream()
|
||||
},
|
||||
})
|
||||
|
||||
const timing = result.execution.output.providerTiming
|
||||
expect(timing?.endTime).toBe(new Date(drainTime).toISOString())
|
||||
expect(timing?.duration).toBe(drainTime - providerStartTime)
|
||||
expect(timing?.timeSegments?.[0]).toEqual({
|
||||
type: 'model',
|
||||
name: 's',
|
||||
startTime: 1,
|
||||
endTime: 2,
|
||||
duration: 1,
|
||||
})
|
||||
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,196 @@
|
||||
import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types'
|
||||
import type { TimeSegment } from '@/providers/types'
|
||||
|
||||
/**
|
||||
* Provider-agnostic assembly of the {@link StreamingExecution} object that every
|
||||
* LLM provider returns from its streaming path. Centralizes the start/end timing,
|
||||
* duration tracking, time-segment wiring, the `success`/`logs`/`metadata` envelope,
|
||||
* and the timing-finalization contract that was previously copy-pasted across
|
||||
* providers. Callers inject only the provider-specific stream iterable (which
|
||||
* writes final content/tokens/cost) via {@link CreateStreamingExecutionOptions.createStream}.
|
||||
*/
|
||||
|
||||
/** Initial cost slice; shape is opaque so providers may include `toolCost`/`pricing`. */
|
||||
type CostSlice = NonNullable<NormalizedBlockOutput['cost']>
|
||||
|
||||
/** Initial token slice written into `execution.output.tokens`. */
|
||||
type TokenSlice = NonNullable<NormalizedBlockOutput['tokens']>
|
||||
|
||||
/**
|
||||
* Tool-call container written into `execution.output.toolCalls`. Providers build
|
||||
* structurally-compatible list items whose `result` field is `unknown`; the
|
||||
* container is widened here (mirroring the providers' former `as StreamingExecution`
|
||||
* cast) and narrowed on assignment.
|
||||
*/
|
||||
type ToolCallSlice = { list: unknown[]; count: number }
|
||||
|
||||
/**
|
||||
* Timing for the no-tools streaming path. The factory builds a single inline
|
||||
* `model` time segment and, when {@link StreamFinalizer.finalizeTiming} runs,
|
||||
* overwrites the top-level `endTime`/`duration` and that segment's
|
||||
* `endTime`/`duration` from the drain timestamp.
|
||||
*/
|
||||
interface SimpleTiming {
|
||||
kind: 'simple'
|
||||
/** Segment label — providers use the model id. */
|
||||
segmentName: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Timing for the post-tool streaming path. The factory emits the pre-built
|
||||
* segments and aggregate counters from the tool-execution loop and, when
|
||||
* {@link StreamFinalizer.finalizeTiming} runs, overwrites only the top-level
|
||||
* `endTime`/`duration` (segments are already finalized by the loop).
|
||||
*/
|
||||
interface AccumulatedTiming {
|
||||
kind: 'accumulated'
|
||||
modelTime: number
|
||||
toolsTime: number
|
||||
firstResponseTime: number
|
||||
iterations: number
|
||||
timeSegments: TimeSegment[]
|
||||
}
|
||||
|
||||
type StreamingTiming = SimpleTiming | AccumulatedTiming
|
||||
|
||||
/** Handles passed to {@link CreateStreamingExecutionOptions.createStream}. */
|
||||
interface StreamFinalizer {
|
||||
/** Live output object — write final `content`/`tokens`/`cost` here on drain. */
|
||||
output: NormalizedBlockOutput
|
||||
/** Overwrites placeholder timing from the drain timestamp. Call once on drain. */
|
||||
finalizeTiming: () => void
|
||||
}
|
||||
|
||||
interface CreateStreamingExecutionOptions {
|
||||
/** Model id echoed into `execution.output.model`. */
|
||||
model: string
|
||||
/** Wall-clock ms when the provider started the request (`Date.now()`). */
|
||||
providerStartTime: number
|
||||
/** ISO form of {@link providerStartTime}. */
|
||||
providerStartTimeISO: string
|
||||
/** Timing shape — `simple` (no tools) or `accumulated` (post-tools). */
|
||||
timing: StreamingTiming
|
||||
/** Initial token counts (zeroed for the simple path, accumulated otherwise). */
|
||||
initialTokens: TokenSlice
|
||||
/** Initial cost (zeroed for the simple path, accumulated otherwise). */
|
||||
initialCost: CostSlice
|
||||
/** Tool-call container, or `undefined` when none were used. */
|
||||
toolCalls?: ToolCallSlice
|
||||
/** Marks `execution.isStreaming = true` when set. */
|
||||
isStreaming?: boolean
|
||||
/**
|
||||
* Builds the provider stream. Receives the live `output` object and a
|
||||
* `finalizeTiming` hook. The provider wires its native stream factory and, in
|
||||
* the drain callback, writes final content/tokens/cost onto `output` then
|
||||
* calls `finalizeTiming()`.
|
||||
*/
|
||||
createStream: (handles: StreamFinalizer) => ReadableStream
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles a fully-wired {@link StreamingExecution}. The provider's stream
|
||||
* (from {@link CreateStreamingExecutionOptions.createStream}) populates the
|
||||
* output and finalizes timing on drain.
|
||||
*/
|
||||
export function createStreamingExecution(
|
||||
options: CreateStreamingExecutionOptions
|
||||
): StreamingExecution {
|
||||
const {
|
||||
model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing,
|
||||
initialTokens,
|
||||
initialCost,
|
||||
toolCalls,
|
||||
isStreaming,
|
||||
createStream,
|
||||
} = options
|
||||
|
||||
const now = Date.now()
|
||||
const nowISO = new Date(now).toISOString()
|
||||
const duration = now - providerStartTime
|
||||
|
||||
const providerTiming: NonNullable<NormalizedBlockOutput['providerTiming']> =
|
||||
timing.kind === 'simple'
|
||||
? {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: nowISO,
|
||||
duration,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: timing.segmentName,
|
||||
startTime: providerStartTime,
|
||||
endTime: now,
|
||||
duration,
|
||||
},
|
||||
],
|
||||
}
|
||||
: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: nowISO,
|
||||
duration,
|
||||
modelTime: timing.modelTime,
|
||||
toolsTime: timing.toolsTime,
|
||||
firstResponseTime: timing.firstResponseTime,
|
||||
iterations: timing.iterations,
|
||||
timeSegments: timing.timeSegments,
|
||||
}
|
||||
|
||||
const output: NormalizedBlockOutput = {
|
||||
content: '',
|
||||
model,
|
||||
tokens: initialTokens,
|
||||
toolCalls: toolCalls as NormalizedBlockOutput['toolCalls'],
|
||||
providerTiming,
|
||||
cost: initialCost,
|
||||
}
|
||||
|
||||
const timingKind = timing.kind
|
||||
const stream = createStream({
|
||||
output,
|
||||
finalizeTiming: () => finalizeTiming(output, providerStartTime, timingKind),
|
||||
})
|
||||
|
||||
return {
|
||||
stream,
|
||||
execution: {
|
||||
success: true,
|
||||
output,
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: nowISO,
|
||||
duration,
|
||||
},
|
||||
...(isStreaming ? { isStreaming: true } : {}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Overwrites the placeholder timing with the drain timestamp. For the simple
|
||||
* path the first time segment is also finalized; for the accumulated path only
|
||||
* the top-level aggregate is touched (segments are pre-finalized by the loop).
|
||||
*/
|
||||
function finalizeTiming(
|
||||
output: NormalizedBlockOutput,
|
||||
providerStartTime: number,
|
||||
kind: StreamingTiming['kind']
|
||||
): void {
|
||||
const streamEndTime = Date.now()
|
||||
const providerTiming = output.providerTiming
|
||||
if (!providerTiming) return
|
||||
|
||||
providerTiming.endTime = new Date(streamEndTime).toISOString()
|
||||
providerTiming.duration = streamEndTime - providerStartTime
|
||||
|
||||
if (kind === 'simple') {
|
||||
const segment = providerTiming.timeSegments?.[0]
|
||||
if (segment) {
|
||||
segment.endTime = streamEndTime
|
||||
segment.duration = streamEndTime - providerStartTime
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,11 +6,13 @@ import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import {
|
||||
checkForForcedToolUsage,
|
||||
createReadableStreamFromOpenAIStream,
|
||||
supportsNativeStructuredOutputs,
|
||||
} from '@/providers/together/utils'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
FunctionCallResponse,
|
||||
@@ -111,14 +113,7 @@ export const togetherProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'together') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -165,71 +160,38 @@ export const togetherProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const end = Date.now()
|
||||
const endISO = new Date(end).toISOString()
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = endISO
|
||||
streamingResult.execution.output.providerTiming.duration = end - providerStartTime
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime = end
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
end - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const costResult = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -487,67 +449,51 @@ export const togetherProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: requestedModel,
|
||||
tokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: requestedModel,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: { input: tokens.input, output: tokens.output, total: tokens.total },
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0 ? { list: toolCalls, count: toolCalls.length } : undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromOpenAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
requestedModel,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
if (request.responseFormat && hasActiveTools) {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
adaptAnthropicToolSchema,
|
||||
adaptOpenAIChatToolSchema,
|
||||
} from '@/providers/tool-schema-adapter'
|
||||
import type { ProviderToolConfig } from '@/providers/types'
|
||||
|
||||
const sampleTool: ProviderToolConfig = {
|
||||
id: 'search_web',
|
||||
name: 'Search Web',
|
||||
description: 'Search the web for a query',
|
||||
params: {},
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { query: { type: 'string', description: 'The query' } },
|
||||
required: ['query'],
|
||||
},
|
||||
}
|
||||
|
||||
const noDescriptionTool: ProviderToolConfig = {
|
||||
id: 'noop',
|
||||
name: 'Noop',
|
||||
description: '',
|
||||
params: {},
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
}
|
||||
|
||||
const emptyParametersTool: ProviderToolConfig = {
|
||||
id: 'ping',
|
||||
name: 'Ping',
|
||||
description: 'Ping the server',
|
||||
params: {},
|
||||
parameters: {} as ProviderToolConfig['parameters'],
|
||||
}
|
||||
|
||||
describe('adaptOpenAIChatToolSchema', () => {
|
||||
it('wraps the tool in the chat function shape', () => {
|
||||
expect(adaptOpenAIChatToolSchema(sampleTool)).toEqual({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'search_web',
|
||||
description: 'Search the web for a query',
|
||||
parameters: sampleTool.parameters,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves an empty description', () => {
|
||||
expect(adaptOpenAIChatToolSchema(noDescriptionTool)).toEqual({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'noop',
|
||||
description: '',
|
||||
parameters: noDescriptionTool.parameters,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('passes through empty parameters unchanged', () => {
|
||||
expect(adaptOpenAIChatToolSchema(emptyParametersTool)).toEqual({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'ping',
|
||||
description: 'Ping the server',
|
||||
parameters: emptyParametersTool.parameters,
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('adaptAnthropicToolSchema', () => {
|
||||
it('produces the Anthropic input_schema shape', () => {
|
||||
expect(adaptAnthropicToolSchema(sampleTool)).toEqual({
|
||||
name: 'search_web',
|
||||
description: 'Search the web for a query',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: sampleTool.parameters.properties,
|
||||
required: sampleTool.parameters.required,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves an empty description', () => {
|
||||
expect(adaptAnthropicToolSchema(noDescriptionTool)).toEqual({
|
||||
name: 'noop',
|
||||
description: '',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: noDescriptionTool.parameters.properties,
|
||||
required: noDescriptionTool.parameters.required,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('passes through empty parameters as undefined properties/required', () => {
|
||||
expect(adaptAnthropicToolSchema(emptyParametersTool)).toEqual({
|
||||
name: 'ping',
|
||||
description: 'Ping the server',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: undefined,
|
||||
required: undefined,
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { ProviderToolConfig } from '@/providers/types'
|
||||
|
||||
/**
|
||||
* OpenAI Chat Completions-style tool definition, shared by every
|
||||
* OpenAI-compatible provider (groq, mistral, together, etc.).
|
||||
*/
|
||||
export interface OpenAIChatToolSchema {
|
||||
type: 'function'
|
||||
function: {
|
||||
name: string
|
||||
description: string
|
||||
parameters: ProviderToolConfig['parameters']
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Anthropic Messages API tool definition.
|
||||
*/
|
||||
export interface AnthropicToolSchema {
|
||||
name: string
|
||||
description: string
|
||||
input_schema: {
|
||||
type: 'object'
|
||||
properties: ProviderToolConfig['parameters']['properties']
|
||||
required: ProviderToolConfig['parameters']['required']
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapts a tool config to the OpenAI Chat Completions function-wrapped shape.
|
||||
*/
|
||||
export function adaptOpenAIChatToolSchema(tool: ProviderToolConfig): OpenAIChatToolSchema {
|
||||
return {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapts a tool config to the Anthropic Messages `input_schema` shape.
|
||||
*/
|
||||
export function adaptAnthropicToolSchema(tool: ProviderToolConfig): AnthropicToolSchema {
|
||||
return {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: tool.parameters.properties,
|
||||
required: tool.parameters.required,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
Message,
|
||||
@@ -126,14 +128,7 @@ export const vllmProvider: ProviderConfig = {
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'vllm') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
@@ -199,80 +194,43 @@ export const vllmProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.content = cleanContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
const streamEndTime = Date.now()
|
||||
const streamEndTimeISO = new Date(streamEndTime).toISOString()
|
||||
|
||||
if (streamingResult.execution.output.providerTiming) {
|
||||
streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO
|
||||
streamingResult.execution.output.providerTiming.duration =
|
||||
streamEndTime - providerStartTime
|
||||
|
||||
if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) {
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].endTime =
|
||||
streamEndTime
|
||||
streamingResult.execution.output.providerTiming.timeSegments[0].duration =
|
||||
streamEndTime - providerStartTime
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
createStream: ({ output, finalizeTiming }) =>
|
||||
createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
},
|
||||
} as StreamingExecution
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
output.content = cleanContent
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
|
||||
finalizeTiming()
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
@@ -556,76 +514,65 @@ export const vllmProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
streamingResult.execution.output.content = cleanContent
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
} as StreamingExecution
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
|
||||
let cleanContent = content
|
||||
if (cleanContent && request.responseFormat) {
|
||||
cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
output.content = cleanContent
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
|
||||
+86
-125
@@ -6,6 +6,8 @@ import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
Message,
|
||||
@@ -79,14 +81,7 @@ export const xAIProvider: ProviderConfig = {
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'xai') as Message[]
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.id,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
},
|
||||
}))
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
if (tools?.length && request.responseFormat) {
|
||||
logger.warn(
|
||||
@@ -125,60 +120,37 @@ export const xAIProvider: ProviderConfig = {
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromXAIStream(streamResponse, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromXAIStream(streamResponse, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
toolCalls: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
timeSegments: [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: providerStartTime,
|
||||
endTime: Date.now(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
],
|
||||
},
|
||||
cost: { input: 0, output: 0, total: 0 },
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
},
|
||||
}
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
return streamingResult
|
||||
}
|
||||
const providerStartTime = Date.now()
|
||||
const providerStartTimeISO = new Date(providerStartTime).toISOString()
|
||||
@@ -520,73 +492,62 @@ export const xAIProvider: ProviderConfig = {
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = {
|
||||
stream: createReadableStreamFromXAIStream(streamResponse as any, (content, usage) => {
|
||||
streamingResult.execution.output.content = content
|
||||
streamingResult.execution.output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
streamingResult.execution.output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
execution: {
|
||||
success: true,
|
||||
output: {
|
||||
content: '',
|
||||
model: request.model,
|
||||
tokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
providerTiming: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
cost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
},
|
||||
logs: [],
|
||||
metadata: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: new Date().toISOString(),
|
||||
duration: Date.now() - providerStartTime,
|
||||
},
|
||||
isStreaming: true,
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
}
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromXAIStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
return streamingResult as StreamingExecution
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
|
||||
Reference in New Issue
Block a user