mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
feat(providers): add NVIDIA NIM and Z.ai providers (#5560)
* v0.6.29: login improvements, posthog telemetry (#4026) * feat(posthog): Add tracking on mothership abort (#4023) Co-authored-by: Theodore Li <theo@sim.ai> * fix(login): fix captcha headers for manual login (#4025) * fix(signup): fix turnstile key loading * fix(login): fix captcha header passing * Catch user already exists, remove login form captcha * feat(providers): add NVIDIA NIM and Z.ai providers - NVIDIA NIM (BYOK): Nemotron model family (70B/Ultra-253B/Super-49B v1.5, Nemotron-3 Nano/Super/Ultra) via integrate.api.nvidia.com's OpenAI-compatible API - Z.ai (hosted): GLM model family (5.2 down to 4-32B) via api.z.ai's OpenAI-compatible API, bare glm-* model ids with no provider prefix, Sim-provided key rotation (ZAI_API_KEY_1/2/3) matching openai/anthropic/google - Z.ai tool_choice is forced to 'auto' since the API only documents auto support; forced/none tool_choice from prepareToolsWithUsageControl is ignored with a warning log instead of being sent to the API * fix(providers): scope zai model routing to the exact catalog Drop the /^glm/ fallback pattern - it would overmatch any unrelated self-hosted "glm-*" model (e.g. a custom vLLM/LiteLLM deployment) and misroute it to Z.ai's hosted, Sim-billed key. Routing now relies solely on the exact model-id match against zai's static catalog. * fix(providers): wire thinking/reasoning_effort into Z.ai requests request.thinkingLevel and request.reasoningEffort were computed but never mapped onto the Z.ai payload, so the thinking toggle and GLM-5.2's reasoning_effort control silently no-op'd and always ran on Z.ai's server-side default instead of the user's selection. * fix(providers): route Z.ai through the workspace BYOK resolver too getApiKeyWithBYOK (the resolver actually used by workspace provider runs, per providers/index.ts) only rotated server keys for openai/anthropic/google/mistral, so GLM calls without a user apiKey failed even with ZAI_API_KEY_1/2/3 configured — getApiKey in providers/utils.ts had zai wired but that's not the codepath workspace runs go through. Add zai to the hosted-check condition, and register it as a BYOKProviderId so a workspace can also bring its own Z.ai key. --------- Co-authored-by: Theodore Li <theodoreqili@gmail.com> Co-authored-by: Siddharth Ganesan <33737564+Sg312@users.noreply.github.com> Co-authored-by: Vikhyath Mondreti <vikhyathvikku@gmail.com> Co-authored-by: Theodore Li <theo@sim.ai>
This commit is contained in:
Waleed
co-authored by
Theodore Li
Siddharth Ganesan
Vikhyath Mondreti
Theodore Li
Theodore Li
Siddharth Ganesan
Vikhyath Mondreti
Theodore Li
parent
7d1c927ab1
commit
2986362b38
@@ -3687,6 +3687,38 @@ export const SakanaIcon = (props: SVGProps<SVGSVGElement>) => (
|
||||
</svg>
|
||||
)
|
||||
|
||||
export const NvidiaIcon = (props: SVGProps<SVGSVGElement>) => (
|
||||
<svg
|
||||
{...props}
|
||||
height='1em'
|
||||
viewBox='30 28 280 172'
|
||||
width='1em'
|
||||
xmlns='http://www.w3.org/2000/svg'
|
||||
>
|
||||
<title>NVIDIA</title>
|
||||
<path
|
||||
d='M82.211,102.414c0,0,22.504-33.203,67.437-36.638V53.73c-49.769,3.997-92.867,46.149-92.867,46.149s24.41,70.565,92.867,77.026v-12.804C99.411,157.781,82.211,102.414,82.211,102.414z M149.648,138.637v11.726c-37.968-6.769-48.507-46.237-48.507-46.237s18.23-20.195,48.507-23.47v12.867c-0.023,0-0.039-0.007-0.058-0.007c-15.891-1.907-28.305,12.938-28.305,12.938S128.243,131.445,149.648,138.637 M149.648,31.512V53.73c1.461-0.112,2.922-0.207,4.391-0.257c56.582-1.907,93.449,46.406,93.449,46.406s-42.343,51.488-86.457,51.488c-4.043,0-7.828-0.375-11.383-1.005v13.739c3.04,0.386,6.192,0.613,9.481,0.613c41.051,0,70.738-20.965,99.484-45.778c4.766,3.817,24.278,13.103,28.289,17.168c-27.332,22.883-91.031,41.329-127.144,41.329c-3.481,0-6.824-0.211-10.11-0.528v19.306h156.032V31.512H149.648z M149.648,80.656V65.777c1.446-0.101,2.903-0.179,4.391-0.226c40.688-1.278,67.382,34.965,67.382,34.965s-28.832,40.043-59.746,40.043c-4.449,0-8.438-0.715-12.028-1.922V93.523c15.84,1.914,19.028,8.911,28.551,24.786l21.18-17.859c0,0-15.461-20.277-41.524-20.277C155.021,80.172,152.31,80.371,149.648,80.656'
|
||||
fill='#77B900'
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
|
||||
export const ZaiIcon = (props: SVGProps<SVGSVGElement>) => (
|
||||
<svg {...props} height='1em' viewBox='0 0 30 30' width='1em' xmlns='http://www.w3.org/2000/svg'>
|
||||
<title>Z.ai</title>
|
||||
<path
|
||||
d='M24.51,28.51H5.49c-2.21,0-4-1.79-4-4V5.49c0-2.21,1.79-4,4-4h19.03c2.21,0,4,1.79,4,4v19.03C28.51,26.72,26.72,28.51,24.51,28.51z'
|
||||
fill='#2D2D2D'
|
||||
/>
|
||||
<path
|
||||
d='M15.47,7.1l-1.3,1.85c-0.2,0.29-0.54,0.47-0.9,0.47h-7.1V7.09C6.16,7.1,15.47,7.1,15.47,7.1z'
|
||||
fill='#FFFFFF'
|
||||
/>
|
||||
<polygon fill='#FFFFFF' points='24.3,7.1 13.14,22.91 5.7,22.91 16.86,7.1' />
|
||||
<path d='M14.53,22.91l1.31-1.86c0.2-0.29,0.54-0.47,0.9-0.47h7.09v2.33H14.53z' fill='#FFFFFF' />
|
||||
</svg>
|
||||
)
|
||||
|
||||
export function MetaIcon(props: SVGProps<SVGSVGElement>) {
|
||||
const id = useId()
|
||||
const gradient1Id = `meta_gradient_1_${id}`
|
||||
|
||||
@@ -204,13 +204,14 @@ export async function getApiKeyWithBYOK(
|
||||
const isClaudeModel = provider === 'anthropic'
|
||||
const isGeminiModel = provider === 'google'
|
||||
const isMistralModel = provider === 'mistral'
|
||||
const isZaiModel = provider === 'zai'
|
||||
|
||||
const byokProviderId = isGeminiModel ? 'google' : (provider as BYOKProviderId)
|
||||
|
||||
if (
|
||||
isHosted &&
|
||||
workspaceId &&
|
||||
(isOpenAIModel || isClaudeModel || isGeminiModel || isMistralModel)
|
||||
(isOpenAIModel || isClaudeModel || isGeminiModel || isMistralModel || isZaiModel)
|
||||
) {
|
||||
const hostedModels = getHostedModels()
|
||||
const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase())
|
||||
|
||||
@@ -6,6 +6,7 @@ export const byokProviderIdSchema = z.enum([
|
||||
'anthropic',
|
||||
'google',
|
||||
'mistral',
|
||||
'zai',
|
||||
'fireworks',
|
||||
'together',
|
||||
'baseten',
|
||||
|
||||
@@ -11,7 +11,8 @@ export function getRotatingApiKey(provider: string): string {
|
||||
provider !== 'openai' &&
|
||||
provider !== 'anthropic' &&
|
||||
provider !== 'gemini' &&
|
||||
provider !== 'cohere'
|
||||
provider !== 'cohere' &&
|
||||
provider !== 'zai'
|
||||
) {
|
||||
throw new Error(`No rotation implemented for provider: ${provider}`)
|
||||
}
|
||||
@@ -34,6 +35,10 @@ export function getRotatingApiKey(provider: string): string {
|
||||
if (env.COHERE_API_KEY_1) keys.push(env.COHERE_API_KEY_1)
|
||||
if (env.COHERE_API_KEY_2) keys.push(env.COHERE_API_KEY_2)
|
||||
if (env.COHERE_API_KEY_3) keys.push(env.COHERE_API_KEY_3)
|
||||
} else if (provider === 'zai') {
|
||||
if (env.ZAI_API_KEY_1) keys.push(env.ZAI_API_KEY_1)
|
||||
if (env.ZAI_API_KEY_2) keys.push(env.ZAI_API_KEY_2)
|
||||
if (env.ZAI_API_KEY_3) keys.push(env.ZAI_API_KEY_3)
|
||||
}
|
||||
|
||||
if (keys.length === 0) {
|
||||
|
||||
@@ -144,6 +144,9 @@ export const env = createEnv({
|
||||
GEMINI_API_KEY_1: z.string().min(1).optional(), // Primary Gemini API key
|
||||
GEMINI_API_KEY_2: z.string().min(1).optional(), // Additional Gemini API key for load balancing
|
||||
GEMINI_API_KEY_3: z.string().min(1).optional(), // Additional Gemini API key for load balancing
|
||||
ZAI_API_KEY_1: z.string().min(1).optional(), // Primary Z.ai API key for load balancing
|
||||
ZAI_API_KEY_2: z.string().min(1).optional(), // Additional Z.ai API key for load balancing
|
||||
ZAI_API_KEY_3: z.string().min(1).optional(), // Additional Z.ai API key for load balancing
|
||||
OLLAMA_URL: z.string().url().optional(), // Ollama local LLM server URL
|
||||
VLLM_BASE_URL: z.string().url().optional(), // vLLM self-hosted base URL (OpenAI-compatible)
|
||||
VLLM_API_KEY: z.string().optional(), // Optional bearer token for vLLM
|
||||
|
||||
@@ -61,11 +61,21 @@ export const TOKENIZATION_CONFIG = {
|
||||
confidence: 'medium',
|
||||
supportedMethods: ['heuristic', 'fallback'],
|
||||
},
|
||||
nvidia: {
|
||||
avgCharsPerToken: 4,
|
||||
confidence: 'medium',
|
||||
supportedMethods: ['heuristic', 'fallback'],
|
||||
},
|
||||
meta: {
|
||||
avgCharsPerToken: 4,
|
||||
confidence: 'medium',
|
||||
supportedMethods: ['heuristic', 'fallback'],
|
||||
},
|
||||
zai: {
|
||||
avgCharsPerToken: 4,
|
||||
confidence: 'medium',
|
||||
supportedMethods: ['heuristic', 'fallback'],
|
||||
},
|
||||
ollama: {
|
||||
avgCharsPerToken: 4,
|
||||
confidence: 'low',
|
||||
|
||||
@@ -36,7 +36,9 @@ export type AttachmentProvider =
|
||||
| 'deepseek'
|
||||
| 'cerebras'
|
||||
| 'sakana'
|
||||
| 'nvidia'
|
||||
| 'meta'
|
||||
| 'zai'
|
||||
|
||||
export interface PreparedProviderAttachment {
|
||||
file: UserFile
|
||||
@@ -124,7 +126,9 @@ const UNSUPPORTED_FILE_PROVIDERS = new Set<AttachmentProvider>([
|
||||
'deepseek',
|
||||
'cerebras',
|
||||
'sakana',
|
||||
'nvidia',
|
||||
'meta',
|
||||
'zai',
|
||||
])
|
||||
|
||||
const PROVIDER_SUPPORTED_LABELS: Record<AttachmentProvider, string> = {
|
||||
@@ -145,7 +149,9 @@ const PROVIDER_SUPPORTED_LABELS: Record<AttachmentProvider, string> = {
|
||||
deepseek: 'no file attachments in the current API adapter',
|
||||
cerebras: 'no file attachments in the current API adapter',
|
||||
sakana: 'no file attachments in the current API adapter',
|
||||
nvidia: 'no file attachments in the current API adapter',
|
||||
meta: 'no file attachments in the current API adapter',
|
||||
zai: 'no file attachments in the current API adapter',
|
||||
}
|
||||
|
||||
export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null {
|
||||
@@ -166,7 +172,9 @@ export function getAttachmentProvider(providerId: ProviderId | string): Attachme
|
||||
if (providerId === 'deepseek') return 'deepseek'
|
||||
if (providerId === 'cerebras') return 'cerebras'
|
||||
if (providerId === 'sakana') return 'sakana'
|
||||
if (providerId === 'nvidia') return 'nvidia'
|
||||
if (providerId === 'meta') return 'meta'
|
||||
if (providerId === 'zai') return 'zai'
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -315,7 +323,9 @@ function isMimeTypeSupportedByProvider(
|
||||
case 'deepseek':
|
||||
case 'cerebras':
|
||||
case 'sakana':
|
||||
case 'nvidia':
|
||||
case 'meta':
|
||||
case 'zai':
|
||||
return false
|
||||
default: {
|
||||
const _exhaustive: never = provider
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
getBaseModelProviders,
|
||||
getHostedModels,
|
||||
orderModelIdsByReleaseDate,
|
||||
PROVIDER_DEFINITIONS,
|
||||
} from '@/providers/models'
|
||||
@@ -134,3 +135,86 @@ describe('sakana provider definition', () => {
|
||||
expect(baseModels['fugu-ultra']).toBe('sakana')
|
||||
})
|
||||
})
|
||||
|
||||
describe('nvidia provider definition', () => {
|
||||
const nvidia = PROVIDER_DEFINITIONS.nvidia
|
||||
|
||||
const expectedModels = [
|
||||
{ id: 'nvidia/llama-3.1-nemotron-70b-instruct', contextWindow: 128000 },
|
||||
{ id: 'nvidia/llama-3.1-nemotron-ultra-253b-v1', contextWindow: 131072 },
|
||||
{ id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5', contextWindow: 131072 },
|
||||
{ id: 'nvidia/nemotron-3-nano-30b-a3b', contextWindow: 262144 },
|
||||
{ id: 'nvidia/nemotron-3-super-120b-a12b', contextWindow: 1048576 },
|
||||
{ id: 'nvidia/nemotron-3-ultra-550b-a55b', contextWindow: 1048576 },
|
||||
]
|
||||
|
||||
it('is registered with the current-gen Super model as the default', () => {
|
||||
expect(nvidia).toBeDefined()
|
||||
expect(nvidia.id).toBe('nvidia')
|
||||
expect(nvidia.defaultModel).toBe('nvidia/nemotron-3-super-120b-a12b')
|
||||
expect(nvidia.modelPatterns).toEqual([/^nvidia\//])
|
||||
})
|
||||
|
||||
it('exposes all six Nemotron models with the documented context windows', () => {
|
||||
expect(nvidia.models.map((m) => m.id)).toEqual(expectedModels.map((m) => m.id))
|
||||
for (const expected of expectedModels) {
|
||||
const model = nvidia.models.find((m) => m.id === expected.id)
|
||||
expect(model?.contextWindow).toBe(expected.contextWindow)
|
||||
}
|
||||
})
|
||||
|
||||
it('routes every nvidia model ID to the nvidia provider', () => {
|
||||
const baseModels = getBaseModelProviders()
|
||||
for (const expected of expectedModels) {
|
||||
expect(baseModels[expected.id]).toBe('nvidia')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('zai provider definition', () => {
|
||||
const zai = PROVIDER_DEFINITIONS.zai
|
||||
|
||||
const expectedModels = [
|
||||
{ id: 'glm-5.2', contextWindow: 1000000 },
|
||||
{ id: 'glm-5.1', contextWindow: 200000 },
|
||||
{ id: 'glm-5', contextWindow: 200000 },
|
||||
{ id: 'glm-5-turbo', contextWindow: 200000 },
|
||||
{ id: 'glm-4.7', contextWindow: 200000 },
|
||||
{ id: 'glm-4.7-flashx', contextWindow: 200000 },
|
||||
{ id: 'glm-4.6', contextWindow: 200000 },
|
||||
{ id: 'glm-4.5', contextWindow: 128000 },
|
||||
{ id: 'glm-4.5-air', contextWindow: 128000 },
|
||||
{ id: 'glm-4.5-x', contextWindow: 128000 },
|
||||
{ id: 'glm-4.5-airx', contextWindow: 128000 },
|
||||
{ id: 'glm-4-32b-0414-128k', contextWindow: 128000 },
|
||||
]
|
||||
|
||||
it('is registered with a bare glm-4.6 as the default model', () => {
|
||||
expect(zai).toBeDefined()
|
||||
expect(zai.id).toBe('zai')
|
||||
expect(zai.defaultModel).toBe('glm-4.6')
|
||||
expect(zai.defaultModel.startsWith('zai/')).toBe(false)
|
||||
// No fallback pattern — an unscoped `/^glm/` would overmatch unrelated self-hosted
|
||||
// "glm-*" models and misroute them to Z.ai's hosted billing.
|
||||
expect(zai.modelPatterns).toEqual([])
|
||||
})
|
||||
|
||||
it('exposes every GLM model with the documented context window', () => {
|
||||
expect(zai.models.map((m) => m.id)).toEqual(expectedModels.map((m) => m.id))
|
||||
for (const expected of expectedModels) {
|
||||
const model = zai.models.find((m) => m.id === expected.id)
|
||||
expect(model?.contextWindow).toBe(expected.contextWindow)
|
||||
}
|
||||
})
|
||||
|
||||
it('routes every bare glm-* model ID to the zai provider', () => {
|
||||
const baseModels = getBaseModelProviders()
|
||||
for (const expected of expectedModels) {
|
||||
expect(baseModels[expected.id]).toBe('zai')
|
||||
}
|
||||
})
|
||||
|
||||
it('is included in getHostedModels since Sim provides the Z.ai key server-side', () => {
|
||||
expect(getHostedModels()).toContain('glm-4.6')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
LitellmIcon,
|
||||
MetaIcon,
|
||||
MistralIcon,
|
||||
NvidiaIcon,
|
||||
OllamaIcon,
|
||||
OpenAIIcon,
|
||||
OpenRouterIcon,
|
||||
@@ -29,6 +30,7 @@ import {
|
||||
VertexIcon,
|
||||
VllmIcon,
|
||||
xAIIcon,
|
||||
ZaiIcon,
|
||||
} from '@/components/icons'
|
||||
import type { ModelPricing, ProviderId } from '@/providers/types'
|
||||
|
||||
@@ -2376,6 +2378,115 @@ export const PROVIDER_DEFINITIONS: Record<string, ProviderDefinition> = {
|
||||
},
|
||||
],
|
||||
},
|
||||
nvidia: {
|
||||
id: 'nvidia',
|
||||
name: 'NVIDIA NIM',
|
||||
description: "NVIDIA's Nemotron models via NIM's OpenAI-compatible API",
|
||||
defaultModel: 'nvidia/nemotron-3-super-120b-a12b',
|
||||
modelPatterns: [/^nvidia\//],
|
||||
icon: NvidiaIcon,
|
||||
color: '#77B900',
|
||||
isReseller: true,
|
||||
contextInformationAvailable: true,
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: 'nvidia/llama-3.1-nemotron-70b-instruct',
|
||||
pricing: {
|
||||
input: 1.2,
|
||||
output: 1.2,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 4096,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2024-10-15',
|
||||
},
|
||||
{
|
||||
id: 'nvidia/llama-3.1-nemotron-ultra-253b-v1',
|
||||
pricing: {
|
||||
input: 0.6,
|
||||
output: 1.8,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 8192,
|
||||
},
|
||||
contextWindow: 131072,
|
||||
releaseDate: '2025-04-07',
|
||||
},
|
||||
{
|
||||
id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5',
|
||||
pricing: {
|
||||
input: 0.4,
|
||||
output: 0.4,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 8192,
|
||||
},
|
||||
contextWindow: 131072,
|
||||
releaseDate: '2025-07-25',
|
||||
},
|
||||
{
|
||||
id: 'nvidia/nemotron-3-nano-30b-a3b',
|
||||
pricing: {
|
||||
input: 0.05,
|
||||
output: 0.2,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 8192,
|
||||
},
|
||||
contextWindow: 262144,
|
||||
releaseDate: '2025-12-15',
|
||||
speedOptimized: true,
|
||||
},
|
||||
{
|
||||
id: 'nvidia/nemotron-3-super-120b-a12b',
|
||||
pricing: {
|
||||
input: 0.08,
|
||||
output: 0.45,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 8192,
|
||||
},
|
||||
contextWindow: 1048576,
|
||||
releaseDate: '2026-03-11',
|
||||
recommended: true,
|
||||
},
|
||||
{
|
||||
id: 'nvidia/nemotron-3-ultra-550b-a55b',
|
||||
pricing: {
|
||||
input: 0.5,
|
||||
output: 2.2,
|
||||
updatedAt: '2026-07-09',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 2 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 8192,
|
||||
},
|
||||
contextWindow: 1048576,
|
||||
releaseDate: '2026-06-04',
|
||||
},
|
||||
],
|
||||
},
|
||||
meta: {
|
||||
id: 'meta',
|
||||
name: 'Meta',
|
||||
@@ -2408,6 +2519,237 @@ export const PROVIDER_DEFINITIONS: Record<string, ProviderDefinition> = {
|
||||
},
|
||||
],
|
||||
},
|
||||
zai: {
|
||||
id: 'zai',
|
||||
name: 'Z.ai',
|
||||
description: "Z.ai's GLM models via an OpenAI-compatible API",
|
||||
defaultModel: 'glm-4.6',
|
||||
// No fallback pattern: `/^glm/` would overmatch any unrelated self-hosted "glm-*" model
|
||||
// (e.g. a custom vLLM/LiteLLM deployment) and misroute it to Z.ai's hosted billing.
|
||||
// Routing for zai's own models relies solely on the exact catalog match in `models`.
|
||||
modelPatterns: [],
|
||||
icon: ZaiIcon,
|
||||
color: '#2D2D2D',
|
||||
contextInformationAvailable: true,
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: 'glm-5.2',
|
||||
pricing: {
|
||||
input: 1.4,
|
||||
output: 4.4,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
reasoningEffort: {
|
||||
values: ['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'],
|
||||
},
|
||||
},
|
||||
contextWindow: 1000000,
|
||||
releaseDate: '2026-06-13',
|
||||
recommended: true,
|
||||
},
|
||||
{
|
||||
id: 'glm-5.1',
|
||||
pricing: {
|
||||
input: 1.4,
|
||||
output: 4.4,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2026-04-07',
|
||||
},
|
||||
{
|
||||
id: 'glm-5',
|
||||
pricing: {
|
||||
input: 1.0,
|
||||
output: 3.2,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2026-02-11',
|
||||
},
|
||||
{
|
||||
id: 'glm-5-turbo',
|
||||
pricing: {
|
||||
input: 1.2,
|
||||
output: 4.0,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2026-03-15',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.7',
|
||||
pricing: {
|
||||
input: 0.6,
|
||||
output: 2.2,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2025-12-22',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.7-flashx',
|
||||
pricing: {
|
||||
input: 0.07,
|
||||
output: 0.4,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2025-12-22',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.6',
|
||||
pricing: {
|
||||
input: 0.6,
|
||||
output: 2.2,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 131072,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 200000,
|
||||
releaseDate: '2025-09-30',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.5',
|
||||
pricing: {
|
||||
input: 0.6,
|
||||
output: 2.2,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 98304,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2025-07-28',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.5-air',
|
||||
pricing: {
|
||||
input: 0.2,
|
||||
output: 1.1,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 98304,
|
||||
thinking: {
|
||||
levels: ['disabled', 'enabled'],
|
||||
default: 'enabled',
|
||||
},
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2025-07-28',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.5-x',
|
||||
pricing: {
|
||||
input: 2.2,
|
||||
output: 8.9,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 98304,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2025-07-28',
|
||||
},
|
||||
{
|
||||
id: 'glm-4.5-airx',
|
||||
pricing: {
|
||||
input: 1.1,
|
||||
output: 4.5,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 98304,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2025-07-28',
|
||||
},
|
||||
{
|
||||
id: 'glm-4-32b-0414-128k',
|
||||
pricing: {
|
||||
input: 0.1,
|
||||
output: 0.1,
|
||||
updatedAt: '2026-07-10',
|
||||
},
|
||||
capabilities: {
|
||||
temperature: { min: 0, max: 1 },
|
||||
toolUsageControl: true,
|
||||
maxOutputTokens: 16384,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
releaseDate: '2025-04-14',
|
||||
},
|
||||
],
|
||||
},
|
||||
mistral: {
|
||||
id: 'mistral',
|
||||
name: 'Mistral AI',
|
||||
@@ -3552,6 +3894,7 @@ export function getHostedModels(): string[] {
|
||||
...getProviderModels('openai'),
|
||||
...getProviderModels('anthropic'),
|
||||
...getProviderModels('google'),
|
||||
...getProviderModels('zai'),
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import OpenAI from 'openai'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createReadableStreamFromNvidiaStream } from '@/providers/nvidia/utils'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
ProviderRequest,
|
||||
ProviderResponse,
|
||||
TimeSegment,
|
||||
} from '@/providers/types'
|
||||
import { ProviderError } from '@/providers/types'
|
||||
import {
|
||||
calculateCost,
|
||||
prepareToolExecution,
|
||||
prepareToolsWithUsageControl,
|
||||
sumToolCosts,
|
||||
trackForcedToolUsage,
|
||||
} from '@/providers/utils'
|
||||
import { executeTool } from '@/tools'
|
||||
|
||||
const logger = createLogger('NvidiaProvider')
|
||||
|
||||
const NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1'
|
||||
|
||||
export const nvidiaProvider: ProviderConfig = {
|
||||
id: 'nvidia',
|
||||
name: 'NVIDIA NIM',
|
||||
description: "NVIDIA's Nemotron models via NIM's OpenAI-compatible API",
|
||||
version: '1.0.0',
|
||||
models: getProviderModels('nvidia'),
|
||||
defaultModel: getProviderDefaultModel('nvidia'),
|
||||
|
||||
executeRequest: async (
|
||||
request: ProviderRequest
|
||||
): Promise<ProviderResponse | StreamingExecution> => {
|
||||
if (!request.apiKey) {
|
||||
throw new Error('API key is required for NVIDIA NIM')
|
||||
}
|
||||
|
||||
const providerStartTime = Date.now()
|
||||
const providerStartTimeISO = new Date(providerStartTime).toISOString()
|
||||
|
||||
try {
|
||||
const nvidia = new OpenAI({
|
||||
apiKey: request.apiKey,
|
||||
baseURL: NVIDIA_BASE_URL,
|
||||
})
|
||||
|
||||
const allMessages = []
|
||||
|
||||
if (request.systemPrompt) {
|
||||
allMessages.push({
|
||||
role: 'system',
|
||||
content: request.systemPrompt,
|
||||
})
|
||||
}
|
||||
|
||||
if (request.context) {
|
||||
allMessages.push({
|
||||
role: 'user',
|
||||
content: request.context,
|
||||
})
|
||||
}
|
||||
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'nvidia')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
model: request.model,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens
|
||||
|
||||
const responseFormatPayload = request.responseFormat
|
||||
? {
|
||||
type: 'json_schema' as const,
|
||||
json_schema: {
|
||||
name: request.responseFormat.name || 'response_schema',
|
||||
schema: request.responseFormat.schema || request.responseFormat,
|
||||
strict: request.responseFormat.strict !== false,
|
||||
},
|
||||
}
|
||||
: undefined
|
||||
|
||||
let preparedTools: ReturnType<typeof prepareToolsWithUsageControl> | null = null
|
||||
let hasActiveTools = false
|
||||
|
||||
if (tools?.length) {
|
||||
preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'openai')
|
||||
const { tools: filteredTools, toolChoice } = preparedTools
|
||||
|
||||
if (filteredTools?.length && toolChoice) {
|
||||
payload.tools = filteredTools
|
||||
payload.tool_choice = toolChoice
|
||||
hasActiveTools = true
|
||||
|
||||
logger.info('NVIDIA NIM request configuration:', {
|
||||
toolCount: filteredTools.length,
|
||||
toolChoice:
|
||||
typeof toolChoice === 'string'
|
||||
? toolChoice
|
||||
: toolChoice.type === 'function'
|
||||
? `force:${toolChoice.function.name}`
|
||||
: 'unknown',
|
||||
model: request.model,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Structured output and tool calling cannot be sent together — OpenAI-compatible
|
||||
// backends reject a request that carries both `response_format` and active
|
||||
// `tools`/`tool_choice`. Defer the schema until after the tool loop completes.
|
||||
const deferResponseFormat = !!responseFormatPayload && hasActiveTools
|
||||
if (responseFormatPayload && !deferResponseFormat) {
|
||||
payload.response_format = responseFormatPayload
|
||||
}
|
||||
|
||||
if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) {
|
||||
logger.info('Using streaming response for NVIDIA NIM request (no tools)')
|
||||
|
||||
const streamResponse = await nvidia.chat.completions.create(
|
||||
{
|
||||
...payload,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
},
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromNvidiaStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
const originalToolChoice = payload.tool_choice
|
||||
const forcedTools = preparedTools?.forcedTools || []
|
||||
let usedForcedTools: string[] = []
|
||||
|
||||
let currentResponse = await nvidia.chat.completions.create(
|
||||
payload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
const firstResponseTime = Date.now() - initialCallTime
|
||||
|
||||
let content = currentResponse.choices[0]?.message?.content || ''
|
||||
|
||||
const tokens = {
|
||||
input: currentResponse.usage?.prompt_tokens || 0,
|
||||
output: currentResponse.usage?.completion_tokens || 0,
|
||||
total: currentResponse.usage?.total_tokens || 0,
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let hasUsedForcedTool = false
|
||||
let modelTime = firstResponseTime
|
||||
let toolsTime = 0
|
||||
|
||||
const timeSegments: TimeSegment[] = [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: initialCallTime,
|
||||
endTime: initialCallTime + firstResponseTime,
|
||||
duration: firstResponseTime,
|
||||
},
|
||||
]
|
||||
|
||||
if (
|
||||
typeof originalToolChoice === 'object' &&
|
||||
currentResponse.choices[0]?.message?.tool_calls
|
||||
) {
|
||||
const toolCallsResponse = currentResponse.choices[0].message.tool_calls
|
||||
const result = trackForcedToolUsage(
|
||||
toolCallsResponse,
|
||||
originalToolChoice,
|
||||
logger,
|
||||
'openai',
|
||||
forcedTools,
|
||||
usedForcedTools
|
||||
)
|
||||
hasUsedForcedTool = result.hasUsedForcedTool
|
||||
usedForcedTools = result.usedForcedTools
|
||||
}
|
||||
|
||||
try {
|
||||
while (iterationCount < MAX_TOOL_ITERATIONS) {
|
||||
if (currentResponse.choices[0]?.message?.content) {
|
||||
content = currentResponse.choices[0].message.content
|
||||
}
|
||||
|
||||
const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls
|
||||
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
toolCallsInResponse,
|
||||
{ model: request.model, provider: 'nvidia' }
|
||||
)
|
||||
|
||||
if (!toolCallsInResponse || toolCallsInResponse.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const toolsStartTime = Date.now()
|
||||
|
||||
const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => {
|
||||
const toolCallStartTime = Date.now()
|
||||
const toolName = toolCall.function.name
|
||||
|
||||
try {
|
||||
const toolArgs = JSON.parse(toolCall.function.arguments)
|
||||
const tool = request.tools?.find((t) => t.id === toolName)
|
||||
|
||||
// Every tool_call in the assistant message must be answered by a matching
|
||||
// `tool` message, or the next request violates the OpenAI message contract.
|
||||
// Emit an error result for an unknown tool rather than dropping it.
|
||||
if (!tool) {
|
||||
const toolCallEndTime = Date.now()
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams: {},
|
||||
result: {
|
||||
success: false,
|
||||
output: undefined,
|
||||
error: `Tool "${toolName}" is not available`,
|
||||
},
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
}
|
||||
|
||||
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
|
||||
const result = await executeTool(toolName, executionParams, {
|
||||
signal: request.abortSignal,
|
||||
})
|
||||
const toolCallEndTime = Date.now()
|
||||
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams,
|
||||
result,
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
} catch (error) {
|
||||
const toolCallEndTime = Date.now()
|
||||
logger.error('Error processing tool call:', { error, toolName })
|
||||
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams: {},
|
||||
result: {
|
||||
success: false,
|
||||
output: undefined,
|
||||
error: getErrorMessage(error, 'Tool execution failed'),
|
||||
},
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const executionResults = await Promise.allSettled(toolExecutionPromises)
|
||||
|
||||
currentMessages.push({
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: toolCallsInResponse.map((tc) => ({
|
||||
id: tc.id,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tc.function.name,
|
||||
arguments: tc.function.arguments,
|
||||
},
|
||||
})),
|
||||
})
|
||||
|
||||
for (const settledResult of executionResults) {
|
||||
if (settledResult.status === 'rejected' || !settledResult.value) continue
|
||||
|
||||
const { toolCall, toolName, toolParams, result, startTime, endTime, duration } =
|
||||
settledResult.value
|
||||
|
||||
timeSegments.push({
|
||||
type: 'tool',
|
||||
name: toolName,
|
||||
startTime: startTime,
|
||||
endTime: endTime,
|
||||
duration: duration,
|
||||
toolCallId: toolCall.id,
|
||||
})
|
||||
|
||||
let resultContent: any
|
||||
if (result.success && result.output) {
|
||||
toolResults.push(result.output)
|
||||
resultContent = result.output
|
||||
} else {
|
||||
resultContent = {
|
||||
error: true,
|
||||
message: result.error || 'Tool execution failed',
|
||||
tool: toolName,
|
||||
}
|
||||
}
|
||||
|
||||
toolCalls.push({
|
||||
name: toolName,
|
||||
arguments: toolParams,
|
||||
startTime: new Date(startTime).toISOString(),
|
||||
endTime: new Date(endTime).toISOString(),
|
||||
duration: duration,
|
||||
result: resultContent,
|
||||
success: result.success,
|
||||
})
|
||||
|
||||
currentMessages.push({
|
||||
role: 'tool',
|
||||
tool_call_id: toolCall.id,
|
||||
content: JSON.stringify(resultContent),
|
||||
})
|
||||
}
|
||||
|
||||
const thisToolsTime = Date.now() - toolsStartTime
|
||||
toolsTime += thisToolsTime
|
||||
|
||||
const nextPayload = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
}
|
||||
|
||||
if (
|
||||
typeof originalToolChoice === 'object' &&
|
||||
hasUsedForcedTool &&
|
||||
forcedTools.length > 0
|
||||
) {
|
||||
const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool))
|
||||
|
||||
if (remainingTools.length > 0) {
|
||||
nextPayload.tool_choice = {
|
||||
type: 'function',
|
||||
function: { name: remainingTools[0] },
|
||||
}
|
||||
logger.info(`Forcing next tool: ${remainingTools[0]}`)
|
||||
} else {
|
||||
nextPayload.tool_choice = 'auto'
|
||||
logger.info('All forced tools have been used, switching to auto tool_choice')
|
||||
}
|
||||
}
|
||||
|
||||
const nextModelStartTime = Date.now()
|
||||
currentResponse = await nvidia.chat.completions.create(
|
||||
nextPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
if (
|
||||
typeof nextPayload.tool_choice === 'object' &&
|
||||
currentResponse.choices[0]?.message?.tool_calls
|
||||
) {
|
||||
const toolCallsResponse = currentResponse.choices[0].message.tool_calls
|
||||
const result = trackForcedToolUsage(
|
||||
toolCallsResponse,
|
||||
nextPayload.tool_choice,
|
||||
logger,
|
||||
'openai',
|
||||
forcedTools,
|
||||
usedForcedTools
|
||||
)
|
||||
hasUsedForcedTool = result.hasUsedForcedTool
|
||||
usedForcedTools = result.usedForcedTools
|
||||
}
|
||||
|
||||
const nextModelEndTime = Date.now()
|
||||
const thisModelTime = nextModelEndTime - nextModelStartTime
|
||||
|
||||
timeSegments.push({
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: nextModelStartTime,
|
||||
endTime: nextModelEndTime,
|
||||
duration: thisModelTime,
|
||||
})
|
||||
|
||||
modelTime += thisModelTime
|
||||
|
||||
if (currentResponse.choices[0]?.message?.content) {
|
||||
content = currentResponse.choices[0].message.content
|
||||
}
|
||||
|
||||
if (currentResponse.usage) {
|
||||
tokens.input += currentResponse.usage.prompt_tokens || 0
|
||||
tokens.output += currentResponse.usage.completion_tokens || 0
|
||||
tokens.total += currentResponse.usage.total_tokens || 0
|
||||
}
|
||||
|
||||
iterationCount++
|
||||
}
|
||||
|
||||
if (iterationCount === MAX_TOOL_ITERATIONS) {
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
currentResponse.choices[0]?.message?.tool_calls,
|
||||
{ model: request.model, provider: 'nvidia' }
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error in NVIDIA NIM request:', { error })
|
||||
throw error
|
||||
}
|
||||
|
||||
if (request.stream) {
|
||||
logger.info('Using streaming for final NVIDIA NIM response after tool processing')
|
||||
|
||||
// The tool loop is complete: this final pass only produces the textual answer.
|
||||
// Force `tool_choice: 'none'` so the model cannot emit fresh tool calls that the
|
||||
// text-only stream adapter would silently drop.
|
||||
const streamingPayload: any = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
tool_choice: 'none',
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
}
|
||||
if (deferResponseFormat && responseFormatPayload) {
|
||||
streamingPayload.response_format = responseFormatPayload
|
||||
streamingPayload.parallel_tool_calls = false
|
||||
}
|
||||
|
||||
const streamResponse = await nvidia.chat.completions.create(
|
||||
streamingPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromNvidiaStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
// Tools were active, so `response_format` was withheld from the loop. Make one final
|
||||
// tool-free call to obtain the structured response now that the tool work is done.
|
||||
if (deferResponseFormat && responseFormatPayload) {
|
||||
logger.info('Applying deferred JSON schema response format after tool processing')
|
||||
|
||||
const finalFormatStartTime = Date.now()
|
||||
const finalPayload: any = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
response_format: responseFormatPayload,
|
||||
tool_choice: 'none',
|
||||
parallel_tool_calls: false,
|
||||
}
|
||||
|
||||
currentResponse = await nvidia.chat.completions.create(
|
||||
finalPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const finalFormatEndTime = Date.now()
|
||||
timeSegments.push({
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: finalFormatStartTime,
|
||||
endTime: finalFormatEndTime,
|
||||
duration: finalFormatEndTime - finalFormatStartTime,
|
||||
})
|
||||
modelTime += finalFormatEndTime - finalFormatStartTime
|
||||
|
||||
const formattedContent = currentResponse.choices[0]?.message?.content
|
||||
if (formattedContent) {
|
||||
content = formattedContent
|
||||
}
|
||||
|
||||
if (currentResponse.usage) {
|
||||
tokens.input += currentResponse.usage.prompt_tokens || 0
|
||||
tokens.output += currentResponse.usage.completion_tokens || 0
|
||||
tokens.total += currentResponse.usage.total_tokens || 0
|
||||
}
|
||||
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
currentResponse.choices[0]?.message?.tool_calls,
|
||||
{ model: request.model, provider: 'nvidia' }
|
||||
)
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
const totalDuration = providerEndTime - providerStartTime
|
||||
|
||||
return {
|
||||
content,
|
||||
model: request.model,
|
||||
tokens,
|
||||
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
||||
toolResults: toolResults.length > 0 ? toolResults : undefined,
|
||||
timing: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: providerEndTimeISO,
|
||||
duration: totalDuration,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
const totalDuration = providerEndTime - providerStartTime
|
||||
|
||||
logger.error('Error in NVIDIA NIM request:', {
|
||||
error,
|
||||
duration: totalDuration,
|
||||
})
|
||||
|
||||
throw new ProviderError(toError(error).message, {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: providerEndTimeISO,
|
||||
duration: totalDuration,
|
||||
})
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
|
||||
import type { CompletionUsage } from 'openai/resources/completions'
|
||||
import { createOpenAICompatibleStream } from '@/providers/utils'
|
||||
|
||||
/**
|
||||
* Creates a ReadableStream from an NVIDIA NIM streaming response.
|
||||
* Uses the shared OpenAI-compatible streaming utility.
|
||||
*/
|
||||
export function createReadableStreamFromNvidiaStream(
|
||||
nvidiaStream: AsyncIterable<ChatCompletionChunk>,
|
||||
onComplete?: (content: string, usage: CompletionUsage) => void
|
||||
): ReadableStream<Uint8Array> {
|
||||
return createOpenAICompatibleStream(nvidiaStream, 'NVIDIA', onComplete)
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import { groqProvider } from '@/providers/groq'
|
||||
import { litellmProvider } from '@/providers/litellm'
|
||||
import { metaProvider } from '@/providers/meta'
|
||||
import { mistralProvider } from '@/providers/mistral'
|
||||
import { nvidiaProvider } from '@/providers/nvidia'
|
||||
import { ollamaProvider } from '@/providers/ollama'
|
||||
import { ollamaCloudProvider } from '@/providers/ollama-cloud'
|
||||
import { openaiProvider } from '@/providers/openai'
|
||||
@@ -23,6 +24,7 @@ import type { ProviderConfig, ProviderId } from '@/providers/types'
|
||||
import { vertexProvider } from '@/providers/vertex'
|
||||
import { vllmProvider } from '@/providers/vllm'
|
||||
import { xAIProvider } from '@/providers/xai'
|
||||
import { zaiProvider } from '@/providers/zai'
|
||||
|
||||
const logger = createLogger('ProviderRegistry')
|
||||
|
||||
@@ -37,7 +39,9 @@ const providerRegistry: Record<ProviderId, ProviderConfig> = {
|
||||
cerebras: cerebrasProvider,
|
||||
groq: groqProvider,
|
||||
sakana: sakanaProvider,
|
||||
nvidia: nvidiaProvider,
|
||||
meta: metaProvider,
|
||||
zai: zaiProvider,
|
||||
vllm: vllmProvider,
|
||||
litellm: litellmProvider,
|
||||
mistral: mistralProvider,
|
||||
|
||||
@@ -12,7 +12,9 @@ export type ProviderId =
|
||||
| 'cerebras'
|
||||
| 'groq'
|
||||
| 'sakana'
|
||||
| 'nvidia'
|
||||
| 'meta'
|
||||
| 'zai'
|
||||
| 'mistral'
|
||||
| 'ollama'
|
||||
| 'ollama-cloud'
|
||||
|
||||
@@ -155,7 +155,9 @@ export const providers: Record<ProviderId, ProviderMetadata> = {
|
||||
cerebras: buildProviderMetadata('cerebras'),
|
||||
groq: buildProviderMetadata('groq'),
|
||||
sakana: buildProviderMetadata('sakana'),
|
||||
nvidia: buildProviderMetadata('nvidia'),
|
||||
meta: buildProviderMetadata('meta'),
|
||||
zai: buildProviderMetadata('zai'),
|
||||
mistral: buildProviderMetadata('mistral'),
|
||||
bedrock: buildProviderMetadata('bedrock'),
|
||||
openrouter: buildProviderMetadata('openrouter'),
|
||||
@@ -900,8 +902,9 @@ export function getApiKey(provider: string, model: string, userProvidedKey?: str
|
||||
const isOpenAIModel = provider === 'openai'
|
||||
const isClaudeModel = provider === 'anthropic'
|
||||
const isGeminiModel = provider === 'google'
|
||||
const isZaiModel = provider === 'zai'
|
||||
|
||||
if (isHosted && (isOpenAIModel || isClaudeModel || isGeminiModel)) {
|
||||
if (isHosted && (isOpenAIModel || isClaudeModel || isGeminiModel || isZaiModel)) {
|
||||
const hostedModels = getHostedModels()
|
||||
const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase())
|
||||
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import OpenAI from 'openai'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { createStreamingExecution } from '@/providers/streaming-execution'
|
||||
import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
ProviderConfig,
|
||||
ProviderRequest,
|
||||
ProviderResponse,
|
||||
TimeSegment,
|
||||
} from '@/providers/types'
|
||||
import { ProviderError } from '@/providers/types'
|
||||
import {
|
||||
calculateCost,
|
||||
prepareToolExecution,
|
||||
prepareToolsWithUsageControl,
|
||||
sumToolCosts,
|
||||
trackForcedToolUsage,
|
||||
} from '@/providers/utils'
|
||||
import { createReadableStreamFromZaiStream } from '@/providers/zai/utils'
|
||||
import { executeTool } from '@/tools'
|
||||
|
||||
const logger = createLogger('ZaiProvider')
|
||||
|
||||
const ZAI_BASE_URL = 'https://api.z.ai/api/paas/v4'
|
||||
|
||||
export const zaiProvider: ProviderConfig = {
|
||||
id: 'zai',
|
||||
name: 'Z.ai',
|
||||
description: "Z.ai's GLM models via an OpenAI-compatible API",
|
||||
version: '1.0.0',
|
||||
models: getProviderModels('zai'),
|
||||
defaultModel: getProviderDefaultModel('zai'),
|
||||
|
||||
executeRequest: async (
|
||||
request: ProviderRequest
|
||||
): Promise<ProviderResponse | StreamingExecution> => {
|
||||
if (!request.apiKey) {
|
||||
throw new Error('API key is required for Z.ai')
|
||||
}
|
||||
|
||||
const providerStartTime = Date.now()
|
||||
const providerStartTimeISO = new Date(providerStartTime).toISOString()
|
||||
|
||||
try {
|
||||
const zai = new OpenAI({
|
||||
apiKey: request.apiKey,
|
||||
baseURL: ZAI_BASE_URL,
|
||||
})
|
||||
|
||||
const allMessages = []
|
||||
|
||||
if (request.systemPrompt) {
|
||||
allMessages.push({
|
||||
role: 'system',
|
||||
content: request.systemPrompt,
|
||||
})
|
||||
}
|
||||
|
||||
if (request.context) {
|
||||
allMessages.push({
|
||||
role: 'user',
|
||||
content: request.context,
|
||||
})
|
||||
}
|
||||
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'zai')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
|
||||
: undefined
|
||||
|
||||
const payload: any = {
|
||||
model: request.model,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens
|
||||
|
||||
// GLM's `thinking` toggle (models where `capabilities.thinking.levels` is
|
||||
// ['disabled', 'enabled']) maps directly to Z.ai's `thinking: { type }` request param.
|
||||
if (request.thinkingLevel === 'enabled' || request.thinkingLevel === 'disabled') {
|
||||
payload.thinking = { type: request.thinkingLevel }
|
||||
}
|
||||
|
||||
// GLM-5.2's `reasoningEffort` capability maps directly to Z.ai's `reasoning_effort`
|
||||
// request param — the only model in this catalog that documents it.
|
||||
if (request.reasoningEffort !== undefined && request.reasoningEffort !== 'auto') {
|
||||
payload.reasoning_effort = request.reasoningEffort
|
||||
}
|
||||
|
||||
// Z.ai's chat-completions API supports `text` and `json_object` response formats but
|
||||
// does not have confirmed `json_schema` support, unlike the shared OpenAI-compatible
|
||||
// template — request a plain JSON-object hint instead of a strict schema.
|
||||
const responseFormatPayload = request.responseFormat
|
||||
? ({ type: 'json_object' as const } as const)
|
||||
: undefined
|
||||
|
||||
let preparedTools: ReturnType<typeof prepareToolsWithUsageControl> | null = null
|
||||
let hasActiveTools = false
|
||||
|
||||
if (tools?.length) {
|
||||
preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'openai')
|
||||
const { tools: filteredTools, toolChoice } = preparedTools
|
||||
|
||||
if (filteredTools?.length && toolChoice) {
|
||||
payload.tools = filteredTools
|
||||
// Z.ai's chat-completions API documents `tool_choice` support for `"auto"` only —
|
||||
// forcing a specific function or disabling tool use via the parameter is rejected.
|
||||
// Ignore any forced/none choice `prepareToolsWithUsageControl` may have produced.
|
||||
payload.tool_choice = 'auto'
|
||||
hasActiveTools = true
|
||||
|
||||
if (preparedTools.forcedTools.length > 0) {
|
||||
logger.warn(
|
||||
"Z.ai does not support forcing a specific tool via tool_choice (API only accepts 'auto') — ignoring force setting and falling back to auto",
|
||||
{ forcedTools: preparedTools.forcedTools, model: request.model }
|
||||
)
|
||||
}
|
||||
|
||||
logger.info('Z.ai request configuration:', {
|
||||
toolCount: filteredTools.length,
|
||||
toolChoice: 'auto',
|
||||
model: request.model,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Structured output and tool calling cannot be sent together — OpenAI-compatible
|
||||
// backends reject a request that carries both `response_format` and active
|
||||
// `tools`/`tool_choice`. Defer the schema until after the tool loop completes.
|
||||
const deferResponseFormat = !!responseFormatPayload && hasActiveTools
|
||||
if (responseFormatPayload && !deferResponseFormat) {
|
||||
payload.response_format = responseFormatPayload
|
||||
}
|
||||
|
||||
if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) {
|
||||
logger.info('Using streaming response for Z.ai request (no tools)')
|
||||
|
||||
const streamResponse = await zai.chat.completions.create(
|
||||
{
|
||||
...payload,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
},
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: { kind: 'simple', segmentName: request.model },
|
||||
initialTokens: { input: 0, output: 0, total: 0 },
|
||||
initialCost: { input: 0, output: 0, total: 0 },
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromZaiStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: usage.prompt_tokens,
|
||||
output: usage.completion_tokens,
|
||||
total: usage.total_tokens,
|
||||
}
|
||||
|
||||
const costResult = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
output.cost = {
|
||||
input: costResult.input,
|
||||
output: costResult.output,
|
||||
total: costResult.total,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
const initialCallTime = Date.now()
|
||||
const originalToolChoice = payload.tool_choice
|
||||
const forcedTools = preparedTools?.forcedTools || []
|
||||
let usedForcedTools: string[] = []
|
||||
|
||||
let currentResponse = await zai.chat.completions.create(
|
||||
payload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
const firstResponseTime = Date.now() - initialCallTime
|
||||
|
||||
let content = currentResponse.choices[0]?.message?.content || ''
|
||||
|
||||
const tokens = {
|
||||
input: currentResponse.usage?.prompt_tokens || 0,
|
||||
output: currentResponse.usage?.completion_tokens || 0,
|
||||
total: currentResponse.usage?.total_tokens || 0,
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let hasUsedForcedTool = false
|
||||
let modelTime = firstResponseTime
|
||||
let toolsTime = 0
|
||||
|
||||
const timeSegments: TimeSegment[] = [
|
||||
{
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: initialCallTime,
|
||||
endTime: initialCallTime + firstResponseTime,
|
||||
duration: firstResponseTime,
|
||||
},
|
||||
]
|
||||
|
||||
if (
|
||||
typeof originalToolChoice === 'object' &&
|
||||
currentResponse.choices[0]?.message?.tool_calls
|
||||
) {
|
||||
const toolCallsResponse = currentResponse.choices[0].message.tool_calls
|
||||
const result = trackForcedToolUsage(
|
||||
toolCallsResponse,
|
||||
originalToolChoice,
|
||||
logger,
|
||||
'openai',
|
||||
forcedTools,
|
||||
usedForcedTools
|
||||
)
|
||||
hasUsedForcedTool = result.hasUsedForcedTool
|
||||
usedForcedTools = result.usedForcedTools
|
||||
}
|
||||
|
||||
try {
|
||||
while (iterationCount < MAX_TOOL_ITERATIONS) {
|
||||
if (currentResponse.choices[0]?.message?.content) {
|
||||
content = currentResponse.choices[0].message.content
|
||||
}
|
||||
|
||||
const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls
|
||||
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
toolCallsInResponse,
|
||||
{ model: request.model, provider: 'zai' }
|
||||
)
|
||||
|
||||
if (!toolCallsInResponse || toolCallsInResponse.length === 0) {
|
||||
break
|
||||
}
|
||||
|
||||
const toolsStartTime = Date.now()
|
||||
|
||||
const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => {
|
||||
const toolCallStartTime = Date.now()
|
||||
const toolName = toolCall.function.name
|
||||
|
||||
try {
|
||||
const toolArgs = JSON.parse(toolCall.function.arguments)
|
||||
const tool = request.tools?.find((t) => t.id === toolName)
|
||||
|
||||
// Every tool_call in the assistant message must be answered by a matching
|
||||
// `tool` message, or the next request violates the OpenAI message contract.
|
||||
// Emit an error result for an unknown tool rather than dropping it.
|
||||
if (!tool) {
|
||||
const toolCallEndTime = Date.now()
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams: {},
|
||||
result: {
|
||||
success: false,
|
||||
output: undefined,
|
||||
error: `Tool "${toolName}" is not available`,
|
||||
},
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
}
|
||||
|
||||
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
|
||||
const result = await executeTool(toolName, executionParams, {
|
||||
signal: request.abortSignal,
|
||||
})
|
||||
const toolCallEndTime = Date.now()
|
||||
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams,
|
||||
result,
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
} catch (error) {
|
||||
const toolCallEndTime = Date.now()
|
||||
logger.error('Error processing tool call:', { error, toolName })
|
||||
|
||||
return {
|
||||
toolCall,
|
||||
toolName,
|
||||
toolParams: {},
|
||||
result: {
|
||||
success: false,
|
||||
output: undefined,
|
||||
error: getErrorMessage(error, 'Tool execution failed'),
|
||||
},
|
||||
startTime: toolCallStartTime,
|
||||
endTime: toolCallEndTime,
|
||||
duration: toolCallEndTime - toolCallStartTime,
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const executionResults = await Promise.allSettled(toolExecutionPromises)
|
||||
|
||||
currentMessages.push({
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: toolCallsInResponse.map((tc) => ({
|
||||
id: tc.id,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tc.function.name,
|
||||
arguments: tc.function.arguments,
|
||||
},
|
||||
})),
|
||||
})
|
||||
|
||||
for (const settledResult of executionResults) {
|
||||
if (settledResult.status === 'rejected' || !settledResult.value) continue
|
||||
|
||||
const { toolCall, toolName, toolParams, result, startTime, endTime, duration } =
|
||||
settledResult.value
|
||||
|
||||
timeSegments.push({
|
||||
type: 'tool',
|
||||
name: toolName,
|
||||
startTime: startTime,
|
||||
endTime: endTime,
|
||||
duration: duration,
|
||||
toolCallId: toolCall.id,
|
||||
})
|
||||
|
||||
let resultContent: any
|
||||
if (result.success && result.output) {
|
||||
toolResults.push(result.output)
|
||||
resultContent = result.output
|
||||
} else {
|
||||
resultContent = {
|
||||
error: true,
|
||||
message: result.error || 'Tool execution failed',
|
||||
tool: toolName,
|
||||
}
|
||||
}
|
||||
|
||||
toolCalls.push({
|
||||
name: toolName,
|
||||
arguments: toolParams,
|
||||
startTime: new Date(startTime).toISOString(),
|
||||
endTime: new Date(endTime).toISOString(),
|
||||
duration: duration,
|
||||
result: resultContent,
|
||||
success: result.success,
|
||||
})
|
||||
|
||||
currentMessages.push({
|
||||
role: 'tool',
|
||||
tool_call_id: toolCall.id,
|
||||
content: JSON.stringify(resultContent),
|
||||
})
|
||||
}
|
||||
|
||||
const thisToolsTime = Date.now() - toolsStartTime
|
||||
toolsTime += thisToolsTime
|
||||
|
||||
const nextPayload = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
}
|
||||
|
||||
if (
|
||||
typeof originalToolChoice === 'object' &&
|
||||
hasUsedForcedTool &&
|
||||
forcedTools.length > 0
|
||||
) {
|
||||
const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool))
|
||||
|
||||
if (remainingTools.length > 0) {
|
||||
nextPayload.tool_choice = {
|
||||
type: 'function',
|
||||
function: { name: remainingTools[0] },
|
||||
}
|
||||
logger.info(`Forcing next tool: ${remainingTools[0]}`)
|
||||
} else {
|
||||
nextPayload.tool_choice = 'auto'
|
||||
logger.info('All forced tools have been used, switching to auto tool_choice')
|
||||
}
|
||||
}
|
||||
|
||||
const nextModelStartTime = Date.now()
|
||||
currentResponse = await zai.chat.completions.create(
|
||||
nextPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
if (
|
||||
typeof nextPayload.tool_choice === 'object' &&
|
||||
currentResponse.choices[0]?.message?.tool_calls
|
||||
) {
|
||||
const toolCallsResponse = currentResponse.choices[0].message.tool_calls
|
||||
const result = trackForcedToolUsage(
|
||||
toolCallsResponse,
|
||||
nextPayload.tool_choice,
|
||||
logger,
|
||||
'openai',
|
||||
forcedTools,
|
||||
usedForcedTools
|
||||
)
|
||||
hasUsedForcedTool = result.hasUsedForcedTool
|
||||
usedForcedTools = result.usedForcedTools
|
||||
}
|
||||
|
||||
const nextModelEndTime = Date.now()
|
||||
const thisModelTime = nextModelEndTime - nextModelStartTime
|
||||
|
||||
timeSegments.push({
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: nextModelStartTime,
|
||||
endTime: nextModelEndTime,
|
||||
duration: thisModelTime,
|
||||
})
|
||||
|
||||
modelTime += thisModelTime
|
||||
|
||||
if (currentResponse.choices[0]?.message?.content) {
|
||||
content = currentResponse.choices[0].message.content
|
||||
}
|
||||
|
||||
if (currentResponse.usage) {
|
||||
tokens.input += currentResponse.usage.prompt_tokens || 0
|
||||
tokens.output += currentResponse.usage.completion_tokens || 0
|
||||
tokens.total += currentResponse.usage.total_tokens || 0
|
||||
}
|
||||
|
||||
iterationCount++
|
||||
}
|
||||
|
||||
if (iterationCount === MAX_TOOL_ITERATIONS) {
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
currentResponse.choices[0]?.message?.tool_calls,
|
||||
{ model: request.model, provider: 'zai' }
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error in Z.ai request:', { error })
|
||||
throw error
|
||||
}
|
||||
|
||||
if (request.stream) {
|
||||
logger.info('Using streaming for final Z.ai response after tool processing')
|
||||
|
||||
// The tool loop is complete: this final pass only produces the textual answer.
|
||||
// Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice`
|
||||
// entirely rather than sending an unsupported `'none'` — with no tools declared,
|
||||
// the model has nothing to call and the text-only stream adapter stays safe.
|
||||
const streamingPayload: any = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
}
|
||||
streamingPayload.tools = undefined
|
||||
streamingPayload.tool_choice = undefined
|
||||
if (deferResponseFormat && responseFormatPayload) {
|
||||
streamingPayload.response_format = responseFormatPayload
|
||||
}
|
||||
|
||||
const streamResponse = await zai.chat.completions.create(
|
||||
streamingPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
|
||||
|
||||
const streamingResult = createStreamingExecution({
|
||||
model: request.model,
|
||||
providerStartTime,
|
||||
providerStartTimeISO,
|
||||
timing: {
|
||||
kind: 'accumulated',
|
||||
modelTime,
|
||||
toolsTime,
|
||||
firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments,
|
||||
},
|
||||
initialTokens: {
|
||||
input: tokens.input,
|
||||
output: tokens.output,
|
||||
total: tokens.total,
|
||||
},
|
||||
initialCost: {
|
||||
input: accumulatedCost.input,
|
||||
output: accumulatedCost.output,
|
||||
toolCost: undefined as number | undefined,
|
||||
total: accumulatedCost.total,
|
||||
},
|
||||
toolCalls:
|
||||
toolCalls.length > 0
|
||||
? {
|
||||
list: toolCalls,
|
||||
count: toolCalls.length,
|
||||
}
|
||||
: undefined,
|
||||
isStreaming: true,
|
||||
createStream: ({ output }) =>
|
||||
createReadableStreamFromZaiStream(streamResponse as any, (content, usage) => {
|
||||
output.content = content
|
||||
output.tokens = {
|
||||
input: tokens.input + usage.prompt_tokens,
|
||||
output: tokens.output + usage.completion_tokens,
|
||||
total: tokens.total + usage.total_tokens,
|
||||
}
|
||||
|
||||
const streamCost = calculateCost(
|
||||
request.model,
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens
|
||||
)
|
||||
const tc = sumToolCosts(toolResults)
|
||||
output.cost = {
|
||||
input: accumulatedCost.input + streamCost.input,
|
||||
output: accumulatedCost.output + streamCost.output,
|
||||
toolCost: tc || undefined,
|
||||
total: accumulatedCost.total + streamCost.total + tc,
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
return streamingResult
|
||||
}
|
||||
|
||||
// Tools were active, so `response_format` was withheld from the loop. Make one final
|
||||
// tool-free call to obtain the structured response now that the tool work is done.
|
||||
if (deferResponseFormat && responseFormatPayload) {
|
||||
logger.info('Applying deferred response_format after tool processing')
|
||||
|
||||
const finalFormatStartTime = Date.now()
|
||||
// Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice`
|
||||
// rather than sending an unsupported `'none'` — with no tools declared, the model
|
||||
// has nothing left to call and simply returns the formatted answer.
|
||||
const finalPayload: any = {
|
||||
...payload,
|
||||
messages: currentMessages,
|
||||
response_format: responseFormatPayload,
|
||||
}
|
||||
finalPayload.tools = undefined
|
||||
finalPayload.tool_choice = undefined
|
||||
|
||||
currentResponse = await zai.chat.completions.create(
|
||||
finalPayload,
|
||||
request.abortSignal ? { signal: request.abortSignal } : undefined
|
||||
)
|
||||
|
||||
const finalFormatEndTime = Date.now()
|
||||
timeSegments.push({
|
||||
type: 'model',
|
||||
name: request.model,
|
||||
startTime: finalFormatStartTime,
|
||||
endTime: finalFormatEndTime,
|
||||
duration: finalFormatEndTime - finalFormatStartTime,
|
||||
})
|
||||
modelTime += finalFormatEndTime - finalFormatStartTime
|
||||
|
||||
const formattedContent = currentResponse.choices[0]?.message?.content
|
||||
if (formattedContent) {
|
||||
content = formattedContent
|
||||
}
|
||||
|
||||
if (currentResponse.usage) {
|
||||
tokens.input += currentResponse.usage.prompt_tokens || 0
|
||||
tokens.output += currentResponse.usage.completion_tokens || 0
|
||||
tokens.total += currentResponse.usage.total_tokens || 0
|
||||
}
|
||||
|
||||
enrichLastModelSegmentFromChatCompletions(
|
||||
timeSegments,
|
||||
currentResponse,
|
||||
currentResponse.choices[0]?.message?.tool_calls,
|
||||
{ model: request.model, provider: 'zai' }
|
||||
)
|
||||
}
|
||||
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
const totalDuration = providerEndTime - providerStartTime
|
||||
|
||||
return {
|
||||
content,
|
||||
model: request.model,
|
||||
tokens,
|
||||
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
||||
toolResults: toolResults.length > 0 ? toolResults : undefined,
|
||||
timing: {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: providerEndTimeISO,
|
||||
duration: totalDuration,
|
||||
modelTime: modelTime,
|
||||
toolsTime: toolsTime,
|
||||
firstResponseTime: firstResponseTime,
|
||||
iterations: iterationCount + 1,
|
||||
timeSegments: timeSegments,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
const providerEndTime = Date.now()
|
||||
const providerEndTimeISO = new Date(providerEndTime).toISOString()
|
||||
const totalDuration = providerEndTime - providerStartTime
|
||||
|
||||
logger.error('Error in Z.ai request:', {
|
||||
error,
|
||||
duration: totalDuration,
|
||||
})
|
||||
|
||||
throw new ProviderError(toError(error).message, {
|
||||
startTime: providerStartTimeISO,
|
||||
endTime: providerEndTimeISO,
|
||||
duration: totalDuration,
|
||||
})
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
|
||||
import type { CompletionUsage } from 'openai/resources/completions'
|
||||
import { createOpenAICompatibleStream } from '@/providers/utils'
|
||||
|
||||
/**
|
||||
* Creates a ReadableStream from a Z.ai streaming response.
|
||||
* Uses the shared OpenAI-compatible streaming utility.
|
||||
*/
|
||||
export function createReadableStreamFromZaiStream(
|
||||
zaiStream: AsyncIterable<ChatCompletionChunk>,
|
||||
onComplete?: (content: string, usage: CompletionUsage) => void
|
||||
): ReadableStream<Uint8Array> {
|
||||
return createOpenAICompatibleStream(zaiStream, 'Z.ai', onComplete)
|
||||
}
|
||||
@@ -7,6 +7,7 @@ export type BYOKProviderId =
|
||||
| 'anthropic'
|
||||
| 'google'
|
||||
| 'mistral'
|
||||
| 'zai'
|
||||
| 'fireworks'
|
||||
| 'together'
|
||||
| 'baseten'
|
||||
|
||||
Reference in New Issue
Block a user