From ea4f2691e34d8327712e9b4da820dc69619d2034 Mon Sep 17 00:00:00 2001 From: Waleed Date: Fri, 10 Jul 2026 14:20:57 -0700 Subject: [PATCH] fix(providers): correct max-tokens param and add schema guidance for NVIDIA/Z.ai (#5569) * fix(providers): correct max-tokens param and add schema guidance for NVIDIA/Z.ai - NVIDIA NIM and Z.ai both document max_tokens for output-length control, not OpenAI's newer max_completion_tokens - the latter was silently ignored by both vLLM-served NIM models and Z.ai's GLM models - Z.ai has no json_schema response_format mode (only text/json_object), so structured-output requests now also inject the expected schema into the system prompt as best-effort guidance, since the request param alone can't enforce field names/types * style(providers): replace inline comments with TSDoc, per CLAUDE.md Consolidated the scattered narrative // comments in nvidia/index.ts and zai/index.ts into a single TSDoc block per provider documenting the provider-specific API quirks; removed the rest where they only restated what the code already shows. Also dropped an inline comment on zai's modelPatterns field in models.ts. * fix(providers): scope Z.ai schema guidance to the response_format call only - schemaGuidance now falls back to the bare responseFormat object when .schema is absent, matching the schema-or-format fallback used elsewhere in the codebase (was silently injecting nothing for callers that pass a bare JSON schema) - guidance is now only appended to the messages sent alongside an actual response_format (the immediate call, or whichever pass deferResponseFormat applies it to) instead of every turn of an active tool loop, where it wrongly told the model to return final JSON instead of continuing to call tools --- apps/sim/providers/models.ts | 3 -- apps/sim/providers/nvidia/index.ts | 18 +++----- apps/sim/providers/zai/index.ts | 69 ++++++++++++++++++------------ 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 86912dda87..e5021a4c72 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -2553,9 +2553,6 @@ export const PROVIDER_DEFINITIONS: Record = { name: 'Z.ai', description: "Z.ai's GLM models via an OpenAI-compatible API", defaultModel: 'glm-4.6', - // No fallback pattern: `/^glm/` would overmatch any unrelated self-hosted "glm-*" model - // (e.g. a custom vLLM/LiteLLM deployment) and misroute it to Z.ai's hosted billing. - // Routing for zai's own models relies solely on the exact catalog match in `models`. modelPatterns: [], icon: ZaiIcon, color: '#2D2D2D', diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 0b147d29b1..ffb7bf0fb2 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -29,6 +29,11 @@ const logger = createLogger('NvidiaProvider') const NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1' +/** + * NVIDIA NIM's Nemotron models via an OpenAI-compatible chat-completions API + * (`integrate.api.nvidia.com`). Output length is capped via `max_tokens`, not OpenAI's newer + * `max_completion_tokens`, which vLLM-served NIM models don't recognize. + */ export const nvidiaProvider: ProviderConfig = { id: 'nvidia', name: 'NVIDIA NIM', @@ -84,7 +89,7 @@ export const nvidiaProvider: ProviderConfig = { } if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens + if (request.maxTokens != null) payload.max_tokens = request.maxTokens const responseFormatPayload = request.responseFormat ? { @@ -122,9 +127,6 @@ export const nvidiaProvider: ProviderConfig = { } } - // Structured output and tool calling cannot be sent together — OpenAI-compatible - // backends reject a request that carries both `response_format` and active - // `tools`/`tool_choice`. Defer the schema until after the tool loop completes. const deferResponseFormat = !!responseFormatPayload && hasActiveTools if (responseFormatPayload && !deferResponseFormat) { payload.response_format = responseFormatPayload @@ -257,9 +259,6 @@ export const nvidiaProvider: ProviderConfig = { const toolArgs = JSON.parse(toolCall.function.arguments) const tool = request.tools?.find((t) => t.id === toolName) - // Every tool_call in the assistant message must be answered by a matching - // `tool` message, or the next request violates the OpenAI message contract. - // Emit an error result for an unknown tool rather than dropping it. if (!tool) { const toolCallEndTime = Date.now() return { @@ -463,9 +462,6 @@ export const nvidiaProvider: ProviderConfig = { if (request.stream) { logger.info('Using streaming for final NVIDIA NIM response after tool processing') - // The tool loop is complete: this final pass only produces the textual answer. - // Force `tool_choice: 'none'` so the model cannot emit fresh tool calls that the - // text-only stream adapter would silently drop. const streamingPayload: any = { ...payload, messages: currentMessages, @@ -543,8 +539,6 @@ export const nvidiaProvider: ProviderConfig = { return streamingResult } - // Tools were active, so `response_format` was withheld from the loop. Make one final - // tool-free call to obtain the structured response now that the tool work is done. if (deferResponseFormat && responseFormatPayload) { logger.info('Applying deferred JSON schema response format after tool processing') diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index d254546d6e..779b18942e 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -29,6 +29,35 @@ const logger = createLogger('ZaiProvider') const ZAI_BASE_URL = 'https://api.z.ai/api/paas/v4' +function buildSchemaGuidance(responseFormat: ProviderRequest['responseFormat']): string { + if (!responseFormat) return '' + const schema = responseFormat.schema || responseFormat + return `\n\nYour response must be valid JSON matching this schema${ + responseFormat.name ? ` ("${responseFormat.name}")` : '' + }:\n${JSON.stringify(schema, null, 2)}` +} + +function withSchemaGuidance(messages: any[], guidance: string): any[] { + if (!guidance) return messages + if (messages[0]?.role === 'system') { + return [{ ...messages[0], content: `${messages[0].content}${guidance}` }, ...messages.slice(1)] + } + return [{ role: 'system', content: guidance.trimStart() }, ...messages] +} + +/** + * Z.ai's GLM models via an OpenAI-compatible chat-completions API (`api.z.ai`), with these + * documented deviations from a standard OpenAI-compatible adapter: + * - Output length is capped via `max_tokens`, not OpenAI's `max_completion_tokens`. + * - `tool_choice` only supports `"auto"` — forcing a specific tool or disabling tool use via + * the parameter is rejected, so any forced/none choice is downgraded to `"auto"` (logged as + * a warning), and a "stop calling tools" pass drops `tools`/`tool_choice` entirely instead of + * sending an unsupported `"none"`. + * - `response_format` only supports `"text"`/`"json_object"`, not `"json_schema"` — the + * expected schema is also injected into the system prompt as best-effort guidance. + * - `thinking: { type }` and `reasoning_effort` map directly from `request.thinkingLevel` and + * `request.reasoningEffort`. + */ export const zaiProvider: ProviderConfig = { id: 'zai', name: 'Z.ai', @@ -84,23 +113,16 @@ export const zaiProvider: ProviderConfig = { } if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens + if (request.maxTokens != null) payload.max_tokens = request.maxTokens - // GLM's `thinking` toggle (models where `capabilities.thinking.levels` is - // ['disabled', 'enabled']) maps directly to Z.ai's `thinking: { type }` request param. if (request.thinkingLevel === 'enabled' || request.thinkingLevel === 'disabled') { payload.thinking = { type: request.thinkingLevel } } - // GLM-5.2's `reasoningEffort` capability maps directly to Z.ai's `reasoning_effort` - // request param — the only model in this catalog that documents it. if (request.reasoningEffort !== undefined && request.reasoningEffort !== 'auto') { payload.reasoning_effort = request.reasoningEffort } - // Z.ai's chat-completions API supports `text` and `json_object` response formats but - // does not have confirmed `json_schema` support, unlike the shared OpenAI-compatible - // template — request a plain JSON-object hint instead of a strict schema. const responseFormatPayload = request.responseFormat ? ({ type: 'json_object' as const } as const) : undefined @@ -114,9 +136,6 @@ export const zaiProvider: ProviderConfig = { if (filteredTools?.length && toolChoice) { payload.tools = filteredTools - // Z.ai's chat-completions API documents `tool_choice` support for `"auto"` only — - // forcing a specific function or disabling tool use via the parameter is rejected. - // Ignore any forced/none choice `prepareToolsWithUsageControl` may have produced. payload.tool_choice = 'auto' hasActiveTools = true @@ -135,12 +154,13 @@ export const zaiProvider: ProviderConfig = { } } - // Structured output and tool calling cannot be sent together — OpenAI-compatible - // backends reject a request that carries both `response_format` and active - // `tools`/`tool_choice`. Defer the schema until after the tool loop completes. const deferResponseFormat = !!responseFormatPayload && hasActiveTools if (responseFormatPayload && !deferResponseFormat) { payload.response_format = responseFormatPayload + payload.messages = withSchemaGuidance( + payload.messages, + buildSchemaGuidance(request.responseFormat) + ) } if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) { @@ -270,9 +290,6 @@ export const zaiProvider: ProviderConfig = { const toolArgs = JSON.parse(toolCall.function.arguments) const tool = request.tools?.find((t) => t.id === toolName) - // Every tool_call in the assistant message must be answered by a matching - // `tool` message, or the next request violates the OpenAI message contract. - // Emit an error result for an unknown tool rather than dropping it. if (!tool) { const toolCallEndTime = Date.now() return { @@ -476,10 +493,6 @@ export const zaiProvider: ProviderConfig = { if (request.stream) { logger.info('Using streaming for final Z.ai response after tool processing') - // The tool loop is complete: this final pass only produces the textual answer. - // Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice` - // entirely rather than sending an unsupported `'none'` — with no tools declared, - // the model has nothing to call and the text-only stream adapter stays safe. const streamingPayload: any = { ...payload, messages: currentMessages, @@ -490,6 +503,10 @@ export const zaiProvider: ProviderConfig = { streamingPayload.tool_choice = undefined if (deferResponseFormat && responseFormatPayload) { streamingPayload.response_format = responseFormatPayload + streamingPayload.messages = withSchemaGuidance( + streamingPayload.messages, + buildSchemaGuidance(request.responseFormat) + ) } const streamResponse = await zai.chat.completions.create( @@ -557,18 +574,16 @@ export const zaiProvider: ProviderConfig = { return streamingResult } - // Tools were active, so `response_format` was withheld from the loop. Make one final - // tool-free call to obtain the structured response now that the tool work is done. if (deferResponseFormat && responseFormatPayload) { logger.info('Applying deferred response_format after tool processing') const finalFormatStartTime = Date.now() - // Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice` - // rather than sending an unsupported `'none'` — with no tools declared, the model - // has nothing left to call and simply returns the formatted answer. const finalPayload: any = { ...payload, - messages: currentMessages, + messages: withSchemaGuidance( + currentMessages, + buildSchemaGuidance(request.responseFormat) + ), response_format: responseFormatPayload, } finalPayload.tools = undefined