From 33bd6005a56e56ed5ce225004e7b4b62298dd5be Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:04:13 -0700 Subject: [PATCH] fix(vscode): revert OpenAI-compatible metadata limit plumbing (#11775) * fix(vscode): stop deriving output limits from model metadata * fix(vscode): send OpenAI-compatible output token limit (#11776) --- .../src/sdk/cline-session-factory.test.ts | 21 ++---- apps/vscode/src/sdk/cline-session-factory.ts | 65 +++---------------- .../extensions/tools/team/delegated-agent.ts | 2 + .../src/runtime/host/local-runtime-host.ts | 1 + .../runtime/orchestration/runtime-builder.ts | 1 + .../src/services/llms/handler-factory.test.ts | 50 +++++++------- .../core/src/services/llms/handler-factory.ts | 2 +- sdk/packages/core/src/types/config.ts | 4 ++ .../llms/src/providers/compat.test.ts | 30 +++++++++ 9 files changed, 77 insertions(+), 99 deletions(-) diff --git a/apps/vscode/src/sdk/cline-session-factory.test.ts b/apps/vscode/src/sdk/cline-session-factory.test.ts index 7c4120055d..fb3d26c41d 100644 --- a/apps/vscode/src/sdk/cline-session-factory.test.ts +++ b/apps/vscode/src/sdk/cline-session-factory.test.ts @@ -411,7 +411,7 @@ describe("buildSessionConfig", () => { expect(config.providerConfig).not.toHaveProperty("apiKey") }) - it("passes OpenAI Compatible custom model metadata through as SDK knownModels", async () => { + it("passes OpenAI Compatible max output tokens as an explicit request limit", async () => { mocks.stateManager.getApiConfiguration.mockReturnValue({ actModeApiProvider: "openai", actModeOpenAiModelId: "custom-reasoner", @@ -423,7 +423,6 @@ describe("buildSessionConfig", () => { maxTokens: 4_096, supportsImages: false, supportsPromptCache: false, - supportsReasoning: true, inputPrice: 0, outputPrice: 0, }, @@ -433,20 +432,10 @@ describe("buildSessionConfig", () => { expect(config.providerId).toBe("openai-compatible") expect(config.modelId).toBe("custom-reasoner") - expect(config.knownModels?.["custom-reasoner"]).toMatchObject({ - id: "custom-reasoner", - name: "Custom Reasoner", - contextWindow: 16_000, - maxInputTokens: 16_000, - maxTokens: 4_096, - capabilities: ["streaming", "tools"], - }) - expect((config.providerConfig as any).knownModels?.["custom-reasoner"]).toMatchObject({ - contextWindow: 16_000, - maxInputTokens: 16_000, - maxTokens: 4_096, - }) - expect((config.providerConfig as any).maxOutputTokens).toBe(4_096) + expect(config.knownModels).toBeUndefined() + expect((config.providerConfig as any).knownModels).toBeUndefined() + expect((config.providerConfig as any).maxOutputTokens).toBeUndefined() + expect((config as any).maxTokensPerTurn).toBe(4_096) }) it("builds structured SAP AI Core config from legacy ApiConfiguration fields", async () => { diff --git a/apps/vscode/src/sdk/cline-session-factory.ts b/apps/vscode/src/sdk/cline-session-factory.ts index c1d5c3710b..f67d3b7f0e 100644 --- a/apps/vscode/src/sdk/cline-session-factory.ts +++ b/apps/vscode/src/sdk/cline-session-factory.ts @@ -16,9 +16,9 @@ import { resolveProviderApiKeyFromSettings, type StartSessionResult, } from "@cline/core" -import { getGeneratedModelsForProvider, MODEL_COLLECTIONS_BY_PROVIDER_ID, type ModelInfo as SdkModelInfo } from "@cline/llms" +import { getGeneratedModelsForProvider, MODEL_COLLECTIONS_BY_PROVIDER_ID } from "@cline/llms" import { buildClineSystemPrompt } from "@cline/shared" -import type { ApiConfiguration, ModelInfo as LegacyModelInfo } from "@shared/api" +import type { ApiConfiguration } from "@shared/api" import type { HistoryItem } from "@shared/HistoryItem" import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, type LanguageDisplay } from "@shared/Languages" import { Logger } from "@shared/services/Logger" @@ -182,52 +182,10 @@ function resolveProviderReasoningConfig(providerId: string): SessionReasoningCon } } -function positiveNumber(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined -} - -function getOpenAiCompatibleModelInfo(config: ApiConfiguration, mode: Mode): LegacyModelInfo | undefined { - return mode === "plan" ? config.planModeOpenAiModelInfo : config.actModeOpenAiModelInfo -} - -const OPENAI_COMPATIBLE_DEFAULT_CAPABILITIES: NonNullable = ["streaming", "tools"] - -function buildOpenAiCompatibleCapabilities(modelInfo: LegacyModelInfo): NonNullable { - const capabilities: NonNullable = [...OPENAI_COMPATIBLE_DEFAULT_CAPABILITIES] - if (modelInfo.supportsImages !== false) { - capabilities.push("images") - } - return capabilities -} - -// VS Code stores OpenAI-compatible custom model metadata in legacy -// ApiConfiguration fields. The SDK runtime enforces context/output limits from -// knownModels, so bridge the selected model into the SDK shape during session -// creation. -function buildOpenAiCompatibleKnownModels( - modelId: string | undefined, - modelInfo: LegacyModelInfo | undefined, -): Record | undefined { - const trimmedModelId = modelId?.trim() - if (!trimmedModelId || !modelInfo) { - return undefined - } - - const contextWindow = positiveNumber(modelInfo.contextWindow) - const maxTokens = positiveNumber(modelInfo.maxTokens) - - return { - [trimmedModelId]: { - id: trimmedModelId, - name: modelInfo.name ?? trimmedModelId, - description: modelInfo.description, - contextWindow, - maxInputTokens: contextWindow, - maxTokens, - capabilities: buildOpenAiCompatibleCapabilities(modelInfo), - temperature: modelInfo.temperature, - }, - } +function resolveOpenAiCompatibleMaxTokens(config: ApiConfiguration | undefined, mode: Mode): number | undefined { + const modelInfo = mode === "plan" ? config?.planModeOpenAiModelInfo : config?.actModeOpenAiModelInfo + const maxTokens = modelInfo?.maxTokens + return typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0 ? maxTokens : undefined } // --------------------------------------------------------------------------- @@ -546,7 +504,6 @@ export async function buildSessionConfig(input: SessionConfigInput): Promise; export interface DelegatedAgentRuntimeConfig @@ -87,6 +88,7 @@ export function createDelegatedAgentConfigProvider( providerConfig: runtimeConfig.providerConfig, knownModels: runtimeConfig.knownModels, thinking: runtimeConfig.thinking, + maxTokensPerTurn: runtimeConfig.maxTokensPerTurn, }), updateConnectionDefaults: (overrides) => { runtimeConfig = { diff --git a/sdk/packages/core/src/runtime/host/local-runtime-host.ts b/sdk/packages/core/src/runtime/host/local-runtime-host.ts index dfafa61e94..055c2bc155 100644 --- a/sdk/packages/core/src/runtime/host/local-runtime-host.ts +++ b/sdk/packages/core/src/runtime/host/local-runtime-host.ts @@ -468,6 +468,7 @@ export class LocalRuntimeHost implements RuntimeHost { thinking: configWithProvider.thinking, reasoningEffort: configWithProvider.reasoningEffort ?? providerConfig.reasoningEffort, + maxTokensPerTurn: configWithProvider.maxTokensPerTurn, systemPrompt: configWithProvider.systemPrompt, maxIterations: configWithProvider.maxIterations, execution: configWithProvider.execution, diff --git a/sdk/packages/core/src/runtime/orchestration/runtime-builder.ts b/sdk/packages/core/src/runtime/orchestration/runtime-builder.ts index 03a11f6dc2..5134235de9 100644 --- a/sdk/packages/core/src/runtime/orchestration/runtime-builder.ts +++ b/sdk/packages/core/src/runtime/orchestration/runtime-builder.ts @@ -485,6 +485,7 @@ export class DefaultRuntimeBuilder implements RuntimeBuilder { providerConfig: config.providerConfig, knownModels: config.knownModels, thinking: config.thinking, + maxTokensPerTurn: config.maxTokensPerTurn, maxIterations: config.maxIterations, hooks, extensions: runtimeExtensions, diff --git a/sdk/packages/core/src/services/llms/handler-factory.test.ts b/sdk/packages/core/src/services/llms/handler-factory.test.ts index 4f5a37e17f..1d3b5414c6 100644 --- a/sdk/packages/core/src/services/llms/handler-factory.test.ts +++ b/sdk/packages/core/src/services/llms/handler-factory.test.ts @@ -125,31 +125,6 @@ describe("createAgentModelFromConfig", () => { ); }); - it("uses providerConfig maxOutputTokens when no direct per-turn override is set", async () => { - const { createAgentModelFromConfig } = await import("./handler-factory"); - - createAgentModelFromConfig( - { - providerId: "openai-compatible", - modelId: "custom-model", - apiKey: "key", - systemPrompt: "", - tools: [], - providerConfig: { - providerId: "openai-compatible", - modelId: "custom-model", - maxOutputTokens: 4_096, - }, - }, - undefined, - ); - - expect(gatewayMock.createAgentModel).toHaveBeenLastCalledWith( - { providerId: "openai-compatible", modelId: "custom-model" }, - { maxTokens: 4_096 }, - ); - }); - it("preserves model capabilities and metadata when configuring gateway models", async () => { const { createAgentModelFromConfig } = await import("./handler-factory"); @@ -225,6 +200,31 @@ describe("createAgentModelFromConfig", () => { }); }); + it("uses explicit per-turn max tokens for gateway request limits", async () => { + const { createAgentModelFromConfig } = await import("./handler-factory"); + + createAgentModelFromConfig( + { + providerId: "openai-compatible", + modelId: "custom-model", + apiKey: "key", + systemPrompt: "", + tools: [], + maxTokensPerTurn: 4_096, + providerConfig: { + providerId: "openai-compatible", + modelId: "custom-model", + }, + }, + undefined, + ); + + expect(gatewayMock.createAgentModel).toHaveBeenLastCalledWith( + { providerId: "openai-compatible", modelId: "custom-model" }, + { maxTokens: 4_096 }, + ); + }); + it("forwards Bedrock AWS settings as gateway provider options", async () => { const { createAgentModelFromConfig } = await import("./handler-factory"); diff --git a/sdk/packages/core/src/services/llms/handler-factory.ts b/sdk/packages/core/src/services/llms/handler-factory.ts index a40bccaeda..c9f60b1f0f 100644 --- a/sdk/packages/core/src/services/llms/handler-factory.ts +++ b/sdk/packages/core/src/services/llms/handler-factory.ts @@ -163,7 +163,7 @@ export function createAgentModelFromConfig( baseUrl: config.baseUrl ?? baseProviderConfig?.baseUrl, headers: config.headers ?? baseProviderConfig?.headers, knownModels: resolveKnownModelsFromConfig(config), - maxOutputTokens: config.maxTokensPerTurn ?? baseProviderConfig?.maxOutputTokens, + maxOutputTokens: config.maxTokensPerTurn, reasoningEffort: config.reasoningEffort, thinkingBudgetTokens: config.thinkingBudgetTokens, thinking: config.thinking, diff --git a/sdk/packages/core/src/types/config.ts b/sdk/packages/core/src/types/config.ts index c74354ccb8..4915b41d11 100644 --- a/sdk/packages/core/src/types/config.ts +++ b/sdk/packages/core/src/types/config.ts @@ -37,6 +37,10 @@ export interface CoreModelConfig { * Explicit reasoning effort override for capable models. */ reasoningEffort?: ProviderConfig["reasoningEffort"]; + /** + * Maximum output tokens per API call. + */ + maxTokensPerTurn?: number; } export interface CoreRuntimeFeatures { diff --git a/sdk/packages/llms/src/providers/compat.test.ts b/sdk/packages/llms/src/providers/compat.test.ts index 84c3f426a4..ca5b4cea0e 100644 --- a/sdk/packages/llms/src/providers/compat.test.ts +++ b/sdk/packages/llms/src/providers/compat.test.ts @@ -394,6 +394,36 @@ describe("createGatewayApiHandler.createMessage", () => { expect(call).not.toHaveProperty("maxOutputTokens"); }); + it("sends configured OpenAI-compatible maxOutputTokens to the provider request", async () => { + streamTextSpy.mockReturnValue({ + fullStream: (async function* () { + yield { type: "finish", finishReason: "stop" }; + })(), + usage: Promise.resolve({ inputTokens: 1, outputTokens: 1 }), + }); + + const handler = createGatewayApiHandler({ + providerId: "openai-compatible", + clientType: "openai-compatible", + modelId: "custom-model", + apiKey: "test-key", + baseUrl: "https://example.com/v1", + maxOutputTokens: 4_096, + }); + + for await (const _chunk of handler.createMessage("", [ + { role: "user", content: "Hello" }, + ])) { + // Drain the stream so the provider request is executed. + } + + expect(streamTextSpy).toHaveBeenCalledWith( + expect.objectContaining({ + maxOutputTokens: 4_096, + }), + ); + }); + it("caps configured maxOutputTokens with the catalog model output limit", async () => { streamTextSpy.mockReturnValue({ fullStream: (async function* () {