diff --git a/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.test.ts b/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.test.ts index 5858051ce2..ec1667d197 100644 --- a/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.test.ts +++ b/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.test.ts @@ -906,6 +906,51 @@ it("derives tool image support metadata from resolved provider model catalog", a expect(runtimeConfig.toolContextMetadata?.telemetry).toBeUndefined(); }); +it.each([ + ["absent", undefined], + ["empty", []], +])("keeps image support enabled when the capability list is %s", async (_label, capabilities) => { + const { deps, configs } = withCapturingFakeRuntime(); + const session = new SessionRuntime( + makeAgentConfig({ + knownModels: { + "claude-3-5-sonnet": { + id: "claude-3-5-sonnet", + ...(capabilities === undefined ? {} : { capabilities }), + }, + }, + }), + deps, + ); + + await session.run("inspect image"); + + expect(configs[0]?.toolContextMetadata).toEqual( + expect.objectContaining({ modelSupportsImages: true }), + ); +}); + +it("disables image support when a populated capability list omits images", async () => { + const { deps, configs } = withCapturingFakeRuntime(); + const session = new SessionRuntime( + makeAgentConfig({ + knownModels: { + "claude-3-5-sonnet": { + id: "claude-3-5-sonnet", + capabilities: ["tools", "prompt-cache"], + }, + }, + }), + deps, + ); + + await session.run("inspect image"); + + expect(configs[0]?.toolContextMetadata).toEqual( + expect.objectContaining({ modelSupportsImages: false }), + ); +}); + describe("SessionRuntime.run", () => { it("invokes the injected AgentRuntime and returns an AgentResult", async () => { const { deps, calls } = withFakeRuntime({ diff --git a/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.ts b/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.ts index a1bfb1db4a..2fdcf3e653 100644 --- a/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.ts +++ b/sdk/packages/core/src/runtime/orchestration/session-runtime-orchestrator.ts @@ -46,6 +46,7 @@ import { type MessageWithMetadata, type ModelInfo, mergeModelOptions, + modelSupportsImageInput, modelSupportsToolCalling, type ToolCallRecord, usesImageGenerationOperation, @@ -890,8 +891,7 @@ export class SessionRuntime { telemetry: this.telemetry, tools, toolContextMetadata: { - modelSupportsImages: - modelInfo?.capabilities?.includes("images") ?? true, + modelSupportsImages: modelSupportsImageInput(modelInfo ?? {}), ...this.config.toolContextMetadata, }, hooks: this.createRuntimeHooks(), diff --git a/sdk/packages/core/src/services/providers/local-provider-registry.ts b/sdk/packages/core/src/services/providers/local-provider-registry.ts index 49bd6cadf9..60646c502b 100644 --- a/sdk/packages/core/src/services/providers/local-provider-registry.ts +++ b/sdk/packages/core/src/services/providers/local-provider-registry.ts @@ -219,6 +219,23 @@ export async function writeModelsFile( } } +/** + * Projects one capability onto `ProviderModel`'s tri-state booleans, where + * `undefined` means "not declared" and drives each picker's own default. + * A missing OR empty capability list carries no signal (see + * `modelHasCapability`), so both must stay `undefined` rather than collapsing + * to a `false` that reads as an authoritative denial. + */ +function declaredCapability( + capabilities: ModelInfo["capabilities"], + capability: ModelCapability, +): boolean | undefined { + if (capabilities === undefined || capabilities.length === 0) { + return undefined; + } + return capabilities.includes(capability); +} + export function toProviderModel( modelId: string, info: Pick< @@ -241,10 +258,15 @@ export function toProviderModel( ...(info.contextWindow !== undefined ? { contextWindow: info.contextWindow } : {}), - supportsAttachments: info.capabilities?.includes("files"), - supportsVision: info.capabilities?.includes("images"), + supportsAttachments: declaredCapability(info.capabilities, "files"), + supportsVision: declaredCapability(info.capabilities, "images"), + // A thinking config is positive evidence on its own; its absence is + // not evidence of absence, so fall back to whatever the capability + // list declares (including "not declared"). supportsReasoning: - info.capabilities?.includes("reasoning") || info.thinkingConfig != null, + info.thinkingConfig != null + ? true + : declaredCapability(info.capabilities, "reasoning"), operationModes: info.operationModes, inputModalities: info.modalities?.input, outputModalities: info.modalities?.output, diff --git a/sdk/packages/core/src/services/providers/local-provider-service.test.ts b/sdk/packages/core/src/services/providers/local-provider-service.test.ts index b8e1ea6aa9..b719c967f5 100644 --- a/sdk/packages/core/src/services/providers/local-provider-service.test.ts +++ b/sdk/packages/core/src/services/providers/local-provider-service.test.ts @@ -959,6 +959,37 @@ describe("addLocalProvider – capabilities", () => { }); }); + it.each([ + ["absent", undefined], + ["empty", [] as const], + ])("leaves capability support undeclared when the list is %s", (_label, capabilities) => { + expect( + toProviderModel("sparse-model", { + name: "Sparse Model", + ...(capabilities === undefined + ? {} + : { capabilities: [...capabilities] }), + }), + ).toMatchObject({ + supportsVision: undefined, + supportsAttachments: undefined, + supportsReasoning: undefined, + }); + }); + + it("reports a populated capability list as authoritative", () => { + expect( + toProviderModel("vision-only", { + name: "Vision Only", + capabilities: ["images"], + }), + ).toMatchObject({ + supportsVision: true, + supportsAttachments: false, + supportsReasoning: false, + }); + }); + it("sets supportsVision and supportsAttachments when capability is 'vision'", async () => { await addLocalProvider(manager, { providerId: "vision-provider", diff --git a/sdk/packages/shared/src/index.browser.ts b/sdk/packages/shared/src/index.browser.ts index 94a4e9d66d..8d38f4c8cd 100644 --- a/sdk/packages/shared/src/index.browser.ts +++ b/sdk/packages/shared/src/index.browser.ts @@ -209,6 +209,7 @@ export { ModelStatusSchema, modelHasCapability, modelProducesImages, + modelSupportsImageInput, modelSupportsToolCalling, supportsChatModalities, type ThinkingConfig, diff --git a/sdk/packages/shared/src/index.ts b/sdk/packages/shared/src/index.ts index 2d453bb112..a0d1d2580d 100644 --- a/sdk/packages/shared/src/index.ts +++ b/sdk/packages/shared/src/index.ts @@ -234,6 +234,7 @@ export { ModelStatusSchema, modelHasCapability, modelProducesImages, + modelSupportsImageInput, modelSupportsToolCalling, supportsChatModalities, type ThinkingConfig, diff --git a/sdk/packages/shared/src/llms/model-info.test.ts b/sdk/packages/shared/src/llms/model-info.test.ts index 51a7d4a8c8..7ff3c366a9 100644 --- a/sdk/packages/shared/src/llms/model-info.test.ts +++ b/sdk/packages/shared/src/llms/model-info.test.ts @@ -3,6 +3,7 @@ import { isChatCompatibleModel, ModelInfoSchema, modelHasCapability, + modelSupportsImageInput, modelSupportsToolCalling, supportsChatModalities, } from "./model-info"; @@ -138,3 +139,17 @@ describe("modelSupportsToolCalling", () => { ).toBe(false); }); }); + +describe("modelSupportsImageInput", () => { + it("fails open when capability metadata is missing or empty", () => { + expect(modelSupportsImageInput({})).toBe(true); + expect(modelSupportsImageInput({ capabilities: [] })).toBe(true); + }); + + it("trusts a populated capability list", () => { + expect(modelSupportsImageInput({ capabilities: ["images"] })).toBe(true); + expect( + modelSupportsImageInput({ capabilities: ["tools", "prompt-cache"] }), + ).toBe(false); + }); +}); diff --git a/sdk/packages/shared/src/llms/model-info.ts b/sdk/packages/shared/src/llms/model-info.ts index 8749f0a4d7..74d33562a4 100644 --- a/sdk/packages/shared/src/llms/model-info.ts +++ b/sdk/packages/shared/src/llms/model-info.ts @@ -220,6 +220,19 @@ export function modelSupportsToolCalling(model: { return modelHasCapability(model, "tools", { assumeWhenUnspecified: true }); } +/** + * Whether a model can receive image parts in a request. Fails open on the + * same grounds as `modelSupportsToolCalling`: a host boundary that reports no + * capabilities at all has not declared the model text-only, and stripping + * images from a vision-capable model loses user content silently. A populated + * list without `images` is authoritative. + */ +export function modelSupportsImageInput(model: { + capabilities?: readonly string[]; +}): boolean { + return modelHasCapability(model, "images", { assumeWhenUnspecified: true }); +} + export const ModelInfoSchema = z.object({ id: z.string(), name: z.string().optional(),