opensource/kilocode
3
0
Fork 0
mirror of https://github.com/Kilo-Org/kilocode.git synced 2026-09-19 01:51:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(vscode): expose custom model image modality

Browse Source
This commit is contained in:
jackson.zhou
2026-07-04 07:32:27 +08:00
committed by jackson-zhou
parent 1fc8f066fd
commit 0b784691f2
27 changed files with 204 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/custom-provider-image-modality.md
+5
View File
@@ -0,0 +1,5 @@
---
"kilo-code": patch
---
Support marking custom provider models as image-capable in VS Code settings.
packages/kilo-vscode/src/shared/custom-provider.ts
+18 -1
View File
@@ -23,6 +23,16 @@ const VariantConfigSchema = z.object({
export type VariantConfig = z.infer<typeof VariantConfigSchema>
// Mirror the CLI provider schema so the UI preserves hand-written configs.
const ModalitySchema = z.enum(["text", "audio", "image", "video", "pdf"])
const ModelModalitiesSchema = z.object({
input: z.array(ModalitySchema).optional(),
output: z.array(ModalitySchema).optional(),
})
export type ModelModalities = z.infer<typeof ModelModalitiesSchema>
export const CustomProviderConfigSchema = z
.object({
npm: z.enum(CUSTOM_PROVIDER_PACKAGES).default(CUSTOM_PROVIDER_PACKAGE),
@@ -47,6 +57,7 @@ export const CustomProviderConfigSchema = z
.object({
name: z.string().trim().min(1).max(200),
reasoning: z.boolean().optional(),
modalities: ModelModalitiesSchema.optional(),
variants: z.record(z.string().trim().min(1), VariantConfigSchema).optional(),
})
.strict(),
@@ -63,7 +74,10 @@ export type SanitizedProviderConfig = {
baseURL: string
headers?: Record<string, string>
}
models: Record<string, { name: string; reasoning?: true; variants?: Record<string, VariantConfig> }>
models: Record<
string,
{ name: string; reasoning?: true; modalities?: ModelModalities; variants?: Record<string, VariantConfig> }
>
}
export type CustomProviderAuthChange = { mode: "preserve" } | { mode: "clear" } | { mode: "set"; key: string }
@@ -134,6 +148,7 @@ export function normalizeCustomProviderConfig(
{
name: model.name.trim(),
...(model.reasoning ? { reasoning: true as const } : {}),
...(model.modalities ? { modalities: model.modalities } : {}),
...(model.variants && Object.keys(model.variants).length > 0 ? { variants: model.variants } : {}),
},
]),
@@ -159,6 +174,7 @@ type ProviderPatch = Omit<SanitizedProviderConfig, "models"> & {
null | {
name: string
reasoning?: true | null
modalities?: ModelModalities | null
variants?: Record<string, VariantConfig | VariantPatch | null>
}
>
@@ -208,6 +224,7 @@ export function withCustomProviderDeletions(existing: unknown, next: SanitizedPr
...newModel,
...(variants ? { variants } : {}),
...(oldModel.reasoning !== undefined && newModel.reasoning === undefined ? { reasoning: null } : {}),
...(oldModel.modalities !== undefined && newModel.modalities === undefined ? { modalities: null } : {}),
}
}
packages/kilo-vscode/tests/unit/custom-provider-dialog-validate.test.ts
+33 -1
View File
@@ -12,7 +12,9 @@ function base(): FormState {
npm: "@ai-sdk/openai-compatible",
baseURL: "https://example.com/v1",
apiKey: "",
models: [{ id: "model-1", name: "Model One", reasoning: false, variants: [] }],
models: [
{ id: "model-1", name: "Model One", reasoning: false, supportsImages: false, modalities: {}, variants: [] },
],
headers: [],
saving: false,
}
@@ -203,4 +205,34 @@ describe("validateCustomProvider – variant name validation", () => {
},
})
})
it("serializes image modality when supportsImages is set", () => {
const form = base()
form.models[0].supportsImages = true
const out = validateCustomProvider(args(form))
expect(out.result).toBeDefined()
const saved = out.result!.config.models["model-1"] as Record<string, unknown>
expect(saved.modalities).toEqual({ input: ["text", "image"] })
})
it("omits modalities when supportsImages is not set on a text-only model", () => {
const form = base()
const out = validateCustomProvider(args(form))
expect(out.result).toBeDefined()
const saved = out.result!.config.models["model-1"] as Record<string, unknown>
expect(saved.modalities).toBeUndefined()
})
it("preserves unsupported UI modalities when toggling image support", () => {
const form = base()
form.models[0].modalities = {
input: ["text", "audio", "image", "video", "pdf"],
output: ["text", "audio"],
}
form.models[0].supportsImages = false
const out = validateCustomProvider(args(form))
expect(out.result).toBeDefined()
const saved = out.result!.config.models["model-1"] as Record<string, unknown>
expect(saved.modalities).toEqual({ input: ["text", "audio", "video", "pdf"], output: ["text", "audio"] })
})
})
packages/kilo-vscode/tests/unit/custom-provider.test.ts
+33
View File
@@ -162,6 +162,39 @@ describe("sanitizeCustomProviderConfig", () => {
})
})
it("preserves core custom model modalities", () => {
const result = sanitizeCustomProviderConfig({
name: "Media Provider",
options: { baseURL: "https://example.com/v1" },
models: {
"model-1": {
name: "Model One",
modalities: {
input: ["text", "audio", "image", "video", "pdf"],
output: ["text", "audio"],
},
},
},
})
expect(result).toEqual({
value: {
npm: "@ai-sdk/openai-compatible",
name: "Media Provider",
options: { baseURL: "https://example.com/v1" },
models: {
"model-1": {
name: "Model One",
modalities: {
input: ["text", "audio", "image", "video", "pdf"],
output: ["text", "audio"],
},
},
},
},
})
})
it("rejects unknown fields", () => {
const result = sanitizeCustomProviderConfig({
name: "Bad Provider",
packages/kilo-vscode/webview-ui/src/components/settings/CustomProviderDialog.tsx
+50 -6
View File
@@ -25,6 +25,8 @@ import { ModelCard } from "./CustomProviderModelCard"
import type {
ChatTemplateArgsValue,
EnableThinkingValue,
Modalities,
Modality,
ModelEntry,
OutputEffortValue,
ReasoningEffortValue,
@@ -54,7 +56,35 @@ function fuzzy(query: string, target: string) {
}
type FetchedModel = { id: string; name: string }
type RawModel = { name?: string; reasoning?: boolean; variants?: Record<string, Record<string, unknown>> }
type RawModel = {
name?: string
reasoning?: boolean
modalities?: { input?: unknown; output?: unknown }
variants?: Record<string, Record<string, unknown>>
}
// Keep this aligned with the CLI provider schema; the UI only exposes image.
const MODES = new Set<Modality>(["text", "audio", "image", "video", "pdf"])
function list(raw: unknown): Modality[] | undefined {
if (!Array.isArray(raw)) return
const set = new Set<Modality>()
raw.forEach((item) => {
if (typeof item === "string" && MODES.has(item as Modality)) set.add(item as Modality)
})
return set.size ? [...set] : undefined
}
function modes(raw: unknown): Modalities {
if (!raw || typeof raw !== "object") return {}
const obj = raw as { input?: unknown; output?: unknown }
const input = list(obj.input)
const output = list(obj.output)
return {
...(input ? { input } : {}),
...(output ? { output } : {}),
}
}
function parseVariant([name, cfg]: [string, Record<string, unknown>]): VariantEntry {
return {
@@ -76,15 +106,20 @@ function parseVariant([name, cfg]: [string, Record<string, unknown>]): VariantEn
}
function initModels(cfg: ProviderConfig | undefined): ModelEntry[] {
if (!cfg?.models || typeof cfg.models !== "object") return [{ id: "", name: "", reasoning: false, variants: [] }]
const empty = { id: "", name: "", reasoning: false, supportsImages: false, modalities: {}, variants: [] }
if (!cfg?.models || typeof cfg.models !== "object") return [{ ...empty }]
const entries = Object.entries(cfg.models)
if (entries.length === 0) return [{ id: "", name: "", reasoning: false, variants: [] }]
if (entries.length === 0) return [{ ...empty }]
return entries.map(([id, model]) => {
const raw = model as RawModel
const modalities = modes(raw.modalities)
const input = modalities.input ?? []
return {
id,
name: raw.name ?? id,
reasoning: raw.reasoning ?? false,
supportsImages: input.includes("image"),
modalities,
variants: Object.entries(raw.variants ?? {}).map(parseVariant),
}
})
@@ -327,7 +362,6 @@ const CustomProviderDialog = (props: CustomProviderDialogProps) => {
// Replace the single empty row or append
const row = form.models[0]
const empty = form.models.length === 1 && !!row && !row.id.trim() && !row.name.trim()
// Dedup against models already in the form (trimmed, case-insensitive). The
// picker is built from a fetch-time snapshot, so a model the user typed
// manually after fetching hasn't been filtered out yet.
@@ -341,7 +375,13 @@ const CustomProviderDialog = (props: CustomProviderDialogProps) => {
return true
})
const defaults = (m: FetchedModel): ModelEntry => ({ ...m, reasoning: false, variants: [] })
const defaults = (m: FetchedModel): ModelEntry => ({
...m,
reasoning: false,
supportsImages: false,
modalities: {},
variants: [],
})
const merged = empty ? toAdd.map(defaults) : [...form.models, ...toAdd.map(defaults)]
if (toAdd.length > 0) {
@@ -396,7 +436,10 @@ const CustomProviderDialog = (props: CustomProviderDialogProps) => {
}
function addModel() {
setForm("models", (v) => [...v, { id: "", name: "", reasoning: false, variants: [] }])
setForm("models", (v) => [
...v,
{ id: "", name: "", reasoning: false, supportsImages: false, modalities: {}, variants: [] },
])
setErrors("models", (v) => [...v, { variants: [] }])
}
@@ -637,6 +680,7 @@ const CustomProviderDialog = (props: CustomProviderDialogProps) => {
onChangeId={(v) => setForm("models", i(), "id", v)}
onChangeName={(v) => setForm("models", i(), "name", v)}
onChangeReasoning={(v) => setForm("models", i(), "reasoning", v)}
onChangeSupportsImages={(v) => setForm("models", i(), "supportsImages", v)}
onRemove={() => removeModel(i())}
onAddVariant={() => addVariant(i())}
onRemoveVariant={(vi) => removeVariant(i(), vi)}
packages/kilo-vscode/webview-ui/src/components/settings/CustomProviderModelCard.tsx
+27
View File
@@ -14,6 +14,12 @@ export type SplitReasoningValue = undefined | boolean
export type ReasoningEffortValue = undefined | "none" | "minimal" | "low" | "medium" | "high" | "xhigh"
export type OutputEffortValue = undefined | "low" | "medium" | "high" | "xhigh" | "max"
export type ChatTemplateArgsValue = undefined | boolean
export type Modality = "text" | "audio" | "image" | "video" | "pdf"
export type Modalities = {
input?: Modality[]
output?: Modality[]
}
export type VariantEntry = {
name: string
@@ -29,6 +35,8 @@ export type ModelEntry = {
id: string
name: string
reasoning: boolean
supportsImages: boolean
modalities: Modalities
variants: VariantEntry[]
}
@@ -296,6 +304,7 @@ type ModelCardProps = {
onChangeId: (val: string) => void
onChangeName: (val: string) => void
onChangeReasoning: (val: boolean) => void
onChangeSupportsImages: (val: boolean) => void
onRemove: () => void
onAddVariant: () => void
onRemoveVariant: (vi: number) => void
@@ -372,6 +381,24 @@ export function ModelCard(props: ModelCardProps) {
{props.t("provider.custom.models.reasoning.label")}
</label>
<label
style={{
display: "flex",
"align-items": "center",
gap: "8px",
cursor: "pointer",
"font-size": "var(--kilo-font-size-13)",
color: "var(--vscode-foreground)",
}}
>
<input
type="checkbox"
checked={props.m.supportsImages}
onChange={(e) => props.onChangeSupportsImages(e.currentTarget.checked)}
/>
{props.t("provider.custom.models.modalities.image")}
</label>
{/* Variants — only available when reasoning is enabled */}
<Show when={props.m.reasoning}>
<Show when={props.m.variants.length > 0}>
packages/kilo-vscode/webview-ui/src/components/settings/CustomProviderValidation.ts
+18 -1
View File
@@ -1,5 +1,5 @@
import type { CustomProviderPackage } from "../../../../src/shared/provider-model"
import type { ModelEntry, VariantEntry } from "./CustomProviderModelCard"
import type { Modalities, ModelEntry, VariantEntry } from "./CustomProviderModelCard"
type Translator = (key: string, params?: Record<string, string>) => string
@@ -115,10 +115,27 @@ function serializeVariant(v: VariantEntry): [string, Record<string, unknown>] {
return [v.name.trim(), cfg]
}
function modalities(m: ModelEntry): Modalities | undefined {
const input = new Set(m.modalities.input ?? [])
const existing = input.size > 0 || (m.modalities.output?.length ?? 0) > 0
if (!existing && !m.supportsImages) return
input.add("text")
if (m.supportsImages) input.add("image")
else input.delete("image")
return {
input: [...input],
...(m.modalities.output?.length ? { output: m.modalities.output } : {}),
}
}
function serializeModel(m: ModelEntry): [string, Record<string, unknown>] {
const ventries = m.reasoning ? m.variants.filter((v) => v.name.trim()).map(serializeVariant) : []
const entry: Record<string, unknown> = { name: m.name.trim() }
const modes = modalities(m)
if (m.reasoning) entry.reasoning = true
if (modes) entry.modalities = modes
if (ventries.length > 0) entry.variants = Object.fromEntries(ventries)
return [m.id.trim(), entry]
}
packages/kilo-vscode/webview-ui/src/i18n/ar.ts
Generated
+1
View File
@@ -939,6 +939,7 @@ export const dict = {
"provider.custom.models.name.label": "الاسم",
"provider.custom.models.name.placeholder": "الاسم المعروض",
"provider.custom.models.reasoning.label": "الاستدلال",
"provider.custom.models.modalities.image": "صورة",
"provider.custom.models.variants.label": "المتغيرات",
"provider.custom.models.variants.add": "إضافة متغير",
"provider.custom.models.variants.remove": "إزالة المتغير",
packages/kilo-vscode/webview-ui/src/i18n/br.ts
Generated
+1
View File
@@ -955,6 +955,7 @@ export const dict = {
"provider.custom.models.name.label": "Nome",
"provider.custom.models.name.placeholder": "Nome de Exibição",
"provider.custom.models.reasoning.label": "Raciocínio",
"provider.custom.models.modalities.image": "Imagem",
"provider.custom.models.variants.label": "Variantes",
"provider.custom.models.variants.add": "Adicionar variante",
"provider.custom.models.variants.remove": "Remover variante",
packages/kilo-vscode/webview-ui/src/i18n/bs.ts
Generated
+1
View File
@@ -998,6 +998,7 @@ export const dict = {
"provider.custom.models.name.label": "Naziv",
"provider.custom.models.name.placeholder": "Naziv za prikaz",
"provider.custom.models.reasoning.label": "Zaključivanje",
"provider.custom.models.modalities.image": "Slika",
"provider.custom.models.variants.label": "Varijante",
"provider.custom.models.variants.add": "Dodaj varijantu",
"provider.custom.models.variants.remove": "Ukloni varijantu",
packages/kilo-vscode/webview-ui/src/i18n/da.ts
Generated
+1
View File
@@ -991,6 +991,7 @@ export const dict = {
"provider.custom.models.name.label": "Navn",
"provider.custom.models.name.placeholder": "Visningsnavn",
"provider.custom.models.reasoning.label": "Ræsonnement",
"provider.custom.models.modalities.image": "Billede",
"provider.custom.models.variants.label": "Varianter",
"provider.custom.models.variants.add": "Tilføj variant",
"provider.custom.models.variants.remove": "Fjern variant",
packages/kilo-vscode/webview-ui/src/i18n/de.ts
+1
View File
@@ -1009,6 +1009,7 @@ export const dict = {
"provider.custom.models.name.label": "Name",
"provider.custom.models.name.placeholder": "Anzeigename",
"provider.custom.models.reasoning.label": "Schlussfolgerung",
"provider.custom.models.modalities.image": "Bild",
"provider.custom.models.variants.label": "Varianten",
"provider.custom.models.variants.add": "Variante hinzufügen",
"provider.custom.models.variants.remove": "Variante entfernen",
packages/kilo-vscode/webview-ui/src/i18n/en.ts
+1
View File
@@ -912,6 +912,7 @@ export const dict = {
"provider.custom.models.name.label": "Name",
"provider.custom.models.name.placeholder": "Display Name",
"provider.custom.models.reasoning.label": "Reasoning",
"provider.custom.models.modalities.image": "Image",
"provider.custom.models.variants.label": "Variants",
"provider.custom.models.variants.add": "Add variant",
"provider.custom.models.variants.remove": "Remove variant",
packages/kilo-vscode/webview-ui/src/i18n/es.ts
Generated
+1
View File
@@ -1001,6 +1001,7 @@ export const dict = {
"provider.custom.models.name.label": "Nombre",
"provider.custom.models.name.placeholder": "Nombre para mostrar",
"provider.custom.models.reasoning.label": "Razonamiento",
"provider.custom.models.modalities.image": "Imagen",
"provider.custom.models.variants.label": "Variantes",
"provider.custom.models.variants.add": "Añadir variante",
"provider.custom.models.variants.remove": "Eliminar variante",
packages/kilo-vscode/webview-ui/src/i18n/fr.ts
Generated
+1
View File
@@ -1007,6 +1007,7 @@ export const dict = {
"provider.custom.models.name.label": "Nom",
"provider.custom.models.name.placeholder": "Nom d'affichage",
"provider.custom.models.reasoning.label": "Raisonnement",
"provider.custom.models.modalities.image": "Image",
"provider.custom.models.variants.label": "Variantes",
"provider.custom.models.variants.add": "Ajouter une variante",
"provider.custom.models.variants.remove": "Supprimer la variante",
packages/kilo-vscode/webview-ui/src/i18n/it.ts
Generated
+1
View File
@@ -767,6 +767,7 @@ export const dict = {
"provider.custom.models.name.label": "Nome",
"provider.custom.models.name.placeholder": "Nome visualizzato",
"provider.custom.models.reasoning.label": "Reasoning",
"provider.custom.models.modalities.image": "Immagine",
"provider.custom.models.variants.label": "Variants",
"provider.custom.models.variants.add": "Aggiungi variante",
"provider.custom.models.variants.remove": "Rimuovi variante",
packages/kilo-vscode/webview-ui/src/i18n/ja.ts
Generated
+1
View File
@@ -988,6 +988,7 @@ export const dict = {
"provider.custom.models.name.label": "名前",
"provider.custom.models.name.placeholder": "表示名",
"provider.custom.models.reasoning.label": "推論",
"provider.custom.models.modalities.image": "画像",
"provider.custom.models.variants.label": "バリアント",
"provider.custom.models.variants.add": "バリアントを追加",
"provider.custom.models.variants.remove": "バリアントを削除",
packages/kilo-vscode/webview-ui/src/i18n/ko.ts
Generated
+1
View File
@@ -946,6 +946,7 @@ export const dict = {
"provider.custom.models.name.label": "이름",
"provider.custom.models.name.placeholder": "표시 이름",
"provider.custom.models.reasoning.label": "추론",
"provider.custom.models.modalities.image": "이미지",
"provider.custom.models.variants.label": "변형",
"provider.custom.models.variants.add": "변형 추가",
"provider.custom.models.variants.remove": "변형 제거",
packages/kilo-vscode/webview-ui/src/i18n/nl.ts
Generated
+1
View File
@@ -949,6 +949,7 @@ export const dict = {
"provider.custom.models.name.label": "Naam",
"provider.custom.models.name.placeholder": "Weergavenaam",
"provider.custom.models.reasoning.label": "Redeneren",
"provider.custom.models.modalities.image": "Afbeelding",
"provider.custom.models.variants.label": "Varianten",
"provider.custom.models.variants.add": "Variant toevoegen",
"provider.custom.models.variants.remove": "Variant verwijderen",
packages/kilo-vscode/webview-ui/src/i18n/no.ts
Generated
+1
View File
@@ -956,6 +956,7 @@ export const dict = {
"provider.custom.models.name.label": "Navn",
"provider.custom.models.name.placeholder": "Visningsnavn",
"provider.custom.models.reasoning.label": "Resonnering",
"provider.custom.models.modalities.image": "Bilde",
"provider.custom.models.variants.label": "Varianter",
"provider.custom.models.variants.add": "Legg til variant",
"provider.custom.models.variants.remove": "Fjern variant",
packages/kilo-vscode/webview-ui/src/i18n/pl.ts
Generated
+1
View File
@@ -954,6 +954,7 @@ export const dict = {
"provider.custom.models.name.label": "Nazwa",
"provider.custom.models.name.placeholder": "Nazwa wyświetlana",
"provider.custom.models.reasoning.label": "Rozumowanie",
"provider.custom.models.modalities.image": "Obraz",
"provider.custom.models.variants.label": "Warianty",
"provider.custom.models.variants.add": "Dodaj wariant",
"provider.custom.models.variants.remove": "Usuń wariant",
packages/kilo-vscode/webview-ui/src/i18n/ru.ts
Generated
+1
View File
@@ -995,6 +995,7 @@ export const dict = {
"provider.custom.models.name.label": "Название",
"provider.custom.models.name.placeholder": "Отображаемое имя",
"provider.custom.models.reasoning.label": "Рассуждение",
"provider.custom.models.modalities.image": "Изображение",
"provider.custom.models.variants.label": "Варианты",
"provider.custom.models.variants.add": "Добавить вариант",
"provider.custom.models.variants.remove": "Удалить вариант",
packages/kilo-vscode/webview-ui/src/i18n/th.ts
Generated
+1
View File
@@ -981,6 +981,7 @@ export const dict = {
"provider.custom.models.name.label": "ชื่อ",
"provider.custom.models.name.placeholder": "ชื่อที่แสดง",
"provider.custom.models.reasoning.label": "การใช้เหตุผล",
"provider.custom.models.modalities.image": "รูปภาพ",
"provider.custom.models.variants.label": "รูปแบบ",
"provider.custom.models.variants.add": "เพิ่มรูปแบบ",
"provider.custom.models.variants.remove": "ลบรูปแบบ",
packages/kilo-vscode/webview-ui/src/i18n/tr.ts
Generated
+1
View File
@@ -946,6 +946,7 @@ export const dict = {
"provider.custom.models.name.label": "Ad",
"provider.custom.models.name.placeholder": "Görünen Ad",
"provider.custom.models.reasoning.label": "Akıl Yürütme",
"provider.custom.models.modalities.image": "Görüntü",
"provider.custom.models.variants.label": "Varyantlar",
"provider.custom.models.variants.add": "Varyant ekle",
"provider.custom.models.variants.remove": "Varyantı kaldır",
packages/kilo-vscode/webview-ui/src/i18n/uk.ts
Generated
+1
View File
@@ -946,6 +946,7 @@ export const dict = {
"provider.custom.models.name.label": "Назва",
"provider.custom.models.name.placeholder": "Відображувана назва",
"provider.custom.models.reasoning.label": "Міркування",
"provider.custom.models.modalities.image": "Зображення",
"provider.custom.models.variants.label": "Варіанти",
"provider.custom.models.variants.add": "Додати варіант",
"provider.custom.models.variants.remove": "Видалити варіант",
packages/kilo-vscode/webview-ui/src/i18n/zh.ts
Generated
+1
View File
@@ -963,6 +963,7 @@ export const dict = {
"provider.custom.models.name.label": "名称",
"provider.custom.models.name.placeholder": "显示名称",
"provider.custom.models.reasoning.label": "推理",
"provider.custom.models.modalities.image": "图片",
"provider.custom.models.variants.label": "变体",
"provider.custom.models.variants.add": "添加变体",
"provider.custom.models.variants.remove": "移除变体",
packages/kilo-vscode/webview-ui/src/i18n/zht.ts
Generated
+1
View File
@@ -927,6 +927,7 @@ export const dict = {
"provider.custom.models.name.label": "名稱",
"provider.custom.models.name.placeholder": "顯示名稱",
"provider.custom.models.reasoning.label": "推理",
"provider.custom.models.modalities.image": "圖片",
"provider.custom.models.variants.label": "變體",
"provider.custom.models.variants.add": "新增變體",
"provider.custom.models.variants.remove": "移除變體",