From 1691aa957bde8ed59799308d9d17ba832a768473 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Wed, 19 Aug 2026 08:40:52 +0000 Subject: [PATCH 1/3] fix(fusion-vision): validate and repair image inputs before forwarding imageInputToUrl currently forwards whatever imageUrl/imageBase64 carries as long as it is not an HTTP(S)/data URL, wrapping it in a blind data:image/png;base64,... envelope. Strict upstreams answer malformed payloads with a 400 that surfaces to the caller as a raw provider error. Failure shapes seen in production: - a local file path passed as imageUrl, or a bare [media_ref:...] id - base64 truncated to length mod 4 == 1 (cut mid-image; padding cannot restore it, the payload is dropped) - an XML/SVG payload (typical vision upstreams accept only jpeg/png/gif/ webp, so relabeling it buys nothing) - base64 truncated to length mod 4 == 2/3 (losslessly repairable by adding = padding) Anything that cannot be made into a well-formed data URL is now dropped with a precise skip reason, and a call left with no usable image fails with a message listing every reason instead of silently proceeding. --- packages/core/src/mcp/fusion-vision-mcp.ts | 109 ++++++++++++++++++--- 1 file changed, 98 insertions(+), 11 deletions(-) diff --git a/packages/core/src/mcp/fusion-vision-mcp.ts b/packages/core/src/mcp/fusion-vision-mcp.ts index 8645ccc7..f330b512 100644 --- a/packages/core/src/mcp/fusion-vision-mcp.ts +++ b/packages/core/src/mcp/fusion-vision-mcp.ts @@ -736,9 +736,11 @@ async function buildImageParts(args: Record, detail: "auto" | " } const parts: JsonValue[] = []; + const skipped: string[] = []; for (const input of inputs) { - const url = await imageInputToUrl(input); - if (!url) { + const result = await imageInputToUrl(input); + if ("skip" in result) { + skipped.push(result.skip); continue; } if (input.label) { @@ -747,36 +749,121 @@ async function buildImageParts(args: Record, detail: "auto" | " parts.push({ image_url: { detail, - url + url: result.url }, type: "image_url" }); } + // Dropping bad images must not look like dropping the argument: surface every + // reason to the caller instead of silently proceeding without the image. + if (parts.length === 0) { + if (skipped.length === 0) { + throw new Error(`${toolName} requires imageUrl, imagePath, imageBase64, or images.`); + } + throw new Error(`${toolName} found no usable image. Skipped: ${skipped.join("; ")}.`); + } return parts; } -async function imageInputToUrl(input: { base64?: string; mimeType?: string; path?: string; url?: string }): Promise { +type ImageInputToUrlResult = { url: string } | { skip: string }; + +/** + * Turn one image input into a data URL ready for the upstream, or say why it cannot + * be used. The upstream (commonly litellm in front of a strict vision provider) + * rejects any image whose payload is not strictly valid base64 and reports that as + * a 400 which surfaces to the caller as a raw provider error -- so anything that + * cannot be made into a well-formed data URL is dropped here instead of forwarded. + * + * Failure shapes seen in production: a local file path passed as imageUrl, a bare + * [media_ref:...] id passed verbatim, a base64 payload whose length mod 4 is 1 + * (irreparably truncated mid-image), and an XML/SVG payload -- which the typical + * upstream rejects outright (supported formats are jpeg/png/gif/webp), so + * relabeling it buys nothing. A remainder of 2 or 3 is repairable by padding. + */ +async function imageInputToUrl(input: { base64?: string; mimeType?: string; path?: string; url?: string }): Promise { if (input.url) { - return input.url; + const url = input.url.trim(); + if (/^https?:\/\//i.test(url)) { + return { url }; + } + if (url.startsWith("data:")) { + const comma = url.indexOf(","); + if (comma < 1) { + return { skip: `malformed data URL (${preview(url)})` }; + } + const header = url.slice(5, comma); + if (!/;base64/i.test(header)) { + return { skip: `data URL is not base64 (${preview(url)})` }; + } + const mimeType = header.split(";")[0] || undefined; + const normalized = normalizeImagePayload(url.slice(comma + 1)); + if (!normalized) { + return { skip: `data URL payload is not usable base64 (${preview(url)})` }; + } + return imageDataUrlOrSkip(normalized, mimeType); + } + // Not an HTTP(S) URL and not a data URL. Either a bare base64 payload (the + // virtual-model tool loop's usual shape; wrapped here because strict gateways + // reject it as an invalid URL) or garbage that must not be forwarded as-is. + const normalized = normalizeImagePayload(url); + if (!normalized) { + return { skip: `imageUrl is neither an HTTP(S) URL, a data URL, nor base64 (${preview(url)})` }; + } + return imageDataUrlOrSkip(normalized, "image/png"); } if (input.base64) { - return toDataUrl(input.base64, input.mimeType || "image/png"); + const normalized = normalizeImagePayload(input.base64); + if (!normalized) { + return { skip: "imageBase64 is not usable base64" }; + } + return imageDataUrlOrSkip(normalized, input.mimeType || "image/png"); } if (!input.path) { - return undefined; + return { skip: "image entry has no url, base64, or path" }; } const buffer = await readFile(input.path); if (buffer.byteLength > maxLocalImageBytes) { throw new Error(`Local image exceeds ${maxLocalImageBytes} bytes: ${input.path}`); } - return toDataUrl(buffer.toString("base64"), input.mimeType || mimeTypeFromPath(input.path)); + return imageDataUrlOrSkip({ payload: buffer.toString("base64"), svg: false }, input.mimeType || mimeTypeFromPath(input.path)); } -function toDataUrl(value: string, mimeType: string): string { - return value.startsWith("data:") ? value : `data:${mimeType};base64,${value}`; +const base64PayloadPattern = /^[A-Za-z0-9+/]*={0,2}$/; +const svgHeadPattern = /^(?:)?\s*(?:<\?xml| 40 ? `${value.slice(0, 37)}…` : value); +}function mimeTypeFromPath(path: string): string { const ext = extname(path).toLowerCase(); if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg"; if (ext === ".webp") return "image/webp"; From fc16be5d416adedc469e40ab2428f2a805a9ac48 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:12:58 +0000 Subject: [PATCH 2/3] fix(fusion-vision): restore data-URL imageBase64, sniff local files, limit media-type labels Review follow-up (three defects): 1. imageBase64 accepted a full data:...;base64,... URL per its schema documentation, but the new validation passed the whole string to the base64 check and rejected it. Data URLs now take the same parsed path as imageUrl via a shared dataUrlResult() helper. 2. Local files (imagePath/images[].path) were forwarded after wrapping buffer bytes without inspecting them, so a .svg on disk went upstream as a fake raster data URL and drew the exact strict-provider 400 this work avoids. File bytes now go through the same content checks as every other input. 3. The data URL media-type label was taken verbatim from the URL header or the mimeType argument. Labels are now restricted to the supported raster types (jpeg/png/gif/webp): an explicit supported label wins, otherwise the format sniffed from the bytes, otherwise image/png. --- packages/core/src/mcp/fusion-vision-mcp.ts | 107 ++++++++++++++++----- 1 file changed, 84 insertions(+), 23 deletions(-) diff --git a/packages/core/src/mcp/fusion-vision-mcp.ts b/packages/core/src/mcp/fusion-vision-mcp.ts index f330b512..e7b561f3 100644 --- a/packages/core/src/mcp/fusion-vision-mcp.ts +++ b/packages/core/src/mcp/fusion-vision-mcp.ts @@ -779,6 +779,7 @@ type ImageInputToUrlResult = { url: string } | { skip: string }; * (irreparably truncated mid-image), and an XML/SVG payload -- which the typical * upstream rejects outright (supported formats are jpeg/png/gif/webp), so * relabeling it buys nothing. A remainder of 2 or 3 is repairable by padding. + * Local files (imagePath/images[].path) go through the same content checks. */ async function imageInputToUrl(input: { base64?: string; mimeType?: string; path?: string; url?: string }): Promise { if (input.url) { @@ -787,20 +788,7 @@ async function imageInputToUrl(input: { base64?: string; mimeType?: string; path return { url }; } if (url.startsWith("data:")) { - const comma = url.indexOf(","); - if (comma < 1) { - return { skip: `malformed data URL (${preview(url)})` }; - } - const header = url.slice(5, comma); - if (!/;base64/i.test(header)) { - return { skip: `data URL is not base64 (${preview(url)})` }; - } - const mimeType = header.split(";")[0] || undefined; - const normalized = normalizeImagePayload(url.slice(comma + 1)); - if (!normalized) { - return { skip: `data URL payload is not usable base64 (${preview(url)})` }; - } - return imageDataUrlOrSkip(normalized, mimeType); + return dataUrlResult(url, input.mimeType); } // Not an HTTP(S) URL and not a data URL. Either a bare base64 payload (the // virtual-model tool loop's usual shape; wrapped here because strict gateways @@ -812,11 +800,18 @@ async function imageInputToUrl(input: { base64?: string; mimeType?: string; path return imageDataUrlOrSkip(normalized, "image/png"); } if (input.base64) { - const normalized = normalizeImagePayload(input.base64); + const value = input.base64.trim(); + if (value.startsWith("data:")) { + // The schema documents imageBase64 as "Single raw base64 image payload or + // data URL"; both shapes must behave alike, so a data URL takes the same + // path as imageUrl above. + return dataUrlResult(value, input.mimeType); + } + const normalized = normalizeImagePayload(value); if (!normalized) { return { skip: "imageBase64 is not usable base64" }; } - return imageDataUrlOrSkip(normalized, input.mimeType || "image/png"); + return imageDataUrlOrSkip(normalized, input.mimeType); } if (!input.path) { return { skip: "image entry has no url, base64, or path" }; @@ -825,20 +820,53 @@ async function imageInputToUrl(input: { base64?: string; mimeType?: string; path if (buffer.byteLength > maxLocalImageBytes) { throw new Error(`Local image exceeds ${maxLocalImageBytes} bytes: ${input.path}`); } - return imageDataUrlOrSkip({ payload: buffer.toString("base64"), svg: false }, input.mimeType || mimeTypeFromPath(input.path)); + // File contents get the same checks as every other input: a .svg on disk must + // not be forwarded as a fake raster data URL. + const normalized = normalizeImagePayload(buffer.toString("base64")); + if (!normalized) { + return { skip: `file at ${preview(input.path)} is not usable image data` }; + } + return imageDataUrlOrSkip(normalized, input.mimeType || mimeTypeFromPath(input.path)); +} + +/** Only these media types are ever emitted on a data URL; strict upstreams validate them. */ +const supportedImageMimeTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]); + +/** + * Parse and validate a full `data:...;base64,...` URL. Shared by the imageUrl and + * imageBase64 fields so both accept the same shapes. Non-base64 data URLs are + * rejected; a header media type outside the supported set is dropped and the + * payload's sniffed type is used instead. + */ +function dataUrlResult(value: string, fallbackMimeType: string | undefined): ImageInputToUrlResult { + const comma = value.indexOf(","); + if (comma < 1) { + return { skip: `malformed data URL (${preview(value)})` }; + } + const header = value.slice(5, comma); + if (!/;base64/i.test(header)) { + return { skip: `data URL is not base64 (${preview(value)})` }; + } + const headerMimeType = header.split(";")[0] || undefined; + const normalized = normalizeImagePayload(value.slice(comma + 1)); + if (!normalized) { + return { skip: `data URL payload is not usable base64 (${preview(value)})` }; + } + return imageDataUrlOrSkip(normalized, headerMimeType || fallbackMimeType); } const base64PayloadPattern = /^[A-Za-z0-9+/]*={0,2}$/; -const svgHeadPattern = /^(?:)?\s*(?:<\?xml|= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return "image/jpeg"; + } + if (buffer.length >= 8 && buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) { + return "image/png"; + } + if (buffer.length >= 6 && buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x38 && (buffer[4] === 0x37 || buffer[4] === 0x39) && buffer[5] === 0x61) { + return "image/gif"; + } + if (buffer.length >= 12 && buffer.subarray(0, 4).toString("latin1") === "RIFF" && buffer.subarray(8, 12).toString("latin1") === "WEBP") { + return "image/webp"; + } + return undefined; +} + +/** + * Pick the media-type label for the data URL. Only the supported raster types are + * ever emitted: an explicit label in that set wins, otherwise the sniffed format, + * otherwise image/png. The label is validated for its own sake -- it is part of the + * data URL a strict upstream checks -- while the bytes decide whether the image is + * decodable at all. + */ +function imageLabel(mimeType: string | undefined, image: { sniffedType?: string }): string { + const explicit = mimeType?.trim().toLowerCase(); + if (explicit && supportedImageMimeTypes.has(explicit)) { + return explicit; + } + return image.sniffedType ?? "image/png"; +} + +function imageDataUrlOrSkip(image: { payload: string; svg: boolean; sniffedType?: string }, mimeType: string | undefined): ImageInputToUrlResult { if (image.svg) { return { skip: "SVG/XML image payload is not supported by the vision upstream (supported: image/jpeg, image/png, image/gif, image/webp)" }; } - return { url: `data:${mimeType || "image/png"};base64,${image.payload}` }; + return { url: `data:${imageLabel(mimeType, image)};base64,${image.payload}` }; } /** Short, quoted head of a value for skip reasons and error messages. */ From 3dab369ce353a54b5181914587159b452c8d8520 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:12:58 +0000 Subject: [PATCH 3/3] test(fusion-vision): regression coverage for data-URL base64, local SVG, label restriction --- packages/core/src/mcp/fusion-vision-mcp.ts | 2 +- .../mcp/fusion-vision-mcp.test.mjs | 171 ++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/packages/core/src/mcp/fusion-vision-mcp.ts b/packages/core/src/mcp/fusion-vision-mcp.ts index e7b561f3..f1db12f2 100644 --- a/packages/core/src/mcp/fusion-vision-mcp.ts +++ b/packages/core/src/mcp/fusion-vision-mcp.ts @@ -77,7 +77,7 @@ const visionTool = { detail: { enum: ["auto", "low", "high"], type: "string" }, imageBase64: { description: "Single raw base64 image payload or data URL.", type: "string" }, imagePath: { description: "Single local image path.", type: "string" }, - imageUrl: { description: "Single HTTP(S) image URL or data URL.", type: "string" }, + imageUrl: { description: "Single HTTP(S) image URL, data URL, or bare base64 payload.", type: "string" }, images: { items: objectSchema({ base64: { type: "string" }, diff --git a/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs b/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs index 9e6cb62f..f13cd6df 100644 --- a/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs +++ b/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs @@ -1,9 +1,13 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import http from "node:http"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import test from "node:test"; +const pngA = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + test("Fusion vision MCP sends the core API key and retries a body-free lightweight usage event", async (t) => { const seen = { providerAuthorization: "", @@ -301,6 +305,173 @@ test("Fusion vision MCP preserves slash-containing model IDs for external runtim assert.deepEqual(seen.usageBody?.target, { model }); }); +test("Fusion vision MCP accepts a full data URL in imageBase64", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageBase64: `data:image/png;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.error, undefined); + assert.equal(response.result?.isError, undefined); + assert.equal(provider.requests, 1); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); +}); + +test("Fusion vision MCP refuses a local SVG file instead of forwarding it as a fake PNG", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const dir = await mkdtemp(path.join(os.tmpdir(), "fusion-vision-svg-")); + t.after(() => rm(dir, { force: true, recursive: true })); + const svgPath = path.join(dir, "logo.svg"); + await writeFile(svgPath, '', "utf8"); + + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { images: [{ path: svgPath }], prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.result?.isError, true); + assert.match(response.result?.content?.[0]?.text, /SVG\/XML/); + assert.equal(provider.requests, 0, "the SVG must never reach the upstream"); +}); + +test("Fusion vision MCP restricts the forwarded media-type label to supported raster types", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const child = spawnVision(t, provider.port); + + const octetStream = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageBase64: pngA, mimeType: "application/octet-stream", prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); + + const textPlain = await sendJsonRpc(child, { + id: 2, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageUrl: `data:text/plain;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); + + const webp = await sendJsonRpc(child, { + id: 3, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageUrl: `data:image/webp;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/webp;base64,${pngA}`); +}); + +test("Fusion vision MCP forwards a local PNG file with a supported label", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const dir = await mkdtemp(path.join(os.tmpdir(), "fusion-vision-png-")); + t.after(() => rm(dir, { force: true, recursive: true })); + const pngPath = path.join(dir, "img.png"); + await writeFile(pngPath, Buffer.from(pngA, "base64")); + + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { images: [{ path: pngPath }], prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.error, undefined); + assert.equal(response.result?.isError, undefined); + assert.equal(provider.requests, 1); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); +}); + +async function serveVision(t, respond) { + let requests = 0; + let lastBody; + const server = http.createServer(async (request, response) => { + const body = readRequestBody(request); + requests += 1; + lastBody = JSON.parse(await body); + if (respond) { + respond(lastBody); + } + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ + choices: [{ message: { content: "vision ok" } }], + usage: { completion_tokens: 3, prompt_tokens: 10, total_tokens: 13 } + })); + }); + try { + await listen(server); + } catch (error) { + if (isLocalListenUnavailable(error)) { + t.skip(`Local HTTP listen is unavailable: ${formatError(error)}`); + return undefined; + } + throw error; + } + t.after(() => server.close()); + assert.ok(server.address() && typeof server.address() === "object"); + return { port: server.address().port, get requests() { return requests; }, get lastBody() { return lastBody; } }; +} + +function spawnVision(t, port, env = {}) { + const child = spawn(process.execPath, [path.join(process.cwd(), ".test-dist", "core", "runtime", "fusion-vision-mcp.js")], { + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: "1", + FUSION_BUILTIN_TOOL_KIND: "vision", + FUSION_TOOL_NAME: "vision_understand", + VISION_API_KEY: "external-key", + VISION_BASE_URL: `http://127.0.0.1:${port}/v1`, + VISION_MODEL: "test-vision", + ...env + }, + stdio: ["pipe", "pipe", "pipe"] + }); + t.after(() => { + if (!child.killed) { + child.kill(); + } + }); + return child; +} function listen(server) { return new Promise((resolve, reject) => { const onError = (error) => {