diff --git a/packages/core/src/mcp/fusion-vision-mcp.ts b/packages/core/src/mcp/fusion-vision-mcp.ts index 8645ccc7..f1db12f2 100644 --- a/packages/core/src/mcp/fusion-vision-mcp.ts +++ b/packages/core/src/mcp/fusion-vision-mcp.ts @@ -77,7 +77,7 @@ const visionTool = { detail: { enum: ["auto", "low", "high"], type: "string" }, imageBase64: { description: "Single raw base64 image payload or data URL.", type: "string" }, imagePath: { description: "Single local image path.", type: "string" }, - imageUrl: { description: "Single HTTP(S) image URL or data URL.", type: "string" }, + imageUrl: { description: "Single HTTP(S) image URL, data URL, or bare base64 payload.", type: "string" }, images: { items: objectSchema({ base64: { type: "string" }, @@ -736,9 +736,11 @@ async function buildImageParts(args: Record, detail: "auto" | " } const parts: JsonValue[] = []; + const skipped: string[] = []; for (const input of inputs) { - const url = await imageInputToUrl(input); - if (!url) { + const result = await imageInputToUrl(input); + if ("skip" in result) { + skipped.push(result.skip); continue; } if (input.label) { @@ -747,36 +749,182 @@ async function buildImageParts(args: Record, detail: "auto" | " parts.push({ image_url: { detail, - url + url: result.url }, type: "image_url" }); } + // Dropping bad images must not look like dropping the argument: surface every + // reason to the caller instead of silently proceeding without the image. + if (parts.length === 0) { + if (skipped.length === 0) { + throw new Error(`${toolName} requires imageUrl, imagePath, imageBase64, or images.`); + } + throw new Error(`${toolName} found no usable image. Skipped: ${skipped.join("; ")}.`); + } return parts; } -async function imageInputToUrl(input: { base64?: string; mimeType?: string; path?: string; url?: string }): Promise { +type ImageInputToUrlResult = { url: string } | { skip: string }; + +/** + * Turn one image input into a data URL ready for the upstream, or say why it cannot + * be used. The upstream (commonly litellm in front of a strict vision provider) + * rejects any image whose payload is not strictly valid base64 and reports that as + * a 400 which surfaces to the caller as a raw provider error -- so anything that + * cannot be made into a well-formed data URL is dropped here instead of forwarded. + * + * Failure shapes seen in production: a local file path passed as imageUrl, a bare + * [media_ref:...] id passed verbatim, a base64 payload whose length mod 4 is 1 + * (irreparably truncated mid-image), and an XML/SVG payload -- which the typical + * upstream rejects outright (supported formats are jpeg/png/gif/webp), so + * relabeling it buys nothing. A remainder of 2 or 3 is repairable by padding. + * Local files (imagePath/images[].path) go through the same content checks. + */ +async function imageInputToUrl(input: { base64?: string; mimeType?: string; path?: string; url?: string }): Promise { if (input.url) { - return input.url; + const url = input.url.trim(); + if (/^https?:\/\//i.test(url)) { + return { url }; + } + if (url.startsWith("data:")) { + return dataUrlResult(url, input.mimeType); + } + // Not an HTTP(S) URL and not a data URL. Either a bare base64 payload (the + // virtual-model tool loop's usual shape; wrapped here because strict gateways + // reject it as an invalid URL) or garbage that must not be forwarded as-is. + const normalized = normalizeImagePayload(url); + if (!normalized) { + return { skip: `imageUrl is neither an HTTP(S) URL, a data URL, nor base64 (${preview(url)})` }; + } + return imageDataUrlOrSkip(normalized, "image/png"); } if (input.base64) { - return toDataUrl(input.base64, input.mimeType || "image/png"); + const value = input.base64.trim(); + if (value.startsWith("data:")) { + // The schema documents imageBase64 as "Single raw base64 image payload or + // data URL"; both shapes must behave alike, so a data URL takes the same + // path as imageUrl above. + return dataUrlResult(value, input.mimeType); + } + const normalized = normalizeImagePayload(value); + if (!normalized) { + return { skip: "imageBase64 is not usable base64" }; + } + return imageDataUrlOrSkip(normalized, input.mimeType); } if (!input.path) { - return undefined; + return { skip: "image entry has no url, base64, or path" }; } const buffer = await readFile(input.path); if (buffer.byteLength > maxLocalImageBytes) { throw new Error(`Local image exceeds ${maxLocalImageBytes} bytes: ${input.path}`); } - return toDataUrl(buffer.toString("base64"), input.mimeType || mimeTypeFromPath(input.path)); + // File contents get the same checks as every other input: a .svg on disk must + // not be forwarded as a fake raster data URL. + const normalized = normalizeImagePayload(buffer.toString("base64")); + if (!normalized) { + return { skip: `file at ${preview(input.path)} is not usable image data` }; + } + return imageDataUrlOrSkip(normalized, input.mimeType || mimeTypeFromPath(input.path)); } -function toDataUrl(value: string, mimeType: string): string { - return value.startsWith("data:") ? value : `data:${mimeType};base64,${value}`; +/** Only these media types are ever emitted on a data URL; strict upstreams validate them. */ +const supportedImageMimeTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]); + +/** + * Parse and validate a full `data:...;base64,...` URL. Shared by the imageUrl and + * imageBase64 fields so both accept the same shapes. Non-base64 data URLs are + * rejected; a header media type outside the supported set is dropped and the + * payload's sniffed type is used instead. + */ +function dataUrlResult(value: string, fallbackMimeType: string | undefined): ImageInputToUrlResult { + const comma = value.indexOf(","); + if (comma < 1) { + return { skip: `malformed data URL (${preview(value)})` }; + } + const header = value.slice(5, comma); + if (!/;base64/i.test(header)) { + return { skip: `data URL is not base64 (${preview(value)})` }; + } + const headerMimeType = header.split(";")[0] || undefined; + const normalized = normalizeImagePayload(value.slice(comma + 1)); + if (!normalized) { + return { skip: `data URL payload is not usable base64 (${preview(value)})` }; + } + return imageDataUrlOrSkip(normalized, headerMimeType || fallbackMimeType); } -function mimeTypeFromPath(path: string): string { +const base64PayloadPattern = /^[A-Za-z0-9+/]*={0,2}$/; +const svgHeadPattern = /^(?:\uFEFF)?\s*(?:<\?xml|= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return "image/jpeg"; + } + if (buffer.length >= 8 && buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) { + return "image/png"; + } + if (buffer.length >= 6 && buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x38 && (buffer[4] === 0x37 || buffer[4] === 0x39) && buffer[5] === 0x61) { + return "image/gif"; + } + if (buffer.length >= 12 && buffer.subarray(0, 4).toString("latin1") === "RIFF" && buffer.subarray(8, 12).toString("latin1") === "WEBP") { + return "image/webp"; + } + return undefined; +} + +/** + * Pick the media-type label for the data URL. Only the supported raster types are + * ever emitted: an explicit label in that set wins, otherwise the sniffed format, + * otherwise image/png. The label is validated for its own sake -- it is part of the + * data URL a strict upstream checks -- while the bytes decide whether the image is + * decodable at all. + */ +function imageLabel(mimeType: string | undefined, image: { sniffedType?: string }): string { + const explicit = mimeType?.trim().toLowerCase(); + if (explicit && supportedImageMimeTypes.has(explicit)) { + return explicit; + } + return image.sniffedType ?? "image/png"; +} + +function imageDataUrlOrSkip(image: { payload: string; svg: boolean; sniffedType?: string }, mimeType: string | undefined): ImageInputToUrlResult { + if (image.svg) { + return { skip: "SVG/XML image payload is not supported by the vision upstream (supported: image/jpeg, image/png, image/gif, image/webp)" }; + } + return { url: `data:${imageLabel(mimeType, image)};base64,${image.payload}` }; +} + +/** Short, quoted head of a value for skip reasons and error messages. */ +function preview(value: string): string { + return JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}…` : value); +}function mimeTypeFromPath(path: string): string { const ext = extname(path).toLowerCase(); if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg"; if (ext === ".webp") return "image/webp"; diff --git a/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs b/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs index 9e6cb62f..f13cd6df 100644 --- a/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs +++ b/packages/core/test/integration/mcp/fusion-vision-mcp.test.mjs @@ -1,9 +1,13 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import http from "node:http"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import test from "node:test"; +const pngA = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; + test("Fusion vision MCP sends the core API key and retries a body-free lightweight usage event", async (t) => { const seen = { providerAuthorization: "", @@ -301,6 +305,173 @@ test("Fusion vision MCP preserves slash-containing model IDs for external runtim assert.deepEqual(seen.usageBody?.target, { model }); }); +test("Fusion vision MCP accepts a full data URL in imageBase64", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageBase64: `data:image/png;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.error, undefined); + assert.equal(response.result?.isError, undefined); + assert.equal(provider.requests, 1); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); +}); + +test("Fusion vision MCP refuses a local SVG file instead of forwarding it as a fake PNG", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const dir = await mkdtemp(path.join(os.tmpdir(), "fusion-vision-svg-")); + t.after(() => rm(dir, { force: true, recursive: true })); + const svgPath = path.join(dir, "logo.svg"); + await writeFile(svgPath, '', "utf8"); + + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { images: [{ path: svgPath }], prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.result?.isError, true); + assert.match(response.result?.content?.[0]?.text, /SVG\/XML/); + assert.equal(provider.requests, 0, "the SVG must never reach the upstream"); +}); + +test("Fusion vision MCP restricts the forwarded media-type label to supported raster types", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const child = spawnVision(t, provider.port); + + const octetStream = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageBase64: pngA, mimeType: "application/octet-stream", prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); + + const textPlain = await sendJsonRpc(child, { + id: 2, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageUrl: `data:text/plain;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); + + const webp = await sendJsonRpc(child, { + id: 3, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { imageUrl: `data:image/webp;base64,${pngA}`, prompt: "Read it." }, + name: "vision_understand" + } + }); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/webp;base64,${pngA}`); +}); + +test("Fusion vision MCP forwards a local PNG file with a supported label", async (t) => { + const provider = await serveVision(t); + if (!provider) { + return; + } + const dir = await mkdtemp(path.join(os.tmpdir(), "fusion-vision-png-")); + t.after(() => rm(dir, { force: true, recursive: true })); + const pngPath = path.join(dir, "img.png"); + await writeFile(pngPath, Buffer.from(pngA, "base64")); + + const child = spawnVision(t, provider.port); + const response = await sendJsonRpc(child, { + id: 1, + jsonrpc: "2.0", + method: "tools/call", + params: { + arguments: { images: [{ path: pngPath }], prompt: "Read it." }, + name: "vision_understand" + } + }); + + assert.equal(response.error, undefined); + assert.equal(response.result?.isError, undefined); + assert.equal(provider.requests, 1); + assert.equal(provider.lastBody?.messages?.[0]?.content?.[1]?.image_url?.url, `data:image/png;base64,${pngA}`); +}); + +async function serveVision(t, respond) { + let requests = 0; + let lastBody; + const server = http.createServer(async (request, response) => { + const body = readRequestBody(request); + requests += 1; + lastBody = JSON.parse(await body); + if (respond) { + respond(lastBody); + } + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ + choices: [{ message: { content: "vision ok" } }], + usage: { completion_tokens: 3, prompt_tokens: 10, total_tokens: 13 } + })); + }); + try { + await listen(server); + } catch (error) { + if (isLocalListenUnavailable(error)) { + t.skip(`Local HTTP listen is unavailable: ${formatError(error)}`); + return undefined; + } + throw error; + } + t.after(() => server.close()); + assert.ok(server.address() && typeof server.address() === "object"); + return { port: server.address().port, get requests() { return requests; }, get lastBody() { return lastBody; } }; +} + +function spawnVision(t, port, env = {}) { + const child = spawn(process.execPath, [path.join(process.cwd(), ".test-dist", "core", "runtime", "fusion-vision-mcp.js")], { + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: "1", + FUSION_BUILTIN_TOOL_KIND: "vision", + FUSION_TOOL_NAME: "vision_understand", + VISION_API_KEY: "external-key", + VISION_BASE_URL: `http://127.0.0.1:${port}/v1`, + VISION_MODEL: "test-vision", + ...env + }, + stdio: ["pipe", "pipe", "pipe"] + }); + t.after(() => { + if (!child.killed) { + child.kill(); + } + }); + return child; +} function listen(server) { return new Promise((resolve, reject) => { const onError = (error) => {