feat(files): let the agent read HEIC photos (#6346)

* feat(files): let the agent read HEIC photos

iPhone photos reach the model as HEIC, which no vision model accepts - the
Claude Messages API takes JPEG, PNG, GIF and WebP only - so the agent saw
nothing. 75 HEIC files are already in production, 64 of them in one workspace
uploaded over the last two days.

sharp cannot cover this: its prebuilt libvips ships libheif with AV1 but not
HEVC (sharp.format.heif.input.fileSuffix is ['.avif']), so a real iPhone photo
fails with 'Security limit exceeded'. Verified against both a HEVC-coded
sample (sharp fails, heic-convert decodes 2.99MB to a 3992x2992 JPEG in
~950ms) and an AV1-coded mif1 sample (sharp decodes it natively).

Decoder selection is capability-based, not brand-based: sharp is always tried
first and the WebAssembly decoder runs only on bytes it could not read. The
container brand cannot identify the codec anyway - mif1 carries either - so
choosing from it would push AV1 files down the slow path. This mirrors how
PhotoPrism layers libvips over libheif.

Also route the image path on the effective MIME type, since a phone upload
commonly stores as application/octet-stream and would otherwise be read as
a binary the model never sees, and stop reporting an undecodable image as
'too large'.

* refactor(files): gate every vision passthrough on model-supported media types

Review found two passthroughs that still handed the model bytes it cannot
decode. The sharp-load-failure branch returned raw HEIF, and the
already-small-enough branch returned raw AVIF, TIFF, BMP or ICO — all of
which isImageFileType accepts and no vision model does.

Gating all three on the existing MODEL_SUPPORTED_IMAGE_MIME_TYPES subsumes
the ad-hoc isHeifContainer re-sniff, and re-encoding an unsupported format
falls out of the resize ladder that was already there.

Also drop two constants that were pure indirection (a one-use alias for
'image/jpeg', and a quality value identical to heic-convert's default), trim
the oversized comments, log successful transcodes so the ratio is visible in
prod, and replace a detection test that could not fail.

* fix(files): read HEIF compatible brands, not just the major brand

A standards-valid HEIF may carry a generic major brand such as isom and
declare heic, heix or mif1 only among the compatible brands that follow the
minor_version at offset 12. Reading bytes 8-11 alone classified those as
non-HEIF, skipping the fallback decode and leaving a small undecodable file
to reach the model as raw bytes.
This commit is contained in:
Waleed
2026-08-06 15:52:33 -07:00
committed by GitHub
parent 85a4cb0c1c
commit 5596640b3b
5 changed files with 241 additions and 23 deletions
+63 -23
View File
@@ -14,7 +14,12 @@ import { recordFileRead } from '@/lib/copilot/request/metrics'
import { markSpanForError } from '@/lib/copilot/request/otel'
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { isImageFileType } from '@/lib/uploads/utils/file-utils'
import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
import {
isImageFileType,
MODEL_SUPPORTED_IMAGE_MIME_TYPES,
resolveEffectiveMimeType,
} from '@/lib/uploads/utils/file-utils'
// Lazy tracer (same pattern as lib/copilot/request/otel.ts).
function getVfsTracer() {
@@ -91,54 +96,82 @@ interface PreparedVisionImage {
* dimension/quality chosen.
*/
async function prepareImageForVision(
buffer: Buffer,
sourceBuffer: Buffer,
claimedType: string
): Promise<PreparedVisionImage | null> {
return getVfsTracer().startActiveSpan(
TraceSpan.CopilotVfsPrepareImage,
{
attributes: {
[TraceAttr.CopilotVfsInputBytes]: buffer.length,
[TraceAttr.CopilotVfsInputBytes]: sourceBuffer.length,
[TraceAttr.CopilotVfsInputMediaTypeClaimed]: claimedType,
},
},
async (span) => {
try {
const mediaType = detectImageMime(buffer, claimedType)
span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, mediaType)
const detectedType = detectImageMime(sourceBuffer, claimedType)
span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, detectedType)
let sharpModule: SharpConstructor
try {
sharpModule = (await import('sharp')).default
} catch (err) {
logger.warn('Failed to load sharp for image preparation', {
mediaType,
mediaType: detectedType,
error: toError(err).message,
})
span.setAttribute(TraceAttr.CopilotVfsSharpLoadFailed, true)
const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES
const fitsWithoutSharp =
MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(detectedType) &&
sourceBuffer.length <= MAX_IMAGE_READ_BYTES
span.setAttribute(
TraceAttr.CopilotVfsOutcome,
fitsWithoutSharp ? 'passthrough_no_sharp' : 'rejected_no_sharp'
)
return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null
return fitsWithoutSharp
? { buffer: sourceBuffer, mediaType: detectedType, resized: false }
: null
}
let metadata: Awaited<ReturnType<ReturnType<typeof sharpModule>['metadata']>>
try {
metadata = await sharpModule(buffer, { limitInputPixels: false }).metadata()
} catch (err) {
logger.warn('Failed to read image metadata for VFS read', {
mediaType,
error: toError(err).message,
})
const readMetadata = (candidate: Buffer) =>
sharpModule(candidate, { limitInputPixels: false })
.metadata()
.catch((err: unknown) => {
logger.warn('Failed to read image metadata for VFS read', {
mediaType: detectedType,
error: toError(err).message,
})
return null
})
// sharp first: its libvips reads everything we accept except HEVC-coded
// HEIF, and it is ~10x faster than the WASM decoder. Capability-based
// rather than brand-based, so AV1-coded `mif1` — which sharp handles
// natively — does not get sent down the slow path.
let buffer = sourceBuffer
let mediaType = detectedType
let metadata = await readMetadata(sourceBuffer)
if (!metadata && isHeifContainer(sourceBuffer)) {
const transcoded = await transcodeHeicToJpeg(sourceBuffer)
if (transcoded) {
buffer = transcoded
mediaType = 'image/jpeg'
metadata = await readMetadata(transcoded)
}
}
if (!metadata) {
span.setAttribute(TraceAttr.CopilotVfsMetadataFailed, true)
const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES
// Bytes the model cannot decode are worse than no image: it describes
// them as empty rather than reporting them as broken.
const passthroughViable =
MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) && buffer.length <= MAX_IMAGE_READ_BYTES
span.setAttribute(
TraceAttr.CopilotVfsOutcome,
fitsWithoutSharp ? 'passthrough_no_metadata' : 'rejected_no_metadata'
passthroughViable ? 'passthrough_no_metadata' : 'rejected_no_metadata'
)
return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null
return passthroughViable ? { buffer, mediaType, resized: false } : null
}
const width = metadata.width ?? 0
@@ -148,11 +181,15 @@ async function prepareImageForVision(
[TraceAttr.CopilotVfsInputHeight]: height,
})
const needsResize =
// A format the model cannot decode has to be re-encoded even when it is
// already small enough — the ladder below emits JPEG or WebP, both of
// which it accepts.
const needsReencode =
!MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) ||
buffer.length > MAX_IMAGE_READ_BYTES ||
width > MAX_IMAGE_DIMENSION ||
height > MAX_IMAGE_DIMENSION
if (!needsResize) {
if (!needsReencode) {
span.setAttributes({
[TraceAttr.CopilotVfsResized]: false,
[TraceAttr.CopilotVfsOutcome]: CopilotVfsOutcome.PassthroughFitsBudget,
@@ -300,14 +337,17 @@ export async function readFileRecord(record: WorkspaceFileRecord): Promise<FileR
},
async (span) => {
try {
if (isImageFileType(record.type)) {
// Resolve against the filename: a phone upload commonly stores as
// `application/octet-stream`, and matching the raw type would route a real
// image down the binary path where the model never sees it.
if (isImageFileType(resolveEffectiveMimeType(record.type, record.name))) {
span.setAttribute(TraceAttr.CopilotVfsReadPath, CopilotVfsReadPath.Image)
const originalBuffer = await fetchWorkspaceFileBuffer(record)
const prepared = await prepareImageForVision(originalBuffer, record.type)
if (!prepared) {
span.setAttribute(TraceAttr.CopilotVfsReadOutcome, CopilotVfsReadOutcome.ImageTooLarge)
return {
content: `[Image too large: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB, limit 5MB after resize/compression)]`,
content: `[Image unavailable: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB). It could not be decoded, or still exceeded the 5MB vision limit after resizing.]`,
totalLines: 1,
}
}
+84
View File
@@ -0,0 +1,84 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
/**
* An ISO-BMFF `ftyp` box: 4-byte size, the `ftyp` marker, the major brand, a
* 4-byte minor version, then any compatible brands.
*/
function ftypHeader(brand: string, compatible: string[] = []): Buffer {
const size = 16 + compatible.length * 4
const header = Buffer.alloc(size)
header.writeUInt32BE(size, 0)
header.write('ftyp', 4, 'ascii')
header.write(brand, 8, 'ascii')
compatible.forEach((entry, index) => header.write(entry, 16 + index * 4, 'ascii'))
return header
}
describe('isHeifContainer', () => {
it.each(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx', 'mif1', 'msf1'])(
'detects the %s brand',
(brand) => {
expect(isHeifContainer(ftypHeader(brand))).toBe(true)
}
)
it.each(['avif', 'avis'])(
'also claims the %s brand — the question is "is this HEIF", not "which codec"',
(brand) => {
expect(isHeifContainer(ftypHeader(brand))).toBe(true)
}
)
it('rejects other image formats', () => {
expect(isHeifContainer(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe(
false
)
expect(isHeifContainer(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe(
false
)
})
it('rejects a HEIF brand that is not behind an ftyp box', () => {
const riff = Buffer.alloc(16)
riff.write('RIFF', 0, 'ascii')
riff.write('heic', 8, 'ascii')
expect(isHeifContainer(riff)).toBe(false)
})
it('rejects an unknown brand in a well-formed ftyp box', () => {
expect(isHeifContainer(ftypHeader('qt '))).toBe(false)
})
it('detects a HEIF brand declared only among the compatible brands', () => {
// Standards-valid: a generic major brand with the HEIF brand listed after it.
expect(isHeifContainer(ftypHeader('isom', ['iso2', 'heic', 'mif1']))).toBe(true)
expect(isHeifContainer(ftypHeader('mp42', ['heix']))).toBe(true)
})
it('rejects a box whose compatible brands are all non-HEIF', () => {
expect(isHeifContainer(ftypHeader('isom', ['iso2', 'mp41', 'mp42']))).toBe(false)
})
it('does not read compatible brands past the declared box size', () => {
const truncated = ftypHeader('isom', ['heic'])
truncated.writeUInt32BE(16, 0)
expect(isHeifContainer(truncated)).toBe(false)
})
it('rejects buffers too short to carry a brand', () => {
expect(isHeifContainer(Buffer.alloc(0))).toBe(false)
expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false)
})
})
describe('transcodeHeicToJpeg', () => {
it('returns null for bytes libheif cannot decode', async () => {
// Also proves the dynamic `heic-convert` import resolves at runtime, which no
// amount of type-checking establishes for a lazily loaded WebAssembly module.
expect(await transcodeHeicToJpeg(ftypHeader('heic'))).toBeNull()
})
})
+77
View File
@@ -0,0 +1,77 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
const logger = createLogger('HeicTranscode')
/**
* ISO-BMFF major brands in the HEIF family. The brand occupies bytes 8-11,
* immediately after the `ftyp` box marker at 4-7.
*
* The list is deliberately broad, `avif` included. It answers "are these bytes
* worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot
* answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1.
*/
const HEIF_BRANDS = new Set([
'heic',
'heix',
'heim',
'heis',
'hevc',
'hevx',
'mif1',
'msf1',
'avif',
'avis',
])
/**
* Whether these bytes are an ISO-BMFF container in the HEIF family.
*
* Sniffed rather than read off the declared type because the common case is a
* `.heic` stored as `application/octet-stream`, where the declared type says
* nothing at all.
*/
export function isHeifContainer(buffer: Buffer): boolean {
if (buffer.length < 12) return false
if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false
if (HEIF_BRANDS.has(buffer.toString('ascii', 8, 12))) return true
// A standards-valid HEIF may carry a generic major brand such as `isom` and name
// the HEIF brand only among the compatible brands, which follow the 4-byte
// minor_version at offset 12 and run to the end of the box. A declared size of 0
// or 1 (the ISO-BMFF size escapes, which `ftyp` does not use) leaves `end` below
// the loop's start, so those simply do not scan.
const end = Math.min(buffer.readUInt32BE(0), buffer.length)
for (let offset = 16; offset + 4 <= end; offset += 4) {
if (HEIF_BRANDS.has(buffer.toString('ascii', offset, offset + 4))) return true
}
return false
}
/**
* Transcode a HEVC-coded HEIF still to JPEG.
*
* Two reasons, neither with a workaround: no vision model accepts HEIC (the Claude
* Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's prebuilt libvips
* ships libheif with AV1 but not HEVC — it decodes AVIF and rejects an iPhone photo.
*
* Returns `null` when the bytes cannot be decoded; never a partial image.
*/
export async function transcodeHeicToJpeg(buffer: Buffer): Promise<Buffer | null> {
try {
const convert = (await import('heic-convert')).default
const jpeg = await convert({ buffer, format: 'JPEG' })
logger.info('Transcoded HEIC image', {
inputBytes: buffer.length,
outputBytes: jpeg.length,
})
return Buffer.from(jpeg)
} catch (error) {
logger.warn('Failed to transcode HEIC image', {
bytes: buffer.length,
brand: buffer.toString('ascii', 8, 12),
error: getErrorMessage(error),
})
return null
}
}
+2
View File
@@ -168,6 +168,7 @@
"google-auth-library": "10.5.0",
"gray-matter": "^4.0.3",
"groq-sdk": "^0.15.0",
"heic-convert": "2.1.0",
"html-to-text": "^9.0.5",
"http-proxy-agent": "7.0.2",
"https-proxy-agent": "7.0.6",
@@ -248,6 +249,7 @@
"@types/archiver": "8.0.0",
"@types/busboy": "1.5.4",
"@types/fluent-ffmpeg": "2.1.28",
"@types/heic-convert": "2.1.1",
"@types/html-to-text": "9.0.4",
"@types/js-yaml": "4.0.9",
"@types/jsdom": "21.1.7",
+15
View File
@@ -1,5 +1,6 @@
{
"lockfileVersion": 1,
"configVersion": 0,
"workspaces": {
"": {
"name": "simstudio",
@@ -269,6 +270,7 @@
"google-auth-library": "10.5.0",
"gray-matter": "^4.0.3",
"groq-sdk": "^0.15.0",
"heic-convert": "2.1.0",
"html-to-text": "^9.0.5",
"http-proxy-agent": "7.0.2",
"https-proxy-agent": "7.0.6",
@@ -349,6 +351,7 @@
"@types/archiver": "8.0.0",
"@types/busboy": "1.5.4",
"@types/fluent-ffmpeg": "2.1.28",
"@types/heic-convert": "2.1.1",
"@types/html-to-text": "9.0.4",
"@types/js-yaml": "4.0.9",
"@types/jsdom": "21.1.7",
@@ -2120,6 +2123,8 @@
"@types/hast": ["@types/hast@3.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ=="],
"@types/heic-convert": ["@types/heic-convert@2.1.1", "", {}, "sha512-+s14762Nf62z9zziIs7ItvAkSUCS3ls4Z5XPT9BlVve3Q3S4DAnf6qekffMTvEWycSUF+kulkGaSN65fK6eKvg=="],
"@types/html-to-text": ["@types/html-to-text@9.0.4", "", {}, "sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ=="],
"@types/http-cache-semantics": ["@types/http-cache-semantics@4.2.0", "", {}, "sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q=="],
@@ -3090,6 +3095,10 @@
"hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="],
"heic-convert": ["heic-convert@2.1.0", "", { "dependencies": { "heic-decode": "^2.0.0", "jpeg-js": "^0.4.4", "pngjs": "^6.0.0" } }, "sha512-1qDuRvEHifTVAj3pFIgkqGgJIr0M3X7cxEPjEp0oG4mo8GFjq99DpCo8Eg3kg17Cy0MTjxpFdoBHOatj7ZVKtg=="],
"heic-decode": ["heic-decode@2.1.0", "", { "dependencies": { "libheif-js": "^1.19.8" } }, "sha512-0fB3O3WMk38+PScbHLVp66jcNhsZ/ErtQ6u2lMYu/YxXgbBtl+oKOhGQHa4RpvE68k8IzbWkABzHnyAIjR758A=="],
"help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="],
"hex-rgb": ["hex-rgb@4.3.0", "", {}, "sha512-Ox1pJVrDCyGHMG9CFg1tmrRUMRPRsAWYc/PinY0XzJU4K7y7vjNoLKIQ7BR5UJMCxNN8EM1MNDmHWA/B3aZUuw=="],
@@ -3244,6 +3253,8 @@
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
"jpeg-js": ["jpeg-js@0.4.4", "", {}, "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg=="],
"js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="],
"js-tokens": ["js-tokens@10.0.0", "", {}, "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q=="],
@@ -3308,6 +3319,8 @@
"libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="],
"libheif-js": ["libheif-js@1.19.8", "", {}, "sha512-vQJWusIxO7wavpON1dusciL8Go9jsIQ+EUrckauFYAiSTjcmLAsuJh3SszLpvkwPci3JcL41ek2n+LUZGFpPIQ=="],
"libmime": ["libmime@5.3.7", "", { "dependencies": { "encoding-japanese": "2.2.0", "iconv-lite": "0.6.3", "libbase64": "1.3.0", "libqp": "2.1.1" } }, "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw=="],
"libqp": ["libqp@2.1.1", "", {}, "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow=="],
@@ -3800,6 +3813,8 @@
"plist": ["plist@3.1.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.8", "base64-js": "^1.5.1", "xmlbuilder": "^15.1.1" } }, "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ=="],
"pngjs": ["pngjs@6.0.0", "", {}, "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg=="],
"points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="],
"points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="],