feat: resize chat image attachments client-side for provider budgets (#24533)

Anthropic rejects inline images over 5,242,880 bytes, but our upload
endpoint accepts images up to 10 MiB — so 5–10 MiB images were
reaching the provider and failing. This adds two layers of
protection: the browser resizes oversized images before upload, and
the server rejects any that still slip through before an upstream
request is issued.

Client-side resizing uses `createImageBitmap` with
`resizeWidth`/`resizeHeight` to clamp the decoded bitmap at decode
time, then iteratively shrinks on an `OffscreenCanvas` (falling back
to `HTMLCanvasElement`) until the output fits the applicable budget.
Anthropic (and Bedrock-hosted Claude — fantasy's bedrock provider is
a thin wrapper around the Anthropic client) uses a ~5 MiB budget;
other providers use a ~10 MiB budget to stay under the server cap.
Doing the resize in the browser avoids decoding attacker-controlled
image bytes in `coderd` (image-bomb DoS surface).

Server-side, `chatFileResolver` now takes a provider string and
looks up the inline-image cap via a new
`chatprovider.InlineImageByteCap`
helper; oversized `image/*` files for capped providers are rejected
with a pre-classified `chaterror` before the SDK call. The backstop
fires for older clients, direct API callers, or any image that was
committed to the composer before the user switched to a stricter
provider.

Attachments commit to composer state synchronously with a new
`"processing"` `UploadState` so paste+Enter can't dispatch before
the resize finishes; the `"uploading"` send gate now covers both
states. Dismissed-while-resizing attachments are tracked in a
`WeakSet` so a late swap can't resurrect a removed file.

Closes CODAGT-215
This commit is contained in:
Dean Sheather
2026-05-08 02:07:33 +10:00
committed by GitHub
parent eef09f3d98
commit e1b1c7ec5b
21 changed files with 2297 additions and 67 deletions
+2 -7
View File
@@ -4343,11 +4343,6 @@ func parseCompactionThresholdKey(key string) (uuid.UUID, error) {
return id, nil
}
const (
// maxChatFileSize is the maximum size of a chat file upload (10 MB).
maxChatFileSize = 10 << 20
)
//nolint:revive // get-return: revive assumes get* must be a getter, but this is an HTTP handler.
func (api *API) getChatSystemPrompt(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
@@ -5832,14 +5827,14 @@ func (api *API) postChatFile(rw http.ResponseWriter, r *http.Request) {
}
}
r.Body = http.MaxBytesReader(rw, r.Body, maxChatFileSize)
r.Body = http.MaxBytesReader(rw, r.Body, codersdk.MaxChatFileSizeBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
httpapi.Write(ctx, rw, http.StatusRequestEntityTooLarge, codersdk.Response{
Message: "File too large.",
Detail: fmt.Sprintf("Maximum file size is %d bytes.", maxChatFileSize),
Detail: fmt.Sprintf("Maximum file size is %d bytes.", codersdk.MaxChatFileSizeBytes),
})
return
}
+40 -5
View File
@@ -19,6 +19,7 @@ import (
"charm.land/fantasy"
"charm.land/fantasy/providers/anthropic"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/shopspring/decimal"
@@ -5161,16 +5162,50 @@ func (p *Server) subscribeChatControl(
return controlCancel
}
// chatFileResolver returns a FileResolver that fetches chat file
// content from the database by ID.
func (p *Server) chatFileResolver() chatprompt.FileResolver {
// Rejects oversize images on capped providers before any upstream
// request is issued.
//
// Gotcha: a historical oversize image bricks the chat on a capped
// provider until the user switches providers back, starts a new
// chat, or edits a message above the offending one (which truncates
// the prompt forward). A future change should skip the file with a
// user-facing warning, but that requires altering the FileResolver
// contract.
func (p *Server) chatFileResolver(provider string) chatprompt.FileResolver {
return func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]chatprompt.FileData, error) {
files, err := p.db.GetChatFilesByIDs(ctx, ids)
if err != nil {
return nil, err
}
imageCap, hasImageCap := chatprovider.InlineImageCapBytes(provider)
normalizedProvider := chatprovider.NormalizeProvider(provider)
result := make(map[uuid.UUID]chatprompt.FileData, len(files))
for _, f := range files {
if hasImageCap &&
strings.HasPrefix(f.Mimetype, "image/") &&
len(f.Data) >= imageCap {
err := xerrors.Errorf(
"image attachment %q is %d bytes; %s inline image limit is %d bytes",
f.Name, len(f.Data),
chatprovider.ProviderDisplayName(normalizedProvider),
imageCap,
)
// User-facing message stays client-agnostic since
// older web clients and direct API callers don't
// auto-resize; the wrapped error above keeps the
// exact byte count for operator logs.
return nil, chaterror.WithClassification(err, chaterror.ClassifiedError{
Kind: codersdk.ChatErrorKindConfig,
Provider: normalizedProvider,
Message: fmt.Sprintf(
"Image attachment exceeds %s's %s inline image limit. Replace it with a smaller image.",
chatprovider.ProviderDisplayName(normalizedProvider),
//nolint:gosec // imageCap is a small positive constant defined in chatprovider.
humanize.IBytes(uint64(imageCap)),
),
Retryable: false,
})
}
result[f.ID] = chatprompt.FileData{
Name: f.Name,
Data: f.Data,
@@ -6478,7 +6513,7 @@ func (p *Server) runChat(
var g2 errgroup.Group
g2.Go(func() error {
var err error
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, messages, p.chatFileResolver(), logger)
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, messages, p.chatFileResolver(modelConfig.Provider), logger)
if err != nil {
return xerrors.Errorf("build chat prompt: %w", err)
}
@@ -7263,7 +7298,7 @@ func (p *Server) runChat(
if compactionOptions != nil {
compactionOptions.HistoryTipMessageID = compactionHistoryTipMessageID
}
reloadedPrompt, err := chatprompt.ConvertMessagesWithFiles(reloadCtx, reloadedMsgs, p.chatFileResolver(), logger)
reloadedPrompt, err := chatprompt.ConvertMessagesWithFiles(reloadCtx, reloadedMsgs, p.chatFileResolver(modelConfig.Provider), logger)
if err != nil {
return nil, xerrors.Errorf("convert reloaded messages: %w", err)
}
+313
View File
@@ -0,0 +1,313 @@
package chatd
import (
"context"
"strconv"
"testing"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/codersdk"
)
// inlineImageCapFor returns the provider's inline image cap. Fails
// the test if the provider has no documented cap.
func inlineImageCapFor(t *testing.T, provider string) int {
t.Helper()
imageCap, ok := chatprovider.InlineImageCapBytes(provider)
require.Truef(t, ok, "expected provider %q to have an inline image cap", provider)
return imageCap
}
// TestChatFileResolver_RejectsOversizedImages is the server-side
// safety net for browser-side resize: oversize images that reach the
// resolver are rejected before any upstream request.
func TestChatFileResolver_RejectsOversizedImages(t *testing.T) {
t.Parallel()
// Computed so the table tracks any future cap retune.
anthropicCap := inlineImageCapFor(t, "anthropic")
tests := []struct {
name string
provider string
mimetype string
size int
expectReject bool
expectProviderID string // classified.Provider after normalization
}{
{
name: "OversizedAnthropicPNG_Rejected",
provider: "anthropic",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: true,
expectProviderID: "anthropic",
},
{
name: "OversizedAnthropicJPEG_Rejected",
provider: "anthropic",
mimetype: "image/jpeg",
size: anthropicCap + 1024,
expectReject: true,
expectProviderID: "anthropic",
},
{
// Boundary is >=: exactly-at-limit is rejected.
// Anthropic's docs say "5 MB maximum" without
// specifying inclusivity, so reject strictly.
name: "AtLimitAnthropicImage_Rejected",
provider: "anthropic",
mimetype: "image/png",
size: anthropicCap,
expectReject: true,
expectProviderID: "anthropic",
},
{
name: "JustUnderLimitAnthropicImage_Accepted",
provider: "anthropic",
mimetype: "image/png",
size: anthropicCap - 1,
expectReject: false,
expectProviderID: "anthropic",
},
{
name: "UndersizedAnthropicImage_Accepted",
provider: "anthropic",
mimetype: "image/png",
size: 1024,
expectReject: false,
expectProviderID: "anthropic",
},
{
// Bedrock reuses Anthropic's cap.
name: "OversizedBedrockPNG_Rejected",
provider: "bedrock",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: true,
expectProviderID: "bedrock",
},
{
name: "OversizedOpenAIImage_Accepted",
provider: "openai",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: false,
expectProviderID: "openai",
},
{
name: "OversizedAnthropicText_Accepted",
provider: "anthropic",
mimetype: "text/plain",
size: anthropicCap + 1,
expectReject: false,
expectProviderID: "anthropic",
},
{
name: "ProviderMixedCase_Rejected",
provider: "Anthropic",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: true,
expectProviderID: "anthropic",
},
{
name: "ProviderAllCaps_Rejected",
provider: "ANTHROPIC",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: true,
expectProviderID: "anthropic",
},
{
name: "ProviderPaddedWhitespace_Rejected",
provider: " anthropic ",
mimetype: "image/png",
size: anthropicCap + 1,
expectReject: true,
expectProviderID: "anthropic",
},
}
// One shared backing buffer sliced per case. The resolver only
// reads len(f.Data), so shared backing is safe and avoids N×max
// allocations in parallel.
maxSize := 0
for _, tc := range tests {
if tc.size > maxSize {
maxSize = tc.size
}
}
sharedData := make([]byte, maxSize)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
server := &Server{db: db}
fileID := uuid.New()
row := database.ChatFile{
ID: fileID,
Name: "attachment.png",
Mimetype: tc.mimetype,
Data: sharedData[:tc.size],
}
db.EXPECT().
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
Return([]database.ChatFile{row}, nil).
Times(1)
resolver := server.chatFileResolver(tc.provider)
got, err := resolver(ctx, []uuid.UUID{fileID})
if tc.expectReject {
require.Error(t, err)
require.Nil(t, got)
// Classification turns the generic upstream error
// into an actionable user-facing message.
classified := chaterror.Classify(err)
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
require.Equal(t, tc.expectProviderID, classified.Provider)
require.False(t, classified.Retryable)
// User-facing message names the provider and shows
// the cap in human units; raw byte count stays in
// the wrapped developer error.
displayName := chatprovider.ProviderDisplayName(tc.expectProviderID)
require.Contains(t, classified.Message, displayName)
imageCap := inlineImageCapFor(t, tc.expectProviderID)
//nolint:gosec // imageCap is a small positive constant defined in chatprovider.
require.Contains(t, classified.Message, humanize.IBytes(uint64(imageCap)))
require.NotContains(
t,
classified.Message,
strconv.Itoa(imageCap),
"user-facing message should not include raw bytes",
)
// Wrapped error preserves exact bytes for logs.
require.Contains(t, err.Error(), strconv.Itoa(imageCap))
return
}
require.NoError(t, err)
require.Contains(t, got, fileID)
require.Equal(t, row.Data, got[fileID].Data)
require.Equal(t, tc.mimetype, got[fileID].MediaType)
})
}
}
// TestChatFileResolver_MultiFileFailsFastOnFirstOversized pins the
// "first bad file aborts the batch" contract.
func TestChatFileResolver_MultiFileFailsFastOnFirstOversized(t *testing.T) {
t.Parallel()
ctx := context.Background()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
server := &Server{db: db}
anthropicCap := inlineImageCapFor(t, "anthropic")
// Shared buffer; ok files take small prefixes.
buf := make([]byte, anthropicCap+1)
okFileA := database.ChatFile{
ID: uuid.New(),
Name: "ok-a.png",
Mimetype: "image/png",
Data: buf[:1024],
}
oversized := database.ChatFile{
ID: uuid.New(),
Name: "too-big.png",
Mimetype: "image/png",
Data: buf,
}
okFileB := database.ChatFile{
ID: uuid.New(),
Name: "ok-b.png",
Mimetype: "image/png",
Data: buf[:1024],
}
ids := []uuid.UUID{okFileA.ID, oversized.ID, okFileB.ID}
db.EXPECT().
GetChatFilesByIDs(gomock.Any(), ids).
Return([]database.ChatFile{okFileA, oversized, okFileB}, nil).
Times(1)
resolver := server.chatFileResolver("anthropic")
got, err := resolver(ctx, ids)
require.Error(t, err)
require.Nil(t, got)
classified := chaterror.Classify(err)
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
// The error must identify the specific offending file so a user
// with several attachments knows which one to replace.
require.Contains(t, err.Error(), oversized.Name)
}
// TestChatFileResolver_PropagatesDBError confirms unrelated database
// failures pass through unchanged (not masked by the size check).
func TestChatFileResolver_PropagatesDBError(t *testing.T) {
t.Parallel()
ctx := context.Background()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
server := &Server{db: db}
sentinel := xerrors.New("boom")
fileID := uuid.New()
db.EXPECT().
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
Return(nil, sentinel).
Times(1)
resolver := server.chatFileResolver("anthropic")
got, err := resolver(ctx, []uuid.UUID{fileID})
require.ErrorIs(t, err, sentinel)
require.Nil(t, got)
}
// TestChatFileResolver_UnknownProviderSkipsCapCheck confirms providers
// without a documented inline cap are never rejected by the backstop.
func TestChatFileResolver_UnknownProviderSkipsCapCheck(t *testing.T) {
t.Parallel()
ctx := context.Background()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
server := &Server{db: db}
fileID := uuid.New()
// Exactly 1 byte above the Anthropic cap is enough to prove
// the backstop is skipped for uncapped providers; no need to
// allocate tens of MiB in CI.
overAnyCap := inlineImageCapFor(t, "anthropic") + 1
row := database.ChatFile{
ID: fileID,
Name: "huge.png",
Mimetype: "image/png",
Data: make([]byte, overAnyCap),
}
db.EXPECT().
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
Return([]database.ChatFile{row}, nil).
Times(1)
resolver := server.chatFileResolver("openrouter")
got, err := resolver(ctx, []uuid.UUID{fileID})
require.NoError(t, err)
require.Contains(t, got, fileID)
}
@@ -83,6 +83,19 @@ func ProviderAllowsAmbientCredentials(provider string) bool {
return NormalizeProvider(provider) == fantasybedrock.Name
}
// InlineImageCapBytes returns the per-image byte cap for inline
// image parts, or (0, false) when no documented cap applies.
// Bedrock shares Anthropic's cap because fantasy's bedrock provider
// wraps the anthropic client.
func InlineImageCapBytes(provider string) (int, bool) {
switch NormalizeProvider(provider) {
case fantasyanthropic.Name, fantasybedrock.Name:
return codersdk.AnthropicInlineImageCapBytes, true
default:
return 0, false
}
}
// ProviderAPIKeys contains API keys for provider calls.
type ProviderAPIKeys struct {
OpenAI string
+9
View File
@@ -33,6 +33,15 @@ const ChatCompactionThresholdKeyPrefix = "chat_compaction_threshold_pct:"
// this limit than to lower it.
const MaxChatFileIDs = 20
// MaxChatFileSizeBytes is the upload-endpoint cap for chat
// attachments.
const MaxChatFileSizeBytes = 10 * 1024 * 1024
// AnthropicInlineImageCapBytes is Anthropic's documented per-image
// wire limit; the same cap applies to Bedrock-hosted Claude. Other
// providers have no documented per-image cap.
const AnthropicInlineImageCapBytes = 5 * 1024 * 1024
// ChatAttachmentMediaType is a media type that is allowed for durable
// chat file storage. The set is intentionally narrow; byte-level
// classification and inline-render rules live alongside the enforcement
+15
View File
@@ -877,6 +877,14 @@ export const AgentSubsystems: AgentSubsystem[] = [
"exectrace",
];
// From codersdk/chats.go
/**
* AnthropicInlineImageCapBytes is Anthropic's documented per-image
* wire limit; the same cap applies to Bedrock-hosted Claude. Other
* providers have no documented per-image cap.
*/
export const AnthropicInlineImageCapBytes = 5242880;
// From codersdk/deployment.go
export interface AppHostResponse {
/**
@@ -4777,6 +4785,13 @@ export interface MatchedProvisioners {
*/
export const MaxChatFileIDs = 20;
// From codersdk/chats.go
/**
* MaxChatFileSizeBytes is the upload-endpoint cap for chat
* attachments.
*/
export const MaxChatFileSizeBytes = 10485760;
// From codersdk/usersecretvalidation.go
/**
* MaxSecretValueSize is the maximum size of a user secret value
@@ -605,3 +605,283 @@ describe("useFileAttachments persistence", () => {
unmount();
});
});
// Synthetic resize swaps the File without invoking the browser's
// decoder, so these tests run in jsdom without resizeImage.ts fakes.
describe("useFileAttachments processResizes", () => {
beforeEach(() => {
localStorage.clear();
});
afterEach(() => {
vi.restoreAllMocks();
});
// Reported size > budget without actually allocating bytes.
const makeOversizeImage = (
name = "photo.png",
type = "image/png",
bytes = 6 * 1024 * 1024,
) => {
const file = new File([new Uint8Array(8)], name, { type });
Object.defineProperty(file, "size", { value: bytes });
return file;
};
it("marks oversize images as processing synchronously on attach", async () => {
const resize = await import("./utils/resizeImage");
// Never-resolving resize so "processing" stays observable.
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() => new Promise(() => undefined),
);
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const file = makeOversizeImage();
act(() => {
result.current.handleAttach([file]);
});
expect(result.current.attachments).toHaveLength(1);
expect(result.current.attachments[0]).toBe(file);
const state = result.current.uploadStates.get(file);
expect(state?.status).toBe("processing");
unmount();
});
it("swaps the original File for the resized replacement when resize succeeds", async () => {
const resize = await import("./utils/resizeImage");
const { API } = await import("#/api/api");
const replacement = new File([new Uint8Array(1024)], "photo.webp", {
type: "image/webp",
});
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(replacement);
vi.spyOn(API.experimental, "uploadChatFile").mockResolvedValue({
id: "file-id",
});
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
await vi.waitFor(() => {
expect(result.current.attachments[0]).toBe(replacement);
});
await vi.waitFor(() => {
expect(result.current.uploadStates.get(replacement)?.status).toBe(
"uploaded",
);
});
expect(result.current.uploadStates.get(original)).toBeUndefined();
unmount();
});
it("falls back to the original File when resize returns null", async () => {
const resize = await import("./utils/resizeImage");
const { API } = await import("#/api/api");
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(null);
vi.spyOn(API.experimental, "uploadChatFile").mockResolvedValue({
id: "file-id",
});
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
// 6 MiB on Anthropic + null resize forces the
// provider-budget-error path.
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const original = makeOversizeImage(
"photo.png",
"image/png",
6 * 1024 * 1024,
);
act(() => {
result.current.handleAttach([original]);
});
await vi.waitFor(() => {
const state = result.current.uploadStates.get(original);
expect(state?.status).toBe("error");
expect(state?.error).toMatch(/Anthropic/);
expect(state?.error).toMatch(/MiB/);
});
unmount();
});
it("freezes the provider snapshot at attach time so a mid-resize provider switch can't mislabel the error", async () => {
const resize = await import("./utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolve) => {
releaseResize = resolve;
}),
);
const { result, rerender, unmount } = renderHook(
({ provider }) => useFileAttachments("org-1", { provider }),
{ initialProps: { provider: "anthropic" } },
);
// Attach on Anthropic (5 MiB budget).
const original = makeOversizeImage(
"photo.gif",
"image/gif",
6 * 1024 * 1024,
);
act(() => {
result.current.handleAttach([original]);
});
// User switches to OpenAI (10 MiB budget) before the
// resize finishes. providerRef.current is now "openai".
rerender({ provider: "openai" });
// Resize gives up (animated GIF; falls back to original).
await act(async () => {
releaseResize(null);
await Promise.resolve();
});
// Error must name Anthropic (the provider whose budget
// rejected the file), not OpenAI (the live ref). The
// budget value in the error must match Anthropic's
// ~5 MiB, not OpenAI's ~10 MiB.
await vi.waitFor(() => {
const state = result.current.uploadStates.get(original);
expect(state?.status).toBe("error");
expect(state?.error).toMatch(/Anthropic/);
expect(state?.error).not.toMatch(/OpenAI/);
expect(state?.error).toMatch(/under 5\.0 MiB/);
});
unmount();
});
it("does not resurrect attachments removed while resize is in flight", async () => {
const resize = await import("./utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolve) => {
releaseResize = resolve;
}),
);
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.attachments).toHaveLength(1);
// Remove the attachment while resize is still pending.
act(() => {
result.current.handleRemoveAttachment(original);
});
expect(result.current.attachments).toHaveLength(0);
// Resolve the pending resize with a swap; the hook must
// NOT resurrect the dismissed attachment.
const replacement = new File([new Uint8Array(512)], "photo.webp", {
type: "image/webp",
});
await act(async () => {
releaseResize(replacement);
await Promise.resolve();
});
expect(result.current.attachments).toHaveLength(0);
expect(result.current.uploadStates.get(replacement)).toBeUndefined();
unmount();
});
it("does not resurrect attachments after resetAttachments fires", async () => {
const resize = await import("./utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolve) => {
releaseResize = resolve;
}),
);
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.attachments).toHaveLength(1);
// Simulate a chat-scope reset (e.g. ChatPageContent's
// editScopeRef effect when navigating between chats)
// while the resize is still pending.
act(() => {
result.current.resetAttachments();
});
expect(result.current.attachments).toHaveLength(0);
// Resolve the pending resize with a swap. The hook must
// not re-add the replacement to attachments or kick off
// an upload against the now-cleared scope.
const replacement = new File([new Uint8Array(512)], "photo.webp", {
type: "image/webp",
});
await act(async () => {
releaseResize(replacement);
await Promise.resolve();
});
expect(result.current.attachments).toHaveLength(0);
expect(result.current.uploadStates.get(replacement)).toBeUndefined();
expect(result.current.uploadStates.get(original)).toBeUndefined();
unmount();
});
it("gates the send-reachable isUploading state on processing", () => {
const resize = import("./utils/resizeImage");
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
// Never-resolving resize keeps the attachment in "processing".
void resize.then((m) =>
vi
.spyOn(m, "resizeImageToMaxBytes")
.mockImplementation(() => new Promise(() => undefined)),
);
const { result, unmount } = renderHook(() =>
useFileAttachments("org-1", { provider: "anthropic" }),
);
const file = makeOversizeImage();
act(() => {
result.current.handleAttach([file]);
});
// Upload state must be "processing" (not "uploaded" or
// undefined). The AgentChatInput send gate treats this the
// same as "uploading" and blocks dispatch.
expect(result.current.uploadStates.get(file)?.status).toBe("processing");
unmount();
});
});
@@ -819,7 +819,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
{/* Warn about invisible Unicode in the message text.
* Unlike the admin/user prompt textareas (which strip
* invisible chars server-side on save), the chat input
* is the user's free-form message — we don't silently
* is the user's free-form message; we don't silently
* mutate it. Instead we surface a warning so the user
* can make an informed decision. This guards against
* social engineering attacks where a user is tricked
@@ -1074,7 +1074,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
)}
</span>
)}{" "}
{/* Badge row — all badges and the pill always
{/* Badge row; all badges and the pill always
* render so the DOM structure never changes.
* Overflow badges use invisible + order-1 to
* hide and reorder via CSS. The pill is invisible
@@ -1107,7 +1107,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
/>
);
})}
{/* Pill — always in the DOM so it permanently
{/* Pill; always in the DOM so it permanently
* reserves layout space. Invisible when nothing
* overflows. CSS order keeps it before order-1
* (overflow) badges. */}
@@ -22,6 +22,7 @@ import { useFileAttachments } from "../hooks/useFileAttachments";
import { parseStoredDraft } from "../utils/draftStorage";
import {
getModelSelectorPlaceholder,
getProviderForModelOption,
hasConfiguredModelsInCatalog,
hasUserFixableProviders,
} from "../utils/modelOptions";
@@ -382,7 +383,10 @@ export const AgentCreateForm: FC<AgentCreateFormProps> = ({
handleAttach,
handleRemoveAttachment,
resetAttachments,
} = useFileAttachments(organizationId || undefined, { persist: true });
} = useFileAttachments(organizationId || undefined, {
persist: true,
provider: getProviderForModelOption(modelOptions, selectedModel),
});
const handleSendWithAttachments = async (message: string) => {
const fileIds: string[] = [];
@@ -153,7 +153,7 @@ export const FileTooLarge: Story = {
file,
{
status: "error",
error: "File too large (12.4 MB). Maximum is 10 MB.",
error: "File too large (12.4 MiB). Maximum is 10 MiB.",
},
],
]),
@@ -17,14 +17,19 @@ import {
} from "../utils/fetchTextAttachment";
export type UploadState = {
status: "pending" | "uploading" | "uploaded" | "error";
// "processing" covers any pre-upload client work (e.g. resize),
// so paste/drop handlers can commit the attachment synchronously
// without the send gate believing it is ready to dispatch.
status: "pending" | "processing" | "uploading" | "uploaded" | "error";
fileId?: string;
error?: string;
draftWarning?: string;
};
export const isUploadInProgress = (state: UploadState | undefined): boolean =>
state?.status === "pending" || state?.status === "uploading";
state?.status === "pending" ||
state?.status === "processing" ||
state?.status === "uploading";
/** Renders an image thumbnail from a pre-created preview URL. */
export const ImageThumbnail: FC<{
@@ -194,6 +199,7 @@ export const AttachmentPreview: FC<{
</button>
)}
{(uploadState?.status === "pending" ||
uploadState?.status === "processing" ||
uploadState?.status === "uploading") && (
<div className="absolute inset-0 flex items-center justify-center rounded-md bg-overlay">
<Spinner className="h-5 w-5 text-white" loading />
@@ -7,6 +7,7 @@ import { useChatDraftAttachments } from "../hooks/useChatDraftAttachments";
import { chatWidthClass, useChatFullWidth } from "../hooks/useChatFullWidth";
import { useFileAttachments } from "../hooks/useFileAttachments";
import { getChatFileURL } from "../utils/chatAttachments";
import { getProviderForModelOption } from "../utils/modelOptions";
import type { ChatDetailError } from "../utils/usageLimitMessage";
import {
AgentChatInput,
@@ -305,8 +306,12 @@ export const ChatPageInput: FC<ChatPageInputProps> = ({
const latestContextUsage = rawUsage
? { ...rawUsage, compressionThreshold, lastInjectedContext }
: rawUsage;
const composeAttachments = useChatDraftAttachments(organizationId, chatId);
const editAttachments = useFileAttachments(organizationId);
const composeAttachments = useChatDraftAttachments(organizationId, chatId, {
provider: getProviderForModelOption(modelOptions, selectedModel),
});
const editAttachments = useFileAttachments(organizationId, {
provider: getProviderForModelOption(modelOptions, selectedModel),
});
const {
setAttachments: setEditAttachments,
setPreviewUrls: setEditPreviewUrls,
@@ -370,7 +370,7 @@ describe("useChatDraftAttachments", () => {
expect(result.current.attachments).toHaveLength(1);
expect(result.current.uploadStates.get(file)).toMatchObject({
status: "error",
error: expect.stringContaining("Maximum is 10 MB"),
error: expect.stringContaining("Maximum is 10 MiB"),
});
expect(localStorage.getItem(storageKey)).toBeNull();
unmount();
@@ -523,4 +523,291 @@ describe("useChatDraftAttachments", () => {
expect(stored[0].clientId).toBe("good");
unmount();
});
describe("compose-path resize", () => {
const makeOversizeImage = () =>
new File([new Uint8Array(5 * 1024 * 1024 + 64 * 1024)], "photo.png", {
type: "image/png",
lastModified: 100,
});
it("commits the original synchronously with status: processing while resize is in flight", async () => {
const resize = await import("../utils/resizeImage");
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() => new Promise<File | null>(() => undefined),
);
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.attachments).toHaveLength(1);
expect(result.current.attachments[0]).toBe(original);
expect(result.current.uploadStates.get(original)).toMatchObject({
status: "processing",
});
// Registry entry is created post-resize.
expect(uploadSpy).not.toHaveBeenCalled();
expect(localStorage.getItem(storageKey)).toBeNull();
unmount();
});
it("swaps the original for a smaller resized File and starts the upload", async () => {
const upload = createDeferred<{ id: string }>();
const uploadSpy = vi
.spyOn(API.experimental, "uploadChatFile")
.mockReturnValue(upload.promise);
const resize = await import("../utils/resizeImage");
const replacement = new File(
[new Uint8Array(2 * 1024 * 1024)],
"photo.webp",
{ type: "image/webp", lastModified: 200 },
);
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolveFn) => {
releaseResize = resolveFn;
}),
);
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.uploadStates.get(original)?.status).toBe(
"processing",
);
await act(async () => {
releaseResize(replacement);
await Promise.resolve();
});
await vi.waitFor(() => {
expect(result.current.attachments).toHaveLength(1);
expect(result.current.attachments[0]).toBe(replacement);
});
expect(result.current.uploadStates.get(original)).toBeUndefined();
expect(uploadSpy).toHaveBeenCalledTimes(1);
expect(uploadSpy).toHaveBeenCalledWith(replacement, orgID);
await act(async () => {
upload.resolve({ id: "file-resized" });
});
await vi.waitFor(() => {
expect(result.current.uploadStates.get(replacement)).toMatchObject({
status: "uploaded",
fileId: "file-resized",
});
});
unmount();
});
it("falls back to the original and surfaces a provider-budget error when resize returns null on Anthropic", async () => {
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
const resize = await import("../utils/resizeImage");
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(null);
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
await vi.waitFor(() => {
const state = result.current.uploadStates.get(original);
expect(state?.status).toBe("error");
expect(state?.error).toMatch(/Anthropic/);
expect(state?.error).toMatch(/MiB/);
});
expect(uploadSpy).not.toHaveBeenCalled();
expect(localStorage.getItem(storageKey)).toBeNull();
unmount();
});
it("does not resurrect attachments removed while resize is in flight", async () => {
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
const resize = await import("../utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolveFn) => {
releaseResize = resolveFn;
}),
);
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.attachments).toHaveLength(1);
act(() => {
result.current.handleRemoveAttachment(original);
});
expect(result.current.attachments).toHaveLength(0);
const replacement = new File(
[new Uint8Array(1 * 1024 * 1024)],
"photo.webp",
{ type: "image/webp" },
);
await act(async () => {
releaseResize(replacement);
await Promise.resolve();
});
expect(result.current.attachments).toHaveLength(0);
expect(uploadSpy).not.toHaveBeenCalled();
expect(localStorage.getItem(storageKey)).toBeNull();
unmount();
});
it("does not resurrect attachments after resetAttachments fires", async () => {
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
const resize = await import("../utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolveFn) => {
releaseResize = resolveFn;
}),
);
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
);
const original = makeOversizeImage();
act(() => {
result.current.handleAttach([original]);
});
expect(result.current.attachments).toHaveLength(1);
act(() => {
result.current.resetAttachments();
});
expect(result.current.attachments).toHaveLength(0);
const replacement = new File(
[new Uint8Array(1 * 1024 * 1024)],
"photo.webp",
{ type: "image/webp" },
);
await act(async () => {
releaseResize(replacement);
await Promise.resolve();
});
expect(result.current.attachments).toHaveLength(0);
expect(uploadSpy).not.toHaveBeenCalled();
expect(localStorage.getItem(storageKey)).toBeNull();
unmount();
});
it("freezes the provider snapshot at attach time so a mid-resize provider switch can't mislabel the error", async () => {
const resize = await import("../utils/resizeImage");
let releaseResize: (value: File | null) => void = () => undefined;
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
() =>
new Promise<File | null>((resolveFn) => {
releaseResize = resolveFn;
}),
);
const { result, rerender, unmount } = renderHook(
({ provider }) => useChatDraftAttachments(orgID, chatID, { provider }),
{ initialProps: { provider: "anthropic" } },
);
// Over Anthropic's 5 MiB but under OpenAI's 10 MiB.
const gif = new File([new Uint8Array(8)], "animated.gif", {
type: "image/gif",
lastModified: 400,
});
Object.defineProperty(gif, "size", { value: 6 * 1024 * 1024 });
act(() => {
result.current.handleAttach([gif]);
});
rerender({ provider: "openai" });
await act(async () => {
releaseResize(null);
await Promise.resolve();
});
// Error must name the provider whose budget rejected
// the file at attach time, not the live provider.
await vi.waitFor(() => {
const state = result.current.uploadStates.get(gif);
expect(state?.status).toBe("error");
expect(state?.error).toMatch(/Anthropic/);
expect(state?.error).not.toMatch(/OpenAI/);
expect(state?.error).toMatch(/under 5\.0 MiB/);
});
unmount();
});
it("uses the default 10 MiB budget for non-Anthropic providers (no resize for sub-10MiB images)", async () => {
const upload = createDeferred<{ id: string }>();
const uploadSpy = vi
.spyOn(API.experimental, "uploadChatFile")
.mockReturnValue(upload.promise);
const resize = await import("../utils/resizeImage");
const resizeSpy = vi
.spyOn(resize, "resizeImageToMaxBytes")
.mockResolvedValue(null);
const { result, unmount } = renderHook(() =>
useChatDraftAttachments(orgID, chatID, { provider: "openai" }),
);
// 7 MiB: over Anthropic's 5 MiB but under the default
// 10 MiB. OpenAI uploads directly without resize.
const file = new File([new Uint8Array(7 * 1024 * 1024)], "medium.png", {
type: "image/png",
lastModified: 300,
});
act(() => {
result.current.handleAttach([file]);
});
expect(resizeSpy).not.toHaveBeenCalled();
await vi.waitFor(() => {
expect(uploadSpy).toHaveBeenCalledTimes(1);
expect(uploadSpy).toHaveBeenCalledWith(file, orgID);
});
await act(async () => {
upload.resolve({ id: "file-direct" });
});
await vi.waitFor(() => {
expect(result.current.uploadStates.get(file)).toMatchObject({
status: "uploaded",
fileId: "file-direct",
});
});
unmount();
});
});
});
@@ -1,5 +1,6 @@
import { useEffect, useRef, useState } from "react";
import { API } from "#/api/api";
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
import type { UploadState } from "../components/AgentChatInput";
import { getChatFileURL } from "../utils/chatAttachments";
import {
@@ -13,9 +14,14 @@ import {
import {
formatAgentAttachmentTooLargeError,
formatAgentAttachmentUploadError,
maxAgentAttachmentSize,
readAgentAttachmentText,
} from "../utils/fileAttachmentLimits";
import {
imageBudgetForProvider,
imageNeedsResize,
providerBudgetError,
} from "../utils/imageBudget";
import { resizeImageToMaxBytes } from "../utils/resizeImage";
const maxTextPreviewSize = 1024 * 1024;
@@ -486,6 +492,7 @@ const queueTextContentReads = (
export function useChatDraftAttachments(
organizationId: string | undefined,
chatId: string | undefined,
options?: { provider?: string },
) {
const [views, setViews] = useState(() =>
hydrateViews(organizationId, chatId),
@@ -493,6 +500,18 @@ export function useChatDraftAttachments(
const viewsRef = useRef(views);
const subscriptionsRef = useRef(new Map<string, () => void>());
const scopeRef = useRef(getDraftScopeKey(organizationId, chatId));
// providerRef lets event-driven handlers (paste/drop) see the
// latest model selection without rebuilding handleAttach. The
// effect-based write keeps React Compiler happy.
const provider = options?.provider;
const providerRef = useRef(provider);
useEffect(() => {
providerRef.current = provider;
}, [provider]);
// clientIds whose resize is in flight but the user removed the
// attachment (or the chat scope changed). processResize checks
// this before swapping in a replacement.
const abandonedResizesRef = useRef<Set<string>>(new Set());
const [subscriber] = useState<UploadRegistrySubscriber>(
() =>
function handleUploadRegistrySnapshot(snapshot: UploadRegistrySnapshot) {
@@ -518,6 +537,11 @@ export function useChatDraftAttachments(
const scopeKey = getDraftScopeKey(organizationId, chatId);
scopeRef.current = scopeKey;
unsubscribeAllEntries(subscriptionsRef);
// Abandon in-flight resizes from the previous scope so
// their callbacks don't register uploads in the new scope.
for (const view of viewsRef.current) {
abandonedResizesRef.current.add(view.clientId);
}
if (!organizationId || !chatId || !scopeKey) {
setViews([]);
return;
@@ -560,9 +584,145 @@ export function useChatDraftAttachments(
}
}, [organizationId, chatId, subscriber]);
// The view enters in "processing" status from handleAttach;
// processResize either swaps in the smaller file and registers
// the upload, or surfaces a too-large error.
const processResize = async (
clientId: string,
original: File,
budget: number,
// Pinned at attach time so a mid-resize provider switch
// can't mislabel the error with the new provider's name.
providerSnapshot: string | undefined,
) => {
let resized: File | null = null;
try {
resized = await resizeImageToMaxBytes(original, budget);
} catch {
resized = null;
}
if (abandonedResizesRef.current.has(clientId)) {
return;
}
if (!organizationId || !chatId) {
return;
}
const scopeKey = getDraftScopeKey(organizationId, chatId);
if (!scopeKey || scopeRef.current !== scopeKey) {
return;
}
const replacement = resized ?? original;
const replaced = replacement !== original;
// Resize failed entirely or couldn't shrink enough; show
// the too-large error instead of uploading and 413-ing.
if (replacement.size > MaxChatFileSizeBytes) {
setViews((prev) =>
prev.map((view) => {
if (view.clientId !== clientId) {
return view;
}
if (replaced) {
revokeBlobPreview(view);
}
const previewState = replaced
? computePreview(replacement, "error")
: {
previewUrl: view.previewUrl,
previewUrlKind: view.previewUrlKind,
};
return {
...view,
file: replacement,
status: "error",
error: formatAgentAttachmentTooLargeError(replacement.size),
previewUrl: previewState.previewUrl,
previewUrlKind: previewState.previewUrlKind,
};
}),
);
return;
}
// Replacement is still over the provider budget (e.g.
// animated GIF on Anthropic that we don't re-encode).
// Surface the error at attach time rather than letting
// the server backstop reject only at send time.
if (replacement.type.startsWith("image/") && replacement.size > budget) {
setViews((prev) =>
prev.map((view) => {
if (view.clientId !== clientId) {
return view;
}
if (replaced) {
revokeBlobPreview(view);
}
const previewState = replaced
? computePreview(replacement, "error")
: {
previewUrl: view.previewUrl,
previewUrlKind: view.previewUrlKind,
};
return {
...view,
file: replacement,
status: "error",
error: providerBudgetError(
providerSnapshot,
replacement.size,
budget,
),
previewUrl: previewState.previewUrl,
previewUrlKind: previewState.previewUrlKind,
};
}),
);
return;
}
// beginUpload below drives the view from pending to uploading
// via subscribers; we set "pending" here so the registry's
// initial snapshot doesn't overwrite our blob preview.
setViews((prev) =>
prev.map((view) => {
if (view.clientId !== clientId) {
return view;
}
if (!replaced) {
return { ...view, status: "pending" };
}
revokeBlobPreview(view);
const nextPreview = computePreview(replacement, "pending");
return {
...view,
file: replacement,
status: "pending",
previewUrl: nextPreview.previewUrl,
previewUrlKind: nextPreview.previewUrlKind,
};
}),
);
const entry = createRegistryEntry(
clientId,
organizationId,
chatId,
replacement,
);
subscribeToEntry(entry, subscriptionsRef, subscriber);
beginUpload(entry);
};
const handleAttach = (files: File[]) => {
const scopeKey = getDraftScopeKey(organizationId, chatId);
// Snapshot provider + budget so a mid-resize switch
// can't relabel the error with the new provider.
const providerSnapshot = providerRef.current;
const budget = imageBudgetForProvider(providerSnapshot);
const entriesToStart: UploadRegistryEntry[] = [];
const resizeJobs: Array<{ clientId: string; file: File }> = [];
const nextViews: DraftAttachmentView[] = [];
for (const file of files) {
const clientId = createClientId();
@@ -571,7 +731,11 @@ export function useChatDraftAttachments(
file,
status: "pending",
};
if (file.size > maxAgentAttachmentSize) {
const needsResize = imageNeedsResize(file, budget);
// Non-image files over the upload cap are rejected.
// Oversized images take the resize pipeline regardless;
// the post-resize check validates the result.
if (file.size > MaxChatFileSizeBytes && !needsResize) {
nextViews.push({
...baseView,
status: "error",
@@ -587,6 +751,18 @@ export function useChatDraftAttachments(
});
continue;
}
if (needsResize) {
// Commit synchronously with "processing" so the
// send gate blocks dispatch until resize finishes.
const view = {
...baseView,
status: "processing" as const,
...computePreview(file, "processing"),
};
nextViews.push(view);
resizeJobs.push({ clientId, file });
continue;
}
const view = { ...baseView, ...computePreview(file, "pending") };
const entry = createRegistryEntry(clientId, organizationId, chatId, file);
subscribeToEntry(entry, subscriptionsRef, subscriber);
@@ -602,6 +778,11 @@ export function useChatDraftAttachments(
setViews,
() => scopeRef.current === scopeKey,
);
// processResize re-checks abandonment + scope before
// mutating state, so it's safe to fire-and-forget.
for (const job of resizeJobs) {
void processResize(job.clientId, job.file, budget, providerSnapshot);
}
};
const handleRemoveAttachment = (attachment: number | File) => {
@@ -613,6 +794,9 @@ export function useChatDraftAttachments(
if (!removed) {
return;
}
// In-flight resize would otherwise swap in a replacement
// after the clear below.
abandonedResizesRef.current.add(removed.clientId);
if (organizationId && chatId) {
removeChatDraftAttachmentRecord(organizationId, chatId, removed.clientId);
}
@@ -629,6 +813,12 @@ export function useChatDraftAttachments(
};
const resetAttachments = () => {
// Abandon all in-flight resizes so they don't swap a
// replacement back in (which would also re-call beginUpload
// against the now-stale scope).
for (const view of viewsRef.current) {
abandonedResizesRef.current.add(view.clientId);
}
if (!organizationId || !chatId) {
setViews([]);
return;
@@ -3,17 +3,23 @@ import {
type SetStateAction,
useEffect,
useEffectEvent,
useRef,
useState,
} from "react";
import { API } from "#/api/api";
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
import type { UploadState } from "../components/AgentChatInput";
import { getChatFileURL } from "../utils/chatAttachments";
import {
formatAgentAttachmentTooLargeError,
formatAgentAttachmentUploadError,
maxAgentAttachmentSize,
readAgentAttachmentText,
} from "../utils/fileAttachmentLimits";
import {
imageBudgetForProvider,
imageNeedsResize,
providerBudgetError,
} from "../utils/imageBudget";
import { resizeImageToMaxBytes } from "../utils/resizeImage";
/** @internal Exported for testing. */
export const persistedAttachmentsStorageKey = "agents.persisted-attachments";
@@ -43,10 +49,9 @@ function restorePersistedAttachments(currentOrgId: string): {
uploadStates: Map<File, UploadState>;
previewUrls: Map<File, string>;
} {
// When the org ID is not yet known (e.g. still loading), skip
// restoration entirely so we don't accidentally prune valid
// entries. The initializer only runs once, so the caller must
// ensure the org ID is available before mounting the hook.
// Skip when org ID isn't loaded yet so we don't prune valid
// entries. The initializer runs once, so callers must wait for
// the org ID before mounting.
if (!currentOrgId) {
return {
attachments: [],
@@ -66,7 +71,6 @@ function restorePersistedAttachments(currentOrgId: string): {
const persisted: PersistedAttachment[] = JSON.parse(stored);
const matched = persisted.filter((p) => p.organizationId === currentOrgId);
// Prune entries that don't match the current org.
if (matched.length !== persisted.length) {
if (matched.length > 0) {
localStorage.setItem(
@@ -84,9 +88,8 @@ function restorePersistedAttachments(currentOrgId: string): {
for (const p of matched) {
if (!p.fileId || !p.fileName) continue;
// Synthetic File used as a Map key only. Its content is
// never read because the existing file_id is reused at
// send time.
// Synthetic File used as a Map key only; the existing
// file_id is reused at send time.
const file = new File([], p.fileName, {
type: p.fileType,
lastModified: p.lastModified,
@@ -173,12 +176,19 @@ interface UseFileAttachmentsReturn {
export function useFileAttachments(
organizationId: string | undefined,
options?: { persist?: boolean },
options?: { persist?: boolean; provider?: string },
): UseFileAttachmentsReturn {
const persist = options?.persist ?? false;
// Restore previously uploaded attachments from localStorage
// when persistence is enabled. Computed once on first render.
// providerRef lets event-driven handlers (paste/drop) see the
// latest model selection without rebuilding handleAttach. The
// effect-based write keeps React Compiler happy.
const provider = options?.provider;
const providerRef = useRef(provider);
useEffect(() => {
providerRef.current = provider;
}, [provider]);
const [restored] = useState(() =>
persist
? restorePersistedAttachments(organizationId ?? "")
@@ -237,11 +247,10 @@ export function useFileAttachments(
if (shouldPersist) {
addPersistedAttachment(file, result.id, organizationId!);
}
// Pre-warm the browser HTTP cache for images so the
// timeline can render them instantly after send. We
// intentionally skip text attachments because the
// composer already has the text content locally.
if (isImage) {
// Pre-warm the HTTP cache so the timeline can
// render the image instantly after send. Text
// content is already local in the composer.
void fetch(getChatFileURL(result.id));
}
} catch (err: unknown) {
@@ -256,21 +265,148 @@ export function useFileAttachments(
})();
};
// Files removed while their resize is in flight. processResizes
// checks this before swapping in a replacement so a dismissed
// file can't be resurrected. WeakSet lets entries get GC'd.
const abandonedResizesRef = useRef<WeakSet<File>>(new WeakSet());
type AttachItem = { file: File; needsResize: boolean };
const processResizes = async (
items: readonly AttachItem[],
budget: number,
// Pinned at attach time so a mid-resize provider switch
// can't mislabel the error with the new provider's name.
providerSnapshot: string | undefined,
) => {
// Sequential so each swap commits before the next starts;
// resizeImageToMaxBytes already serializes decode work.
for (const { file: original, needsResize } of items) {
if (!needsResize) continue;
let resized: File | null = null;
try {
resized = await resizeImageToMaxBytes(original, budget);
} catch {
resized = null;
}
// Skip if the user removed this attachment while
// resizing; updates here would resurrect it.
if (abandonedResizesRef.current.has(original)) {
continue;
}
const replacement = resized ?? original;
const replaced = replacement !== original;
// Functional updaters: if a racing removal cleared the
// original, every updater below becomes a no-op.
setAttachments((prev) => {
const idx = prev.indexOf(original);
if (idx === -1 || !replaced) return prev;
const next = prev.slice();
next[idx] = replacement;
return next;
});
setPreviewUrls((prev) => {
// Skip when no replacement happened so we don't
// revoke the original's still-in-use blob URL.
if (!prev.has(original) || !replaced) return prev;
const next = new Map(prev);
const oldUrl = next.get(original);
if (oldUrl?.startsWith("blob:")) URL.revokeObjectURL(oldUrl);
next.delete(original);
if (replacement.type !== "text/plain") {
next.set(replacement, URL.createObjectURL(replacement));
}
return next;
});
setUploadStates((prev) => {
// Skip when no replacement: startUpload below
// overwrites "processing" with "uploading".
if (!prev.has(original) || !replaced) return prev;
const next = new Map(prev);
next.delete(original);
return next;
});
// Resize failed and the original still exceeds the
// server cap; show the too-large error instead of
// kicking off an upload that will 413.
if (replacement.size > MaxChatFileSizeBytes) {
setUploadStates((prev) =>
new Map(prev).set(replacement, {
status: "error" as const,
error: formatAgentAttachmentTooLargeError(replacement.size),
}),
);
continue;
}
// Replacement is still over the provider budget (e.g.
// animated GIF on Anthropic that we don't re-encode).
// Surface the error at attach time rather than letting
// the server backstop reject only at send time.
if (replacement.type.startsWith("image/") && replacement.size > budget) {
setUploadStates((prev) =>
new Map(prev).set(replacement, {
status: "error" as const,
error: providerBudgetError(
providerSnapshot,
replacement.size,
budget,
),
}),
);
continue;
}
startUpload(replacement);
}
};
const handleAttach = (files: File[]) => {
// Originals enter state with a "processing" status so the
// send gate blocks dispatch until processResizes finishes.
// Snapshot provider + budget so a mid-resize switch can't
// relabel the error with the new provider.
const providerSnapshot = providerRef.current;
const budget = imageBudgetForProvider(providerSnapshot);
const items: AttachItem[] = files.map((file) => ({
file,
needsResize: imageNeedsResize(file, budget),
}));
setAttachments((prev) => [...prev, ...files]);
setPreviewUrls((prev) => {
const next = new Map(prev);
for (const file of files) {
for (const { file } of items) {
if (file.type !== "text/plain") {
next.set(file, URL.createObjectURL(file));
}
}
return next;
});
// Read text content for preview, but skip oversized files.
for (const file of files) {
if (file.type === "text/plain" && file.size <= maxAgentAttachmentSize) {
void readAgentAttachmentText(file)
setUploadStates((prev) => {
const next = new Map(prev);
for (const { file, needsResize } of items) {
if (file.size > MaxChatFileSizeBytes && !needsResize) {
next.set(file, {
status: "error" as const,
error: formatAgentAttachmentTooLargeError(file.size),
});
} else if (needsResize) {
next.set(file, { status: "processing" });
}
}
return next;
});
for (const { file } of items) {
if (file.type === "text/plain" && file.size <= MaxChatFileSizeBytes) {
// Some test environments lack File.prototype.text.
const readText =
typeof file.text === "function"
? file.text()
: new Response(file).text();
void readText
.then((content) => {
setTextContents((prev) => {
const next = new Map(prev);
@@ -283,30 +419,30 @@ export function useFileAttachments(
});
}
}
for (const file of files) {
if (file.size > maxAgentAttachmentSize) {
setUploadStates((prev) =>
new Map(prev).set(file, {
status: "error" as const,
error: formatAgentAttachmentTooLargeError(file.size),
}),
);
} else {
startUpload(file);
}
for (const { file, needsResize } of items) {
if (needsResize) continue;
if (file.size > MaxChatFileSizeBytes) continue; // already marked as error above
startUpload(file);
}
void processResizes(items, budget, providerSnapshot);
};
const handleRemoveAttachment = (attachment: number | File) => {
// Resolve the file to remove and perform localStorage side
// effects before entering state updaters. React may call
// updaters more than once (StrictMode, React Compiler), so
// they must stay pure.
// Side effects (localStorage, abandonment) happen here;
// React may call updaters multiple times under StrictMode
// or React Compiler, so they must stay pure.
const idx =
typeof attachment === "number"
? attachment
: attachments.indexOf(attachment);
const removed = idx >= 0 ? attachments[idx] : undefined;
if (removed) {
// In-flight resize would otherwise resurrect this file
// by swapping in a replacement after the clear below.
abandonedResizesRef.current.add(removed);
}
if (persist && removed) {
const state = uploadStates.get(removed);
if (state?.status === "uploaded" && state.fileId) {
@@ -344,6 +480,12 @@ export function useFileAttachments(
};
const resetAttachments = () => {
// Abandon all in-flight resizes so they don't swap a
// replacement back in (which would also re-call startUpload
// against the now-stale scope).
for (const file of attachments) {
abandonedResizesRef.current.add(file);
}
for (const [, url] of previewUrls) {
if (url.startsWith("blob:")) URL.revokeObjectURL(url);
}
@@ -365,9 +507,9 @@ export function useFileAttachments(
handleRemoveAttachment,
startUpload,
resetAttachments,
// Raw setters exposed for ChatPageContent to pre-populate
// attachments from existing chat messages. These bypass
// localStorage persistence. Only use when persist is false.
// Raw setters bypass localStorage persistence; only use
// when persist is false (e.g. ChatPageContent pre-populating
// attachments from existing chat messages).
setAttachments,
setPreviewUrls,
setUploadStates,
@@ -1,9 +1,8 @@
import { getErrorDetail, getErrorMessage } from "#/api/errors";
export const maxAgentAttachmentSize = 10 * 1024 * 1024;
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
export const formatAgentAttachmentTooLargeError = (fileSize: number): string =>
`File too large (${(fileSize / 1024 / 1024).toFixed(1)} MB). Maximum is ${maxAgentAttachmentSize / 1024 / 1024} MB.`;
`File too large (${(fileSize / 1024 / 1024).toFixed(1)} MiB). Maximum is ${MaxChatFileSizeBytes / 1024 / 1024} MiB.`;
export const formatAgentAttachmentUploadError = (error: unknown): string => {
const message = getErrorMessage(error, "Upload failed");
@@ -0,0 +1,112 @@
import { describe, expect, it } from "vitest";
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
import {
formatMiB,
imageBudgetForProvider,
imageNeedsResize,
providerBudgetError,
} from "./imageBudget";
const ANTHROPIC_BUDGET = 5 * 1024 * 1024 - 16 * 1024;
const DEFAULT_BUDGET = MaxChatFileSizeBytes - 16 * 1024;
describe("imageBudgetForProvider", () => {
it("returns the Anthropic budget for direct Anthropic", () => {
expect(imageBudgetForProvider("anthropic")).toBe(ANTHROPIC_BUDGET);
});
it("returns the Anthropic budget for Bedrock (Anthropic-compatible)", () => {
expect(imageBudgetForProvider("bedrock")).toBe(ANTHROPIC_BUDGET);
});
it("returns the default budget for OpenAI", () => {
expect(imageBudgetForProvider("openai")).toBe(DEFAULT_BUDGET);
});
it("returns the default budget for unknown providers", () => {
expect(imageBudgetForProvider("brand-new-provider")).toBe(DEFAULT_BUDGET);
});
it("returns the default budget when provider is undefined", () => {
expect(imageBudgetForProvider(undefined)).toBe(DEFAULT_BUDGET);
});
// Mirrors server-side chatprovider.NormalizeProvider so a
// caller passing a case/whitespace variant gets the strict
// budget instead of silently falling through to the default.
it.each([
"Anthropic",
"ANTHROPIC",
" anthropic ",
"\tanthropic\n",
"AnThRoPiC",
])("normalizes case/whitespace before matching strict providers (%s)", (input) => {
expect(imageBudgetForProvider(input)).toBe(ANTHROPIC_BUDGET);
});
it("normalizes Bedrock variants too", () => {
expect(imageBudgetForProvider("Bedrock")).toBe(ANTHROPIC_BUDGET);
expect(imageBudgetForProvider(" BEDROCK ")).toBe(ANTHROPIC_BUDGET);
});
});
describe("imageNeedsResize", () => {
const oversize = (type: string, bytes: number): File => {
const f = new File([new Uint8Array(8)], `f.${type.split("/")[1]}`, {
type,
});
Object.defineProperty(f, "size", { value: bytes });
return f;
};
it("returns true for an over-budget image", () => {
expect(
imageNeedsResize(oversize("image/png", 6 * 1024 * 1024), 5 * 1024 * 1024),
).toBe(true);
});
it("returns false for an under-budget image", () => {
expect(
imageNeedsResize(oversize("image/png", 1 * 1024 * 1024), 5 * 1024 * 1024),
).toBe(false);
});
it("returns false for non-image files even when oversized", () => {
expect(
imageNeedsResize(
oversize("text/plain", 6 * 1024 * 1024),
5 * 1024 * 1024,
),
).toBe(false);
});
});
describe("formatMiB", () => {
it("renders one decimal place", () => {
expect(formatMiB(5 * 1024 * 1024)).toBe("5.0");
expect(formatMiB(5 * 1024 * 1024 + 512 * 1024)).toBe("5.5");
expect(formatMiB(0)).toBe("0.0");
});
});
describe("providerBudgetError", () => {
it("uses the provider's display label and MiB units", () => {
const message = providerBudgetError(
"anthropic",
6 * 1024 * 1024,
ANTHROPIC_BUDGET,
);
expect(message).toMatch(/Anthropic/);
expect(message).toMatch(/6\.0 MiB/);
expect(message).toMatch(/5\.0 MiB/);
});
it("falls back to a generic label when provider is undefined", () => {
const message = providerBudgetError(
undefined,
6 * 1024 * 1024,
ANTHROPIC_BUDGET,
);
expect(message).toMatch(/this provider/);
});
});
@@ -0,0 +1,46 @@
import {
AnthropicInlineImageCapBytes,
MaxChatFileSizeBytes,
} from "#/api/typesGenerated";
import { formatProviderLabel } from "./modelOptions";
// Budgets sit below the wire limits to leave room for encoder framing
// overhead, so a file at exactly the budget is still under the
// server's hard cap.
const FRAMING_MARGIN_BYTES = 16 * 1024;
const DEFAULT_IMAGE_BUDGET_BYTES = MaxChatFileSizeBytes - FRAMING_MARGIN_BYTES;
const ANTHROPIC_IMAGE_BUDGET_BYTES =
AnthropicInlineImageCapBytes - FRAMING_MARGIN_BYTES;
// Must mirror chatprovider.InlineImageCapBytes on the server.
const ANTHROPIC_STRICT_BUDGET_PROVIDERS: ReadonlySet<string> = new Set([
"anthropic",
"bedrock",
]);
// Inputs are normalized to match chatprovider.NormalizeProvider on
// the server, so callers don't have to pre-normalize.
export function imageBudgetForProvider(provider: string | undefined): number {
const normalized = provider?.trim().toLowerCase();
if (normalized && ANTHROPIC_STRICT_BUDGET_PROVIDERS.has(normalized)) {
return ANTHROPIC_IMAGE_BUDGET_BYTES;
}
return DEFAULT_IMAGE_BUDGET_BYTES;
}
export function formatMiB(bytes: number): string {
return (bytes / 1024 / 1024).toFixed(1);
}
export function providerBudgetError(
provider: string | undefined,
actualBytes: number,
budgetBytes: number,
): string {
const label = provider ? formatProviderLabel(provider) : "this provider";
return `Image too large for ${label} (${formatMiB(actualBytes)} MiB). Inline images must be under ${formatMiB(budgetBytes)} MiB on this provider.`;
}
export function imageNeedsResize(file: File, budget: number): boolean {
return file.type.startsWith("image/") && file.size > budget;
}
@@ -210,6 +210,16 @@ export const getModelOptionsFromConfigs = (
});
};
// getProviderForModelOption returns the provider string for the
// currently-selected model option, or undefined when the selection
// is not (yet) in the options list. Extracted so resize/budget logic
// has one place to resolve provider from the selector state.
export const getProviderForModelOption = (
modelOptions: readonly ModelSelectorOption[],
selectedModel: string,
): string | undefined =>
modelOptions.find((option) => option.id === selectedModel)?.provider;
export const formatProviderLabel = (provider: string): string => {
const normalized = provider.trim().toLowerCase();
switch (normalized) {
@@ -0,0 +1,460 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { resizeImageToMaxBytes } from "./resizeImage";
// jsdom (the default vitest environment) does not implement
// createImageBitmap / OffscreenCanvas, so the re-encode codepaths
// cannot run against real browser decoders. The "with stubbed
// decoders" block below installs a deterministic fake so the shrink
// loop runs in CI; the "with real decoders" block only runs when
// actual browser APIs are available (e.g. a future Playwright-based
// vitest project).
const canDecodeImages =
typeof createImageBitmap === "function" &&
typeof OffscreenCanvas === "function";
const describeIfDecode = canDecodeImages ? describe : describe.skip;
// Minimum byte count the fake decoder reports for a blob at given
// dimensions. Sized so the shrink loop runs a realistic number of
// iterations within the module's MAX_SHRINK_ITERATIONS=8 budget.
const FAKE_BYTES_PER_PIXEL = 0.5;
// Returns a fake encoded blob whose size is proportional to (width
// * height * quality) so the shrink loop can observe convergence.
function fakeEncodedSize(
width: number,
height: number,
quality: number,
): number {
return Math.max(
64,
Math.round(width * height * quality * FAKE_BYTES_PER_PIXEL),
);
}
describe("resizeImageToMaxBytes", () => {
it("returns non-image files unchanged", async () => {
const file = new File([new Uint8Array([1, 2, 3])], "notes.txt", {
type: "text/plain",
});
const result = await resizeImageToMaxBytes(file, 1024);
expect(result).toBe(file);
});
it("returns GIFs unchanged even when oversize", async () => {
// Canvas re-encoding flattens animation; we hand the
// original back instead.
const bytes = new Uint8Array(8 * 1024 * 1024);
const file = new File([bytes], "clip.gif", { type: "image/gif" });
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
expect(result).toBe(file);
});
it("returns the original image unchanged when already under budget", async () => {
const bytes = new Uint8Array(1024);
const file = new File([bytes], "tiny.png", { type: "image/png" });
const result = await resizeImageToMaxBytes(file, 4096);
expect(result).toBe(file);
});
it("returns null for an unsupported image MIME that would need resizing", async () => {
// Unsupported MIME + over budget: refuse rather than
// silently produce garbage.
const bytes = new Uint8Array(2 * 1024 * 1024);
const file = new File([bytes], "diagram.svg", {
type: "image/svg+xml",
});
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
expect(result).toBeNull();
});
it("accepts image/jpg alias alongside image/jpeg", async () => {
// Pins the non-IANA `image/jpg` alias in RESIZABLE_MIME_TYPES.
// The under-budget passthrough is enough to prove acceptance;
// the over-budget case in the stubbed-decoder block proves
// the encode pipeline runs.
const under = new File([new Uint8Array(512)], "icon.jpg", {
type: "image/jpg",
});
const result = await resizeImageToMaxBytes(under, 4096);
expect(result).toBe(under);
});
it("returns an under-budget unsupported-MIME image unchanged", async () => {
// Under-budget unsupported MIMEs pass through; the contract
// is "give me something <= maxBytes" and we already have
// that.
const bytes = new Uint8Array(512);
const file = new File([bytes], "icon.bmp", {
type: "image/bmp",
});
const result = await resizeImageToMaxBytes(file, 4096);
expect(result).toBe(file);
});
});
describe("resizeImageToMaxBytes with stubbed decoders", () => {
// Each test installs its own fakes; track per-test state on a
// shared object so the stubs can read the active configuration.
interface StubState {
srcWidth: number;
srcHeight: number;
decodeThrows: boolean;
convertBlobType: string;
decodeCalls: number;
encodeCalls: Array<{ width: number; height: number; quality: number }>;
}
let state: StubState;
beforeEach(() => {
state = {
srcWidth: 4096,
srcHeight: 4096,
decodeThrows: false,
convertBlobType: "image/webp",
decodeCalls: 0,
encodeCalls: [],
};
// Fake createImageBitmap matching the HTML spec output rules:
// - both resize dims => stretch (no aspect-ratio preservation).
// - only resizeWidth => width exact, height proportional.
// UPSCALES if resizeWidth > source width.
// - only resizeHeight => mirror of above.
// - neither => source dimensions unchanged.
//
// Critical that the fake doesn't cap at source dimensions:
// real browsers follow the spec and upscale, so production
// code must handle that. A capped fake would mask the bug.
vi.stubGlobal(
"createImageBitmap",
vi.fn(
async (
_blob: Blob,
options?: {
resizeWidth?: number;
resizeHeight?: number;
},
) => {
state.decodeCalls++;
if (state.decodeThrows) {
throw new Error("decode boom");
}
const srcW = state.srcWidth;
const srcH = state.srcHeight;
const rW = options?.resizeWidth;
const rH = options?.resizeHeight;
let w: number;
let h: number;
if (rW !== undefined && rH !== undefined) {
// Spec: stretch-to-fit, no source clamp.
w = rW;
h = rH;
} else if (rW !== undefined) {
w = rW;
h = Math.max(1, Math.round((srcH * rW) / srcW));
} else if (rH !== undefined) {
h = rH;
w = Math.max(1, Math.round((srcW * rH) / srcH));
} else {
w = srcW;
h = srcH;
}
return {
width: w,
height: h,
close: vi.fn(),
} as unknown as ImageBitmap;
},
),
);
// Fake Image (for probeNaturalDimensions in production
// code). jsdom exposes `Image` but does not load blob URLs,
// so we stub it with a synthetic implementation that reports
// state.srcWidth/Height and fires onload on microtask.
class FakeImage {
onload: (() => void) | null = null;
onerror: (() => void) | null = null;
naturalWidth = 0;
naturalHeight = 0;
private _src = "";
get src() {
return this._src;
}
set src(url: string) {
this._src = url;
queueMicrotask(() => {
if (state.decodeThrows) {
this.onerror?.();
return;
}
this.naturalWidth = state.srcWidth;
this.naturalHeight = state.srcHeight;
this.onload?.();
});
}
}
vi.stubGlobal("Image", FakeImage);
// convertToBlob size scales with width*height*quality so
// the shrink loop converges.
class FakeOffscreenCanvas {
width: number;
height: number;
constructor(w: number, h: number) {
this.width = w;
this.height = h;
}
getContext() {
return {
drawImage: () => undefined,
};
}
async convertToBlob(opts?: { quality?: number }): Promise<Blob> {
const quality = opts?.quality ?? 1;
state.encodeCalls.push({
width: this.width,
height: this.height,
quality,
});
const size = fakeEncodedSize(this.width, this.height, quality);
return new Blob([new Uint8Array(size)], {
type: state.convertBlobType,
});
}
}
vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas);
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("re-encodes an oversized image down to the requested byte budget", async () => {
// Forces at least a few shrink iterations before convergence.
const file = new File([new Uint8Array(6 * 1024 * 1024)], "big.png", {
type: "image/png",
});
const budget = 512 * 1024;
const result = await resizeImageToMaxBytes(file, budget);
expect(result).not.toBeNull();
if (!result) return;
expect(result.size).toBeLessThanOrEqual(budget);
expect(result.type).toBe("image/webp");
expect(result.name.endsWith(".webp")).toBe(true);
expect(state.encodeCalls.length).toBeGreaterThan(0);
});
it("decoder never stretches non-square sources", async () => {
// 4:1 source with long axis above MAX_INITIAL_DIMENSION.
// If decodeToBitmap passes both resize dims, the spec
// would force 8192x8192 output and the ratio assertion
// would fail.
state.srcWidth = 16_000;
state.srcHeight = 4_000;
const file = new File([new Uint8Array(6 * 1024 * 1024)], "wide.png", {
type: "image/png",
});
await resizeImageToMaxBytes(file, 32 * 1024);
expect(state.encodeCalls.length).toBeGreaterThan(0);
const first = state.encodeCalls[0];
const ratio = first.width / first.height;
expect(ratio).toBeGreaterThanOrEqual(4 - 0.05);
expect(ratio).toBeLessThanOrEqual(4 + 0.05);
expect(first.width).toBeLessThanOrEqual(8192);
expect(first.height).toBeLessThanOrEqual(8192);
});
it("keeps the bitmap within MAX_INITIAL_DIMENSION on both axes for extreme portraits", async () => {
// Regression: passing resizeWidth: MAX on a 2000x60000
// source would upscale to 8192x245760, blowing past
// Chromium's ~268M pixel limit. Probe must pick
// resizeHeight only.
state.srcWidth = 2000;
state.srcHeight = 60_000;
const file = new File([new Uint8Array(6 * 1024 * 1024)], "tall.png", {
type: "image/png",
});
await resizeImageToMaxBytes(file, 64 * 1024);
for (const call of state.encodeCalls) {
expect(call.width).toBeLessThanOrEqual(8192);
expect(call.height).toBeLessThanOrEqual(8192);
}
const first = state.encodeCalls[0];
const sourceRatio = 2000 / 60_000;
const bitmapRatio = first.width / first.height;
expect(bitmapRatio).toBeGreaterThan(sourceRatio * 0.99);
expect(bitmapRatio).toBeLessThan(sourceRatio * 1.01);
});
it("stays within MAX_INITIAL_DIMENSION when source exceeds it on both axes", async () => {
state.srcWidth = 60_000;
state.srcHeight = 40_000;
const file = new File([new Uint8Array(6 * 1024 * 1024)], "huge.png", {
type: "image/png",
});
await resizeImageToMaxBytes(file, 256 * 1024);
for (const call of state.encodeCalls) {
expect(call.width).toBeLessThanOrEqual(8192);
expect(call.height).toBeLessThanOrEqual(8192);
}
});
it("skips createImageBitmap resize options entirely when source is already under clamp", async () => {
// Regression: passing resizeWidth: MAX on a 1920x1080
// source would upscale to 8192x4608 (spec: output width
// is exactly resizeWidth). Probe must skip resize options
// when the source already fits.
state.srcWidth = 1920;
state.srcHeight = 1080;
const file = new File([new Uint8Array(6 * 1024 * 1024)], "shot.png", {
type: "image/png",
});
await resizeImageToMaxBytes(file, 256 * 1024);
const first = state.encodeCalls[0];
expect(first.width).toBe(1920);
expect(first.height).toBe(1080);
});
it("tries the fallback quality pass when shrink iterations saturate", async () => {
// Tiny source + unreachable 1-byte budget exhausts the main
// loop and forces FALLBACK_QUALITY (0.7).
state.srcWidth = 64;
state.srcHeight = 64;
const file = new File([new Uint8Array(1024 * 1024)], "tiny.png", {
type: "image/png",
});
const result = await resizeImageToMaxBytes(file, 1);
expect(result).toBeNull();
const last = state.encodeCalls[state.encodeCalls.length - 1];
expect(last.quality).toBeCloseTo(0.7, 5);
});
it("returns null (does not throw) when decode fails", async () => {
state.decodeThrows = true;
const file = new File([new Uint8Array(1024 * 1024)], "broken.png", {
type: "image/png",
});
const result = await resizeImageToMaxBytes(file, 4096);
expect(result).toBeNull();
});
it("fake createImageBitmap matches HTML spec output-dimension rules", async () => {
// Pins fake createImageBitmap behavior: stretch with both
// dims, proportional scale with one, including upscale
// when a resize dim exceeds the source. A fake that capped
// at source dimensions or scaled uniformly would mask real
// decoder bugs in production code.
state.srcWidth = 4000;
state.srcHeight = 1000;
const blob = new Blob([new Uint8Array(8)], { type: "image/png" });
const createBitmap = (
globalThis as unknown as {
createImageBitmap: (
blob: Blob,
opts?: { resizeWidth?: number; resizeHeight?: number },
) => Promise<ImageBitmap>;
}
).createImageBitmap;
// Stretch.
const stretched = await createBitmap(blob, {
resizeWidth: 800,
resizeHeight: 800,
});
expect(stretched.width).toBe(800);
expect(stretched.height).toBe(800);
// Downscale by width.
const downWidth = await createBitmap(blob, { resizeWidth: 800 });
expect(downWidth.width).toBe(800);
expect(downWidth.height).toBe(200);
// Downscale by height.
const downHeight = await createBitmap(blob, { resizeHeight: 200 });
expect(downHeight.height).toBe(200);
expect(downHeight.width).toBe(800);
// Upscale by width: spec allows; a capped fake would
// report (4000, 1000) and mask production decoder bugs.
const upWidth = await createBitmap(blob, { resizeWidth: 8000 });
expect(upWidth.width).toBe(8000);
expect(upWidth.height).toBe(2000);
// Natural.
const natural = await createBitmap(blob);
expect(natural.width).toBe(4000);
expect(natural.height).toBe(1000);
});
it("keeps the File type honest when the encoder falls back to PNG", async () => {
// Some browsers without WebP encode return a PNG; the
// File's labelled type must match the actual content.
state.convertBlobType = "image/png";
const file = new File([new Uint8Array(2 * 1024 * 1024)], "photo.png", {
type: "image/png",
});
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
expect(result).not.toBeNull();
if (!result) return;
expect(result.type).toBe("image/png");
expect(result.name.endsWith(".webp")).toBe(false);
});
it("re-encodes an oversized image/jpg through the resize pipeline", async () => {
// Over-budget: ensures the image/jpg alias passes the
// allowlist gate and enters the encode pipeline. Removing
// the alias from RESIZABLE_MIME_TYPES would short-circuit
// to null here.
const file = new File([new Uint8Array(3 * 1024 * 1024)], "photo.jpg", {
type: "image/jpg",
});
const result = await resizeImageToMaxBytes(file, 512 * 1024);
expect(result).not.toBeNull();
expect(state.encodeCalls.length).toBeGreaterThan(0);
if (!result) return;
expect(result.size).toBeLessThanOrEqual(512 * 1024);
});
});
describeIfDecode("resizeImageToMaxBytes with real decoders", () => {
it("returns null (does not throw) for a corrupt image blob", async () => {
// Real decoder only; jsdom <img> fallback never fires
// onload/onerror on a corrupt blob and would hang.
const bytes = new Uint8Array([0x00, 0x01, 0x02, 0x03]);
const file = new File([bytes], "broken.png", { type: "image/png" });
const result = await resizeImageToMaxBytes(file, 1);
expect(result).toBeNull();
});
it("re-encodes a large PNG down to the requested byte budget", async () => {
const canvas = new OffscreenCanvas(1024, 1024);
const ctx = canvas.getContext("2d");
if (!ctx) throw new Error("no 2d ctx");
// Noise pattern so compression doesn't trivialize the
// byte count before the shrink path runs.
const data = ctx.createImageData(1024, 1024);
for (let i = 0; i < data.data.length; i += 4) {
data.data[i] = (i * 13) & 0xff;
data.data[i + 1] = (i * 7) & 0xff;
data.data[i + 2] = (i * 19) & 0xff;
data.data[i + 3] = 0xff;
}
ctx.putImageData(data, 0, 0);
const sourceBlob = await canvas.convertToBlob({ type: "image/png" });
const file = new File([sourceBlob], "large.png", {
type: "image/png",
});
const budget = 64 * 1024;
const result = await resizeImageToMaxBytes(file, budget);
expect(result).not.toBeNull();
if (!result) return;
expect(result.size).toBeLessThan(budget);
expect(result.type).toBe("image/webp");
expect(result.name.endsWith(".webp")).toBe(true);
});
});
@@ -0,0 +1,309 @@
/**
* Browser-side image re-encoding to a caller-supplied byte budget.
* Plain TS (no React) so it can be used by any upload pipeline.
*/
// Formats we re-encode. GIFs are excluded so we don't flatten
// animation; image/jpg is a non-IANA alias for image/jpeg some
// OSes emit.
const RESIZABLE_MIME_TYPES = new Set([
"image/png",
"image/jpeg",
"image/jpg",
"image/webp",
]);
// Per-axis clamp applied at decode so a low-byte image with
// pathologically large pixel extent can't OOM the tab. 8192 stays
// under Safari/Chrome canvas limits while preserving detail for
// typical screenshots.
const MAX_INITIAL_DIMENSION = 8192;
// 8 iterations × DIMENSION_STEP per axis gives ~3% of original
// pixels, plenty of headroom for any modestly-oversize image.
const MAX_SHRINK_ITERATIONS = 8;
const DIMENSION_STEP = 0.8;
// 0.85 is near-lossless for screenshots; 0.7 is a hail-mary if
// dimension shrinking alone didn't fit the budget.
const INITIAL_QUALITY = 0.85;
const FALLBACK_QUALITY = 0.7;
// Time-bound the legacy <img> decode fallback so a blob that fires
// neither onload nor onerror can't wedge the module queue.
const FALLBACK_DECODE_TIMEOUT_MS = 10_000;
// Sequential queue: pasting many images won't spawn parallel
// decode pipelines.
let queue: Promise<unknown> = Promise.resolve();
function enqueue<T>(fn: () => Promise<T>): Promise<T> {
// Chain off both settlement branches so the next task runs
// after a rejection too; the .catch below detaches rejection
// from the shared tail.
const next = queue.then(fn, fn);
queue = next.catch(() => undefined);
return next;
}
/**
* Re-encode `file` as WebP, iteratively shrinking until the output
* is at or below `maxBytes`.
*
* - Returns the original `file` unchanged when it is already within
* the budget and its MIME type is in our resizable set (no need
* to pay the decode cost).
* - Returns the original `file` unchanged for animated formats like
* GIF where canvas re-encoding would destroy the animation.
* - Returns a new `File` (WebP, renamed to `.webp`) when resizing
* succeeded.
* - Returns `null` when the file cannot be decoded or no iteration
* fit the budget; callers fall back to the original file.
*/
export async function resizeImageToMaxBytes(
file: File,
maxBytes: number,
): Promise<File | null> {
if (!file.type.startsWith("image/")) {
return file;
}
// GIFs return as-is so we don't flatten animation.
if (file.type === "image/gif") {
return file;
}
// Already under budget; return as-is regardless of MIME (the
// function's contract is "give me something <= maxBytes" and
// we already have that).
if (file.size <= maxBytes) {
return file;
}
// Over budget but unsupported MIME (e.g. image/bmp): refuse
// rather than silently produce a black canvas or wrong file.
if (!RESIZABLE_MIME_TYPES.has(file.type)) {
return null;
}
return enqueue(() => shrinkOnce(file, maxBytes));
}
async function shrinkOnce(file: File, maxBytes: number): Promise<File | null> {
let bitmap: ImageBitmap | null = null;
try {
bitmap = await decodeToBitmap(file);
} catch {
return null;
}
if (!bitmap) {
return null;
}
try {
// decodeToBitmap already clamped to MAX_INITIAL_DIMENSION
// per axis, so we start the shrink loop from the bitmap's
// reported dimensions.
let width = bitmap.width;
let height = bitmap.height;
if (width <= 0 || height <= 0) {
return null;
}
for (let i = 0; i < MAX_SHRINK_ITERATIONS; i++) {
const blob = await encodeWebP(bitmap, width, height, INITIAL_QUALITY);
if (blob && blob.size <= maxBytes) {
return toWebPFile(file, blob);
}
// Guard against tiny images that can't shrink further.
if (width <= 1 || height <= 1) {
break;
}
width = Math.max(1, Math.round(width * DIMENSION_STEP));
height = Math.max(1, Math.round(height * DIMENSION_STEP));
}
// Last-ditch attempt at the smallest dimensions with a
// lower quality, for photographic images where dimension
// shrinking alone saturated.
const fallbackBlob = await encodeWebP(
bitmap,
width,
height,
FALLBACK_QUALITY,
);
if (fallbackBlob && fallbackBlob.size <= maxBytes) {
return toWebPFile(file, fallbackBlob);
}
return null;
} catch {
return null;
} finally {
bitmap.close?.();
}
}
async function decodeToBitmap(file: File): Promise<ImageBitmap | null> {
// createImageBitmap's HTML-spec output rules:
// - both resize dims => stretch (destroys aspect ratio).
// - only resizeWidth => width is exact, height proportional.
// Per spec this UPSCALES if resizeWidth > natural width.
// - both omitted => source's natural size.
//
// Upscaling can blow past Chromium's ~268M-pixel decode limit
// (e.g. a 1080x5000 screenshot with resizeWidth: 8192 becomes
// ~310M pixels). Probe natural dimensions first to pick the
// smallest resize option that fits.
if (typeof createImageBitmap !== "function") {
return await decodeViaImgFallback(file);
}
const natural = await probeNaturalDimensions(file);
if (!natural) {
return await decodeViaImgFallback(file);
}
const { width, height } = natural;
if (width <= 0 || height <= 0) {
return null;
}
// Pick the smallest resize that fits MAX_INITIAL_DIMENSION
// without upscaling. Already-small sources pass no options.
const needsWidthClamp = width > MAX_INITIAL_DIMENSION;
const needsHeightClamp = height > MAX_INITIAL_DIMENSION;
if (!needsWidthClamp && !needsHeightClamp) {
return await createImageBitmap(file);
}
// Clamp the longer axis; the shorter axis scales with it.
if (width >= height) {
return await createImageBitmap(file, {
resizeWidth: MAX_INITIAL_DIMENSION,
resizeQuality: "medium",
});
}
return await createImageBitmap(file, {
resizeHeight: MAX_INITIAL_DIMENSION,
resizeQuality: "medium",
});
}
// Reads natural dimensions via <img> without allocating a full
// ImageBitmap. Returns null on decode/timeout failure.
async function probeNaturalDimensions(
file: File,
): Promise<{ width: number; height: number } | null> {
return await new Promise<{ width: number; height: number } | null>(
(resolve) => {
const url = URL.createObjectURL(file);
const img = new Image();
let settled = false;
const cleanup = () => {
settled = true;
URL.revokeObjectURL(url);
};
const timer = setTimeout(() => {
if (settled) return;
cleanup();
resolve(null);
}, FALLBACK_DECODE_TIMEOUT_MS);
img.onload = () => {
if (settled) return;
clearTimeout(timer);
cleanup();
resolve({ width: img.naturalWidth, height: img.naturalHeight });
};
img.onerror = () => {
if (settled) return;
clearTimeout(timer);
cleanup();
resolve(null);
};
img.src = url;
},
);
}
async function decodeViaImgFallback(file: File): Promise<ImageBitmap | null> {
// Decode via <img> + Blob URL. Reached only on browsers
// without createImageBitmap (very old Safari, embedded
// webviews); time-bounded so a stuck decoder can't wedge the
// queue. No decode-time clamp on this path.
return await new Promise<ImageBitmap | null>((resolve, reject) => {
const url = URL.createObjectURL(file);
const img = new Image();
let settled = false;
const cleanup = () => {
settled = true;
URL.revokeObjectURL(url);
};
const timer = setTimeout(() => {
if (settled) return;
cleanup();
reject(new Error("image decode timed out"));
}, FALLBACK_DECODE_TIMEOUT_MS);
img.onload = () => {
if (settled) return;
clearTimeout(timer);
cleanup();
// HTMLImageElement is a valid CanvasImageSource;
// width/height are all we need downstream.
resolve(img as unknown as ImageBitmap);
};
img.onerror = () => {
if (settled) return;
clearTimeout(timer);
cleanup();
reject(new Error("image decode failed"));
};
img.src = url;
});
}
async function encodeWebP(
source: ImageBitmap,
width: number,
height: number,
quality: number,
): Promise<Blob | null> {
// OffscreenCanvas's convertToBlob is fully async and doesn't
// need the canvas laid out.
if (typeof OffscreenCanvas === "function") {
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d");
if (!ctx) {
return null;
}
ctx.drawImage(source, 0, 0, width, height);
try {
return await canvas.convertToBlob({ type: "image/webp", quality });
} catch {
return null;
}
}
// Fallback for environments without OffscreenCanvas.
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d");
if (!ctx) {
return null;
}
ctx.drawImage(source as CanvasImageSource, 0, 0, width, height);
return await new Promise<Blob | null>((resolve) => {
canvas.toBlob((blob) => resolve(blob), "image/webp", quality);
});
}
function toWebPFile(original: File, blob: Blob): File {
// Match the extension to the actual content type; some upload
// handlers still key behavior off the extension.
const dot = original.name.lastIndexOf(".");
const baseName = dot > 0 ? original.name.slice(0, dot) : original.name;
const webpName = `${baseName || "image"}.webp`;
// Use blob.type: canvas encoders fall back to PNG on browsers
// without WebP support; this keeps the File's labelled type
// matching its actual content.
const effectiveType = blob.type || "image/webp";
const effectiveName =
effectiveType === "image/webp" ? webpName : original.name;
return new File([blob], effectiveName, {
type: effectiveType,
lastModified: original.lastModified,
});
}