mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: resize chat image attachments client-side for provider budgets (#24533)
Anthropic rejects inline images over 5,242,880 bytes, but our upload endpoint accepts images up to 10 MiB — so 5–10 MiB images were reaching the provider and failing. This adds two layers of protection: the browser resizes oversized images before upload, and the server rejects any that still slip through before an upstream request is issued. Client-side resizing uses `createImageBitmap` with `resizeWidth`/`resizeHeight` to clamp the decoded bitmap at decode time, then iteratively shrinks on an `OffscreenCanvas` (falling back to `HTMLCanvasElement`) until the output fits the applicable budget. Anthropic (and Bedrock-hosted Claude — fantasy's bedrock provider is a thin wrapper around the Anthropic client) uses a ~5 MiB budget; other providers use a ~10 MiB budget to stay under the server cap. Doing the resize in the browser avoids decoding attacker-controlled image bytes in `coderd` (image-bomb DoS surface). Server-side, `chatFileResolver` now takes a provider string and looks up the inline-image cap via a new `chatprovider.InlineImageByteCap` helper; oversized `image/*` files for capped providers are rejected with a pre-classified `chaterror` before the SDK call. The backstop fires for older clients, direct API callers, or any image that was committed to the composer before the user switched to a stricter provider. Attachments commit to composer state synchronously with a new `"processing"` `UploadState` so paste+Enter can't dispatch before the resize finishes; the `"uploading"` send gate now covers both states. Dismissed-while-resizing attachments are tracked in a `WeakSet` so a late swap can't resurrect a removed file. Closes CODAGT-215
This commit is contained in:
+2
-7
@@ -4343,11 +4343,6 @@ func parseCompactionThresholdKey(key string) (uuid.UUID, error) {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
const (
|
||||
// maxChatFileSize is the maximum size of a chat file upload (10 MB).
|
||||
maxChatFileSize = 10 << 20
|
||||
)
|
||||
|
||||
//nolint:revive // get-return: revive assumes get* must be a getter, but this is an HTTP handler.
|
||||
func (api *API) getChatSystemPrompt(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
@@ -5832,14 +5827,14 @@ func (api *API) postChatFile(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(rw, r.Body, maxChatFileSize)
|
||||
r.Body = http.MaxBytesReader(rw, r.Body, codersdk.MaxChatFileSizeBytes)
|
||||
data, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesErr) {
|
||||
httpapi.Write(ctx, rw, http.StatusRequestEntityTooLarge, codersdk.Response{
|
||||
Message: "File too large.",
|
||||
Detail: fmt.Sprintf("Maximum file size is %d bytes.", maxChatFileSize),
|
||||
Detail: fmt.Sprintf("Maximum file size is %d bytes.", codersdk.MaxChatFileSizeBytes),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
+40
-5
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"charm.land/fantasy"
|
||||
"charm.land/fantasy/providers/anthropic"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/google/uuid"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/shopspring/decimal"
|
||||
@@ -5161,16 +5162,50 @@ func (p *Server) subscribeChatControl(
|
||||
return controlCancel
|
||||
}
|
||||
|
||||
// chatFileResolver returns a FileResolver that fetches chat file
|
||||
// content from the database by ID.
|
||||
func (p *Server) chatFileResolver() chatprompt.FileResolver {
|
||||
// Rejects oversize images on capped providers before any upstream
|
||||
// request is issued.
|
||||
//
|
||||
// Gotcha: a historical oversize image bricks the chat on a capped
|
||||
// provider until the user switches providers back, starts a new
|
||||
// chat, or edits a message above the offending one (which truncates
|
||||
// the prompt forward). A future change should skip the file with a
|
||||
// user-facing warning, but that requires altering the FileResolver
|
||||
// contract.
|
||||
func (p *Server) chatFileResolver(provider string) chatprompt.FileResolver {
|
||||
return func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]chatprompt.FileData, error) {
|
||||
files, err := p.db.GetChatFilesByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
imageCap, hasImageCap := chatprovider.InlineImageCapBytes(provider)
|
||||
normalizedProvider := chatprovider.NormalizeProvider(provider)
|
||||
result := make(map[uuid.UUID]chatprompt.FileData, len(files))
|
||||
for _, f := range files {
|
||||
if hasImageCap &&
|
||||
strings.HasPrefix(f.Mimetype, "image/") &&
|
||||
len(f.Data) >= imageCap {
|
||||
err := xerrors.Errorf(
|
||||
"image attachment %q is %d bytes; %s inline image limit is %d bytes",
|
||||
f.Name, len(f.Data),
|
||||
chatprovider.ProviderDisplayName(normalizedProvider),
|
||||
imageCap,
|
||||
)
|
||||
// User-facing message stays client-agnostic since
|
||||
// older web clients and direct API callers don't
|
||||
// auto-resize; the wrapped error above keeps the
|
||||
// exact byte count for operator logs.
|
||||
return nil, chaterror.WithClassification(err, chaterror.ClassifiedError{
|
||||
Kind: codersdk.ChatErrorKindConfig,
|
||||
Provider: normalizedProvider,
|
||||
Message: fmt.Sprintf(
|
||||
"Image attachment exceeds %s's %s inline image limit. Replace it with a smaller image.",
|
||||
chatprovider.ProviderDisplayName(normalizedProvider),
|
||||
//nolint:gosec // imageCap is a small positive constant defined in chatprovider.
|
||||
humanize.IBytes(uint64(imageCap)),
|
||||
),
|
||||
Retryable: false,
|
||||
})
|
||||
}
|
||||
result[f.ID] = chatprompt.FileData{
|
||||
Name: f.Name,
|
||||
Data: f.Data,
|
||||
@@ -6478,7 +6513,7 @@ func (p *Server) runChat(
|
||||
var g2 errgroup.Group
|
||||
g2.Go(func() error {
|
||||
var err error
|
||||
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, messages, p.chatFileResolver(), logger)
|
||||
prompt, err = chatprompt.ConvertMessagesWithFiles(ctx, messages, p.chatFileResolver(modelConfig.Provider), logger)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("build chat prompt: %w", err)
|
||||
}
|
||||
@@ -7263,7 +7298,7 @@ func (p *Server) runChat(
|
||||
if compactionOptions != nil {
|
||||
compactionOptions.HistoryTipMessageID = compactionHistoryTipMessageID
|
||||
}
|
||||
reloadedPrompt, err := chatprompt.ConvertMessagesWithFiles(reloadCtx, reloadedMsgs, p.chatFileResolver(), logger)
|
||||
reloadedPrompt, err := chatprompt.ConvertMessagesWithFiles(reloadCtx, reloadedMsgs, p.chatFileResolver(modelConfig.Provider), logger)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("convert reloaded messages: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
package chatd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/mock/gomock"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbmock"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
)
|
||||
|
||||
// inlineImageCapFor returns the provider's inline image cap. Fails
|
||||
// the test if the provider has no documented cap.
|
||||
func inlineImageCapFor(t *testing.T, provider string) int {
|
||||
t.Helper()
|
||||
imageCap, ok := chatprovider.InlineImageCapBytes(provider)
|
||||
require.Truef(t, ok, "expected provider %q to have an inline image cap", provider)
|
||||
return imageCap
|
||||
}
|
||||
|
||||
// TestChatFileResolver_RejectsOversizedImages is the server-side
|
||||
// safety net for browser-side resize: oversize images that reach the
|
||||
// resolver are rejected before any upstream request.
|
||||
func TestChatFileResolver_RejectsOversizedImages(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Computed so the table tracks any future cap retune.
|
||||
anthropicCap := inlineImageCapFor(t, "anthropic")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
provider string
|
||||
mimetype string
|
||||
size int
|
||||
expectReject bool
|
||||
expectProviderID string // classified.Provider after normalization
|
||||
}{
|
||||
{
|
||||
name: "OversizedAnthropicPNG_Rejected",
|
||||
provider: "anthropic",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "OversizedAnthropicJPEG_Rejected",
|
||||
provider: "anthropic",
|
||||
mimetype: "image/jpeg",
|
||||
size: anthropicCap + 1024,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
// Boundary is >=: exactly-at-limit is rejected.
|
||||
// Anthropic's docs say "5 MB maximum" without
|
||||
// specifying inclusivity, so reject strictly.
|
||||
name: "AtLimitAnthropicImage_Rejected",
|
||||
provider: "anthropic",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "JustUnderLimitAnthropicImage_Accepted",
|
||||
provider: "anthropic",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap - 1,
|
||||
expectReject: false,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "UndersizedAnthropicImage_Accepted",
|
||||
provider: "anthropic",
|
||||
mimetype: "image/png",
|
||||
size: 1024,
|
||||
expectReject: false,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
// Bedrock reuses Anthropic's cap.
|
||||
name: "OversizedBedrockPNG_Rejected",
|
||||
provider: "bedrock",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: true,
|
||||
expectProviderID: "bedrock",
|
||||
},
|
||||
{
|
||||
name: "OversizedOpenAIImage_Accepted",
|
||||
provider: "openai",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: false,
|
||||
expectProviderID: "openai",
|
||||
},
|
||||
{
|
||||
name: "OversizedAnthropicText_Accepted",
|
||||
provider: "anthropic",
|
||||
mimetype: "text/plain",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: false,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "ProviderMixedCase_Rejected",
|
||||
provider: "Anthropic",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "ProviderAllCaps_Rejected",
|
||||
provider: "ANTHROPIC",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
{
|
||||
name: "ProviderPaddedWhitespace_Rejected",
|
||||
provider: " anthropic ",
|
||||
mimetype: "image/png",
|
||||
size: anthropicCap + 1,
|
||||
expectReject: true,
|
||||
expectProviderID: "anthropic",
|
||||
},
|
||||
}
|
||||
|
||||
// One shared backing buffer sliced per case. The resolver only
|
||||
// reads len(f.Data), so shared backing is safe and avoids N×max
|
||||
// allocations in parallel.
|
||||
maxSize := 0
|
||||
for _, tc := range tests {
|
||||
if tc.size > maxSize {
|
||||
maxSize = tc.size
|
||||
}
|
||||
}
|
||||
sharedData := make([]byte, maxSize)
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
db := dbmock.NewMockStore(ctrl)
|
||||
server := &Server{db: db}
|
||||
|
||||
fileID := uuid.New()
|
||||
row := database.ChatFile{
|
||||
ID: fileID,
|
||||
Name: "attachment.png",
|
||||
Mimetype: tc.mimetype,
|
||||
Data: sharedData[:tc.size],
|
||||
}
|
||||
db.EXPECT().
|
||||
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
|
||||
Return([]database.ChatFile{row}, nil).
|
||||
Times(1)
|
||||
|
||||
resolver := server.chatFileResolver(tc.provider)
|
||||
got, err := resolver(ctx, []uuid.UUID{fileID})
|
||||
|
||||
if tc.expectReject {
|
||||
require.Error(t, err)
|
||||
require.Nil(t, got)
|
||||
// Classification turns the generic upstream error
|
||||
// into an actionable user-facing message.
|
||||
classified := chaterror.Classify(err)
|
||||
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
|
||||
require.Equal(t, tc.expectProviderID, classified.Provider)
|
||||
require.False(t, classified.Retryable)
|
||||
// User-facing message names the provider and shows
|
||||
// the cap in human units; raw byte count stays in
|
||||
// the wrapped developer error.
|
||||
displayName := chatprovider.ProviderDisplayName(tc.expectProviderID)
|
||||
require.Contains(t, classified.Message, displayName)
|
||||
imageCap := inlineImageCapFor(t, tc.expectProviderID)
|
||||
//nolint:gosec // imageCap is a small positive constant defined in chatprovider.
|
||||
require.Contains(t, classified.Message, humanize.IBytes(uint64(imageCap)))
|
||||
require.NotContains(
|
||||
t,
|
||||
classified.Message,
|
||||
strconv.Itoa(imageCap),
|
||||
"user-facing message should not include raw bytes",
|
||||
)
|
||||
// Wrapped error preserves exact bytes for logs.
|
||||
require.Contains(t, err.Error(), strconv.Itoa(imageCap))
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, got, fileID)
|
||||
require.Equal(t, row.Data, got[fileID].Data)
|
||||
require.Equal(t, tc.mimetype, got[fileID].MediaType)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatFileResolver_MultiFileFailsFastOnFirstOversized pins the
|
||||
// "first bad file aborts the batch" contract.
|
||||
func TestChatFileResolver_MultiFileFailsFastOnFirstOversized(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
db := dbmock.NewMockStore(ctrl)
|
||||
server := &Server{db: db}
|
||||
|
||||
anthropicCap := inlineImageCapFor(t, "anthropic")
|
||||
// Shared buffer; ok files take small prefixes.
|
||||
buf := make([]byte, anthropicCap+1)
|
||||
okFileA := database.ChatFile{
|
||||
ID: uuid.New(),
|
||||
Name: "ok-a.png",
|
||||
Mimetype: "image/png",
|
||||
Data: buf[:1024],
|
||||
}
|
||||
oversized := database.ChatFile{
|
||||
ID: uuid.New(),
|
||||
Name: "too-big.png",
|
||||
Mimetype: "image/png",
|
||||
Data: buf,
|
||||
}
|
||||
okFileB := database.ChatFile{
|
||||
ID: uuid.New(),
|
||||
Name: "ok-b.png",
|
||||
Mimetype: "image/png",
|
||||
Data: buf[:1024],
|
||||
}
|
||||
ids := []uuid.UUID{okFileA.ID, oversized.ID, okFileB.ID}
|
||||
|
||||
db.EXPECT().
|
||||
GetChatFilesByIDs(gomock.Any(), ids).
|
||||
Return([]database.ChatFile{okFileA, oversized, okFileB}, nil).
|
||||
Times(1)
|
||||
|
||||
resolver := server.chatFileResolver("anthropic")
|
||||
got, err := resolver(ctx, ids)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, got)
|
||||
classified := chaterror.Classify(err)
|
||||
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
|
||||
// The error must identify the specific offending file so a user
|
||||
// with several attachments knows which one to replace.
|
||||
require.Contains(t, err.Error(), oversized.Name)
|
||||
}
|
||||
|
||||
// TestChatFileResolver_PropagatesDBError confirms unrelated database
|
||||
// failures pass through unchanged (not masked by the size check).
|
||||
func TestChatFileResolver_PropagatesDBError(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
db := dbmock.NewMockStore(ctrl)
|
||||
server := &Server{db: db}
|
||||
|
||||
sentinel := xerrors.New("boom")
|
||||
fileID := uuid.New()
|
||||
db.EXPECT().
|
||||
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
|
||||
Return(nil, sentinel).
|
||||
Times(1)
|
||||
|
||||
resolver := server.chatFileResolver("anthropic")
|
||||
got, err := resolver(ctx, []uuid.UUID{fileID})
|
||||
require.ErrorIs(t, err, sentinel)
|
||||
require.Nil(t, got)
|
||||
}
|
||||
|
||||
// TestChatFileResolver_UnknownProviderSkipsCapCheck confirms providers
|
||||
// without a documented inline cap are never rejected by the backstop.
|
||||
func TestChatFileResolver_UnknownProviderSkipsCapCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
db := dbmock.NewMockStore(ctrl)
|
||||
server := &Server{db: db}
|
||||
|
||||
fileID := uuid.New()
|
||||
// Exactly 1 byte above the Anthropic cap is enough to prove
|
||||
// the backstop is skipped for uncapped providers; no need to
|
||||
// allocate tens of MiB in CI.
|
||||
overAnyCap := inlineImageCapFor(t, "anthropic") + 1
|
||||
row := database.ChatFile{
|
||||
ID: fileID,
|
||||
Name: "huge.png",
|
||||
Mimetype: "image/png",
|
||||
Data: make([]byte, overAnyCap),
|
||||
}
|
||||
db.EXPECT().
|
||||
GetChatFilesByIDs(gomock.Any(), []uuid.UUID{fileID}).
|
||||
Return([]database.ChatFile{row}, nil).
|
||||
Times(1)
|
||||
|
||||
resolver := server.chatFileResolver("openrouter")
|
||||
got, err := resolver(ctx, []uuid.UUID{fileID})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, got, fileID)
|
||||
}
|
||||
@@ -83,6 +83,19 @@ func ProviderAllowsAmbientCredentials(provider string) bool {
|
||||
return NormalizeProvider(provider) == fantasybedrock.Name
|
||||
}
|
||||
|
||||
// InlineImageCapBytes returns the per-image byte cap for inline
|
||||
// image parts, or (0, false) when no documented cap applies.
|
||||
// Bedrock shares Anthropic's cap because fantasy's bedrock provider
|
||||
// wraps the anthropic client.
|
||||
func InlineImageCapBytes(provider string) (int, bool) {
|
||||
switch NormalizeProvider(provider) {
|
||||
case fantasyanthropic.Name, fantasybedrock.Name:
|
||||
return codersdk.AnthropicInlineImageCapBytes, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
// ProviderAPIKeys contains API keys for provider calls.
|
||||
type ProviderAPIKeys struct {
|
||||
OpenAI string
|
||||
|
||||
@@ -33,6 +33,15 @@ const ChatCompactionThresholdKeyPrefix = "chat_compaction_threshold_pct:"
|
||||
// this limit than to lower it.
|
||||
const MaxChatFileIDs = 20
|
||||
|
||||
// MaxChatFileSizeBytes is the upload-endpoint cap for chat
|
||||
// attachments.
|
||||
const MaxChatFileSizeBytes = 10 * 1024 * 1024
|
||||
|
||||
// AnthropicInlineImageCapBytes is Anthropic's documented per-image
|
||||
// wire limit; the same cap applies to Bedrock-hosted Claude. Other
|
||||
// providers have no documented per-image cap.
|
||||
const AnthropicInlineImageCapBytes = 5 * 1024 * 1024
|
||||
|
||||
// ChatAttachmentMediaType is a media type that is allowed for durable
|
||||
// chat file storage. The set is intentionally narrow; byte-level
|
||||
// classification and inline-render rules live alongside the enforcement
|
||||
|
||||
Generated
+15
@@ -877,6 +877,14 @@ export const AgentSubsystems: AgentSubsystem[] = [
|
||||
"exectrace",
|
||||
];
|
||||
|
||||
// From codersdk/chats.go
|
||||
/**
|
||||
* AnthropicInlineImageCapBytes is Anthropic's documented per-image
|
||||
* wire limit; the same cap applies to Bedrock-hosted Claude. Other
|
||||
* providers have no documented per-image cap.
|
||||
*/
|
||||
export const AnthropicInlineImageCapBytes = 5242880;
|
||||
|
||||
// From codersdk/deployment.go
|
||||
export interface AppHostResponse {
|
||||
/**
|
||||
@@ -4777,6 +4785,13 @@ export interface MatchedProvisioners {
|
||||
*/
|
||||
export const MaxChatFileIDs = 20;
|
||||
|
||||
// From codersdk/chats.go
|
||||
/**
|
||||
* MaxChatFileSizeBytes is the upload-endpoint cap for chat
|
||||
* attachments.
|
||||
*/
|
||||
export const MaxChatFileSizeBytes = 10485760;
|
||||
|
||||
// From codersdk/usersecretvalidation.go
|
||||
/**
|
||||
* MaxSecretValueSize is the maximum size of a user secret value
|
||||
|
||||
@@ -605,3 +605,283 @@ describe("useFileAttachments persistence", () => {
|
||||
unmount();
|
||||
});
|
||||
});
|
||||
|
||||
// Synthetic resize swaps the File without invoking the browser's
|
||||
// decoder, so these tests run in jsdom without resizeImage.ts fakes.
|
||||
describe("useFileAttachments processResizes", () => {
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
// Reported size > budget without actually allocating bytes.
|
||||
const makeOversizeImage = (
|
||||
name = "photo.png",
|
||||
type = "image/png",
|
||||
bytes = 6 * 1024 * 1024,
|
||||
) => {
|
||||
const file = new File([new Uint8Array(8)], name, { type });
|
||||
Object.defineProperty(file, "size", { value: bytes });
|
||||
return file;
|
||||
};
|
||||
|
||||
it("marks oversize images as processing synchronously on attach", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
// Never-resolving resize so "processing" stays observable.
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() => new Promise(() => undefined),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const file = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([file]);
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
expect(result.current.attachments[0]).toBe(file);
|
||||
const state = result.current.uploadStates.get(file);
|
||||
expect(state?.status).toBe("processing");
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("swaps the original File for the resized replacement when resize succeeds", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
const { API } = await import("#/api/api");
|
||||
const replacement = new File([new Uint8Array(1024)], "photo.webp", {
|
||||
type: "image/webp",
|
||||
});
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(replacement);
|
||||
vi.spyOn(API.experimental, "uploadChatFile").mockResolvedValue({
|
||||
id: "file-id",
|
||||
});
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(result.current.attachments[0]).toBe(replacement);
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(result.current.uploadStates.get(replacement)?.status).toBe(
|
||||
"uploaded",
|
||||
);
|
||||
});
|
||||
expect(result.current.uploadStates.get(original)).toBeUndefined();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("falls back to the original File when resize returns null", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
const { API } = await import("#/api/api");
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(null);
|
||||
vi.spyOn(API.experimental, "uploadChatFile").mockResolvedValue({
|
||||
id: "file-id",
|
||||
});
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
|
||||
|
||||
// 6 MiB on Anthropic + null resize forces the
|
||||
// provider-budget-error path.
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const original = makeOversizeImage(
|
||||
"photo.png",
|
||||
"image/png",
|
||||
6 * 1024 * 1024,
|
||||
);
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const state = result.current.uploadStates.get(original);
|
||||
expect(state?.status).toBe("error");
|
||||
expect(state?.error).toMatch(/Anthropic/);
|
||||
expect(state?.error).toMatch(/MiB/);
|
||||
});
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("freezes the provider snapshot at attach time so a mid-resize provider switch can't mislabel the error", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolve) => {
|
||||
releaseResize = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, rerender, unmount } = renderHook(
|
||||
({ provider }) => useFileAttachments("org-1", { provider }),
|
||||
{ initialProps: { provider: "anthropic" } },
|
||||
);
|
||||
|
||||
// Attach on Anthropic (5 MiB budget).
|
||||
const original = makeOversizeImage(
|
||||
"photo.gif",
|
||||
"image/gif",
|
||||
6 * 1024 * 1024,
|
||||
);
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
|
||||
// User switches to OpenAI (10 MiB budget) before the
|
||||
// resize finishes. providerRef.current is now "openai".
|
||||
rerender({ provider: "openai" });
|
||||
|
||||
// Resize gives up (animated GIF; falls back to original).
|
||||
await act(async () => {
|
||||
releaseResize(null);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// Error must name Anthropic (the provider whose budget
|
||||
// rejected the file), not OpenAI (the live ref). The
|
||||
// budget value in the error must match Anthropic's
|
||||
// ~5 MiB, not OpenAI's ~10 MiB.
|
||||
await vi.waitFor(() => {
|
||||
const state = result.current.uploadStates.get(original);
|
||||
expect(state?.status).toBe("error");
|
||||
expect(state?.error).toMatch(/Anthropic/);
|
||||
expect(state?.error).not.toMatch(/OpenAI/);
|
||||
expect(state?.error).toMatch(/under 5\.0 MiB/);
|
||||
});
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("does not resurrect attachments removed while resize is in flight", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolve) => {
|
||||
releaseResize = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
|
||||
// Remove the attachment while resize is still pending.
|
||||
act(() => {
|
||||
result.current.handleRemoveAttachment(original);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
|
||||
// Resolve the pending resize with a swap; the hook must
|
||||
// NOT resurrect the dismissed attachment.
|
||||
const replacement = new File([new Uint8Array(512)], "photo.webp", {
|
||||
type: "image/webp",
|
||||
});
|
||||
await act(async () => {
|
||||
releaseResize(replacement);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
expect(result.current.uploadStates.get(replacement)).toBeUndefined();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("does not resurrect attachments after resetAttachments fires", async () => {
|
||||
const resize = await import("./utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolve) => {
|
||||
releaseResize = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
|
||||
// Simulate a chat-scope reset (e.g. ChatPageContent's
|
||||
// editScopeRef effect when navigating between chats)
|
||||
// while the resize is still pending.
|
||||
act(() => {
|
||||
result.current.resetAttachments();
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
|
||||
// Resolve the pending resize with a swap. The hook must
|
||||
// not re-add the replacement to attachments or kick off
|
||||
// an upload against the now-cleared scope.
|
||||
const replacement = new File([new Uint8Array(512)], "photo.webp", {
|
||||
type: "image/webp",
|
||||
});
|
||||
await act(async () => {
|
||||
releaseResize(replacement);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
expect(result.current.uploadStates.get(replacement)).toBeUndefined();
|
||||
expect(result.current.uploadStates.get(original)).toBeUndefined();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("gates the send-reachable isUploading state on processing", () => {
|
||||
const resize = import("./utils/resizeImage");
|
||||
vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response());
|
||||
// Never-resolving resize keeps the attachment in "processing".
|
||||
void resize.then((m) =>
|
||||
vi
|
||||
.spyOn(m, "resizeImageToMaxBytes")
|
||||
.mockImplementation(() => new Promise(() => undefined)),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useFileAttachments("org-1", { provider: "anthropic" }),
|
||||
);
|
||||
|
||||
const file = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([file]);
|
||||
});
|
||||
|
||||
// Upload state must be "processing" (not "uploaded" or
|
||||
// undefined). The AgentChatInput send gate treats this the
|
||||
// same as "uploading" and blocks dispatch.
|
||||
expect(result.current.uploadStates.get(file)?.status).toBe("processing");
|
||||
unmount();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -819,7 +819,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
|
||||
{/* Warn about invisible Unicode in the message text.
|
||||
* Unlike the admin/user prompt textareas (which strip
|
||||
* invisible chars server-side on save), the chat input
|
||||
* is the user's free-form message — we don't silently
|
||||
* is the user's free-form message; we don't silently
|
||||
* mutate it. Instead we surface a warning so the user
|
||||
* can make an informed decision. This guards against
|
||||
* social engineering attacks where a user is tricked
|
||||
@@ -1074,7 +1074,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
|
||||
)}
|
||||
</span>
|
||||
)}{" "}
|
||||
{/* Badge row — all badges and the pill always
|
||||
{/* Badge row; all badges and the pill always
|
||||
* render so the DOM structure never changes.
|
||||
* Overflow badges use invisible + order-1 to
|
||||
* hide and reorder via CSS. The pill is invisible
|
||||
@@ -1107,7 +1107,7 @@ export const AgentChatInput: FC<AgentChatInputProps> = ({
|
||||
/>
|
||||
);
|
||||
})}
|
||||
{/* Pill — always in the DOM so it permanently
|
||||
{/* Pill; always in the DOM so it permanently
|
||||
* reserves layout space. Invisible when nothing
|
||||
* overflows. CSS order keeps it before order-1
|
||||
* (overflow) badges. */}
|
||||
|
||||
@@ -22,6 +22,7 @@ import { useFileAttachments } from "../hooks/useFileAttachments";
|
||||
import { parseStoredDraft } from "../utils/draftStorage";
|
||||
import {
|
||||
getModelSelectorPlaceholder,
|
||||
getProviderForModelOption,
|
||||
hasConfiguredModelsInCatalog,
|
||||
hasUserFixableProviders,
|
||||
} from "../utils/modelOptions";
|
||||
@@ -382,7 +383,10 @@ export const AgentCreateForm: FC<AgentCreateFormProps> = ({
|
||||
handleAttach,
|
||||
handleRemoveAttachment,
|
||||
resetAttachments,
|
||||
} = useFileAttachments(organizationId || undefined, { persist: true });
|
||||
} = useFileAttachments(organizationId || undefined, {
|
||||
persist: true,
|
||||
provider: getProviderForModelOption(modelOptions, selectedModel),
|
||||
});
|
||||
|
||||
const handleSendWithAttachments = async (message: string) => {
|
||||
const fileIds: string[] = [];
|
||||
|
||||
@@ -153,7 +153,7 @@ export const FileTooLarge: Story = {
|
||||
file,
|
||||
{
|
||||
status: "error",
|
||||
error: "File too large (12.4 MB). Maximum is 10 MB.",
|
||||
error: "File too large (12.4 MiB). Maximum is 10 MiB.",
|
||||
},
|
||||
],
|
||||
]),
|
||||
|
||||
@@ -17,14 +17,19 @@ import {
|
||||
} from "../utils/fetchTextAttachment";
|
||||
|
||||
export type UploadState = {
|
||||
status: "pending" | "uploading" | "uploaded" | "error";
|
||||
// "processing" covers any pre-upload client work (e.g. resize),
|
||||
// so paste/drop handlers can commit the attachment synchronously
|
||||
// without the send gate believing it is ready to dispatch.
|
||||
status: "pending" | "processing" | "uploading" | "uploaded" | "error";
|
||||
fileId?: string;
|
||||
error?: string;
|
||||
draftWarning?: string;
|
||||
};
|
||||
|
||||
export const isUploadInProgress = (state: UploadState | undefined): boolean =>
|
||||
state?.status === "pending" || state?.status === "uploading";
|
||||
state?.status === "pending" ||
|
||||
state?.status === "processing" ||
|
||||
state?.status === "uploading";
|
||||
|
||||
/** Renders an image thumbnail from a pre-created preview URL. */
|
||||
export const ImageThumbnail: FC<{
|
||||
@@ -194,6 +199,7 @@ export const AttachmentPreview: FC<{
|
||||
</button>
|
||||
)}
|
||||
{(uploadState?.status === "pending" ||
|
||||
uploadState?.status === "processing" ||
|
||||
uploadState?.status === "uploading") && (
|
||||
<div className="absolute inset-0 flex items-center justify-center rounded-md bg-overlay">
|
||||
<Spinner className="h-5 w-5 text-white" loading />
|
||||
|
||||
@@ -7,6 +7,7 @@ import { useChatDraftAttachments } from "../hooks/useChatDraftAttachments";
|
||||
import { chatWidthClass, useChatFullWidth } from "../hooks/useChatFullWidth";
|
||||
import { useFileAttachments } from "../hooks/useFileAttachments";
|
||||
import { getChatFileURL } from "../utils/chatAttachments";
|
||||
import { getProviderForModelOption } from "../utils/modelOptions";
|
||||
import type { ChatDetailError } from "../utils/usageLimitMessage";
|
||||
import {
|
||||
AgentChatInput,
|
||||
@@ -305,8 +306,12 @@ export const ChatPageInput: FC<ChatPageInputProps> = ({
|
||||
const latestContextUsage = rawUsage
|
||||
? { ...rawUsage, compressionThreshold, lastInjectedContext }
|
||||
: rawUsage;
|
||||
const composeAttachments = useChatDraftAttachments(organizationId, chatId);
|
||||
const editAttachments = useFileAttachments(organizationId);
|
||||
const composeAttachments = useChatDraftAttachments(organizationId, chatId, {
|
||||
provider: getProviderForModelOption(modelOptions, selectedModel),
|
||||
});
|
||||
const editAttachments = useFileAttachments(organizationId, {
|
||||
provider: getProviderForModelOption(modelOptions, selectedModel),
|
||||
});
|
||||
const {
|
||||
setAttachments: setEditAttachments,
|
||||
setPreviewUrls: setEditPreviewUrls,
|
||||
|
||||
@@ -370,7 +370,7 @@ describe("useChatDraftAttachments", () => {
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
expect(result.current.uploadStates.get(file)).toMatchObject({
|
||||
status: "error",
|
||||
error: expect.stringContaining("Maximum is 10 MB"),
|
||||
error: expect.stringContaining("Maximum is 10 MiB"),
|
||||
});
|
||||
expect(localStorage.getItem(storageKey)).toBeNull();
|
||||
unmount();
|
||||
@@ -523,4 +523,291 @@ describe("useChatDraftAttachments", () => {
|
||||
expect(stored[0].clientId).toBe("good");
|
||||
unmount();
|
||||
});
|
||||
|
||||
describe("compose-path resize", () => {
|
||||
const makeOversizeImage = () =>
|
||||
new File([new Uint8Array(5 * 1024 * 1024 + 64 * 1024)], "photo.png", {
|
||||
type: "image/png",
|
||||
lastModified: 100,
|
||||
});
|
||||
|
||||
it("commits the original synchronously with status: processing while resize is in flight", async () => {
|
||||
const resize = await import("../utils/resizeImage");
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() => new Promise<File | null>(() => undefined),
|
||||
);
|
||||
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
|
||||
);
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
expect(result.current.attachments[0]).toBe(original);
|
||||
expect(result.current.uploadStates.get(original)).toMatchObject({
|
||||
status: "processing",
|
||||
});
|
||||
// Registry entry is created post-resize.
|
||||
expect(uploadSpy).not.toHaveBeenCalled();
|
||||
expect(localStorage.getItem(storageKey)).toBeNull();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("swaps the original for a smaller resized File and starts the upload", async () => {
|
||||
const upload = createDeferred<{ id: string }>();
|
||||
const uploadSpy = vi
|
||||
.spyOn(API.experimental, "uploadChatFile")
|
||||
.mockReturnValue(upload.promise);
|
||||
const resize = await import("../utils/resizeImage");
|
||||
const replacement = new File(
|
||||
[new Uint8Array(2 * 1024 * 1024)],
|
||||
"photo.webp",
|
||||
{ type: "image/webp", lastModified: 200 },
|
||||
);
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolveFn) => {
|
||||
releaseResize = resolveFn;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
|
||||
);
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
expect(result.current.uploadStates.get(original)?.status).toBe(
|
||||
"processing",
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
releaseResize(replacement);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
expect(result.current.attachments[0]).toBe(replacement);
|
||||
});
|
||||
expect(result.current.uploadStates.get(original)).toBeUndefined();
|
||||
expect(uploadSpy).toHaveBeenCalledTimes(1);
|
||||
expect(uploadSpy).toHaveBeenCalledWith(replacement, orgID);
|
||||
|
||||
await act(async () => {
|
||||
upload.resolve({ id: "file-resized" });
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(result.current.uploadStates.get(replacement)).toMatchObject({
|
||||
status: "uploaded",
|
||||
fileId: "file-resized",
|
||||
});
|
||||
});
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("falls back to the original and surfaces a provider-budget error when resize returns null on Anthropic", async () => {
|
||||
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
|
||||
const resize = await import("../utils/resizeImage");
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockResolvedValue(null);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
|
||||
);
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
const state = result.current.uploadStates.get(original);
|
||||
expect(state?.status).toBe("error");
|
||||
expect(state?.error).toMatch(/Anthropic/);
|
||||
expect(state?.error).toMatch(/MiB/);
|
||||
});
|
||||
|
||||
expect(uploadSpy).not.toHaveBeenCalled();
|
||||
expect(localStorage.getItem(storageKey)).toBeNull();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("does not resurrect attachments removed while resize is in flight", async () => {
|
||||
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
|
||||
const resize = await import("../utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolveFn) => {
|
||||
releaseResize = resolveFn;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
|
||||
);
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
|
||||
act(() => {
|
||||
result.current.handleRemoveAttachment(original);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
|
||||
const replacement = new File(
|
||||
[new Uint8Array(1 * 1024 * 1024)],
|
||||
"photo.webp",
|
||||
{ type: "image/webp" },
|
||||
);
|
||||
await act(async () => {
|
||||
releaseResize(replacement);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
expect(uploadSpy).not.toHaveBeenCalled();
|
||||
expect(localStorage.getItem(storageKey)).toBeNull();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("does not resurrect attachments after resetAttachments fires", async () => {
|
||||
const uploadSpy = vi.spyOn(API.experimental, "uploadChatFile");
|
||||
const resize = await import("../utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolveFn) => {
|
||||
releaseResize = resolveFn;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "anthropic" }),
|
||||
);
|
||||
const original = makeOversizeImage();
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([original]);
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(1);
|
||||
|
||||
act(() => {
|
||||
result.current.resetAttachments();
|
||||
});
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
|
||||
const replacement = new File(
|
||||
[new Uint8Array(1 * 1024 * 1024)],
|
||||
"photo.webp",
|
||||
{ type: "image/webp" },
|
||||
);
|
||||
await act(async () => {
|
||||
releaseResize(replacement);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.attachments).toHaveLength(0);
|
||||
expect(uploadSpy).not.toHaveBeenCalled();
|
||||
expect(localStorage.getItem(storageKey)).toBeNull();
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("freezes the provider snapshot at attach time so a mid-resize provider switch can't mislabel the error", async () => {
|
||||
const resize = await import("../utils/resizeImage");
|
||||
let releaseResize: (value: File | null) => void = () => undefined;
|
||||
vi.spyOn(resize, "resizeImageToMaxBytes").mockImplementation(
|
||||
() =>
|
||||
new Promise<File | null>((resolveFn) => {
|
||||
releaseResize = resolveFn;
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, rerender, unmount } = renderHook(
|
||||
({ provider }) => useChatDraftAttachments(orgID, chatID, { provider }),
|
||||
{ initialProps: { provider: "anthropic" } },
|
||||
);
|
||||
|
||||
// Over Anthropic's 5 MiB but under OpenAI's 10 MiB.
|
||||
const gif = new File([new Uint8Array(8)], "animated.gif", {
|
||||
type: "image/gif",
|
||||
lastModified: 400,
|
||||
});
|
||||
Object.defineProperty(gif, "size", { value: 6 * 1024 * 1024 });
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([gif]);
|
||||
});
|
||||
|
||||
rerender({ provider: "openai" });
|
||||
|
||||
await act(async () => {
|
||||
releaseResize(null);
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// Error must name the provider whose budget rejected
|
||||
// the file at attach time, not the live provider.
|
||||
await vi.waitFor(() => {
|
||||
const state = result.current.uploadStates.get(gif);
|
||||
expect(state?.status).toBe("error");
|
||||
expect(state?.error).toMatch(/Anthropic/);
|
||||
expect(state?.error).not.toMatch(/OpenAI/);
|
||||
expect(state?.error).toMatch(/under 5\.0 MiB/);
|
||||
});
|
||||
unmount();
|
||||
});
|
||||
|
||||
it("uses the default 10 MiB budget for non-Anthropic providers (no resize for sub-10MiB images)", async () => {
|
||||
const upload = createDeferred<{ id: string }>();
|
||||
const uploadSpy = vi
|
||||
.spyOn(API.experimental, "uploadChatFile")
|
||||
.mockReturnValue(upload.promise);
|
||||
const resize = await import("../utils/resizeImage");
|
||||
const resizeSpy = vi
|
||||
.spyOn(resize, "resizeImageToMaxBytes")
|
||||
.mockResolvedValue(null);
|
||||
|
||||
const { result, unmount } = renderHook(() =>
|
||||
useChatDraftAttachments(orgID, chatID, { provider: "openai" }),
|
||||
);
|
||||
// 7 MiB: over Anthropic's 5 MiB but under the default
|
||||
// 10 MiB. OpenAI uploads directly without resize.
|
||||
const file = new File([new Uint8Array(7 * 1024 * 1024)], "medium.png", {
|
||||
type: "image/png",
|
||||
lastModified: 300,
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.handleAttach([file]);
|
||||
});
|
||||
|
||||
expect(resizeSpy).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => {
|
||||
expect(uploadSpy).toHaveBeenCalledTimes(1);
|
||||
expect(uploadSpy).toHaveBeenCalledWith(file, orgID);
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
upload.resolve({ id: "file-direct" });
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(result.current.uploadStates.get(file)).toMatchObject({
|
||||
status: "uploaded",
|
||||
fileId: "file-direct",
|
||||
});
|
||||
});
|
||||
unmount();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { API } from "#/api/api";
|
||||
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
|
||||
import type { UploadState } from "../components/AgentChatInput";
|
||||
import { getChatFileURL } from "../utils/chatAttachments";
|
||||
import {
|
||||
@@ -13,9 +14,14 @@ import {
|
||||
import {
|
||||
formatAgentAttachmentTooLargeError,
|
||||
formatAgentAttachmentUploadError,
|
||||
maxAgentAttachmentSize,
|
||||
readAgentAttachmentText,
|
||||
} from "../utils/fileAttachmentLimits";
|
||||
import {
|
||||
imageBudgetForProvider,
|
||||
imageNeedsResize,
|
||||
providerBudgetError,
|
||||
} from "../utils/imageBudget";
|
||||
import { resizeImageToMaxBytes } from "../utils/resizeImage";
|
||||
|
||||
const maxTextPreviewSize = 1024 * 1024;
|
||||
|
||||
@@ -486,6 +492,7 @@ const queueTextContentReads = (
|
||||
export function useChatDraftAttachments(
|
||||
organizationId: string | undefined,
|
||||
chatId: string | undefined,
|
||||
options?: { provider?: string },
|
||||
) {
|
||||
const [views, setViews] = useState(() =>
|
||||
hydrateViews(organizationId, chatId),
|
||||
@@ -493,6 +500,18 @@ export function useChatDraftAttachments(
|
||||
const viewsRef = useRef(views);
|
||||
const subscriptionsRef = useRef(new Map<string, () => void>());
|
||||
const scopeRef = useRef(getDraftScopeKey(organizationId, chatId));
|
||||
// providerRef lets event-driven handlers (paste/drop) see the
|
||||
// latest model selection without rebuilding handleAttach. The
|
||||
// effect-based write keeps React Compiler happy.
|
||||
const provider = options?.provider;
|
||||
const providerRef = useRef(provider);
|
||||
useEffect(() => {
|
||||
providerRef.current = provider;
|
||||
}, [provider]);
|
||||
// clientIds whose resize is in flight but the user removed the
|
||||
// attachment (or the chat scope changed). processResize checks
|
||||
// this before swapping in a replacement.
|
||||
const abandonedResizesRef = useRef<Set<string>>(new Set());
|
||||
const [subscriber] = useState<UploadRegistrySubscriber>(
|
||||
() =>
|
||||
function handleUploadRegistrySnapshot(snapshot: UploadRegistrySnapshot) {
|
||||
@@ -518,6 +537,11 @@ export function useChatDraftAttachments(
|
||||
const scopeKey = getDraftScopeKey(organizationId, chatId);
|
||||
scopeRef.current = scopeKey;
|
||||
unsubscribeAllEntries(subscriptionsRef);
|
||||
// Abandon in-flight resizes from the previous scope so
|
||||
// their callbacks don't register uploads in the new scope.
|
||||
for (const view of viewsRef.current) {
|
||||
abandonedResizesRef.current.add(view.clientId);
|
||||
}
|
||||
if (!organizationId || !chatId || !scopeKey) {
|
||||
setViews([]);
|
||||
return;
|
||||
@@ -560,9 +584,145 @@ export function useChatDraftAttachments(
|
||||
}
|
||||
}, [organizationId, chatId, subscriber]);
|
||||
|
||||
// The view enters in "processing" status from handleAttach;
|
||||
// processResize either swaps in the smaller file and registers
|
||||
// the upload, or surfaces a too-large error.
|
||||
const processResize = async (
|
||||
clientId: string,
|
||||
original: File,
|
||||
budget: number,
|
||||
// Pinned at attach time so a mid-resize provider switch
|
||||
// can't mislabel the error with the new provider's name.
|
||||
providerSnapshot: string | undefined,
|
||||
) => {
|
||||
let resized: File | null = null;
|
||||
try {
|
||||
resized = await resizeImageToMaxBytes(original, budget);
|
||||
} catch {
|
||||
resized = null;
|
||||
}
|
||||
|
||||
if (abandonedResizesRef.current.has(clientId)) {
|
||||
return;
|
||||
}
|
||||
if (!organizationId || !chatId) {
|
||||
return;
|
||||
}
|
||||
const scopeKey = getDraftScopeKey(organizationId, chatId);
|
||||
if (!scopeKey || scopeRef.current !== scopeKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const replacement = resized ?? original;
|
||||
const replaced = replacement !== original;
|
||||
|
||||
// Resize failed entirely or couldn't shrink enough; show
|
||||
// the too-large error instead of uploading and 413-ing.
|
||||
if (replacement.size > MaxChatFileSizeBytes) {
|
||||
setViews((prev) =>
|
||||
prev.map((view) => {
|
||||
if (view.clientId !== clientId) {
|
||||
return view;
|
||||
}
|
||||
if (replaced) {
|
||||
revokeBlobPreview(view);
|
||||
}
|
||||
const previewState = replaced
|
||||
? computePreview(replacement, "error")
|
||||
: {
|
||||
previewUrl: view.previewUrl,
|
||||
previewUrlKind: view.previewUrlKind,
|
||||
};
|
||||
return {
|
||||
...view,
|
||||
file: replacement,
|
||||
status: "error",
|
||||
error: formatAgentAttachmentTooLargeError(replacement.size),
|
||||
previewUrl: previewState.previewUrl,
|
||||
previewUrlKind: previewState.previewUrlKind,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Replacement is still over the provider budget (e.g.
|
||||
// animated GIF on Anthropic that we don't re-encode).
|
||||
// Surface the error at attach time rather than letting
|
||||
// the server backstop reject only at send time.
|
||||
if (replacement.type.startsWith("image/") && replacement.size > budget) {
|
||||
setViews((prev) =>
|
||||
prev.map((view) => {
|
||||
if (view.clientId !== clientId) {
|
||||
return view;
|
||||
}
|
||||
if (replaced) {
|
||||
revokeBlobPreview(view);
|
||||
}
|
||||
const previewState = replaced
|
||||
? computePreview(replacement, "error")
|
||||
: {
|
||||
previewUrl: view.previewUrl,
|
||||
previewUrlKind: view.previewUrlKind,
|
||||
};
|
||||
return {
|
||||
...view,
|
||||
file: replacement,
|
||||
status: "error",
|
||||
error: providerBudgetError(
|
||||
providerSnapshot,
|
||||
replacement.size,
|
||||
budget,
|
||||
),
|
||||
previewUrl: previewState.previewUrl,
|
||||
previewUrlKind: previewState.previewUrlKind,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// beginUpload below drives the view from pending to uploading
|
||||
// via subscribers; we set "pending" here so the registry's
|
||||
// initial snapshot doesn't overwrite our blob preview.
|
||||
setViews((prev) =>
|
||||
prev.map((view) => {
|
||||
if (view.clientId !== clientId) {
|
||||
return view;
|
||||
}
|
||||
if (!replaced) {
|
||||
return { ...view, status: "pending" };
|
||||
}
|
||||
revokeBlobPreview(view);
|
||||
const nextPreview = computePreview(replacement, "pending");
|
||||
return {
|
||||
...view,
|
||||
file: replacement,
|
||||
status: "pending",
|
||||
previewUrl: nextPreview.previewUrl,
|
||||
previewUrlKind: nextPreview.previewUrlKind,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const entry = createRegistryEntry(
|
||||
clientId,
|
||||
organizationId,
|
||||
chatId,
|
||||
replacement,
|
||||
);
|
||||
subscribeToEntry(entry, subscriptionsRef, subscriber);
|
||||
beginUpload(entry);
|
||||
};
|
||||
|
||||
const handleAttach = (files: File[]) => {
|
||||
const scopeKey = getDraftScopeKey(organizationId, chatId);
|
||||
// Snapshot provider + budget so a mid-resize switch
|
||||
// can't relabel the error with the new provider.
|
||||
const providerSnapshot = providerRef.current;
|
||||
const budget = imageBudgetForProvider(providerSnapshot);
|
||||
const entriesToStart: UploadRegistryEntry[] = [];
|
||||
const resizeJobs: Array<{ clientId: string; file: File }> = [];
|
||||
const nextViews: DraftAttachmentView[] = [];
|
||||
for (const file of files) {
|
||||
const clientId = createClientId();
|
||||
@@ -571,7 +731,11 @@ export function useChatDraftAttachments(
|
||||
file,
|
||||
status: "pending",
|
||||
};
|
||||
if (file.size > maxAgentAttachmentSize) {
|
||||
const needsResize = imageNeedsResize(file, budget);
|
||||
// Non-image files over the upload cap are rejected.
|
||||
// Oversized images take the resize pipeline regardless;
|
||||
// the post-resize check validates the result.
|
||||
if (file.size > MaxChatFileSizeBytes && !needsResize) {
|
||||
nextViews.push({
|
||||
...baseView,
|
||||
status: "error",
|
||||
@@ -587,6 +751,18 @@ export function useChatDraftAttachments(
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (needsResize) {
|
||||
// Commit synchronously with "processing" so the
|
||||
// send gate blocks dispatch until resize finishes.
|
||||
const view = {
|
||||
...baseView,
|
||||
status: "processing" as const,
|
||||
...computePreview(file, "processing"),
|
||||
};
|
||||
nextViews.push(view);
|
||||
resizeJobs.push({ clientId, file });
|
||||
continue;
|
||||
}
|
||||
const view = { ...baseView, ...computePreview(file, "pending") };
|
||||
const entry = createRegistryEntry(clientId, organizationId, chatId, file);
|
||||
subscribeToEntry(entry, subscriptionsRef, subscriber);
|
||||
@@ -602,6 +778,11 @@ export function useChatDraftAttachments(
|
||||
setViews,
|
||||
() => scopeRef.current === scopeKey,
|
||||
);
|
||||
// processResize re-checks abandonment + scope before
|
||||
// mutating state, so it's safe to fire-and-forget.
|
||||
for (const job of resizeJobs) {
|
||||
void processResize(job.clientId, job.file, budget, providerSnapshot);
|
||||
}
|
||||
};
|
||||
|
||||
const handleRemoveAttachment = (attachment: number | File) => {
|
||||
@@ -613,6 +794,9 @@ export function useChatDraftAttachments(
|
||||
if (!removed) {
|
||||
return;
|
||||
}
|
||||
// In-flight resize would otherwise swap in a replacement
|
||||
// after the clear below.
|
||||
abandonedResizesRef.current.add(removed.clientId);
|
||||
if (organizationId && chatId) {
|
||||
removeChatDraftAttachmentRecord(organizationId, chatId, removed.clientId);
|
||||
}
|
||||
@@ -629,6 +813,12 @@ export function useChatDraftAttachments(
|
||||
};
|
||||
|
||||
const resetAttachments = () => {
|
||||
// Abandon all in-flight resizes so they don't swap a
|
||||
// replacement back in (which would also re-call beginUpload
|
||||
// against the now-stale scope).
|
||||
for (const view of viewsRef.current) {
|
||||
abandonedResizesRef.current.add(view.clientId);
|
||||
}
|
||||
if (!organizationId || !chatId) {
|
||||
setViews([]);
|
||||
return;
|
||||
|
||||
@@ -3,17 +3,23 @@ import {
|
||||
type SetStateAction,
|
||||
useEffect,
|
||||
useEffectEvent,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { API } from "#/api/api";
|
||||
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
|
||||
import type { UploadState } from "../components/AgentChatInput";
|
||||
import { getChatFileURL } from "../utils/chatAttachments";
|
||||
import {
|
||||
formatAgentAttachmentTooLargeError,
|
||||
formatAgentAttachmentUploadError,
|
||||
maxAgentAttachmentSize,
|
||||
readAgentAttachmentText,
|
||||
} from "../utils/fileAttachmentLimits";
|
||||
import {
|
||||
imageBudgetForProvider,
|
||||
imageNeedsResize,
|
||||
providerBudgetError,
|
||||
} from "../utils/imageBudget";
|
||||
import { resizeImageToMaxBytes } from "../utils/resizeImage";
|
||||
|
||||
/** @internal Exported for testing. */
|
||||
export const persistedAttachmentsStorageKey = "agents.persisted-attachments";
|
||||
@@ -43,10 +49,9 @@ function restorePersistedAttachments(currentOrgId: string): {
|
||||
uploadStates: Map<File, UploadState>;
|
||||
previewUrls: Map<File, string>;
|
||||
} {
|
||||
// When the org ID is not yet known (e.g. still loading), skip
|
||||
// restoration entirely so we don't accidentally prune valid
|
||||
// entries. The initializer only runs once, so the caller must
|
||||
// ensure the org ID is available before mounting the hook.
|
||||
// Skip when org ID isn't loaded yet so we don't prune valid
|
||||
// entries. The initializer runs once, so callers must wait for
|
||||
// the org ID before mounting.
|
||||
if (!currentOrgId) {
|
||||
return {
|
||||
attachments: [],
|
||||
@@ -66,7 +71,6 @@ function restorePersistedAttachments(currentOrgId: string): {
|
||||
const persisted: PersistedAttachment[] = JSON.parse(stored);
|
||||
const matched = persisted.filter((p) => p.organizationId === currentOrgId);
|
||||
|
||||
// Prune entries that don't match the current org.
|
||||
if (matched.length !== persisted.length) {
|
||||
if (matched.length > 0) {
|
||||
localStorage.setItem(
|
||||
@@ -84,9 +88,8 @@ function restorePersistedAttachments(currentOrgId: string): {
|
||||
|
||||
for (const p of matched) {
|
||||
if (!p.fileId || !p.fileName) continue;
|
||||
// Synthetic File used as a Map key only. Its content is
|
||||
// never read because the existing file_id is reused at
|
||||
// send time.
|
||||
// Synthetic File used as a Map key only; the existing
|
||||
// file_id is reused at send time.
|
||||
const file = new File([], p.fileName, {
|
||||
type: p.fileType,
|
||||
lastModified: p.lastModified,
|
||||
@@ -173,12 +176,19 @@ interface UseFileAttachmentsReturn {
|
||||
|
||||
export function useFileAttachments(
|
||||
organizationId: string | undefined,
|
||||
options?: { persist?: boolean },
|
||||
options?: { persist?: boolean; provider?: string },
|
||||
): UseFileAttachmentsReturn {
|
||||
const persist = options?.persist ?? false;
|
||||
|
||||
// Restore previously uploaded attachments from localStorage
|
||||
// when persistence is enabled. Computed once on first render.
|
||||
// providerRef lets event-driven handlers (paste/drop) see the
|
||||
// latest model selection without rebuilding handleAttach. The
|
||||
// effect-based write keeps React Compiler happy.
|
||||
const provider = options?.provider;
|
||||
const providerRef = useRef(provider);
|
||||
useEffect(() => {
|
||||
providerRef.current = provider;
|
||||
}, [provider]);
|
||||
|
||||
const [restored] = useState(() =>
|
||||
persist
|
||||
? restorePersistedAttachments(organizationId ?? "")
|
||||
@@ -237,11 +247,10 @@ export function useFileAttachments(
|
||||
if (shouldPersist) {
|
||||
addPersistedAttachment(file, result.id, organizationId!);
|
||||
}
|
||||
// Pre-warm the browser HTTP cache for images so the
|
||||
// timeline can render them instantly after send. We
|
||||
// intentionally skip text attachments because the
|
||||
// composer already has the text content locally.
|
||||
if (isImage) {
|
||||
// Pre-warm the HTTP cache so the timeline can
|
||||
// render the image instantly after send. Text
|
||||
// content is already local in the composer.
|
||||
void fetch(getChatFileURL(result.id));
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
@@ -256,21 +265,148 @@ export function useFileAttachments(
|
||||
})();
|
||||
};
|
||||
|
||||
// Files removed while their resize is in flight. processResizes
|
||||
// checks this before swapping in a replacement so a dismissed
|
||||
// file can't be resurrected. WeakSet lets entries get GC'd.
|
||||
const abandonedResizesRef = useRef<WeakSet<File>>(new WeakSet());
|
||||
|
||||
type AttachItem = { file: File; needsResize: boolean };
|
||||
|
||||
const processResizes = async (
|
||||
items: readonly AttachItem[],
|
||||
budget: number,
|
||||
// Pinned at attach time so a mid-resize provider switch
|
||||
// can't mislabel the error with the new provider's name.
|
||||
providerSnapshot: string | undefined,
|
||||
) => {
|
||||
// Sequential so each swap commits before the next starts;
|
||||
// resizeImageToMaxBytes already serializes decode work.
|
||||
for (const { file: original, needsResize } of items) {
|
||||
if (!needsResize) continue;
|
||||
let resized: File | null = null;
|
||||
try {
|
||||
resized = await resizeImageToMaxBytes(original, budget);
|
||||
} catch {
|
||||
resized = null;
|
||||
}
|
||||
|
||||
// Skip if the user removed this attachment while
|
||||
// resizing; updates here would resurrect it.
|
||||
if (abandonedResizesRef.current.has(original)) {
|
||||
continue;
|
||||
}
|
||||
const replacement = resized ?? original;
|
||||
const replaced = replacement !== original;
|
||||
|
||||
// Functional updaters: if a racing removal cleared the
|
||||
// original, every updater below becomes a no-op.
|
||||
setAttachments((prev) => {
|
||||
const idx = prev.indexOf(original);
|
||||
if (idx === -1 || !replaced) return prev;
|
||||
const next = prev.slice();
|
||||
next[idx] = replacement;
|
||||
return next;
|
||||
});
|
||||
setPreviewUrls((prev) => {
|
||||
// Skip when no replacement happened so we don't
|
||||
// revoke the original's still-in-use blob URL.
|
||||
if (!prev.has(original) || !replaced) return prev;
|
||||
const next = new Map(prev);
|
||||
const oldUrl = next.get(original);
|
||||
if (oldUrl?.startsWith("blob:")) URL.revokeObjectURL(oldUrl);
|
||||
next.delete(original);
|
||||
if (replacement.type !== "text/plain") {
|
||||
next.set(replacement, URL.createObjectURL(replacement));
|
||||
}
|
||||
return next;
|
||||
});
|
||||
setUploadStates((prev) => {
|
||||
// Skip when no replacement: startUpload below
|
||||
// overwrites "processing" with "uploading".
|
||||
if (!prev.has(original) || !replaced) return prev;
|
||||
const next = new Map(prev);
|
||||
next.delete(original);
|
||||
return next;
|
||||
});
|
||||
|
||||
// Resize failed and the original still exceeds the
|
||||
// server cap; show the too-large error instead of
|
||||
// kicking off an upload that will 413.
|
||||
if (replacement.size > MaxChatFileSizeBytes) {
|
||||
setUploadStates((prev) =>
|
||||
new Map(prev).set(replacement, {
|
||||
status: "error" as const,
|
||||
error: formatAgentAttachmentTooLargeError(replacement.size),
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Replacement is still over the provider budget (e.g.
|
||||
// animated GIF on Anthropic that we don't re-encode).
|
||||
// Surface the error at attach time rather than letting
|
||||
// the server backstop reject only at send time.
|
||||
if (replacement.type.startsWith("image/") && replacement.size > budget) {
|
||||
setUploadStates((prev) =>
|
||||
new Map(prev).set(replacement, {
|
||||
status: "error" as const,
|
||||
error: providerBudgetError(
|
||||
providerSnapshot,
|
||||
replacement.size,
|
||||
budget,
|
||||
),
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
startUpload(replacement);
|
||||
}
|
||||
};
|
||||
|
||||
const handleAttach = (files: File[]) => {
|
||||
// Originals enter state with a "processing" status so the
|
||||
// send gate blocks dispatch until processResizes finishes.
|
||||
// Snapshot provider + budget so a mid-resize switch can't
|
||||
// relabel the error with the new provider.
|
||||
const providerSnapshot = providerRef.current;
|
||||
const budget = imageBudgetForProvider(providerSnapshot);
|
||||
const items: AttachItem[] = files.map((file) => ({
|
||||
file,
|
||||
needsResize: imageNeedsResize(file, budget),
|
||||
}));
|
||||
|
||||
setAttachments((prev) => [...prev, ...files]);
|
||||
setPreviewUrls((prev) => {
|
||||
const next = new Map(prev);
|
||||
for (const file of files) {
|
||||
for (const { file } of items) {
|
||||
if (file.type !== "text/plain") {
|
||||
next.set(file, URL.createObjectURL(file));
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
// Read text content for preview, but skip oversized files.
|
||||
for (const file of files) {
|
||||
if (file.type === "text/plain" && file.size <= maxAgentAttachmentSize) {
|
||||
void readAgentAttachmentText(file)
|
||||
setUploadStates((prev) => {
|
||||
const next = new Map(prev);
|
||||
for (const { file, needsResize } of items) {
|
||||
if (file.size > MaxChatFileSizeBytes && !needsResize) {
|
||||
next.set(file, {
|
||||
status: "error" as const,
|
||||
error: formatAgentAttachmentTooLargeError(file.size),
|
||||
});
|
||||
} else if (needsResize) {
|
||||
next.set(file, { status: "processing" });
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
|
||||
for (const { file } of items) {
|
||||
if (file.type === "text/plain" && file.size <= MaxChatFileSizeBytes) {
|
||||
// Some test environments lack File.prototype.text.
|
||||
const readText =
|
||||
typeof file.text === "function"
|
||||
? file.text()
|
||||
: new Response(file).text();
|
||||
void readText
|
||||
.then((content) => {
|
||||
setTextContents((prev) => {
|
||||
const next = new Map(prev);
|
||||
@@ -283,30 +419,30 @@ export function useFileAttachments(
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const file of files) {
|
||||
if (file.size > maxAgentAttachmentSize) {
|
||||
setUploadStates((prev) =>
|
||||
new Map(prev).set(file, {
|
||||
status: "error" as const,
|
||||
error: formatAgentAttachmentTooLargeError(file.size),
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
startUpload(file);
|
||||
}
|
||||
|
||||
for (const { file, needsResize } of items) {
|
||||
if (needsResize) continue;
|
||||
if (file.size > MaxChatFileSizeBytes) continue; // already marked as error above
|
||||
startUpload(file);
|
||||
}
|
||||
|
||||
void processResizes(items, budget, providerSnapshot);
|
||||
};
|
||||
|
||||
const handleRemoveAttachment = (attachment: number | File) => {
|
||||
// Resolve the file to remove and perform localStorage side
|
||||
// effects before entering state updaters. React may call
|
||||
// updaters more than once (StrictMode, React Compiler), so
|
||||
// they must stay pure.
|
||||
// Side effects (localStorage, abandonment) happen here;
|
||||
// React may call updaters multiple times under StrictMode
|
||||
// or React Compiler, so they must stay pure.
|
||||
const idx =
|
||||
typeof attachment === "number"
|
||||
? attachment
|
||||
: attachments.indexOf(attachment);
|
||||
const removed = idx >= 0 ? attachments[idx] : undefined;
|
||||
if (removed) {
|
||||
// In-flight resize would otherwise resurrect this file
|
||||
// by swapping in a replacement after the clear below.
|
||||
abandonedResizesRef.current.add(removed);
|
||||
}
|
||||
if (persist && removed) {
|
||||
const state = uploadStates.get(removed);
|
||||
if (state?.status === "uploaded" && state.fileId) {
|
||||
@@ -344,6 +480,12 @@ export function useFileAttachments(
|
||||
};
|
||||
|
||||
const resetAttachments = () => {
|
||||
// Abandon all in-flight resizes so they don't swap a
|
||||
// replacement back in (which would also re-call startUpload
|
||||
// against the now-stale scope).
|
||||
for (const file of attachments) {
|
||||
abandonedResizesRef.current.add(file);
|
||||
}
|
||||
for (const [, url] of previewUrls) {
|
||||
if (url.startsWith("blob:")) URL.revokeObjectURL(url);
|
||||
}
|
||||
@@ -365,9 +507,9 @@ export function useFileAttachments(
|
||||
handleRemoveAttachment,
|
||||
startUpload,
|
||||
resetAttachments,
|
||||
// Raw setters exposed for ChatPageContent to pre-populate
|
||||
// attachments from existing chat messages. These bypass
|
||||
// localStorage persistence. Only use when persist is false.
|
||||
// Raw setters bypass localStorage persistence; only use
|
||||
// when persist is false (e.g. ChatPageContent pre-populating
|
||||
// attachments from existing chat messages).
|
||||
setAttachments,
|
||||
setPreviewUrls,
|
||||
setUploadStates,
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { getErrorDetail, getErrorMessage } from "#/api/errors";
|
||||
|
||||
export const maxAgentAttachmentSize = 10 * 1024 * 1024;
|
||||
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
|
||||
|
||||
export const formatAgentAttachmentTooLargeError = (fileSize: number): string =>
|
||||
`File too large (${(fileSize / 1024 / 1024).toFixed(1)} MB). Maximum is ${maxAgentAttachmentSize / 1024 / 1024} MB.`;
|
||||
`File too large (${(fileSize / 1024 / 1024).toFixed(1)} MiB). Maximum is ${MaxChatFileSizeBytes / 1024 / 1024} MiB.`;
|
||||
|
||||
export const formatAgentAttachmentUploadError = (error: unknown): string => {
|
||||
const message = getErrorMessage(error, "Upload failed");
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MaxChatFileSizeBytes } from "#/api/typesGenerated";
|
||||
import {
|
||||
formatMiB,
|
||||
imageBudgetForProvider,
|
||||
imageNeedsResize,
|
||||
providerBudgetError,
|
||||
} from "./imageBudget";
|
||||
|
||||
const ANTHROPIC_BUDGET = 5 * 1024 * 1024 - 16 * 1024;
|
||||
const DEFAULT_BUDGET = MaxChatFileSizeBytes - 16 * 1024;
|
||||
|
||||
describe("imageBudgetForProvider", () => {
|
||||
it("returns the Anthropic budget for direct Anthropic", () => {
|
||||
expect(imageBudgetForProvider("anthropic")).toBe(ANTHROPIC_BUDGET);
|
||||
});
|
||||
|
||||
it("returns the Anthropic budget for Bedrock (Anthropic-compatible)", () => {
|
||||
expect(imageBudgetForProvider("bedrock")).toBe(ANTHROPIC_BUDGET);
|
||||
});
|
||||
|
||||
it("returns the default budget for OpenAI", () => {
|
||||
expect(imageBudgetForProvider("openai")).toBe(DEFAULT_BUDGET);
|
||||
});
|
||||
|
||||
it("returns the default budget for unknown providers", () => {
|
||||
expect(imageBudgetForProvider("brand-new-provider")).toBe(DEFAULT_BUDGET);
|
||||
});
|
||||
|
||||
it("returns the default budget when provider is undefined", () => {
|
||||
expect(imageBudgetForProvider(undefined)).toBe(DEFAULT_BUDGET);
|
||||
});
|
||||
|
||||
// Mirrors server-side chatprovider.NormalizeProvider so a
|
||||
// caller passing a case/whitespace variant gets the strict
|
||||
// budget instead of silently falling through to the default.
|
||||
it.each([
|
||||
"Anthropic",
|
||||
"ANTHROPIC",
|
||||
" anthropic ",
|
||||
"\tanthropic\n",
|
||||
"AnThRoPiC",
|
||||
])("normalizes case/whitespace before matching strict providers (%s)", (input) => {
|
||||
expect(imageBudgetForProvider(input)).toBe(ANTHROPIC_BUDGET);
|
||||
});
|
||||
|
||||
it("normalizes Bedrock variants too", () => {
|
||||
expect(imageBudgetForProvider("Bedrock")).toBe(ANTHROPIC_BUDGET);
|
||||
expect(imageBudgetForProvider(" BEDROCK ")).toBe(ANTHROPIC_BUDGET);
|
||||
});
|
||||
});
|
||||
|
||||
describe("imageNeedsResize", () => {
|
||||
const oversize = (type: string, bytes: number): File => {
|
||||
const f = new File([new Uint8Array(8)], `f.${type.split("/")[1]}`, {
|
||||
type,
|
||||
});
|
||||
Object.defineProperty(f, "size", { value: bytes });
|
||||
return f;
|
||||
};
|
||||
|
||||
it("returns true for an over-budget image", () => {
|
||||
expect(
|
||||
imageNeedsResize(oversize("image/png", 6 * 1024 * 1024), 5 * 1024 * 1024),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for an under-budget image", () => {
|
||||
expect(
|
||||
imageNeedsResize(oversize("image/png", 1 * 1024 * 1024), 5 * 1024 * 1024),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for non-image files even when oversized", () => {
|
||||
expect(
|
||||
imageNeedsResize(
|
||||
oversize("text/plain", 6 * 1024 * 1024),
|
||||
5 * 1024 * 1024,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatMiB", () => {
|
||||
it("renders one decimal place", () => {
|
||||
expect(formatMiB(5 * 1024 * 1024)).toBe("5.0");
|
||||
expect(formatMiB(5 * 1024 * 1024 + 512 * 1024)).toBe("5.5");
|
||||
expect(formatMiB(0)).toBe("0.0");
|
||||
});
|
||||
});
|
||||
|
||||
describe("providerBudgetError", () => {
|
||||
it("uses the provider's display label and MiB units", () => {
|
||||
const message = providerBudgetError(
|
||||
"anthropic",
|
||||
6 * 1024 * 1024,
|
||||
ANTHROPIC_BUDGET,
|
||||
);
|
||||
expect(message).toMatch(/Anthropic/);
|
||||
expect(message).toMatch(/6\.0 MiB/);
|
||||
expect(message).toMatch(/5\.0 MiB/);
|
||||
});
|
||||
|
||||
it("falls back to a generic label when provider is undefined", () => {
|
||||
const message = providerBudgetError(
|
||||
undefined,
|
||||
6 * 1024 * 1024,
|
||||
ANTHROPIC_BUDGET,
|
||||
);
|
||||
expect(message).toMatch(/this provider/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import {
|
||||
AnthropicInlineImageCapBytes,
|
||||
MaxChatFileSizeBytes,
|
||||
} from "#/api/typesGenerated";
|
||||
import { formatProviderLabel } from "./modelOptions";
|
||||
|
||||
// Budgets sit below the wire limits to leave room for encoder framing
|
||||
// overhead, so a file at exactly the budget is still under the
|
||||
// server's hard cap.
|
||||
const FRAMING_MARGIN_BYTES = 16 * 1024;
|
||||
const DEFAULT_IMAGE_BUDGET_BYTES = MaxChatFileSizeBytes - FRAMING_MARGIN_BYTES;
|
||||
const ANTHROPIC_IMAGE_BUDGET_BYTES =
|
||||
AnthropicInlineImageCapBytes - FRAMING_MARGIN_BYTES;
|
||||
|
||||
// Must mirror chatprovider.InlineImageCapBytes on the server.
|
||||
const ANTHROPIC_STRICT_BUDGET_PROVIDERS: ReadonlySet<string> = new Set([
|
||||
"anthropic",
|
||||
"bedrock",
|
||||
]);
|
||||
|
||||
// Inputs are normalized to match chatprovider.NormalizeProvider on
|
||||
// the server, so callers don't have to pre-normalize.
|
||||
export function imageBudgetForProvider(provider: string | undefined): number {
|
||||
const normalized = provider?.trim().toLowerCase();
|
||||
if (normalized && ANTHROPIC_STRICT_BUDGET_PROVIDERS.has(normalized)) {
|
||||
return ANTHROPIC_IMAGE_BUDGET_BYTES;
|
||||
}
|
||||
return DEFAULT_IMAGE_BUDGET_BYTES;
|
||||
}
|
||||
|
||||
export function formatMiB(bytes: number): string {
|
||||
return (bytes / 1024 / 1024).toFixed(1);
|
||||
}
|
||||
|
||||
export function providerBudgetError(
|
||||
provider: string | undefined,
|
||||
actualBytes: number,
|
||||
budgetBytes: number,
|
||||
): string {
|
||||
const label = provider ? formatProviderLabel(provider) : "this provider";
|
||||
return `Image too large for ${label} (${formatMiB(actualBytes)} MiB). Inline images must be under ${formatMiB(budgetBytes)} MiB on this provider.`;
|
||||
}
|
||||
|
||||
export function imageNeedsResize(file: File, budget: number): boolean {
|
||||
return file.type.startsWith("image/") && file.size > budget;
|
||||
}
|
||||
@@ -210,6 +210,16 @@ export const getModelOptionsFromConfigs = (
|
||||
});
|
||||
};
|
||||
|
||||
// getProviderForModelOption returns the provider string for the
|
||||
// currently-selected model option, or undefined when the selection
|
||||
// is not (yet) in the options list. Extracted so resize/budget logic
|
||||
// has one place to resolve provider from the selector state.
|
||||
export const getProviderForModelOption = (
|
||||
modelOptions: readonly ModelSelectorOption[],
|
||||
selectedModel: string,
|
||||
): string | undefined =>
|
||||
modelOptions.find((option) => option.id === selectedModel)?.provider;
|
||||
|
||||
export const formatProviderLabel = (provider: string): string => {
|
||||
const normalized = provider.trim().toLowerCase();
|
||||
switch (normalized) {
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resizeImageToMaxBytes } from "./resizeImage";
|
||||
|
||||
// jsdom (the default vitest environment) does not implement
|
||||
// createImageBitmap / OffscreenCanvas, so the re-encode codepaths
|
||||
// cannot run against real browser decoders. The "with stubbed
|
||||
// decoders" block below installs a deterministic fake so the shrink
|
||||
// loop runs in CI; the "with real decoders" block only runs when
|
||||
// actual browser APIs are available (e.g. a future Playwright-based
|
||||
// vitest project).
|
||||
const canDecodeImages =
|
||||
typeof createImageBitmap === "function" &&
|
||||
typeof OffscreenCanvas === "function";
|
||||
|
||||
const describeIfDecode = canDecodeImages ? describe : describe.skip;
|
||||
|
||||
// Minimum byte count the fake decoder reports for a blob at given
|
||||
// dimensions. Sized so the shrink loop runs a realistic number of
|
||||
// iterations within the module's MAX_SHRINK_ITERATIONS=8 budget.
|
||||
const FAKE_BYTES_PER_PIXEL = 0.5;
|
||||
|
||||
// Returns a fake encoded blob whose size is proportional to (width
|
||||
// * height * quality) so the shrink loop can observe convergence.
|
||||
function fakeEncodedSize(
|
||||
width: number,
|
||||
height: number,
|
||||
quality: number,
|
||||
): number {
|
||||
return Math.max(
|
||||
64,
|
||||
Math.round(width * height * quality * FAKE_BYTES_PER_PIXEL),
|
||||
);
|
||||
}
|
||||
|
||||
describe("resizeImageToMaxBytes", () => {
|
||||
it("returns non-image files unchanged", async () => {
|
||||
const file = new File([new Uint8Array([1, 2, 3])], "notes.txt", {
|
||||
type: "text/plain",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 1024);
|
||||
expect(result).toBe(file);
|
||||
});
|
||||
|
||||
it("returns GIFs unchanged even when oversize", async () => {
|
||||
// Canvas re-encoding flattens animation; we hand the
|
||||
// original back instead.
|
||||
const bytes = new Uint8Array(8 * 1024 * 1024);
|
||||
const file = new File([bytes], "clip.gif", { type: "image/gif" });
|
||||
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
|
||||
expect(result).toBe(file);
|
||||
});
|
||||
|
||||
it("returns the original image unchanged when already under budget", async () => {
|
||||
const bytes = new Uint8Array(1024);
|
||||
const file = new File([bytes], "tiny.png", { type: "image/png" });
|
||||
const result = await resizeImageToMaxBytes(file, 4096);
|
||||
expect(result).toBe(file);
|
||||
});
|
||||
|
||||
it("returns null for an unsupported image MIME that would need resizing", async () => {
|
||||
// Unsupported MIME + over budget: refuse rather than
|
||||
// silently produce garbage.
|
||||
const bytes = new Uint8Array(2 * 1024 * 1024);
|
||||
const file = new File([bytes], "diagram.svg", {
|
||||
type: "image/svg+xml",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("accepts image/jpg alias alongside image/jpeg", async () => {
|
||||
// Pins the non-IANA `image/jpg` alias in RESIZABLE_MIME_TYPES.
|
||||
// The under-budget passthrough is enough to prove acceptance;
|
||||
// the over-budget case in the stubbed-decoder block proves
|
||||
// the encode pipeline runs.
|
||||
const under = new File([new Uint8Array(512)], "icon.jpg", {
|
||||
type: "image/jpg",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(under, 4096);
|
||||
expect(result).toBe(under);
|
||||
});
|
||||
|
||||
it("returns an under-budget unsupported-MIME image unchanged", async () => {
|
||||
// Under-budget unsupported MIMEs pass through; the contract
|
||||
// is "give me something <= maxBytes" and we already have
|
||||
// that.
|
||||
const bytes = new Uint8Array(512);
|
||||
const file = new File([bytes], "icon.bmp", {
|
||||
type: "image/bmp",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 4096);
|
||||
expect(result).toBe(file);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resizeImageToMaxBytes with stubbed decoders", () => {
|
||||
// Each test installs its own fakes; track per-test state on a
|
||||
// shared object so the stubs can read the active configuration.
|
||||
interface StubState {
|
||||
srcWidth: number;
|
||||
srcHeight: number;
|
||||
decodeThrows: boolean;
|
||||
convertBlobType: string;
|
||||
decodeCalls: number;
|
||||
encodeCalls: Array<{ width: number; height: number; quality: number }>;
|
||||
}
|
||||
|
||||
let state: StubState;
|
||||
|
||||
beforeEach(() => {
|
||||
state = {
|
||||
srcWidth: 4096,
|
||||
srcHeight: 4096,
|
||||
decodeThrows: false,
|
||||
convertBlobType: "image/webp",
|
||||
decodeCalls: 0,
|
||||
encodeCalls: [],
|
||||
};
|
||||
|
||||
// Fake createImageBitmap matching the HTML spec output rules:
|
||||
// - both resize dims => stretch (no aspect-ratio preservation).
|
||||
// - only resizeWidth => width exact, height proportional.
|
||||
// UPSCALES if resizeWidth > source width.
|
||||
// - only resizeHeight => mirror of above.
|
||||
// - neither => source dimensions unchanged.
|
||||
//
|
||||
// Critical that the fake doesn't cap at source dimensions:
|
||||
// real browsers follow the spec and upscale, so production
|
||||
// code must handle that. A capped fake would mask the bug.
|
||||
vi.stubGlobal(
|
||||
"createImageBitmap",
|
||||
vi.fn(
|
||||
async (
|
||||
_blob: Blob,
|
||||
options?: {
|
||||
resizeWidth?: number;
|
||||
resizeHeight?: number;
|
||||
},
|
||||
) => {
|
||||
state.decodeCalls++;
|
||||
if (state.decodeThrows) {
|
||||
throw new Error("decode boom");
|
||||
}
|
||||
const srcW = state.srcWidth;
|
||||
const srcH = state.srcHeight;
|
||||
const rW = options?.resizeWidth;
|
||||
const rH = options?.resizeHeight;
|
||||
let w: number;
|
||||
let h: number;
|
||||
if (rW !== undefined && rH !== undefined) {
|
||||
// Spec: stretch-to-fit, no source clamp.
|
||||
w = rW;
|
||||
h = rH;
|
||||
} else if (rW !== undefined) {
|
||||
w = rW;
|
||||
h = Math.max(1, Math.round((srcH * rW) / srcW));
|
||||
} else if (rH !== undefined) {
|
||||
h = rH;
|
||||
w = Math.max(1, Math.round((srcW * rH) / srcH));
|
||||
} else {
|
||||
w = srcW;
|
||||
h = srcH;
|
||||
}
|
||||
return {
|
||||
width: w,
|
||||
height: h,
|
||||
close: vi.fn(),
|
||||
} as unknown as ImageBitmap;
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Fake Image (for probeNaturalDimensions in production
|
||||
// code). jsdom exposes `Image` but does not load blob URLs,
|
||||
// so we stub it with a synthetic implementation that reports
|
||||
// state.srcWidth/Height and fires onload on microtask.
|
||||
class FakeImage {
|
||||
onload: (() => void) | null = null;
|
||||
onerror: (() => void) | null = null;
|
||||
naturalWidth = 0;
|
||||
naturalHeight = 0;
|
||||
private _src = "";
|
||||
get src() {
|
||||
return this._src;
|
||||
}
|
||||
set src(url: string) {
|
||||
this._src = url;
|
||||
queueMicrotask(() => {
|
||||
if (state.decodeThrows) {
|
||||
this.onerror?.();
|
||||
return;
|
||||
}
|
||||
this.naturalWidth = state.srcWidth;
|
||||
this.naturalHeight = state.srcHeight;
|
||||
this.onload?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("Image", FakeImage);
|
||||
|
||||
// convertToBlob size scales with width*height*quality so
|
||||
// the shrink loop converges.
|
||||
class FakeOffscreenCanvas {
|
||||
width: number;
|
||||
height: number;
|
||||
constructor(w: number, h: number) {
|
||||
this.width = w;
|
||||
this.height = h;
|
||||
}
|
||||
getContext() {
|
||||
return {
|
||||
drawImage: () => undefined,
|
||||
};
|
||||
}
|
||||
async convertToBlob(opts?: { quality?: number }): Promise<Blob> {
|
||||
const quality = opts?.quality ?? 1;
|
||||
state.encodeCalls.push({
|
||||
width: this.width,
|
||||
height: this.height,
|
||||
quality,
|
||||
});
|
||||
const size = fakeEncodedSize(this.width, this.height, quality);
|
||||
return new Blob([new Uint8Array(size)], {
|
||||
type: state.convertBlobType,
|
||||
});
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("re-encodes an oversized image down to the requested byte budget", async () => {
|
||||
// Forces at least a few shrink iterations before convergence.
|
||||
const file = new File([new Uint8Array(6 * 1024 * 1024)], "big.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
const budget = 512 * 1024;
|
||||
const result = await resizeImageToMaxBytes(file, budget);
|
||||
expect(result).not.toBeNull();
|
||||
if (!result) return;
|
||||
expect(result.size).toBeLessThanOrEqual(budget);
|
||||
expect(result.type).toBe("image/webp");
|
||||
expect(result.name.endsWith(".webp")).toBe(true);
|
||||
|
||||
expect(state.encodeCalls.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("decoder never stretches non-square sources", async () => {
|
||||
// 4:1 source with long axis above MAX_INITIAL_DIMENSION.
|
||||
// If decodeToBitmap passes both resize dims, the spec
|
||||
// would force 8192x8192 output and the ratio assertion
|
||||
// would fail.
|
||||
state.srcWidth = 16_000;
|
||||
state.srcHeight = 4_000;
|
||||
const file = new File([new Uint8Array(6 * 1024 * 1024)], "wide.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
await resizeImageToMaxBytes(file, 32 * 1024);
|
||||
expect(state.encodeCalls.length).toBeGreaterThan(0);
|
||||
const first = state.encodeCalls[0];
|
||||
const ratio = first.width / first.height;
|
||||
expect(ratio).toBeGreaterThanOrEqual(4 - 0.05);
|
||||
expect(ratio).toBeLessThanOrEqual(4 + 0.05);
|
||||
expect(first.width).toBeLessThanOrEqual(8192);
|
||||
expect(first.height).toBeLessThanOrEqual(8192);
|
||||
});
|
||||
|
||||
it("keeps the bitmap within MAX_INITIAL_DIMENSION on both axes for extreme portraits", async () => {
|
||||
// Regression: passing resizeWidth: MAX on a 2000x60000
|
||||
// source would upscale to 8192x245760, blowing past
|
||||
// Chromium's ~268M pixel limit. Probe must pick
|
||||
// resizeHeight only.
|
||||
state.srcWidth = 2000;
|
||||
state.srcHeight = 60_000;
|
||||
const file = new File([new Uint8Array(6 * 1024 * 1024)], "tall.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
await resizeImageToMaxBytes(file, 64 * 1024);
|
||||
for (const call of state.encodeCalls) {
|
||||
expect(call.width).toBeLessThanOrEqual(8192);
|
||||
expect(call.height).toBeLessThanOrEqual(8192);
|
||||
}
|
||||
const first = state.encodeCalls[0];
|
||||
const sourceRatio = 2000 / 60_000;
|
||||
const bitmapRatio = first.width / first.height;
|
||||
expect(bitmapRatio).toBeGreaterThan(sourceRatio * 0.99);
|
||||
expect(bitmapRatio).toBeLessThan(sourceRatio * 1.01);
|
||||
});
|
||||
|
||||
it("stays within MAX_INITIAL_DIMENSION when source exceeds it on both axes", async () => {
|
||||
state.srcWidth = 60_000;
|
||||
state.srcHeight = 40_000;
|
||||
const file = new File([new Uint8Array(6 * 1024 * 1024)], "huge.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
await resizeImageToMaxBytes(file, 256 * 1024);
|
||||
for (const call of state.encodeCalls) {
|
||||
expect(call.width).toBeLessThanOrEqual(8192);
|
||||
expect(call.height).toBeLessThanOrEqual(8192);
|
||||
}
|
||||
});
|
||||
|
||||
it("skips createImageBitmap resize options entirely when source is already under clamp", async () => {
|
||||
// Regression: passing resizeWidth: MAX on a 1920x1080
|
||||
// source would upscale to 8192x4608 (spec: output width
|
||||
// is exactly resizeWidth). Probe must skip resize options
|
||||
// when the source already fits.
|
||||
state.srcWidth = 1920;
|
||||
state.srcHeight = 1080;
|
||||
const file = new File([new Uint8Array(6 * 1024 * 1024)], "shot.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
await resizeImageToMaxBytes(file, 256 * 1024);
|
||||
const first = state.encodeCalls[0];
|
||||
expect(first.width).toBe(1920);
|
||||
expect(first.height).toBe(1080);
|
||||
});
|
||||
|
||||
it("tries the fallback quality pass when shrink iterations saturate", async () => {
|
||||
// Tiny source + unreachable 1-byte budget exhausts the main
|
||||
// loop and forces FALLBACK_QUALITY (0.7).
|
||||
state.srcWidth = 64;
|
||||
state.srcHeight = 64;
|
||||
const file = new File([new Uint8Array(1024 * 1024)], "tiny.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 1);
|
||||
expect(result).toBeNull();
|
||||
const last = state.encodeCalls[state.encodeCalls.length - 1];
|
||||
expect(last.quality).toBeCloseTo(0.7, 5);
|
||||
});
|
||||
|
||||
it("returns null (does not throw) when decode fails", async () => {
|
||||
state.decodeThrows = true;
|
||||
const file = new File([new Uint8Array(1024 * 1024)], "broken.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 4096);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("fake createImageBitmap matches HTML spec output-dimension rules", async () => {
|
||||
// Pins fake createImageBitmap behavior: stretch with both
|
||||
// dims, proportional scale with one, including upscale
|
||||
// when a resize dim exceeds the source. A fake that capped
|
||||
// at source dimensions or scaled uniformly would mask real
|
||||
// decoder bugs in production code.
|
||||
state.srcWidth = 4000;
|
||||
state.srcHeight = 1000;
|
||||
const blob = new Blob([new Uint8Array(8)], { type: "image/png" });
|
||||
const createBitmap = (
|
||||
globalThis as unknown as {
|
||||
createImageBitmap: (
|
||||
blob: Blob,
|
||||
opts?: { resizeWidth?: number; resizeHeight?: number },
|
||||
) => Promise<ImageBitmap>;
|
||||
}
|
||||
).createImageBitmap;
|
||||
|
||||
// Stretch.
|
||||
const stretched = await createBitmap(blob, {
|
||||
resizeWidth: 800,
|
||||
resizeHeight: 800,
|
||||
});
|
||||
expect(stretched.width).toBe(800);
|
||||
expect(stretched.height).toBe(800);
|
||||
|
||||
// Downscale by width.
|
||||
const downWidth = await createBitmap(blob, { resizeWidth: 800 });
|
||||
expect(downWidth.width).toBe(800);
|
||||
expect(downWidth.height).toBe(200);
|
||||
|
||||
// Downscale by height.
|
||||
const downHeight = await createBitmap(blob, { resizeHeight: 200 });
|
||||
expect(downHeight.height).toBe(200);
|
||||
expect(downHeight.width).toBe(800);
|
||||
|
||||
// Upscale by width: spec allows; a capped fake would
|
||||
// report (4000, 1000) and mask production decoder bugs.
|
||||
const upWidth = await createBitmap(blob, { resizeWidth: 8000 });
|
||||
expect(upWidth.width).toBe(8000);
|
||||
expect(upWidth.height).toBe(2000);
|
||||
|
||||
// Natural.
|
||||
const natural = await createBitmap(blob);
|
||||
expect(natural.width).toBe(4000);
|
||||
expect(natural.height).toBe(1000);
|
||||
});
|
||||
|
||||
it("keeps the File type honest when the encoder falls back to PNG", async () => {
|
||||
// Some browsers without WebP encode return a PNG; the
|
||||
// File's labelled type must match the actual content.
|
||||
state.convertBlobType = "image/png";
|
||||
const file = new File([new Uint8Array(2 * 1024 * 1024)], "photo.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 1024 * 1024);
|
||||
expect(result).not.toBeNull();
|
||||
if (!result) return;
|
||||
expect(result.type).toBe("image/png");
|
||||
expect(result.name.endsWith(".webp")).toBe(false);
|
||||
});
|
||||
|
||||
it("re-encodes an oversized image/jpg through the resize pipeline", async () => {
|
||||
// Over-budget: ensures the image/jpg alias passes the
|
||||
// allowlist gate and enters the encode pipeline. Removing
|
||||
// the alias from RESIZABLE_MIME_TYPES would short-circuit
|
||||
// to null here.
|
||||
const file = new File([new Uint8Array(3 * 1024 * 1024)], "photo.jpg", {
|
||||
type: "image/jpg",
|
||||
});
|
||||
const result = await resizeImageToMaxBytes(file, 512 * 1024);
|
||||
expect(result).not.toBeNull();
|
||||
expect(state.encodeCalls.length).toBeGreaterThan(0);
|
||||
if (!result) return;
|
||||
expect(result.size).toBeLessThanOrEqual(512 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describeIfDecode("resizeImageToMaxBytes with real decoders", () => {
|
||||
it("returns null (does not throw) for a corrupt image blob", async () => {
|
||||
// Real decoder only; jsdom <img> fallback never fires
|
||||
// onload/onerror on a corrupt blob and would hang.
|
||||
const bytes = new Uint8Array([0x00, 0x01, 0x02, 0x03]);
|
||||
const file = new File([bytes], "broken.png", { type: "image/png" });
|
||||
const result = await resizeImageToMaxBytes(file, 1);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("re-encodes a large PNG down to the requested byte budget", async () => {
|
||||
const canvas = new OffscreenCanvas(1024, 1024);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("no 2d ctx");
|
||||
// Noise pattern so compression doesn't trivialize the
|
||||
// byte count before the shrink path runs.
|
||||
const data = ctx.createImageData(1024, 1024);
|
||||
for (let i = 0; i < data.data.length; i += 4) {
|
||||
data.data[i] = (i * 13) & 0xff;
|
||||
data.data[i + 1] = (i * 7) & 0xff;
|
||||
data.data[i + 2] = (i * 19) & 0xff;
|
||||
data.data[i + 3] = 0xff;
|
||||
}
|
||||
ctx.putImageData(data, 0, 0);
|
||||
const sourceBlob = await canvas.convertToBlob({ type: "image/png" });
|
||||
const file = new File([sourceBlob], "large.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
|
||||
const budget = 64 * 1024;
|
||||
const result = await resizeImageToMaxBytes(file, budget);
|
||||
expect(result).not.toBeNull();
|
||||
if (!result) return;
|
||||
expect(result.size).toBeLessThan(budget);
|
||||
expect(result.type).toBe("image/webp");
|
||||
expect(result.name.endsWith(".webp")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* Browser-side image re-encoding to a caller-supplied byte budget.
|
||||
* Plain TS (no React) so it can be used by any upload pipeline.
|
||||
*/
|
||||
|
||||
// Formats we re-encode. GIFs are excluded so we don't flatten
|
||||
// animation; image/jpg is a non-IANA alias for image/jpeg some
|
||||
// OSes emit.
|
||||
const RESIZABLE_MIME_TYPES = new Set([
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/webp",
|
||||
]);
|
||||
|
||||
// Per-axis clamp applied at decode so a low-byte image with
|
||||
// pathologically large pixel extent can't OOM the tab. 8192 stays
|
||||
// under Safari/Chrome canvas limits while preserving detail for
|
||||
// typical screenshots.
|
||||
const MAX_INITIAL_DIMENSION = 8192;
|
||||
|
||||
// 8 iterations × DIMENSION_STEP per axis gives ~3% of original
|
||||
// pixels, plenty of headroom for any modestly-oversize image.
|
||||
const MAX_SHRINK_ITERATIONS = 8;
|
||||
const DIMENSION_STEP = 0.8;
|
||||
|
||||
// 0.85 is near-lossless for screenshots; 0.7 is a hail-mary if
|
||||
// dimension shrinking alone didn't fit the budget.
|
||||
const INITIAL_QUALITY = 0.85;
|
||||
const FALLBACK_QUALITY = 0.7;
|
||||
|
||||
// Time-bound the legacy <img> decode fallback so a blob that fires
|
||||
// neither onload nor onerror can't wedge the module queue.
|
||||
const FALLBACK_DECODE_TIMEOUT_MS = 10_000;
|
||||
|
||||
// Sequential queue: pasting many images won't spawn parallel
|
||||
// decode pipelines.
|
||||
let queue: Promise<unknown> = Promise.resolve();
|
||||
|
||||
function enqueue<T>(fn: () => Promise<T>): Promise<T> {
|
||||
// Chain off both settlement branches so the next task runs
|
||||
// after a rejection too; the .catch below detaches rejection
|
||||
// from the shared tail.
|
||||
const next = queue.then(fn, fn);
|
||||
queue = next.catch(() => undefined);
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-encode `file` as WebP, iteratively shrinking until the output
|
||||
* is at or below `maxBytes`.
|
||||
*
|
||||
* - Returns the original `file` unchanged when it is already within
|
||||
* the budget and its MIME type is in our resizable set (no need
|
||||
* to pay the decode cost).
|
||||
* - Returns the original `file` unchanged for animated formats like
|
||||
* GIF where canvas re-encoding would destroy the animation.
|
||||
* - Returns a new `File` (WebP, renamed to `.webp`) when resizing
|
||||
* succeeded.
|
||||
* - Returns `null` when the file cannot be decoded or no iteration
|
||||
* fit the budget; callers fall back to the original file.
|
||||
*/
|
||||
export async function resizeImageToMaxBytes(
|
||||
file: File,
|
||||
maxBytes: number,
|
||||
): Promise<File | null> {
|
||||
if (!file.type.startsWith("image/")) {
|
||||
return file;
|
||||
}
|
||||
// GIFs return as-is so we don't flatten animation.
|
||||
if (file.type === "image/gif") {
|
||||
return file;
|
||||
}
|
||||
// Already under budget; return as-is regardless of MIME (the
|
||||
// function's contract is "give me something <= maxBytes" and
|
||||
// we already have that).
|
||||
if (file.size <= maxBytes) {
|
||||
return file;
|
||||
}
|
||||
// Over budget but unsupported MIME (e.g. image/bmp): refuse
|
||||
// rather than silently produce a black canvas or wrong file.
|
||||
if (!RESIZABLE_MIME_TYPES.has(file.type)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return enqueue(() => shrinkOnce(file, maxBytes));
|
||||
}
|
||||
|
||||
async function shrinkOnce(file: File, maxBytes: number): Promise<File | null> {
|
||||
let bitmap: ImageBitmap | null = null;
|
||||
try {
|
||||
bitmap = await decodeToBitmap(file);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!bitmap) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// decodeToBitmap already clamped to MAX_INITIAL_DIMENSION
|
||||
// per axis, so we start the shrink loop from the bitmap's
|
||||
// reported dimensions.
|
||||
let width = bitmap.width;
|
||||
let height = bitmap.height;
|
||||
if (width <= 0 || height <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (let i = 0; i < MAX_SHRINK_ITERATIONS; i++) {
|
||||
const blob = await encodeWebP(bitmap, width, height, INITIAL_QUALITY);
|
||||
if (blob && blob.size <= maxBytes) {
|
||||
return toWebPFile(file, blob);
|
||||
}
|
||||
// Guard against tiny images that can't shrink further.
|
||||
if (width <= 1 || height <= 1) {
|
||||
break;
|
||||
}
|
||||
width = Math.max(1, Math.round(width * DIMENSION_STEP));
|
||||
height = Math.max(1, Math.round(height * DIMENSION_STEP));
|
||||
}
|
||||
|
||||
// Last-ditch attempt at the smallest dimensions with a
|
||||
// lower quality, for photographic images where dimension
|
||||
// shrinking alone saturated.
|
||||
const fallbackBlob = await encodeWebP(
|
||||
bitmap,
|
||||
width,
|
||||
height,
|
||||
FALLBACK_QUALITY,
|
||||
);
|
||||
if (fallbackBlob && fallbackBlob.size <= maxBytes) {
|
||||
return toWebPFile(file, fallbackBlob);
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
bitmap.close?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function decodeToBitmap(file: File): Promise<ImageBitmap | null> {
|
||||
// createImageBitmap's HTML-spec output rules:
|
||||
// - both resize dims => stretch (destroys aspect ratio).
|
||||
// - only resizeWidth => width is exact, height proportional.
|
||||
// Per spec this UPSCALES if resizeWidth > natural width.
|
||||
// - both omitted => source's natural size.
|
||||
//
|
||||
// Upscaling can blow past Chromium's ~268M-pixel decode limit
|
||||
// (e.g. a 1080x5000 screenshot with resizeWidth: 8192 becomes
|
||||
// ~310M pixels). Probe natural dimensions first to pick the
|
||||
// smallest resize option that fits.
|
||||
if (typeof createImageBitmap !== "function") {
|
||||
return await decodeViaImgFallback(file);
|
||||
}
|
||||
const natural = await probeNaturalDimensions(file);
|
||||
if (!natural) {
|
||||
return await decodeViaImgFallback(file);
|
||||
}
|
||||
const { width, height } = natural;
|
||||
if (width <= 0 || height <= 0) {
|
||||
return null;
|
||||
}
|
||||
// Pick the smallest resize that fits MAX_INITIAL_DIMENSION
|
||||
// without upscaling. Already-small sources pass no options.
|
||||
const needsWidthClamp = width > MAX_INITIAL_DIMENSION;
|
||||
const needsHeightClamp = height > MAX_INITIAL_DIMENSION;
|
||||
if (!needsWidthClamp && !needsHeightClamp) {
|
||||
return await createImageBitmap(file);
|
||||
}
|
||||
// Clamp the longer axis; the shorter axis scales with it.
|
||||
if (width >= height) {
|
||||
return await createImageBitmap(file, {
|
||||
resizeWidth: MAX_INITIAL_DIMENSION,
|
||||
resizeQuality: "medium",
|
||||
});
|
||||
}
|
||||
return await createImageBitmap(file, {
|
||||
resizeHeight: MAX_INITIAL_DIMENSION,
|
||||
resizeQuality: "medium",
|
||||
});
|
||||
}
|
||||
|
||||
// Reads natural dimensions via <img> without allocating a full
|
||||
// ImageBitmap. Returns null on decode/timeout failure.
|
||||
async function probeNaturalDimensions(
|
||||
file: File,
|
||||
): Promise<{ width: number; height: number } | null> {
|
||||
return await new Promise<{ width: number; height: number } | null>(
|
||||
(resolve) => {
|
||||
const url = URL.createObjectURL(file);
|
||||
const img = new Image();
|
||||
let settled = false;
|
||||
const cleanup = () => {
|
||||
settled = true;
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}, FALLBACK_DECODE_TIMEOUT_MS);
|
||||
img.onload = () => {
|
||||
if (settled) return;
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
resolve({ width: img.naturalWidth, height: img.naturalHeight });
|
||||
};
|
||||
img.onerror = () => {
|
||||
if (settled) return;
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
resolve(null);
|
||||
};
|
||||
img.src = url;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function decodeViaImgFallback(file: File): Promise<ImageBitmap | null> {
|
||||
// Decode via <img> + Blob URL. Reached only on browsers
|
||||
// without createImageBitmap (very old Safari, embedded
|
||||
// webviews); time-bounded so a stuck decoder can't wedge the
|
||||
// queue. No decode-time clamp on this path.
|
||||
return await new Promise<ImageBitmap | null>((resolve, reject) => {
|
||||
const url = URL.createObjectURL(file);
|
||||
const img = new Image();
|
||||
let settled = false;
|
||||
const cleanup = () => {
|
||||
settled = true;
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
cleanup();
|
||||
reject(new Error("image decode timed out"));
|
||||
}, FALLBACK_DECODE_TIMEOUT_MS);
|
||||
img.onload = () => {
|
||||
if (settled) return;
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
// HTMLImageElement is a valid CanvasImageSource;
|
||||
// width/height are all we need downstream.
|
||||
resolve(img as unknown as ImageBitmap);
|
||||
};
|
||||
img.onerror = () => {
|
||||
if (settled) return;
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
reject(new Error("image decode failed"));
|
||||
};
|
||||
img.src = url;
|
||||
});
|
||||
}
|
||||
|
||||
async function encodeWebP(
|
||||
source: ImageBitmap,
|
||||
width: number,
|
||||
height: number,
|
||||
quality: number,
|
||||
): Promise<Blob | null> {
|
||||
// OffscreenCanvas's convertToBlob is fully async and doesn't
|
||||
// need the canvas laid out.
|
||||
if (typeof OffscreenCanvas === "function") {
|
||||
const canvas = new OffscreenCanvas(width, height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
return null;
|
||||
}
|
||||
ctx.drawImage(source, 0, 0, width, height);
|
||||
try {
|
||||
return await canvas.convertToBlob({ type: "image/webp", quality });
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback for environments without OffscreenCanvas.
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
return null;
|
||||
}
|
||||
ctx.drawImage(source as CanvasImageSource, 0, 0, width, height);
|
||||
return await new Promise<Blob | null>((resolve) => {
|
||||
canvas.toBlob((blob) => resolve(blob), "image/webp", quality);
|
||||
});
|
||||
}
|
||||
|
||||
function toWebPFile(original: File, blob: Blob): File {
|
||||
// Match the extension to the actual content type; some upload
|
||||
// handlers still key behavior off the extension.
|
||||
const dot = original.name.lastIndexOf(".");
|
||||
const baseName = dot > 0 ? original.name.slice(0, dot) : original.name;
|
||||
const webpName = `${baseName || "image"}.webp`;
|
||||
// Use blob.type: canvas encoders fall back to PNG on browsers
|
||||
// without WebP support; this keeps the File's labelled type
|
||||
// matching its actual content.
|
||||
const effectiveType = blob.type || "image/webp";
|
||||
const effectiveName =
|
||||
effectiveType === "image/webp" ? webpName : original.name;
|
||||
return new File([blob], effectiveName, {
|
||||
type: effectiveType,
|
||||
lastModified: original.lastModified,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user