opensource/sim
3
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-08-31 01:11:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(uploads): require an explicit byte ceiling on workspace-file downloads (#6985)

Browse Source 
* fix(uploads): require an explicit byte ceiling on workspace-file downloads

Workspace files are admitted at 5 GB because they stream straight to object
storage, but a tool that pulls one back to hand it to a third party buffers the
whole thing in the shared app process. maxBytes was optional on every download
helper, so 51 call sites had silently inherited "unbounded".

Make maxBytes required on all five entry points so a new call site cannot
inherit it again, and give each existing site a ceiling: the destination's own
documented limit where the route already declared one, otherwise the 100 MB this
codebase already uses for buffered work.

Multi-attachment routes were the worse case — Gmail, Outlook, SendGrid and SMTP
downloaded every attachment via Promise.all and only summed the sizes once they
were all resident, so their pre-check on declared sizes protected nothing. Add
downloadServableFilesWithinBudget, which walks the list against a shrinking
budget, and use the same running budget in Slack, Jira, Discord and Quiver.

* fix(uploads): bound Sim-page asset inlining before the bytes are resident

The ceiling on the rendered page checked the finished document, by which point
renderSimPageDocumentWithAssets had already downloaded every referenced image
concurrently with no per-download limit and base64-inlined them — so the
allocation the check exists to prevent had already happened.

Pick the inline set from recorded sizes before fetching anything, against a
per-document budget as well as the existing per-image one, and give each
download its own ceiling in case a row understates its object. An image that
does not fit keeps its URL reference, exactly as an oversized one already did.

* improvement(uploads): charge the page-render budget by delivered bytes

Planning the inline set from recorded sizes left the aggregate ceiling resting
on metadata being accurate, and needed a paragraph explaining why that was safe.
Downloading one image at a time and subtracting what each download actually
returned needs no such argument: the budget cannot be exceeded whatever a row
says, and the peak is one image rather than the sum of them.

Also drop two rough edges the first pass left behind — an SFTP total-size check
that became unreachable once the download carried the remaining budget, and a
Dataverse error helper whose optional size argument existed only to paper over
one caller that had not passed it.
This commit is contained in:
Vikhyath Mondreti
2026-08-22 14:05:59 -07:00
committed by GitHub
parent 28cfa4361a
commit 8c7a2f1df0
54 changed files with 863 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/sim/app/api/tools/agiloft/attach/route.ts
+9 -2
View File
@@ -6,7 +6,9 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import type { RawFileInput } from '@/lib/uploads/utils/file-schemas'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -75,13 +77,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let fileBuffer: Buffer
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 })
return NextResponse.json(
{ success: false, error: toError(error).message },
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
const resolvedFileName = data.fileName || userFile.name || 'attachment'
apps/sim/app/api/tools/box/upload/route.ts
+6 -2
View File
@@ -5,7 +5,9 @@ import { boxUploadContract } from '@/lib/api/contracts/storage-transfer'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -55,14 +57,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = result.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
fileName = validatedData.fileName || userFile.name
apps/sim/app/api/tools/brex/upload-receipt/route.test.ts
+19 -4
View File
@@ -27,6 +27,7 @@ vi.mock('@/app/api/files/authorization', () => ({
assertToolFileAccess: mockAssertToolFileAccess,
}))
import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { POST } from '@/app/api/tools/brex/upload-receipt/route'
const mockFetch = vi.fn()
@@ -194,11 +195,25 @@ describe('POST /api/tools/brex/upload-receipt', () => {
expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled()
})
it('asks the downloader for at most 50 MB', async () => {
await POST(createMockRequest('POST', baseBody))
expect(mockDownloadFileFromStorage).toHaveBeenCalledWith(
expect.anything(),
expect.any(String),
expect.anything(),
{ maxBytes: 50 * 1024 * 1024 }
)
})
it('rejects files over the 50 MB limit', async () => {
mockDownloadFileFromStorage.mockResolvedValueOnce({
buffer: Buffer.alloc(50 * 1024 * 1024 + 1),
contentType: 'application/pdf',
})
mockDownloadFileFromStorage.mockRejectedValueOnce(
new PayloadSizeLimitError({
label: 'storage file download',
maxBytes: 50 * 1024 * 1024,
observedBytes: 50 * 1024 * 1024 + 1,
})
)
const response = await POST(createMockRequest('POST', baseBody))
expect(response.status).toBe(400)
apps/sim/app/api/tools/brex/upload-receipt/route.ts
+10 -7
View File
@@ -9,6 +9,7 @@ import {
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -51,23 +52,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let fileBuffer: Buffer
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_RECEIPT_SIZE_BYTES,
})
fileBuffer = resolved.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
return NextResponse.json(
{ success: false, error: 'Receipt file exceeds the 50 MB limit' },
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download receipt file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error') },
{ status: 500 }
)
}
if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) {
return NextResponse.json(
{ success: false, error: 'Receipt file exceeds the 50 MB limit' },
{ status: 400 }
)
}
const effectiveReceiptName = receiptName || userFile.name
const endpoint = expenseId
apps/sim/app/api/tools/confluence/upload-attachment/route.ts
+11 -2
View File
@@ -5,7 +5,9 @@ import { confluenceUploadAttachmentContract } from '@/lib/api/contracts/selector
import { parseRequest } from '@/lib/api/server'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -94,7 +96,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let fileBuffer: Buffer
let resolvedContentType: string
try {
const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger)
const servable = await downloadServableFileFromStorage(
userFile,
'confluence-upload',
logger,
{
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
}
)
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
@@ -105,7 +114,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{
error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/daytona/upload/route.ts
+10 -1
View File
@@ -5,6 +5,7 @@ import { daytonaUploadFileContract } from '@/lib/api/contracts/tools/daytona'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -62,11 +63,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`)
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_UPLOAD_SIZE_BYTES,
})
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
return NextResponse.json(
{ success: false, error: 'File exceeds upload limit of 100MB' },
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
apps/sim/app/api/tools/discord/send-message/route.ts
+8 -8
View File
@@ -6,9 +6,11 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { validateNumericId } from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -146,12 +148,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
userFiles.map(async (file, i) => {
logger.info(`[${requestId}] Downloading file ${i}: ${file.name}`)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, {
totalMaxBytes: MAX_BUFFERED_TRANSFER_BYTES,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
@@ -161,7 +161,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/dropbox/upload/route.ts
+6 -2
View File
@@ -5,8 +5,10 @@ import { dropboxUploadContract } from '@/lib/api/contracts/storage-transfer'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { httpHeaderSafeJson } from '@/lib/core/utils/validation'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = result.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
fileName = userFile.name
apps/sim/app/api/tools/elevenlabs/audio/route.ts
+4 -1
View File
@@ -20,6 +20,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -153,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(file, requestId, logger)
const buffer = await downloadFileFromStorage(file, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
const ext = file.name.split('.').pop()?.toLowerCase() || ''
source = {
buffer,
apps/sim/app/api/tools/firecrawl/parse/route.ts
+5 -1
View File
@@ -11,6 +11,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -85,7 +86,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
logger,
{
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
}
)
const formData = new FormData()
apps/sim/app/api/tools/gmail/draft/route.ts
+16 -21
View File
@@ -5,9 +5,10 @@ import { gmailDraftContract } from '@/lib/api/contracts/tools/google'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
@@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
apps/sim/app/api/tools/gmail/edit-draft/route.ts
+16 -21
View File
@@ -5,9 +5,10 @@ import { gmailEditDraftContract } from '@/lib/api/contracts/tools/google'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
@@ -93,17 +94,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -114,18 +121,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
apps/sim/app/api/tools/gmail/send/route.ts
+16 -21
View File
@@ -5,9 +5,10 @@ import { gmailSendContract } from '@/lib/api/contracts/tools/google'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
@@ -97,17 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -118,18 +125,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
apps/sim/app/api/tools/google_drive/upload/route.ts
+6 -2
View File
@@ -6,7 +6,9 @@ import { googleDriveUploadContract } from '@/lib/api/contracts/tools/google'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -124,7 +126,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
@@ -136,7 +140,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
success: false,
error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/jira/add-attachment/route.ts
+8 -1
View File
@@ -5,6 +5,7 @@ import { jiraAddAttachmentContract } from '@/lib/api/contracts/selectors/jira'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -44,6 +45,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
(await getJiraCloudId(validatedData.domain, validatedData.accessToken))
const formData = new FormData()
// Every attachment lands in the same multipart body, so the ceiling covers the
// set rather than each file on its own.
let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES
for (const file of userFiles) {
const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger)
@@ -51,7 +55,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let buffer: Buffer
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(file, requestId, logger)
const result = await downloadServableFileFromStorage(file, requestId, logger, {
maxBytes: remainingBytes,
})
buffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
@@ -59,6 +65,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (notReady) return notReady
throw error
}
remainingBytes -= buffer.length
const blob = new Blob([new Uint8Array(buffer)], {
type: downloadedContentType || file.type || 'application/octet-stream',
})
apps/sim/app/api/tools/jupyter/upload/route.ts
+6 -2
View File
@@ -10,7 +10,9 @@ import {
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -58,14 +60,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = result.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
fileName = data.fileName || userFile.name
apps/sim/app/api/tools/linq/upload/route.ts
+17 -8
View File
@@ -5,6 +5,7 @@ import { linqUploadAttachmentContract } from '@/lib/api/contracts/tools/communic
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -19,6 +20,16 @@ const logger = createLogger('LinqUploadAttachmentAPI')
/** Linq pre-upload caps attachments at 100MB. */
const MAX_SIZE_BYTES = 100 * 1024 * 1024
function fileTooLargeError(sizeBytes: number): NextResponse {
return NextResponse.json(
{
success: false,
error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`,
},
{ status: 400 }
)
}
/**
* Upload a file to Linq as a reusable attachment.
*
@@ -61,12 +72,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
let resolvedContentTypeFromStorage: string
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_SIZE_BYTES,
})
buffer = resolved.buffer
resolvedContentTypeFromStorage = resolved.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error))
return fileTooLargeError(error.observedBytes ?? userFile.size)
logger.error(`[${requestId}] Failed to download Linq attachment file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error occurred') },
@@ -93,13 +108,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
return NextResponse.json({ success: false, error: 'File is empty' }, { status: 400 })
}
if (sizeBytes > MAX_SIZE_BYTES) {
return NextResponse.json(
{
success: false,
error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`,
},
{ status: 400 }
)
return fileTooLargeError(sizeBytes)
}
logger.info(`[${requestId}] Registering Linq attachment`, {
apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts
+18 -8
View File
@@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -20,6 +21,17 @@ const logger = createLogger('DataverseUploadFileAPI')
/** Dataverse Web API's absolute ceiling for a single-request (non-chunked) file column upload. */
const DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES = 128 * 1024 * 1024
function uploadTooLargeError(observedBytes: number): NextResponse {
const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`,
},
{ status: 400 }
)
}
export const POST = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
@@ -77,11 +89,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES,
})
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error))
return uploadTooLargeError(error.observedBytes ?? userFile.size)
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
@@ -100,13 +116,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (fileBuffer.length > DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES) {
const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2)
logger.warn(`[${requestId}] File too large for single-request upload: ${sizeMB}MB`)
return NextResponse.json(
{
success: false,
error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`,
},
{ status: 400 }
)
return uploadTooLargeError(fileBuffer.length)
}
const baseUrl = getDataverseBaseUrl(validatedData.environmentUrl)
apps/sim/app/api/tools/mistral/parse/route.ts
+5 -1
View File
@@ -15,6 +15,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import {
extractStorageKey,
isInternalFileUrl,
@@ -157,7 +158,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
logger,
{
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
}
)
base64 = buffer.toString('base64')
if (contentType && contentType !== 'application/octet-stream') {
apps/sim/app/api/tools/onedrive/upload/route.ts
+25 -11
View File
@@ -8,6 +8,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import {
getExtensionFromMimeType,
@@ -24,6 +25,20 @@ const logger = createLogger('OneDriveUploadAPI')
const MICROSOFT_GRAPH_BASE = 'https://graph.microsoft.com/v1.0'
/** Microsoft Graph's ceiling for a simple (non-chunked) drive-item upload. */
const MAX_SIMPLE_UPLOAD_BYTES = 250 * 1024 * 1024
function fileTooLargeError(observedBytes: number): NextResponse {
const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`,
},
{ status: 400 }
)
}
/** Microsoft Graph DriveItem response */
interface OneDriveFileData {
id: string
@@ -115,12 +130,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_SIMPLE_UPLOAD_BYTES,
})
fileBuffer = result.buffer
mimeType = result.contentType || userFile.type || 'application/octet-stream'
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
return fileTooLargeError(error.observedBytes ?? userFile.size)
}
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{
@@ -132,17 +152,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
}
const maxSize = 250 * 1024 * 1024
if (fileBuffer.length > maxSize) {
const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2)
logger.warn(`[${requestId}] File too large: ${sizeMB}MB`)
return NextResponse.json(
{
success: false,
error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`,
},
{ status: 400 }
if (fileBuffer.length > MAX_SIMPLE_UPLOAD_BYTES) {
logger.warn(
`[${requestId}] File too large: ${(fileBuffer.length / (1024 * 1024)).toFixed(2)}MB`
)
return fileTooLargeError(fileBuffer.length)
}
let fileName = validatedData.fileName
apps/sim/app/api/tools/outlook/draft/route.ts
+16 -21
View File
@@ -5,9 +5,10 @@ import { outlookDraftContract } from '@/lib/api/contracts/tools/microsoft'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`,
},
{ status: 400 }
)
}
const attachmentObjects = attachments.map((file, i) => ({
'@odata.type': '#microsoft.graph.fileAttachment',
name: file.name,
apps/sim/app/api/tools/outlook/send/route.ts
+16 -21
View File
@@ -5,9 +5,10 @@ import { outlookSendContract } from '@/lib/api/contracts/tools/microsoft'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -110,17 +111,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -131,18 +138,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`,
},
{ status: 400 }
)
}
const attachmentObjects = attachments.map((file, i) => ({
'@odata.type': '#microsoft.graph.fileAttachment',
name: file.name,
apps/sim/app/api/tools/persona/import-accounts/route.ts
+6 -2
View File
@@ -5,7 +5,9 @@ import { personaImportAccountsContract } from '@/lib/api/contracts/tools/persona
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -58,7 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let buffer: Buffer
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
buffer = resolved.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
@@ -66,7 +70,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.error(`[${requestId}] Failed to download Persona import file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Internal server error') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/quiver/image-to-svg/route.ts
+7 -2
View File
@@ -11,6 +11,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import type { RawFileInput } from '@/lib/uploads/utils/file-schemas'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -81,7 +82,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger)
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
apiImage = { base64: buffer.toString('base64') }
} else {
return NextResponse.json(
@@ -114,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger)
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
apiImage = { base64: buffer.toString('base64') }
} else {
return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 })
apps/sim/app/api/tools/quiver/text-to-svg/route.ts
+12 -2
View File
@@ -11,6 +11,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import type { RawFileInput } from '@/lib/uploads/utils/file-schemas'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -59,6 +60,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
const apiReferences: Array<{ url: string } | { base64: string }> = []
// Every reference is buffered and base64'd before the list is sliced to 4, so the
// budget has to span the whole loop rather than bound each file on its own.
let referenceBudget = MAX_BUFFERED_TRANSFER_BYTES
if (data.references) {
const rawRefs = Array.isArray(data.references) ? data.references : [data.references]
@@ -86,7 +90,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger)
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, {
maxBytes: referenceBudget,
})
referenceBudget -= buffer.length
apiReferences.push({ base64: buffer.toString('base64') })
}
}
@@ -107,7 +114,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger)
const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, {
maxBytes: referenceBudget,
})
referenceBudget -= buffer.length
apiReferences.push({ base64: buffer.toString('base64') })
}
}
apps/sim/app/api/tools/s3/put-object/route.ts
+6 -2
View File
@@ -6,7 +6,9 @@ import { awsS3PutObjectContract } from '@/lib/api/contracts/tools/aws/s3-put-obj
import { parseToolRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -84,7 +86,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
uploadBody = result.buffer
downloadedContentType = result.contentType
} catch (error) {
@@ -92,7 +96,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/sendgrid/send-mail/route.ts
+22 -22
View File
@@ -5,9 +5,10 @@ import { sendGridSendMailContract } from '@/lib/api/contracts/tools/communicatio
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -15,6 +16,9 @@ export const dynamic = 'force-dynamic'
const logger = createLogger('SendGridSendMailAPI')
/** SendGrid rejects a message whose total attachment payload exceeds 30MB. */
const MAX_ATTACHMENT_TOTAL_BYTES = 30 * 1024 * 1024
export const POST = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
@@ -109,17 +113,26 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
userFiles.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, {
totalMaxBytes: MAX_ATTACHMENT_TOTAL_BYTES,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = (
(error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) /
(1024 * 1024)
).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -130,19 +143,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
const maxSize = 30 * 1024 * 1024
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`,
},
{ status: 400 }
)
}
const sendGridAttachments = userFiles.map((file, i) => ({
content: resolved[i].buffer.toString('base64'),
filename: file.name,
apps/sim/app/api/tools/servicenow/upload-attachment/route.ts
+6 -2
View File
@@ -6,7 +6,9 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -56,7 +58,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let fileBuffer: Buffer
let resolvedContentType: string
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
@@ -65,7 +69,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/sftp/upload/route.ts
+12 -8
View File
@@ -5,6 +5,7 @@ import { sftpUploadContract } from '@/lib/api/contracts/storage-transfer'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -109,16 +110,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.info(
`[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)`
)
const { buffer } = await downloadServableFileFromStorage(file, requestId, logger)
const { buffer } = await downloadServableFileFromStorage(file, requestId, logger, {
maxBytes: maxSize - resolvedTotal,
})
resolvedTotal += buffer.length
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{ success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` },
{ status: 400 }
)
}
const safeFileName = sanitizeFileName(file.name)
const fullRemotePath = remotePath.endsWith('/')
@@ -155,6 +151,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const observed = resolvedTotal + (error.observedBytes ?? file.size)
const sizeMB = (observed / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{ success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` },
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error)
throw new Error(
`Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
apps/sim/app/api/tools/sharepoint/upload/route.ts
+25 -19
View File
@@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -88,36 +89,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
logger.info(`[${requestId}] Uploading file: ${userFile.name}`)
const fileName = validatedData.fileName || userFile.name
const folderPath = validatedData.folderPath?.trim() || ''
const skipOversized = (size: number) => {
logger.warn(
`[${requestId}] File ${fileName} is ${(size / (1024 * 1024)).toFixed(2)}MB, exceeds 250MB limit`
)
skippedFiles.push({
name: fileName,
size,
limit: MAX_SHAREPOINT_UPLOAD_BYTES,
reason: 'File exceeds the 250 MB Microsoft Graph small upload limit',
})
}
let buffer: Buffer
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES,
})
buffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
// An oversized file is skipped rather than failing the whole batch, exactly as
// it was when the size was only discovered after the download.
if (isPayloadSizeLimitError(error)) {
skipOversized(error.observedBytes ?? userFile.size)
continue
}
throw error
}
const fileName = validatedData.fileName || userFile.name
const folderPath = validatedData.folderPath?.trim() || ''
const fileSizeMB = buffer.length / (1024 * 1024)
if (buffer.length > MAX_SHAREPOINT_UPLOAD_BYTES) {
logger.warn(
`[${requestId}] File ${fileName} is ${fileSizeMB.toFixed(2)}MB, exceeds 250MB limit`
)
skippedFiles.push({
name: fileName,
size: buffer.length,
limit: MAX_SHAREPOINT_UPLOAD_BYTES,
reason: 'File exceeds the 250 MB Microsoft Graph small upload limit',
})
continue
}
let uploadPath = ''
if (folderPath) {
const normalizedPath = folderPath.startsWith('/') ? folderPath : `/${folderPath}`
apps/sim/app/api/tools/slack/utils.ts
+7 -1
View File
@@ -1,5 +1,6 @@
import type { Logger } from '@sim/logger'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization'
@@ -80,6 +81,9 @@ async function uploadFilesToSlack(
const userFiles = processFilesToUserFiles(files, requestId, logger)
const uploadedFileIds: string[] = []
const uploadedFiles: ToolFileData[] = []
// One share can carry several files, so the ceiling spans the set: each file may
// only use what its predecessors left.
let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES
for (const userFile of userFiles) {
logger.info(`[${requestId}] Uploading file: ${userFile.name}`)
@@ -92,8 +96,10 @@ async function uploadFilesToSlack(
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
logger,
{ maxBytes: remainingBytes }
)
remainingBytes -= buffer.length
const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', {
method: 'POST',
apps/sim/app/api/tools/smtp/send/route.ts
+16 -21
View File
@@ -7,10 +7,11 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { validateDatabaseHost } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { getSmtpEhloName } from '@/lib/messaging/email/ehlo'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -132,17 +133,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, {
totalMaxBytes: maxSize,
label: 'Total attachment size',
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
@@ -153,18 +160,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
content: resolved[i].buffer,
apps/sim/app/api/tools/square/catalog-image/route.ts
+4 -1
View File
@@ -7,6 +7,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -52,7 +53,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
const fileName = validatedData.fileName || userFile.name
const mimeType = userFile.type || 'application/octet-stream'
apps/sim/app/api/tools/stt/route.ts
+6 -2
View File
@@ -117,7 +117,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
audioBuffer = await downloadFileFromStorage(file, requestId, logger)
audioBuffer = await downloadFileFromStorage(file, requestId, logger, {
maxBytes: MAX_FILE_SIZE,
})
audioFileName = file.name
// file.type may be missing if the file came from a block that doesn't preserve it
// Infer from filename extension as fallback
@@ -143,7 +145,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
audioBuffer = await downloadFileFromStorage(file, requestId, logger)
audioBuffer = await downloadFileFromStorage(file, requestId, logger, {
maxBytes: MAX_FILE_SIZE,
})
audioFileName = file.name
const ext = file.name.split('.').pop()?.toLowerCase() || ''
apps/sim/app/api/tools/supabase/storage-upload/route.ts
+6 -2
View File
@@ -6,7 +6,9 @@ import { parseToolRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { validateSupabaseProjectId } from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
@@ -152,7 +154,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let buffer: Buffer
let resolvedContentType: string
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
buffer = resolved.buffer
resolvedContentType = resolved.contentType
} catch (error) {
@@ -161,7 +165,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Internal server error') },
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/app/api/tools/telegram/send-document/route.ts
+14 -1
View File
@@ -5,6 +5,7 @@ import { telegramSendDocumentContract } from '@/lib/api/contracts/tools/communic
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -97,12 +98,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let buffer: Buffer
let contentType: string
try {
const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger)
const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: maxSize,
})
buffer = downloaded.buffer
contentType = downloaded.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
const sizeMB = ((error.observedBytes ?? userFile.size) / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error)
return NextResponse.json(
{
apps/sim/app/api/tools/textract/shared.ts
+5 -1
View File
@@ -12,6 +12,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import type { RawFileInput } from '@/lib/uploads/utils/file-utils'
import {
extractStorageKey,
@@ -160,7 +161,10 @@ export async function resolveDocumentInput(
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
logger,
{
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
}
)
const resolvedContentType = contentType || userFile.type || 'application/octet-stream'
apps/sim/app/api/tools/uptimerobot/server-utils.ts
+9 -1
View File
@@ -1,5 +1,6 @@
import type { Logger } from '@sim/logger'
import { NextResponse } from 'next/server'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
@@ -47,7 +48,14 @@ async function appendPspImage(
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const { buffer, contentType } = await downloadServableFileFromStorage(userFile, requestId, logger)
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger,
{
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
}
)
const mimeType = contentType || userFile.type || 'application/octet-stream'
form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name)
return null
apps/sim/app/api/tools/vanta/upload/route.ts
+7 -1
View File
@@ -5,6 +5,7 @@ import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
@@ -72,7 +73,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_UPLOAD_SIZE_BYTES,
})
fileBuffer = resolved.buffer
fileName = params.fileName || userFile.name
mimeType =
@@ -80,6 +83,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (isPayloadSizeLimitError(error)) {
return uploadSizeError(error.observedBytes ?? userFile.size)
}
logger.error(`[${requestId}] Failed to download Vanta upload file`, {
error: getErrorMessage(error),
})
apps/sim/app/api/tools/vision/analyze/route.ts
+9 -1
View File
@@ -16,6 +16,7 @@ import {
isModelSafeWorkspaceFileKey,
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import {
extractStorageKey,
isInternalFileUrl,
@@ -125,7 +126,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
// The three providers this route serves disagree too much for a single
// route-wide image limit to be right (Anthropic: 10MB base64 per image;
// OpenAI: 512MB total request payload), and picking the lowest would reject
// images the others accept. So bound the buffer we hold and let each provider
// reject what it will not take, with its own message.
const buffer = await downloadFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
base64 = buffer.toString('base64')
bufferLength = buffer.length
}
apps/sim/app/api/tools/wordpress/upload/route.ts
+6 -2
View File
@@ -5,7 +5,9 @@ import { wordpressUploadContract } from '@/lib/api/contracts/storage-transfer'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import {
getFileExtension,
getMimeTypeFromExtension,
@@ -93,7 +95,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
let resolvedContentType: string
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
@@ -105,7 +109,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
success: false,
error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
apps/sim/lib/collab-doc/persist.ts
+9 -2
View File
@@ -7,6 +7,7 @@ import {
getWorkspaceFile,
updateWorkspaceFileContent,
} from '@/lib/uploads/contexts/workspace'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { collabDocStateSourceHash, hashMarkdown, saveCollabDocState } from './collab-state'
import { canonicalizeYDoc, yDocToFileMarkdown } from './converter'
@@ -94,7 +95,11 @@ export async function persistFileDoc(
// nothing to clobber, so this reports the file's CURRENT durable version rather than conflicting on a
// stale `expectedVersion`: it resynchronizes the relay's If-Match token instead of stranding it.
if (record.size === markdownBuffer.length) {
const current = await fetchWorkspaceFileBuffer(record).catch(() => null)
// A byte-for-byte equality check: anything longer than what we are comparing against
// cannot match, so the buffer we are about to compare is itself the ceiling.
const current = await fetchWorkspaceFileBuffer(record, {
maxBytes: markdownBuffer.length,
}).catch(() => null)
if (current?.equals(markdownBuffer)) {
// Still refresh the cached snapshot: the markdown is unchanged (so its `sourceHash` tag stays
// valid) but the doc state may have just been canonicalized, and a cold open should seed from
@@ -191,7 +196,9 @@ async function recoverFromVersionConflict(
try {
const current = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true })
if (!current) return { status: 'missing' }
const durable = await fetchWorkspaceFileBuffer(current)
const durable = await fetchWorkspaceFileBuffer(current, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
if (hashMarkdown(durable) !== (await collabDocStateSourceHash(fileId))) return conflict()
logger.info(
`Persist token for file ${fileId} was stale, not the file; re-syncing and writing the projection`
apps/sim/lib/copilot/tools/handlers/materialize-file.ts
+6 -1
View File
@@ -39,6 +39,7 @@ import { hasCloudStorage, headObject } from '@/lib/uploads/core/storage-service'
import { toLegacyWorkspaceFileSize } from '@/lib/uploads/shared/types'
import { isArchiveFileName } from '@/lib/uploads/utils/file-utils'
import { parseWorkflowJson } from '@/lib/workflows/operations/import-export'
import { MAX_IMPORT_BODY_BYTES } from '@/lib/workflows/operations/import-workflow'
import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils'
import { deduplicateWorkflowName } from '@/lib/workflows/utils'
import { admitCreateWorkspaceFile } from '@/lib/workspace-files/application/create-workspace-file'
@@ -260,7 +261,11 @@ async function executeImport(
}
}
const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row))
// The bytes are headed straight for `parseWorkflowJson`, so the import body ceiling is
// the real limit here — a larger file could not be imported even if it were read.
const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row), {
maxBytes: MAX_IMPORT_BODY_BYTES,
})
const content = buffer.toString('utf-8')
let parsed: unknown
apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts
+2 -2
View File
@@ -1659,7 +1659,7 @@ export async function getWorkspaceFile(
*/
export async function fetchServableWorkspaceFileBuffer(
fileRecord: WorkspaceFileRecord,
options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {}
options: { maxBytes: number; signal?: AbortSignal; requestId?: string }
): Promise<{ buffer: Buffer; contentType: string }> {
const { downloadServableFileFromStorage } = await import('@/lib/uploads/utils/file-utils.server')
@@ -1685,7 +1685,7 @@ export async function fetchServableWorkspaceFileBuffer(
*/
export async function fetchWorkspaceFileBuffer(
fileRecord: WorkspaceFileRecord,
options: { maxBytes?: number } = {}
options: { maxBytes: number }
): Promise<Buffer> {
logger.info(`Downloading workspace file: ${fileRecord.name}`)
apps/sim/lib/uploads/shared/types.ts
+19
View File
@@ -25,6 +25,25 @@ export const MAX_WORKSPACE_FORMDATA_FILE_SIZE = 100 * 1024 * 1024
/** Maximum size accepted by the knowledge-document parsing pipeline. */
export const MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE = 100 * 1024 * 1024
/**
* Default ceiling for a read that holds the whole file resident as one `Buffer`.
*
* Workspace files are admitted at {@link MAX_WORKSPACE_FILE_SIZE} (5 GB) because they
* are streamed straight to object storage and never sit in the app process. A tool
* that pulls one back to hand it to a third party does not stream — it buffers, then
* usually copies again (base64, `Blob`, multipart), so peak resident memory is a
* multiple of the file. Sharing one ceiling keeps that multiple bounded no matter how
* many blocks run concurrently.
*
* 100 MB is the value this codebase already converged on for buffered work
* ({@link MAX_WORKSPACE_FORMDATA_FILE_SIZE}, `MAX_ARCHIVE_BYTES`, the 100 MB
* `maxResponseBytes` on the STT URL branch, and the ClickUp/Vanta/Daytona/Linq/SFTP
* upload routes). Use a destination's own documented limit instead whenever it is
* lower — failing here beats a slow round trip to a provider that will reject it.
* Genuinely large transfers belong on `downloadFileStream`, not on a bigger ceiling.
*/
export const MAX_BUFFERED_TRANSFER_BYTES = 100 * 1024 * 1024
/**
* Rejection wording shared by every surface that admits a knowledge document.
*
apps/sim/lib/uploads/utils/file-utils.server.test.ts
+116 -1
View File
@@ -33,9 +33,12 @@ vi.mock('@/app/api/files/authorization', () => ({
}))
import { createLogger } from '@sim/logger'
import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import {
downloadFileFromStorage,
downloadServableFileFromStorage,
downloadServableFilesWithinBudget,
} from '@/lib/uploads/utils/file-utils.server'
import type { UserFile } from '@/executor/types'
@@ -61,7 +64,9 @@ describe('downloadFileFromStorage context derivation', () => {
context: 'og-images',
}
await downloadFileFromStorage(userFile, 'req-1', createLogger('test'))
await downloadFileFromStorage(userFile, 'req-1', createLogger('test'), {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
expect(mockDownloadFile).toHaveBeenCalledTimes(1)
expect(mockDownloadFile).toHaveBeenCalledWith(
@@ -83,6 +88,7 @@ describe('downloadFileFromStorage context derivation', () => {
const filePrincipal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' }
await downloadServableFileFromStorage(userFile, 'req-1', createLogger('test'), {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
filePrincipal,
})
@@ -91,3 +97,112 @@ describe('downloadFileFromStorage context derivation', () => {
)
})
})
describe('downloadFileFromStorage size ceiling', () => {
const logger = createLogger('test')
const fileOfSize = (size: number): UserFile => ({
id: 'f1',
name: 'clip.wav',
url: '',
size,
type: 'audio/wav',
key: 'workspace/ws-1/1700000000000-abc1234-clip.wav',
})
beforeEach(() => {
vi.clearAllMocks()
mockParseWorkspaceFileKey.mockReturnValue(null)
})
it('rejects on the declared size before moving any bytes', async () => {
await expect(
downloadFileFromStorage(fileOfSize(2048), 'req-1', logger, { maxBytes: 1024 })
).rejects.toThrow(PayloadSizeLimitError)
expect(mockDownloadFile).not.toHaveBeenCalled()
})
it('rejects on the delivered bytes when the declared size understated them', async () => {
mockDownloadFile.mockResolvedValue(Buffer.alloc(2048))
await expect(
downloadFileFromStorage(fileOfSize(1), 'req-1', logger, { maxBytes: 1024 })
).rejects.toThrow(PayloadSizeLimitError)
})
it('forwards the ceiling to the storage layer so a provider can stop mid-stream', async () => {
mockDownloadFile.mockResolvedValue(Buffer.alloc(512))
await downloadFileFromStorage(fileOfSize(512), 'req-1', logger, { maxBytes: 1024 })
expect(mockDownloadFile).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 1024 }))
})
})
describe('downloadServableFilesWithinBudget', () => {
const logger = createLogger('test')
const fileOfSize = (name: string, size: number): UserFile => ({
id: name,
name,
url: '',
size,
type: 'application/octet-stream',
key: `workspace/ws-1/1700000000000-abc1234-${name}`,
})
beforeEach(() => {
vi.clearAllMocks()
mockParseWorkspaceFileKey.mockReturnValue(null)
mockDownloadFile.mockImplementation(async ({ key }) =>
Buffer.alloc(key.endsWith('big.bin') ? 900 : 400)
)
})
it('spends the budget across the list rather than per file', async () => {
const resolved = await downloadServableFilesWithinBudget(
[fileOfSize('a.bin', 400), fileOfSize('b.bin', 400)],
'req-1',
logger,
{ totalMaxBytes: 1000, label: 'Total attachment size' }
)
expect(resolved.map((r) => r.buffer.length)).toEqual([400, 400])
// The second file was only offered what the first left behind.
expect(mockDownloadFile).toHaveBeenNthCalledWith(2, expect.objectContaining({ maxBytes: 600 }))
})
it('rejects the combined size even when every file is individually under the limit', async () => {
const failure = await downloadServableFilesWithinBudget(
[fileOfSize('a.bin', 400), fileOfSize('b.bin', 400), fileOfSize('c.bin', 400)],
'req-1',
logger,
{ totalMaxBytes: 1000, label: 'Total attachment size' }
).catch((error) => error)
// Restated in the caller's terms: the whole set against the whole budget, not the
// third file against the 200 bytes the first two happened to leave.
expect(failure).toBeInstanceOf(PayloadSizeLimitError)
expect(failure).toMatchObject({
label: 'Total attachment size',
maxBytes: 1000,
observedBytes: 1200,
})
// The third file's declared size already exceeds what the first two left, so it is
// refused without fetching its bytes — the whole set is never resident at once.
expect(mockDownloadFile).toHaveBeenCalledTimes(2)
})
it('refuses the next file on its declared size once the budget is spent', async () => {
await expect(
downloadServableFilesWithinBudget(
[fileOfSize('big.bin', 900), fileOfSize('a.bin', 400)],
'req-1',
logger,
{ totalMaxBytes: 1000, label: 'Total attachment size' }
)
).rejects.toThrow(PayloadSizeLimitError)
expect(mockDownloadFile).toHaveBeenCalledTimes(1)
})
})
apps/sim/lib/uploads/utils/file-utils.server.ts
+85 -18
View File
@@ -9,6 +9,8 @@ import {
import {
assertKnownSizeWithinLimit,
consumeOrCancelBody,
isPayloadSizeLimitError,
PayloadSizeLimitError,
readResponseToBufferWithLimit,
} from '@/lib/core/utils/stream-limits'
import { StorageService } from '@/lib/uploads'
@@ -335,29 +337,38 @@ export async function resolveInternalFileUrl(
}
/**
* Downloads a file from storage (execution or regular)
* Downloads a file from storage (execution or regular) into a single resident buffer.
*
* `maxBytes` is required, not optional. Workspace files are admitted at 5 GB, so a
* caller that forgets a ceiling inherits "unbounded" and can allocate gigabytes inside
* the shared app process. Making the parameter mandatory means a new call site has to
* name its limit — see {@link MAX_BUFFERED_TRANSFER_BYTES} for the default, and prefer
* the destination's own documented limit whenever it is lower. The size is checked
* twice: against the declared size before any bytes move, and against the delivered
* buffer, because the declared size comes from the caller and may be a lie.
*
* @param userFile - UserFile object
* @param requestId - Request ID for logging
* @param logger - Logger instance
* @param options.maxBytes - Hard ceiling; throws `PayloadSizeLimitError` when exceeded
* @returns Buffer containing file data
*/
export async function downloadFileFromStorage(
userFile: UserFile,
requestId: string,
logger: Logger,
options: { maxBytes?: number } = {}
options: { maxBytes: number }
): Promise<Buffer> {
const { maxBytes } = options
let buffer: Buffer
if (options.maxBytes !== undefined && userFile.size > options.maxBytes) {
assertKnownSizeWithinLimit(userFile.size, options.maxBytes, 'storage file download')
}
assertKnownSizeWithinLimit(userFile.size, maxBytes, 'storage file download')
if (isExecutionFile(userFile)) {
logger.info(`[${requestId}] Downloading from execution storage: ${userFile.key}`)
const { downloadExecutionFile } = await import(
'@/lib/uploads/contexts/execution/execution-file-manager'
)
buffer = await downloadExecutionFile(userFile, { maxBytes: options.maxBytes })
buffer = await downloadExecutionFile(userFile, { maxBytes })
} else if (userFile.key) {
const context = resolveTrustedFileContext(userFile.key, userFile.context)
logger.info(`[${requestId}] Downloading from ${context} storage: ${userFile.key}`)
@@ -366,15 +377,13 @@ export async function downloadFileFromStorage(
buffer = await downloadFile({
key: userFile.key,
context,
maxBytes: options.maxBytes,
maxBytes,
})
} else {
throw new Error('File has no key - cannot download')
}
if (options.maxBytes !== undefined) {
assertKnownSizeWithinLimit(buffer.length, options.maxBytes, 'storage file download')
}
assertKnownSizeWithinLimit(buffer.length, maxBytes, 'storage file download')
return buffer
}
@@ -404,17 +413,21 @@ export interface ServableFile {
*
* Throws `DocCompileUserError` when a generated doc's artifact is not ready (still
* compiling) — callers should surface a retryable error rather than attach source.
*
* `maxBytes` is required for the reason given on {@link downloadFileFromStorage}: it
* bounds the source read. A compiled artifact is re-checked against the same ceiling
* below, since rendering can grow a small source into a large document.
*/
export async function downloadServableFileFromStorage(
userFile: UserFile,
requestId: string,
logger: Logger,
options: {
maxBytes?: number
maxBytes: number
signal?: AbortSignal
ownerKey?: string
filePrincipal?: Principal
} = {}
}
): Promise<ServableFile> {
const buffer = await downloadFileFromStorage(userFile, requestId, logger, {
maxBytes: options.maxBytes,
@@ -430,10 +443,14 @@ export async function downloadServableFileFromStorage(
const workspaceId = userFile.key
? (parseWorkspaceFileKey(userFile.key) ?? undefined)
: undefined
return {
buffer: Buffer.from(await renderSimPageDocumentWithAssets(text, { workspaceId }), 'utf8'),
contentType: 'text/html',
}
const rendered = Buffer.from(
await renderSimPageDocumentWithAssets(text, { workspaceId }),
'utf8'
)
// Rendering inlines referenced assets, so a source well under the ceiling can
// resolve to a document well over it.
assertKnownSizeWithinLimit(rendered.length, options.maxBytes, 'servable page render')
return { buffer: rendered, contentType: 'text/html' }
}
}
@@ -462,8 +479,58 @@ export async function downloadServableFileFromStorage(
// Re-check: the raw download enforced maxBytes on the source, but a generated doc
// resolves to a larger artifact.
if (options.maxBytes !== undefined && resolved.buffer.length > options.maxBytes) {
assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download')
assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download')
return resolved
}
/**
* Resolve every file of a multi-attachment request while bounding their COMBINED
* resident size.
*
* A per-file ceiling is not enough when a request carries an array: N attachments
* each just under the limit still cost N times the limit, and downloading them with
* `Promise.all` makes that the peak. Routes that pre-checked `sum(file.size)` were
* not protected either — the declared sizes come from the caller, which is why the
* downloader re-checks the delivered bytes.
*
* So this walks the list in order against a shrinking budget: each file may only use
* what the previous ones left. Sequential is the point — it is what keeps the peak at
* `totalMaxBytes` instead of the sum, and attachment lists are short enough that the
* lost parallelism does not register next to the provider round trip that follows.
*
* The overrun surfaces as a `PayloadSizeLimitError` restated in the caller's terms:
* the per-file failure underneath reports one file against whatever budget was left,
* which would read as a nonsense limit in a "total attachment size" message. `label`
* and `totalMaxBytes` are the caller's, and `observedBytes` is what the set needed.
*/
export async function downloadServableFilesWithinBudget(
userFiles: readonly UserFile[],
requestId: string,
logger: Logger,
options: { totalMaxBytes: number; label: string; signal?: AbortSignal }
): Promise<ServableFile[]> {
const resolved: ServableFile[] = []
let spent = 0
for (const userFile of userFiles) {
logger.info(`[${requestId}] Downloading ${userFile.name} (${userFile.size} bytes)`)
let servable: ServableFile
try {
servable = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: options.totalMaxBytes - spent,
signal: options.signal,
})
} catch (error) {
if (!isPayloadSizeLimitError(error)) throw error
throw new PayloadSizeLimitError({
label: options.label,
maxBytes: options.totalMaxBytes,
observedBytes: spent + (error.observedBytes ?? userFile.size),
})
}
spent += servable.buffer.length
resolved.push(servable)
}
return resolved
apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts
+1 -1
View File
@@ -13,7 +13,7 @@ const logger = createLogger('FetchServableWorkspaceFileBuffer')
export async function fetchAuthorizedServableWorkspaceFileBuffer(
fileRecord: WorkspaceFileRecord,
filePrincipal: Principal,
options: { maxBytes?: number; signal?: AbortSignal; requestId?: string } = {}
options: { maxBytes: number; signal?: AbortSignal; requestId?: string }
): Promise<{ buffer: Buffer; contentType: string }> {
return downloadServableFileFromStorage(
{
apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts
+4 -1
View File
@@ -26,6 +26,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({
resolveEffectiveWorkspacePermission: mocks.resolvePermission,
}))
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key'
const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' }
@@ -75,7 +76,9 @@ describe('readWorkspaceFileContentByKey', () => {
expect(mocks.getFile).toHaveBeenCalledWith(file.workspaceId, file.id, {
throwOnError: true,
})
expect(mocks.fetchContent).toHaveBeenCalledWith(file)
expect(mocks.fetchContent).toHaveBeenCalledWith(file, {
maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
})
})
it('rejects a stale key instead of serving the file current at the same ID', async () => {
apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts
+5 -1
View File
@@ -8,6 +8,7 @@ import {
type WorkspaceFileRecord,
} from '@/lib/uploads/contexts/workspace'
import { getFileMetadataByKey } from '@/lib/uploads/server/metadata'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
import { fileOperations } from '@/lib/workspace-files/application/operations'
@@ -33,7 +34,10 @@ async function executeReadWorkspaceFileContentByKey({
throwOnError: true,
})
if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found')
return { file, content: await fetchWorkspaceFileBuffer(file) }
return {
file,
content: await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }),
}
}
export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({
apps/sim/lib/workspace-files/application/read-workspace-file-content.ts
+4 -1
View File
@@ -10,6 +10,7 @@ import {
getBoundWorkspaceFileSecretProvenance,
type WorkspaceFileSecretProvenance,
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
import { fileOperations } from '@/lib/workspace-files/application/operations'
import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context'
@@ -43,7 +44,9 @@ async function executeReadWorkspaceFileContent({
throwOnError: true,
})
if (!file) throw new OrchestrationError('not_found', 'File not found')
const content = await fetchWorkspaceFileBuffer(file, { maxBytes: input.maxBytes })
const content = await fetchWorkspaceFileBuffer(file, {
maxBytes: input.maxBytes ?? MAX_BUFFERED_TRANSFER_BYTES,
})
const secretProvenance = input.includeSecretProvenance
? await getBoundWorkspaceFileSecretProvenance(context.workspaceId, {
fileId: file.id,
apps/sim/lib/workspace-files/page-document.server.test.ts
+121
View File
@@ -0,0 +1,121 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockDownloadFile, mockGetFileMetadataById, mockRenderSimPageDocument } = vi.hoisted(() => ({
mockDownloadFile: vi.fn(),
mockGetFileMetadataById: vi.fn(),
mockRenderSimPageDocument: vi.fn(),
}))
vi.mock('@/lib/uploads/core/storage-service', () => ({
downloadFile: mockDownloadFile,
}))
vi.mock('@/lib/uploads/server/metadata', () => ({
getFileMetadataById: mockGetFileMetadataById,
}))
vi.mock('@/lib/workspace-files/page-document', () => ({
renderSimPageDocument: mockRenderSimPageDocument,
}))
import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server'
const WORKSPACE_ID = 'ws-1'
const MB = 1024 * 1024
function imageRecord(id: string, size: number) {
return {
id,
key: `workspace/${WORKSPACE_ID}/${id}.png`,
context: 'workspace',
workspaceId: WORKSPACE_ID,
contentType: 'image/png',
size,
sizeBytes: size,
}
}
function documentReferencing(ids: string[]) {
return ids.map((id) => `<img src="/api/files/view/${id}">`).join('')
}
describe('renderSimPageDocumentWithAssets memory bounds', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('charges the budget by delivered bytes, not by what the metadata claimed', async () => {
// Every row claims to be tiny; the objects are 8MB each. The budget must still
// stop at 32MB — planning off the recorded size would admit all six.
const ids = ['a', 'b', 'c', 'd', 'e', 'f']
mockRenderSimPageDocument.mockReturnValue(documentReferencing(ids))
mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024))
mockDownloadFile.mockImplementation(async () => Buffer.alloc(8 * MB))
const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID })
expect(mockDownloadFile).toHaveBeenCalledTimes(4)
// The images past the budget keep their URL reference rather than failing the render.
expect(html).toContain('src="/api/files/view/e"')
expect(html).toContain('src="/api/files/view/f"')
})
it('offers each download only what the budget has left', async () => {
mockRenderSimPageDocument.mockReturnValue(documentReferencing(['a', 'b']))
mockGetFileMetadataById.mockImplementation(async (id: string) => imageRecord(id, 1024))
mockDownloadFile.mockImplementation(async () => Buffer.alloc(30 * MB))
await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID })
expect(mockDownloadFile).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ maxBytes: 8 * MB })
)
// 30MB delivered leaves 2MB, which is below the per-image limit and becomes the cap.
expect(mockDownloadFile).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ maxBytes: 2 * MB })
)
})
it('never offers a download more than the per-image limit', async () => {
mockRenderSimPageDocument.mockReturnValue(documentReferencing(['solo']))
mockGetFileMetadataById.mockResolvedValue(imageRecord('solo', 1024))
mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes'))
await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID })
expect(mockDownloadFile).toHaveBeenCalledWith(
expect.objectContaining({ maxBytes: 8 * MB, context: 'workspace' })
)
})
it('keeps the URL reference when a capped download rejects', async () => {
mockRenderSimPageDocument.mockReturnValue(documentReferencing(['big']))
mockGetFileMetadataById.mockResolvedValue(imageRecord('big', 1024))
mockDownloadFile.mockRejectedValue(new Error('storage download exceeds maximum size'))
const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID })
expect(html).toContain('src="/api/files/view/big"')
})
it('inlines images that fit and leaves cross-workspace references alone', async () => {
mockRenderSimPageDocument.mockReturnValue(documentReferencing(['mine', 'theirs']))
mockGetFileMetadataById.mockImplementation(async (id: string) =>
id === 'mine'
? imageRecord('mine', 1024)
: { ...imageRecord('theirs', 1024), workspaceId: 'ws-2' }
)
mockDownloadFile.mockResolvedValue(Buffer.from('png-bytes'))
const html = await renderSimPageDocumentWithAssets('source', { workspaceId: WORKSPACE_ID })
expect(mockDownloadFile).toHaveBeenCalledTimes(1)
expect(html).toContain(`data:image/png;base64,${Buffer.from('png-bytes').toString('base64')}`)
expect(html).toContain('src="/api/files/view/theirs"')
})
})
apps/sim/lib/workspace-files/page-document.server.ts
+39 -15
View File
@@ -5,6 +5,13 @@ import { renderSimPageDocument } from '@/lib/workspace-files/page-document'
/** Images past this size stay as URL references rather than bloating the document. */
const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024
/**
* Ceiling on everything a single document inlines. A per-image limit does not bound
* the page on its own — N images each just under it still cost N times it. Images
* that do not fit what is left keep their URL reference, exactly like an oversized one.
*/
const MAX_INLINE_TOTAL_BYTES = 32 * 1024 * 1024
const IMAGE_SRC = /src="[^"]*\/api\/files\/view\/([^"]+)"/g
/**
@@ -24,24 +31,41 @@ export async function renderSimPageDocumentWithAssets(
const ids = [...new Set([...documentHtml.matchAll(IMAGE_SRC)].map((match) => match[1]))]
if (ids.length === 0 || !options.workspaceId) return documentHtml
const inlined = new Map<string, string>()
await Promise.all(
const candidates = await Promise.all(
ids.map(async (id) => {
try {
const record = await getFileMetadataById(id)
if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId)
return
const bytes = await downloadFile({ key: record.key, context: 'workspace' })
if (bytes.length > MAX_INLINE_IMAGE_BYTES) return
const mime = record.contentType?.startsWith('image/')
? record.contentType
: 'application/octet-stream'
inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`)
} catch {
// A missing or unreadable image keeps its URL reference.
}
const record = await getFileMetadataById(id).catch(() => null)
if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId)
return null
return { id, record }
})
)
// One image at a time, charged against the budget by what each download actually
// delivered. Fetching them concurrently made the peak the sum of every image rather
// than the largest one, and the ceiling on the finished document could only observe
// that after the fact. Each download is given whatever the budget has left, so an
// image that does not fit is refused by the read itself instead of after it lands.
const inlined = new Map<string, string>()
let remaining = MAX_INLINE_TOTAL_BYTES
for (const candidate of candidates) {
if (!candidate) continue
if (remaining === 0) break
const { id, record } = candidate
try {
const bytes = await downloadFile({
key: record.key,
context: 'workspace',
maxBytes: Math.min(MAX_INLINE_IMAGE_BYTES, remaining),
})
remaining -= bytes.length
const mime = record.contentType?.startsWith('image/')
? record.contentType
: 'application/octet-stream'
inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`)
} catch {
// A missing, unreadable or too-large image keeps its URL reference.
}
}
if (inlined.size === 0) return documentHtml
return documentHtml.replace(IMAGE_SRC, (match, id: string) => {
const dataUri = inlined.get(id)
apps/sim/tools/microsoft_teams/server-utils.ts
+14 -5
View File
@@ -5,6 +5,7 @@
*/
import type { Logger } from '@sim/logger'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization'
@@ -79,11 +80,19 @@ export async function uploadFilesForTeamsMessage(params: {
throw new FileAccessDeniedError()
}
// Download file from storage
const { buffer, contentType } = await downloadServableFileFromStorage(file, requestId, log)
if (buffer.length > MAX_TEAMS_FILE_SIZE) {
const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2)
// Download file from storage, bounded by the same 4MB limit checked above — the
// declared size is the caller's claim, so the ceiling has to reach the read itself.
let buffer: Buffer
let contentType: string
try {
const servable = await downloadServableFileFromStorage(file, requestId, log, {
maxBytes: MAX_TEAMS_FILE_SIZE,
})
buffer = servable.buffer
contentType = servable.contentType
} catch (error) {
if (!isPayloadSizeLimitError(error)) throw error
const sizeMB = ((error.observedBytes ?? file.size) / (1024 * 1024)).toFixed(2)
throw new Error(
`File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.`
)