fix(uploads): attach compiled binary for AI-generated docs, not source (#5266)

* fix(uploads): attach compiled binary for AI-generated docs, not source

AI-generated documents (pdf/docx/pptx/xlsx) created in Chat are stored as
their generation source, with the rendered binary in a separate
content-addressed artifact store. Read/preview paths swap in the binary, but
attachment/upload/provider paths downloaded the raw source — so a generated
PDF emailed via Gmail (and 30+ other tools) arrived as the generator script
renamed .pdf.

- Add shared resolveServableDocBytes resolver + downloadServableFileFromStorage
  wrapper; the file-serve route now delegates to the same resolver so the two
  paths resolve identically.
- Migrate ~34 attachment/upload/parse tool routes + the LLM provider attachment
  path to the servable download; media-only tools and source-editing paths keep
  the raw download intentionally.
- Surface a retryable 409 (shared docNotReadyResponse) when a doc artifact is
  still compiling instead of shipping source.

* fix(uploads): return retryable 409 for not-ready docs in slack/teams sends

The slack send-message and teams write_channel/write_chat routes call
download helpers that can throw DocCompileUserError while a generated doc is
still compiling. Map it to the shared docNotReadyResponse 409 (matching the
other migrated tool routes) instead of a generic 500. The provider attachment
path is internal LLM execution (no HTTP response), so it intentionally
propagates the typed error.

* fix(uploads): not-ready 409 for uptimerobot, real MIME for non-doc, xlsx tests

Address review findings:
- uptimerobot create-psp/update-psp now map DocCompileUserError to the shared
  409 (Greptile + Cursor flagged the gap alongside slack/teams).
- downloadServableFileFromStorage returns the extension-derived MIME
  (getMimeTypeFromExtension) for non-doc files instead of an empty string when
  userFile.type is unset.
- Add resolveServableDocBytes tests for the three xlsx branches (binary ZIP
  passthrough, not-ready throw under E2B+beta, no-workspaceId raw passthrough).

* fix(uploads): enforce attachment size limits on resolved bytes

Size limits were checked against userFile.size (source metadata) before
resolution, but a generated doc resolves to a larger compiled binary — so a
small-source doc could pass the pre-check yet exceed the service limit. Add a
post-resolution check on the actual resolved bytes (mirroring docusign/vanta)
across gmail send/draft/edit-draft, smtp, outlook send/draft, telegram, sftp,
and teams; the cheap source pre-check stays as an early reject.

* chore(uploads): drop extraneous inline comments from servable-file changes

* fix(sftp): enforce 100MB cap on cumulative resolved bytes, not per-file

The SFTP batch upload checked each resolved file against the 100MB cap
individually, so multiple resolved attachments could each pass while their
combined size exceeded the limit. Accumulate resolved bytes across the loop
and reject once the running total exceeds the cap.

* fix(sendgrid): reject attachments exceeding SendGrid's 30MB limit on resolved bytes

SendGrid had no attachment-size guard, so a generated doc resolving to a large
compiled binary could be sent and fail opaquely at the API. Add a post-resolution
total-size check (30MB, SendGrid's documented message limit) matching the
gmail/smtp/outlook routes.
This commit is contained in:
Waleed
2026-06-29 13:01:56 -07:00
committed by GitHub
parent a6196e0c86
commit 950c2602c2
51 changed files with 1129 additions and 332 deletions
+13 -108
View File
@@ -1,18 +1,14 @@
import { readFile } from 'fs/promises'
import { createLogger } from '@sim/logger'
import { sha256Hex } from '@sim/security/hash'
import type { NextRequest } from 'next/server'
import { NextResponse } from 'next/server'
import { fileServeParamsSchema, fileServeQuerySchema } from '@/lib/api/contracts/storage-transfer'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import {
DocCompileUserError,
getE2BDocFormat,
loadCompiledDocByExt,
resolveServableDocBytes,
} from '@/lib/copilot/tools/server/files/doc-compile'
import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads'
import type { StorageContext } from '@/lib/uploads/config'
import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
@@ -26,47 +22,14 @@ import {
findLocalFile,
getContentType,
} from '@/app/api/files/utils'
import type { SandboxTaskId } from '@/sandbox-tasks/registry'
const logger = createLogger('FilesServeAPI')
const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04])
const PDF_MAGIC = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d]) // %PDF-
interface CompilableFormat {
magic: Buffer
taskId: SandboxTaskId
contentType: string
}
const COMPILABLE_FORMATS: Record<string, CompilableFormat> = {
'.pptx': {
magic: ZIP_MAGIC,
taskId: 'pptx-generate',
contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
},
'.docx': {
magic: ZIP_MAGIC,
taskId: 'docx-generate',
contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
},
'.pdf': {
magic: PDF_MAGIC,
taskId: 'pdf-generate',
contentType: 'application/pdf',
},
}
const MAX_COMPILED_DOC_CACHE = 10
const compiledDocCache = new Map<string, Buffer>()
function compiledCacheSet(key: string, buffer: Buffer): void {
if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) {
compiledDocCache.delete(compiledDocCache.keys().next().value as string)
}
compiledDocCache.set(key, buffer)
}
/**
* Resolves the bytes + content type to serve for a stored file via the shared
* {@link resolveServableDocBytes} (generated docs → compiled artifact). `raw=1`
* bypasses resolution and serves the stored source as-is.
*/
async function compileDocumentIfNeeded(
buffer: Buffer,
filename: string,
@@ -76,71 +39,13 @@ async function compileDocumentIfNeeded(
signal: AbortSignal | undefined
): Promise<{ buffer: Buffer; contentType: string }> {
if (raw) return { buffer, contentType: getContentType(filename) }
const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase()
const extNoDot = ext.replace(/^\./, '')
const format = COMPILABLE_FORMATS[ext]
// Already a binary file (uploaded or pre-compiled)? Serve as-is.
if (format) {
const magicLen = format.magic.length
if (buffer.length >= magicLen && buffer.subarray(0, magicLen).equals(format.magic)) {
return { buffer, contentType: getContentType(filename) }
}
}
// .xlsx is a ZIP container with no JS compile path. An uploaded/binary xlsx
// must short-circuit here (it isn't in COMPILABLE_FORMATS) — otherwise every
// xlsx open would utf-8-decode the whole binary and do an always-miss S3 GET.
// Only a Python-source xlsx (UTF-8 text, no ZIP magic) falls through.
if (
extNoDot === 'xlsx' &&
buffer.length >= ZIP_MAGIC.length &&
buffer.subarray(0, ZIP_MAGIC.length).equals(ZIP_MAGIC)
) {
return { buffer, contentType: getContentType(filename) }
}
// Generated docs render from a content-addressed compiled binary that is built
// exactly ONCE per edit_content/create (at write time) and stored in S3. Serve
// only LOADS it — it must never compile, or it would re-run E2B on every preview
// fetch, including against the incomplete source mid-generation. A hit returns
// the (possibly partial) committed doc; a miss in the E2B regime means the doc
// is still being generated → 409, and the client polls until the artifact lands.
if (workspaceId && (format || extNoDot === 'xlsx')) {
const source = buffer.toString('utf-8')
// Load the prebuilt artifact directly from S3 (content-addressed). No extra
// in-memory layer here: the store is the source of truth, the client (react
// query) already caches the bytes, and this branch never recomputes.
const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot)
if (stored) {
return { buffer: stored.buffer, contentType: stored.contentType }
}
if (isE2BDocEnabled && (await getE2BDocFormat(filename))) {
// Artifact not built yet (still generating, or the source didn't compile at
// write time). Signal "not ready" without compiling — handled as 409.
throw new DocCompileUserError('Document is still being generated')
}
}
if (!format) return { buffer, contentType: getContentType(filename) }
// E2B disabled and no stored artifact → compile JS source via isolated-vm.
const code = buffer.toString('utf-8')
const cacheKey = sha256Hex(`${ext}${code}${workspaceId ?? ''}`)
const cached = compiledDocCache.get(cacheKey)
if (cached) {
return { buffer: cached, contentType: format.contentType }
}
const compiled = await runSandboxTask(
format.taskId,
{ code, workspaceId: workspaceId || '' },
{ ownerKey, signal }
)
compiledCacheSet(cacheKey, compiled)
return { buffer: compiled, contentType: format.contentType }
return resolveServableDocBytes({
rawBuffer: buffer,
fileName: filename,
workspaceId,
ownerKey,
signal,
})
}
const STORAGE_KEY_PREFIX_RE = /^\d{13}-[a-z0-9]{7}-/
@@ -17,7 +17,8 @@ import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -90,13 +91,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
}
files = await Promise.all(
userFiles.map(async (userFile) => ({
bytes: await downloadFileFromStorage(userFile, requestId, logger, {
maxBytes: A2A_MAX_FILE_BYTES,
}),
name: userFile.name,
mediaType: userFile.type || 'application/octet-stream',
}))
userFiles.map(async (userFile) => {
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger,
{ maxBytes: A2A_MAX_FILE_BYTES }
)
return {
bytes: buffer,
name: userFile.name,
mediaType: contentType || userFile.type || 'application/octet-stream',
}
})
)
}
@@ -130,6 +137,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
output,
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] A2A send failed`, { error: getErrorMessage(error) })
return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 })
}
@@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({
processFilesToUserFiles: mockProcessFilesToUserFiles,
}))
vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
downloadFileFromStorage: mockDownloadFileFromStorage,
downloadServableFileFromStorage: mockDownloadFileFromStorage,
}))
vi.mock('@/app/api/files/authorization', () => ({
assertToolFileAccess: mockAssertToolFileAccess,
@@ -77,7 +77,10 @@ beforeEach(() => {
{ key: 's3://bucket/file.txt', name: 'file.txt', size: 5, type: 'text/plain' },
])
mockAssertToolFileAccess.mockResolvedValue(null)
mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('hello'))
mockDownloadFileFromStorage.mockResolvedValue({
buffer: Buffer.from('hello'),
contentType: 'application/octet-stream',
})
})
describe('POST /api/tools/agiloft/attach', () => {
+14 -2
View File
@@ -9,7 +9,8 @@ import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import type { RawFileInput } from '@/lib/uploads/utils/file-schemas'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { buildAttachFileUrl } from '@/tools/agiloft/utils'
import {
@@ -74,7 +75,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
let fileBuffer: Buffer
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 })
}
const resolvedFileName = data.fileName || userFile.name || 'attachment'
let resolvedIP: string
+13 -2
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -53,7 +54,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = result.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
fileName = validatedData.fileName || userFile.name
} else if (validatedData.fileContent) {
logger.info(`[${requestId}] Using legacy base64 content input`)
@@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({
processFilesToUserFiles: mockProcessFilesToUserFiles,
}))
vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
downloadFileFromStorage: mockDownloadFileFromStorage,
downloadServableFileFromStorage: mockDownloadFileFromStorage,
}))
vi.mock('@/app/api/files/authorization', () => ({
assertToolFileAccess: mockAssertToolFileAccess,
@@ -65,7 +65,10 @@ beforeEach(() => {
{ key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5, type: 'application/pdf' },
])
mockAssertToolFileAccess.mockResolvedValue(null)
mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('receipt-bytes'))
mockDownloadFileFromStorage.mockResolvedValue({
buffer: Buffer.from('receipt-bytes'),
contentType: 'application/pdf',
})
})
describe('POST /api/tools/brex/upload-receipt', () => {
@@ -192,7 +195,10 @@ describe('POST /api/tools/brex/upload-receipt', () => {
})
it('rejects files over the 50 MB limit', async () => {
mockDownloadFileFromStorage.mockResolvedValueOnce(Buffer.alloc(50 * 1024 * 1024 + 1))
mockDownloadFileFromStorage.mockResolvedValueOnce({
buffer: Buffer.alloc(50 * 1024 * 1024 + 1),
contentType: 'application/pdf',
})
const response = await POST(createMockRequest('POST', baseBody))
expect(response.status).toBe(400)
@@ -11,7 +11,8 @@ import {
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { BREX_API_BASE, buildBrexHeaders } from '@/tools/brex/utils'
@@ -48,7 +49,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
let fileBuffer: Buffer
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = resolved.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download receipt file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error') },
{ status: 500 }
)
}
if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) {
return NextResponse.json(
{ success: false, error: 'Receipt file exceeds the 50 MB limit' },
@@ -7,7 +7,8 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { getConfluenceCloudId } from '@/tools/confluence/utils'
import { parseAtlassianErrorMessage } from '@/tools/jira/utils'
@@ -91,9 +92,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
let fileBuffer: Buffer
let resolvedContentType: string
try {
fileBuffer = await downloadFileFromStorage(userFile, 'confluence-upload', logger)
const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger)
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error('Failed to download file from storage:', error)
return NextResponse.json(
{
@@ -104,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
const uploadFileName = fileName || userFile.name || 'attachment'
const mimeType = userFile.type || 'application/octet-stream'
const mimeType = resolvedContentType || userFile.type || 'application/octet-stream'
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/child/attachment`
+14 -2
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { DAYTONA_TOOLBOX_BASE_URL, extractDaytonaError } from '@/tools/daytona/utils'
@@ -60,7 +61,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`)
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
fileName = params.fileName || userFile.name
} else if (params.fileContent) {
logger.info(`[${requestId}] Using legacy base64 content input`)
@@ -8,7 +8,8 @@ import { validateNumericId } from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -143,31 +144,39 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
userFiles.map(async (file, i) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
userFiles.map(async (file, i) => {
logger.info(`[${requestId}] Downloading file ${i}: ${file.name}`)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
for (let i = 0; i < userFiles.length; i++) {
const userFile = userFiles[i]
const buffer = buffers[i]
const buffer = resolved[i].buffer
const mimeType = resolved[i].contentType || userFile.type || 'application/octet-stream'
logger.info(`[${requestId}] Added file ${i}: ${userFile.name} (${buffer.length} bytes)`)
filesOutput.push({
name: userFile.name,
mimeType: userFile.type || 'application/octet-stream',
mimeType,
data: buffer.toString('base64'),
size: buffer.length,
})
const blob = new Blob([new Uint8Array(buffer)], { type: userFile.type })
const blob = new Blob([new Uint8Array(buffer)], { type: mimeType })
formData.append(`files[${i}]`, blob, userFile.name)
}
+12 -4
View File
@@ -17,7 +17,8 @@ import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot'
import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
import { FileInputSchema } from '@/lib/uploads/utils/file-schemas'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
const logger = createLogger('DocuSignAPI')
@@ -228,14 +229,21 @@ async function handleSendEnvelope(
{ status: 413 }
)
}
const buffer = await downloadFileFromStorage(userFile, 'docusign-send', logger, {
maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES,
})
const { buffer } = await downloadServableFileFromStorage(
userFile,
'docusign-send',
logger,
{
maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES,
}
)
assertKnownSizeWithinLimit(buffer.length, MAX_DOCUSIGN_DOCUMENT_BYTES, 'DocuSign document')
documentBase64 = buffer.toString('base64')
documentName = userFile.name
}
} catch (fileError) {
const notReady = docNotReadyResponse(fileError)
if (notReady) return notReady
logger.error('Failed to process file for DocuSign envelope', { fileError })
return NextResponse.json(
{
+13 -2
View File
@@ -8,7 +8,8 @@ import { generateRequestId } from '@/lib/core/utils/request'
import { httpHeaderSafeJson } from '@/lib/core/utils/validation'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -56,7 +57,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = result.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
fileName = userFile.name
} else if (validatedData.fileContent) {
// Legacy: base64 string input (backwards compatibility)
+8 -2
View File
@@ -31,7 +31,11 @@ import {
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { getFileMetadataByKey } from '@/lib/uploads/server/metadata'
import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import {
downloadFileFromStorage,
downloadServableFileFromStorage,
} from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { performMoveWorkspaceFileItems } from '@/lib/workspace-files/orchestration'
import {
assertActiveWorkspaceAccess,
@@ -306,7 +310,7 @@ const extractUserFileTextContent = async (
userFile: UserFile,
requestId: string
): Promise<string> => {
const buffer = await downloadFileFromStorage(userFile, requestId, logger, {
const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, {
maxBytes: MAX_GET_CONTENT_FILE_BYTES,
})
@@ -1038,6 +1042,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 403 }
)
}
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
if (error instanceof ShareValidationError) {
return NextResponse.json({ success: false, error: error.message }, { status: 400 })
}
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -47,11 +48,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
)
const formData = new FormData()
const blob = new Blob([new Uint8Array(buffer)], {
type: userFile.type || 'application/octet-stream',
type: contentType || userFile.type || 'application/octet-stream',
})
formData.append('file', blob, userFile.name)
@@ -88,6 +93,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
output: firecrawlData.data ?? firecrawlData,
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error in Firecrawl parse:`, error)
return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 })
}
+35 -15
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
base64UrlEncode,
@@ -94,26 +95,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: file.type || 'application/octet-stream',
content: buffers[i],
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
content: resolved[i].buffer,
}))
const mimeMessage = buildMimeMessage({
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
base64UrlEncode,
@@ -90,26 +91,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: file.type || 'application/octet-stream',
content: buffers[i],
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
content: resolved[i].buffer,
}))
const mimeMessage = buildMimeMessage({
+35 -15
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
base64UrlEncode,
@@ -94,26 +95,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
mimeType: file.type || 'application/octet-stream',
content: buffers[i],
mimeType: resolved[i].contentType || file.type || 'application/octet-stream',
content: resolved[i].buffer,
}))
const mimeMessage = buildMimeMessage({
@@ -8,7 +8,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
GOOGLE_WORKSPACE_MIME_TYPES,
@@ -120,10 +121,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
let fileBuffer: Buffer
let downloadedContentType = ''
try {
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file:`, error)
return NextResponse.json(
{
@@ -134,8 +140,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
let uploadMimeType = validatedData.mimeType || userFile.type || 'application/octet-stream'
const requestedMimeType = validatedData.mimeType || userFile.type || 'application/octet-stream'
let uploadMimeType =
validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream'
const requestedMimeType =
validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream'
if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) {
uploadMimeType = SOURCE_MIME_TYPES[requestedMimeType] || 'text/plain'
@@ -6,7 +6,8 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils'
@@ -47,9 +48,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
for (const file of userFiles) {
const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(file, requestId, logger)
let buffer: Buffer
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(file, requestId, logger)
buffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
throw error
}
const blob = new Blob([new Uint8Array(buffer)], {
type: file.type || 'application/octet-stream',
type: downloadedContentType || file.type || 'application/octet-stream',
})
formData.append('file', blob, file.name)
}
+19 -3
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { extractLinqError, LINQ_API_BASE, linqHeaders } from '@/tools/linq/utils'
@@ -58,9 +59,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const userFile = userFiles[0]
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
buffer = await downloadFileFromStorage(userFile, requestId, logger)
let resolvedContentTypeFromStorage: string
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
buffer = resolved.buffer
resolvedContentTypeFromStorage = resolved.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download Linq attachment file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error occurred') },
{ status: 500 }
)
}
if (!resolvedFilename) resolvedFilename = userFile.name
if (!resolvedContentType) resolvedContentType = userFile.type || 'application/octet-stream'
if (!resolvedContentType)
resolvedContentType =
resolvedContentTypeFromStorage || userFile.type || 'application/octet-stream'
} else if (fileContent) {
buffer = Buffer.from(fileContent, 'base64')
if (!resolvedFilename) resolvedFilename = 'file'
@@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation.
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -71,7 +72,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = servable.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
} else if (validatedData.fileContent) {
fileBuffer = Buffer.from(validatedData.fileContent, 'base64')
} else {
@@ -7,6 +7,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { FileAccessDeniedError } from '@/app/api/files/authorization'
import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils'
import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types'
@@ -167,6 +168,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (error instanceof FileAccessDeniedError) {
return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 })
}
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error sending Teams channel message:`, error)
return NextResponse.json(
{
@@ -7,6 +7,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { FileAccessDeniedError } from '@/app/api/files/authorization'
import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils'
import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types'
@@ -164,6 +165,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (error instanceof FileAccessDeniedError) {
return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 })
}
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error sending Teams chat message:`, error)
return NextResponse.json(
{
+13 -2
View File
@@ -12,9 +12,10 @@ import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import {
downloadFileFromStorage,
downloadServableFileFromStorage,
resolveInternalFileUrl,
} from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -124,8 +125,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (!base64) {
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
)
base64 = buffer.toString('base64')
if (contentType && contentType !== 'application/octet-stream') {
mimeType = contentType
}
}
const base64Payload = base64.startsWith('data:')
? base64
@@ -265,6 +273,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
output: mistralData,
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error in Mistral parse:`, error)
return NextResponse.json(
@@ -13,7 +13,8 @@ import {
getExtensionFromMimeType,
processSingleFileToUserFile,
} from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { normalizeExcelValues } from '@/tools/onedrive/utils'
@@ -114,8 +115,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
try {
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = result.buffer
mimeType = result.contentType || userFile.type || 'application/octet-stream'
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{
@@ -125,8 +130,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 500 }
)
}
mimeType = userFile.type || 'application/octet-stream'
}
const maxSize = 250 * 1024 * 1024
+35 -15
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -107,27 +108,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`,
},
{ status: 400 }
)
}
const attachmentObjects = attachments.map((file, i) => ({
'@odata.type': '#microsoft.graph.fileAttachment',
name: file.name,
contentType: file.type || 'application/octet-stream',
contentBytes: buffers[i].toString('base64'),
contentType: resolved[i].contentType || file.type || 'application/octet-stream',
contentBytes: resolved[i].buffer.toString('base64'),
}))
logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`)
+35 -15
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -107,27 +108,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`,
},
{ status: 400 }
)
}
const attachmentObjects = attachments.map((file, i) => ({
'@odata.type': '#microsoft.graph.fileAttachment',
name: file.name,
contentType: file.type || 'application/octet-stream',
contentBytes: buffers[i].toString('base64'),
contentType: resolved[i].contentType || file.type || 'application/octet-stream',
contentBytes: resolved[i].buffer.toString('base64'),
}))
logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`)
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess, FileAccessDeniedError } from '@/app/api/files/authorization'
import {
buildPersonaHeaders,
@@ -55,7 +56,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
let buffer: Buffer
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
buffer = resolved.buffer
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download Persona import file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Internal server error') },
{ status: 500 }
)
}
logger.info(`[${requestId}] Importing accounts into Persona`, {
fileName: userFile.name,
+20 -4
View File
@@ -8,7 +8,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -81,10 +82,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
uploadBody = result.buffer
downloadedContentType = result.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
uploadBody = buffer
uploadContentType = validatedData.contentType || userFile.type || 'application/octet-stream'
uploadContentType =
validatedData.contentType ||
downloadedContentType ||
userFile.type ||
'application/octet-stream'
} else if (validatedData.content) {
uploadBody = Buffer.from(validatedData.content, 'utf-8')
uploadContentType = validatedData.contentType || 'text/plain'
@@ -1,5 +1,5 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { type NextRequest, NextResponse } from 'next/server'
import { getValidationErrorMessage, isZodError } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
@@ -7,7 +7,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation.
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
assertSafeExternalUrl,
@@ -208,9 +209,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const userFile = userFiles[0]
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
let fileBuffer: Buffer
let resolvedContentType: string
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = resolved.buffer
resolvedContentType = resolved.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download Concur receipt file:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error') },
{ status: 500 }
)
}
const fileName = userFile.name
const mimeType = inferMimeType(fileName, userFile.type)
const mimeType = inferMimeType(fileName, resolvedContentType || userFile.type)
const allowedForOperation =
uploadReq.operation === 'create_quick_expense_with_image'
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -106,26 +107,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
userFiles.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
userFiles.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
const maxSize = 30 * 1024 * 1024
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`,
},
{ status: 400 }
)
}
const sendGridAttachments = userFiles.map((file, i) => ({
content: buffers[i].toString('base64'),
content: resolved[i].buffer.toString('base64'),
filename: file.name,
type: file.type || 'application/octet-stream',
type: resolved[i].contentType || file.type || 'application/octet-stream',
disposition: 'attachment',
}))
@@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation.
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import type { ServiceNowAttachment } from '@/tools/servicenow/types'
import { createBasicAuthHeader } from '@/tools/servicenow/utils'
@@ -52,8 +53,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const contentType = userFile.type || 'application/octet-stream'
const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
let fileBuffer: Buffer
let resolvedContentType: string
try {
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file from storage:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
const contentType = resolvedContentType || userFile.type || 'application/octet-stream'
const baseUrl = body.instanceUrl.trim().replace(/\/$/, '')
const uploadParams = new URLSearchParams({
+15 -2
View File
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
createSftpConnection,
@@ -95,6 +96,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
let resolvedTotal = 0
for (const file of userFiles) {
try {
const denied = await assertToolFileAccess(
@@ -107,7 +109,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.info(
`[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)`
)
const buffer = await downloadFileFromStorage(file, requestId, logger)
const { buffer } = await downloadServableFileFromStorage(file, requestId, logger)
resolvedTotal += buffer.length
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{ success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` },
{ status: 400 }
)
}
const safeFileName = sanitizeFileName(file.name)
const fullRemotePath = remotePath.endsWith('/')
@@ -142,6 +153,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger.info(`[${requestId}] Uploaded ${safeFileName} to ${sanitizedRemotePath}`)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error)
throw new Error(
`Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
@@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation.
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types'
import type { SharepointSkippedFile, SharepointUploadError } from '@/tools/sharepoint/types'
@@ -87,7 +88,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (denied) return denied
logger.info(`[${requestId}] Uploading file: ${userFile.name}`)
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
let buffer: Buffer
let downloadedContentType = ''
try {
const result = await downloadServableFileFromStorage(userFile, requestId, logger)
buffer = result.buffer
downloadedContentType = result.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
throw error
}
const fileName = validatedData.fileName || userFile.name
const folderPath = validatedData.folderPath?.trim() || ''
@@ -135,7 +146,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
method: 'PUT',
headers: {
Authorization: `Bearer ${validatedData.accessToken}`,
'Content-Type': userFile.type || 'application/octet-stream',
'Content-Type': downloadedContentType || userFile.type || 'application/octet-stream',
},
body: buffer,
},
@@ -156,7 +167,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
method: 'PUT',
headers: {
Authorization: `Bearer ${validatedData.accessToken}`,
'Content-Type': userFile.type || 'application/octet-stream',
'Content-Type':
downloadedContentType || userFile.type || 'application/octet-stream',
},
body: buffer,
},
@@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { FileAccessDeniedError } from '@/app/api/files/authorization'
import { sendSlackMessage } from '@/app/api/tools/slack/utils'
@@ -72,6 +73,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
if (error instanceof FileAccessDeniedError) {
return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 })
}
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error sending Slack message:`, error)
return NextResponse.json(
{
+7 -3
View File
@@ -1,7 +1,7 @@
import type { Logger } from '@sim/logger'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization'
import type { ToolFileData } from '@/tools/types'
@@ -89,7 +89,11 @@ async function uploadFilesToSlack(
throw new FileAccessDeniedError()
}
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
const { buffer, contentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
)
const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', {
method: 'POST',
@@ -131,7 +135,7 @@ async function uploadFilesToSlack(
// Only add to uploadedFiles after successful upload to keep arrays in sync
uploadedFiles.push({
name: userFile.name,
mimeType: userFile.type || 'application/octet-stream',
mimeType: contentType || userFile.type || 'application/octet-stream',
data: buffer.toString('base64'),
size: buffer.length,
})
+35 -15
View File
@@ -9,7 +9,8 @@ import { validateDatabaseHost } from '@/lib/core/security/input-validation.serve
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -127,26 +128,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = accessResults.find((r) => r !== null)
if (denied) return denied
const buffers = await Promise.all(
attachments.map(async (file) => {
try {
let resolved: Array<{ buffer: Buffer; contentType: string }>
try {
resolved = await Promise.all(
attachments.map(async (file) => {
logger.info(
`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`
)
return await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error)
throw new Error(
`Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}`
)
}
})
)
return await downloadServableFileFromStorage(file, requestId, logger)
})
)
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download an attachment:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0)
if (resolvedTotal > maxSize) {
const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`,
},
{ status: 400 }
)
}
const attachmentBuffers = attachments.map((file, i) => ({
filename: file.name,
content: buffers[i],
contentType: file.type || 'application/octet-stream',
content: resolved[i].buffer,
contentType: resolved[i].contentType || file.type || 'application/octet-stream',
}))
logger.info(`[${requestId}] Processed ${attachmentBuffers.length} attachment(s)`)
@@ -8,7 +8,8 @@ import { validateSupabaseProjectId } from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -147,10 +148,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
let buffer: Buffer
let resolvedContentType: string
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
buffer = resolved.buffer
resolvedContentType = resolved.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Internal server error') },
{ status: 500 }
)
}
uploadBody = buffer
uploadContentType = validatedData.contentType || userFile.type || 'application/octet-stream'
uploadContentType =
validatedData.contentType ||
resolvedContentType ||
userFile.type ||
'application/octet-stream'
}
let fullPath = validatedData.fileName
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { convertMarkdownToHTML } from '@/tools/telegram/utils'
@@ -93,11 +94,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
let buffer: Buffer
let contentType: string
try {
const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger)
buffer = downloaded.buffer
contentType = downloaded.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error)
return NextResponse.json(
{
success: false,
error: `Failed to download document: ${getErrorMessage(error, 'Unknown error')}`,
},
{ status: 500 }
)
}
if (buffer.length > maxSize) {
const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2)
return NextResponse.json(
{
success: false,
error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`,
},
{ status: 400 }
)
}
const resolvedMimeType = contentType || userFile.type || 'application/octet-stream'
const filesOutput = [
{
name: userFile.name,
mimeType: userFile.type || 'application/octet-stream',
mimeType: resolvedMimeType,
data: buffer.toString('base64'),
size: buffer.length,
},
@@ -108,7 +139,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const formData = new FormData()
formData.append('chat_id', validatedData.chatId)
const blob = new Blob([new Uint8Array(buffer)], { type: userFile.type })
const blob = new Blob([new Uint8Array(buffer)], { type: resolvedMimeType })
formData.append('document', blob, userFile.name)
if (validatedData.caption) {
+11 -3
View File
@@ -16,9 +16,10 @@ import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import {
downloadFileFromStorage,
downloadServableFileFromStorage,
resolveInternalFileUrl,
} from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -437,9 +438,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
const { buffer, contentType: resolvedContentType } = await downloadServableFileFromStorage(
userFile,
requestId,
logger
)
bytes = buffer.toString('base64')
contentType = userFile.type || 'application/octet-stream'
contentType = resolvedContentType || userFile.type || 'application/octet-stream'
isPdf = contentType.includes('pdf') || userFile.name?.toLowerCase().endsWith('.pdf')
} else if (validatedData.filePath) {
let fileUrl = validatedData.filePath
@@ -616,6 +621,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
},
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Error in Textract parse:`, error)
return NextResponse.json(
@@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils'
export const dynamic = 'force-dynamic'
@@ -39,6 +40,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger,
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Unexpected error creating status page:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error') },
@@ -1,7 +1,7 @@
import type { Logger } from '@sim/logger'
import { NextResponse } from 'next/server'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { mapPsp, UPTIMEROBOT_API_BASE } from '@/tools/uptimerobot/types'
@@ -47,8 +47,8 @@ async function appendPspImage(
const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger)
if (denied) return denied
const buffer = await downloadFileFromStorage(userFile, requestId, logger)
const mimeType = userFile.type || 'application/octet-stream'
const { buffer, contentType } = await downloadServableFileFromStorage(userFile, requestId, logger)
const mimeType = contentType || userFile.type || 'application/octet-stream'
form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name)
return null
}
@@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils'
export const dynamic = 'force-dynamic'
@@ -39,6 +40,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
logger,
})
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Unexpected error updating status page:`, error)
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Unknown error') },
+20 -5
View File
@@ -1,5 +1,5 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { type NextRequest, NextResponse } from 'next/server'
import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta'
import { parseRequest } from '@/lib/api/server'
@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import {
asVantaRecord,
@@ -70,9 +71,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
return uploadSizeError(userFile.size)
}
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
fileName = params.fileName || userFile.name
mimeType = userFile.type || params.mimeType || 'application/octet-stream'
try {
const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = resolved.buffer
fileName = params.fileName || userFile.name
mimeType =
resolved.contentType || userFile.type || params.mimeType || 'application/octet-stream'
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download Vanta upload file`, {
error: getErrorMessage(error),
})
return NextResponse.json(
{ success: false, error: getErrorMessage(error, 'Failed to download file') },
{ status: 500 }
)
}
} else if (params.fileContent) {
fileBuffer = Buffer.from(params.fileContent, 'base64')
fileName = params.fileName || 'file'
@@ -11,7 +11,8 @@ import {
getMimeTypeFromExtension,
processSingleFileToUserFile,
} from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
import { assertToolFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
@@ -89,10 +90,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
})
let fileBuffer: Buffer
let resolvedContentType: string
try {
fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
fileBuffer = servable.buffer
resolvedContentType = servable.contentType
} catch (error) {
const notReady = docNotReadyResponse(error)
if (notReady) return notReady
logger.error(`[${requestId}] Failed to download file:`, error)
return NextResponse.json(
{
@@ -104,7 +110,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
const filename = validatedData.filename || userFile.name
const mimeType = userFile.type || getMimeTypeFromExtension(getFileExtension(filename))
const mimeType =
resolvedContentType || userFile.type || getMimeTypeFromExtension(getFileExtension(filename))
logger.info(`[${requestId}] Uploading to WordPress`, {
siteId: validatedData.siteId,
@@ -1,11 +1,16 @@
import { createLogger } from '@sim/logger'
import { sha256Hex } from '@sim/security/hash'
import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
import { executeInE2B, executeShellInE2B, type SandboxFile } from '@/lib/execution/e2b'
import { CodeLanguage } from '@/lib/execution/languages'
import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
import {
fetchWorkspaceFileBuffer,
getWorkspaceFile,
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { getContentType } from '@/app/api/files/utils'
import type { SandboxTaskId } from '@/sandbox-tasks/registry'
import { loadCompiledDoc, storeCompiledDoc } from './doc-compiled-store'
const logger = createLogger('CopilotDocCompile')
@@ -449,3 +454,102 @@ export async function resolveServableDoc(
const artifact = await loadCompiledDocByExt(workspaceId, storedBytes.toString('utf-8'), fmt.ext)
return artifact ? { kind: 'artifact', ...artifact } : { kind: 'unavailable' }
}
interface CompilableFormat {
magic: Buffer
taskId: SandboxTaskId
contentType: string
}
const COMPILABLE_FORMATS: Record<string, CompilableFormat> = {
'.pptx': { magic: ZIP_MAGIC, taskId: 'pptx-generate', contentType: PPTX_MIME },
'.docx': { magic: ZIP_MAGIC, taskId: 'docx-generate', contentType: DOCX_MIME },
'.pdf': { magic: PDF_MAGIC, taskId: 'pdf-generate', contentType: PDF_MIME },
}
const MAX_COMPILED_DOC_CACHE = 10
const compiledDocCache = new Map<string, Buffer>()
function compiledCacheSet(key: string, buffer: Buffer): void {
if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) {
compiledDocCache.delete(compiledDocCache.keys().next().value as string)
}
compiledDocCache.set(key, buffer)
}
/**
* Resolves the bytes a consumer should actually serve/attach for a stored file —
* the single source of truth shared by the file-serve route and every tool that
* downloads a workspace file (email attachments, uploads, provider file inputs).
*
* Generated docs (pdf/docx/pptx/xlsx) store their GENERATION SOURCE as the primary
* file; the rendered binary lives in a separate content-addressed artifact store.
* A naive raw-byte read therefore hands out source text under a `.pdf` name — the
* corruption every non-serve consumer used to ship. The file-serve route and the
* attachment download helper share this one function so they resolve identically.
* (The public read-only share route uses the non-compiling {@link resolveServableDoc}
* variant, which returns `unavailable` instead of throwing.) The swap:
*
* - Bytes already carry the format magic (`%PDF`/ZIP) → real uploaded/binary file,
* serve as-is.
* - Generated-doc source → load the content-addressed compiled artifact.
* - Artifact missing in the E2B regime → the doc is still being generated; throw
* {@link DocCompileUserError} so callers signal "not ready / retry" instead of
* shipping source.
* - E2B disabled → compile the committed JS source via isolated-vm (cached).
* - Non-doc files → pass through with the extension-derived content type.
*
* It never falls back to attaching the raw source bytes for a generated doc.
*/
export async function resolveServableDocBytes(args: {
rawBuffer: Buffer
fileName: string
workspaceId: string | undefined
ownerKey?: string
signal?: AbortSignal
}): Promise<{ buffer: Buffer; contentType: string }> {
const { rawBuffer, fileName, workspaceId, ownerKey, signal } = args
const ext = fileName.slice(fileName.lastIndexOf('.')).toLowerCase()
const extNoDot = ext.replace(/^\./, '')
const format = COMPILABLE_FORMATS[ext]
// xlsx isn't in COMPILABLE_FORMATS (no isolated-vm path), so match its ZIP magic
// explicitly alongside the table-driven formats.
const magic = format?.magic ?? (extNoDot === 'xlsx' ? ZIP_MAGIC : undefined)
if (magic && bufferStartsWith(rawBuffer, magic)) {
return { buffer: rawBuffer, contentType: getContentType(fileName) }
}
if (!format && extNoDot !== 'xlsx') {
return { buffer: rawBuffer, contentType: getContentType(fileName) }
}
const source = rawBuffer.toString('utf-8')
if (workspaceId) {
const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot)
if (stored) {
return { buffer: stored.buffer, contentType: stored.contentType }
}
if (isE2BDocEnabled && (await getE2BDocFormat(fileName))) {
throw new DocCompileUserError('Document is still being generated')
}
}
// Reaches here only for xlsx, which has no isolated-vm fallback.
if (!format) return { buffer: rawBuffer, contentType: getContentType(fileName) }
const cacheKey = sha256Hex(`${ext}${source}${workspaceId ?? ''}`)
const cached = compiledDocCache.get(cacheKey)
if (cached) {
return { buffer: cached, contentType: format.contentType }
}
const compiled = await runSandboxTask(
format.taskId,
{ code: source, workspaceId: workspaceId || '' },
{ ownerKey, signal }
)
compiledCacheSet(cacheKey, compiled)
return { buffer: compiled, contentType: format.contentType }
}
@@ -0,0 +1,183 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { e2bFlag, betaFlag, mockLoadCompiledDoc, mockRunSandboxTask } = vi.hoisted(() => ({
e2bFlag: { value: true },
betaFlag: { value: false },
mockLoadCompiledDoc: vi.fn(),
mockRunSandboxTask: vi.fn(),
}))
vi.mock('@/lib/execution/e2b', () => ({
executeInE2B: vi.fn(),
executeShellInE2B: vi.fn(),
}))
vi.mock('@/lib/execution/languages', () => ({
CodeLanguage: { javascript: 'javascript', python: 'python' },
}))
vi.mock('@/lib/execution/sandbox/run-task', () => ({
runSandboxTask: mockRunSandboxTask,
}))
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
getWorkspaceFile: vi.fn(),
fetchWorkspaceFileBuffer: vi.fn(),
}))
vi.mock('./doc-compiled-store', () => ({
loadCompiledDoc: mockLoadCompiledDoc,
storeCompiledDoc: vi.fn(),
}))
vi.mock('@/lib/core/config/feature-flags', () => ({
isFeatureEnabled: vi.fn(async () => betaFlag.value),
}))
vi.mock('@/lib/core/config/env-flags', () => ({
get isE2BDocEnabled() {
return e2bFlag.value
},
}))
vi.mock('@/app/api/files/utils', () => ({
getContentType: (name: string) =>
name.endsWith('.pdf')
? 'application/pdf'
: name.endsWith('.txt')
? 'text/plain'
: 'application/octet-stream',
}))
import { DocCompileUserError, resolveServableDocBytes } from './doc-compile'
const WORKSPACE_ID = '550e8400-e29b-41d4-a716-446655440000'
const PDF_MAGIC = Buffer.from('%PDF-1.7\n...binary...')
const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas\n# generates a PDF', 'utf-8')
const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x00, 0x01])
const XLSX_SOURCE = Buffer.from('from openpyxl import Workbook\n# generates an xlsx', 'utf-8')
describe('resolveServableDocBytes', () => {
beforeEach(() => {
vi.clearAllMocks()
e2bFlag.value = true
betaFlag.value = false
})
it('swaps generated-doc source for the compiled artifact + binary content type', async () => {
const artifact = Buffer.from('%PDF-compiled-binary')
mockLoadCompiledDoc.mockResolvedValue(artifact)
const result = await resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(artifact)
expect(result.contentType).toBe('application/pdf')
expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
WORKSPACE_ID,
PDF_SOURCE.toString('utf-8'),
'pdf'
)
})
it('passes through a real binary PDF (carries the %PDF magic) without an artifact lookup', async () => {
const result = await resolveServableDocBytes({
rawBuffer: PDF_MAGIC,
fileName: 'uploaded.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(PDF_MAGIC)
expect(result.contentType).toBe('application/pdf')
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('throws DocCompileUserError when a generated doc artifact is not ready (E2B regime)', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = true
await expect(
resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
).rejects.toBeInstanceOf(DocCompileUserError)
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
it('compiles via the sandbox when E2B is disabled and no artifact is stored', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = false
const compiled = Buffer.from('%PDF-isolated-vm-binary')
mockRunSandboxTask.mockResolvedValue(compiled)
const result = await resolveServableDocBytes({
rawBuffer: PDF_SOURCE,
fileName: 'report.pdf',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(compiled)
expect(result.contentType).toBe('application/pdf')
expect(mockRunSandboxTask).toHaveBeenCalledWith(
'pdf-generate',
{ code: PDF_SOURCE.toString('utf-8'), workspaceId: WORKSPACE_ID },
expect.objectContaining({})
)
})
it('passes non-doc files through untouched with their extension content type', async () => {
const text = Buffer.from('hello world', 'utf-8')
const result = await resolveServableDocBytes({
rawBuffer: text,
fileName: 'notes.txt',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(text)
expect(result.contentType).toBe('text/plain')
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('passes through a real binary XLSX (ZIP magic) without an artifact lookup', async () => {
const result = await resolveServableDocBytes({
rawBuffer: ZIP_MAGIC,
fileName: 'sheet.xlsx',
workspaceId: WORKSPACE_ID,
})
expect(result.buffer).toBe(ZIP_MAGIC)
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
})
it('throws when a generated XLSX artifact is not ready (E2B + mothership-beta enabled)', async () => {
mockLoadCompiledDoc.mockResolvedValue(null)
e2bFlag.value = true
betaFlag.value = true
await expect(
resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: WORKSPACE_ID,
})
).rejects.toBeInstanceOf(DocCompileUserError)
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
it('returns raw XLSX source when there is no workspaceId (xlsx has no isolated-vm path)', async () => {
betaFlag.value = true
const result = await resolveServableDocBytes({
rawBuffer: XLSX_SOURCE,
fileName: 'sheet.xlsx',
workspaceId: undefined,
})
expect(result.buffer).toBe(XLSX_SOURCE)
expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
expect(mockRunSandboxTask).not.toHaveBeenCalled()
})
})
@@ -17,6 +17,8 @@ import { StorageService } from '@/lib/uploads'
import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils'
import {
extractStorageKey,
getFileExtension,
getMimeTypeFromExtension,
inferContextFromKey,
isInternalFileUrl,
processSingleFileToUserFile,
@@ -300,3 +302,68 @@ export async function downloadFileFromStorage(
return buffer
}
/**
* Result of {@link downloadServableFileFromStorage}: the bytes a consumer should
* actually attach/upload, plus the content type that matches those bytes.
*/
export interface ServableFile {
buffer: Buffer
contentType: string
}
/**
* Downloads a workspace file and resolves it to its SERVABLE bytes — the variant
* every tool that hands a file to an external service (email attachments, chat
* uploads, provider file inputs) should use instead of {@link downloadFileFromStorage}.
*
* AI-generated docs (pdf/docx/pptx/xlsx) store their generation SOURCE as the
* primary file and keep the rendered binary in a separate content-addressed
* artifact store. A raw download therefore yields source text under a `.pdf`
* name — the file the recipient cannot open. This swaps in the compiled artifact
* (and the correct binary content type) via the same resolver the file-serve
* route uses, so the serve and attachment paths resolve identically. Non-doc files
* and real uploaded binaries pass through unchanged, carrying `userFile.type` when set.
*
* Throws `DocCompileUserError` when a generated doc's artifact is not ready (still
* compiling) — callers should surface a retryable error rather than attach source.
*/
export async function downloadServableFileFromStorage(
userFile: UserFile,
requestId: string,
logger: Logger,
options: { maxBytes?: number; signal?: AbortSignal; ownerKey?: string } = {}
): Promise<ServableFile> {
const buffer = await downloadFileFromStorage(userFile, requestId, logger, {
maxBytes: options.maxBytes,
})
// Cheap pre-filter so only generated-doc candidates pay for the heavier resolver
// import below.
const ext = getFileExtension(userFile.name)
if (ext !== 'pdf' && ext !== 'docx' && ext !== 'pptx' && ext !== 'xlsx') {
return { buffer, contentType: userFile.type || getMimeTypeFromExtension(ext) }
}
const { parseWorkspaceFileKey } = await import(
'@/lib/uploads/contexts/workspace/workspace-file-manager'
)
const workspaceId = userFile.key ? (parseWorkspaceFileKey(userFile.key) ?? undefined) : undefined
const { resolveServableDocBytes } = await import('@/lib/copilot/tools/server/files/doc-compile')
const resolved = await resolveServableDocBytes({
rawBuffer: buffer,
fileName: userFile.name,
workspaceId,
ownerKey: options.ownerKey,
signal: options.signal,
})
// Re-check: the raw download enforced maxBytes on the source, but a generated doc
// resolves to a larger artifact.
if (options.maxBytes !== undefined && resolved.buffer.length > options.maxBytes) {
assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download')
}
return resolved
}
@@ -0,0 +1,23 @@
import { NextResponse } from 'next/server'
import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile'
/**
* Canonical retryable response for an attachment/upload whose generated-document
* artifact is still compiling. Returns the 409 when `error` is a
* {@link DocCompileUserError} (thrown by `downloadServableFileFromStorage`),
* otherwise `null` so the caller falls through to its own error handling. Shared
* by every tool route that downloads workspace files so the status, body shape,
* and user-facing copy stay identical instead of being re-typed per route.
*/
export function docNotReadyResponse(error: unknown): NextResponse | null {
if (error instanceof DocCompileUserError) {
return NextResponse.json(
{
success: false,
error: 'A document is still being generated. Wait for it to finish, then try again.',
},
{ status: 409 }
)
}
return null
}
+10 -3
View File
@@ -5,7 +5,7 @@ import { sleep } from '@sim/utils/helpers'
import type { StorageContext } from '@/lib/uploads'
import { StorageService } from '@/lib/uploads'
import { inferContextFromKey } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { verifyFileAccess } from '@/app/api/files/authorization'
import type { UserFile } from '@/executor/types'
import {
@@ -178,8 +178,15 @@ function groupUploadableFiles(messages: Message[] | undefined): UserFile[][] {
* object storage works. Bounded by the provider's attachment ceiling.
*/
async function downloadFileForUpload(file: UserFile, maxBytes: number): Promise<Blob> {
const buffer = await downloadFileFromStorage(file, 'provider-file-upload', logger, { maxBytes })
return new Blob([new Uint8Array(buffer)], { type: file.type || inferAttachmentMimeType(file) })
const { buffer, contentType } = await downloadServableFileFromStorage(
file,
'provider-file-upload',
logger,
{ maxBytes }
)
return new Blob([new Uint8Array(buffer)], {
type: contentType || file.type || inferAttachmentMimeType(file),
})
}
/**
+13 -4
View File
@@ -6,7 +6,7 @@
import type { Logger } from '@sim/logger'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization'
import type { UserFile } from '@/executor/types'
import type { GraphApiErrorResponse, GraphDriveItem } from '@/tools/microsoft_teams/types'
@@ -80,10 +80,19 @@ export async function uploadFilesForTeamsMessage(params: {
}
// Download file from storage
const buffer = await downloadFileFromStorage(file, requestId, log)
const { buffer, contentType } = await downloadServableFileFromStorage(file, requestId, log)
if (buffer.length > MAX_TEAMS_FILE_SIZE) {
const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2)
throw new Error(
`File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.`
)
}
const resolvedMimeType = contentType || file.type || 'application/octet-stream'
filesOutput.push({
name: file.name,
mimeType: file.type || 'application/octet-stream',
mimeType: resolvedMimeType,
data: buffer.toString('base64'),
size: buffer.length,
})
@@ -102,7 +111,7 @@ export async function uploadFilesForTeamsMessage(params: {
method: 'PUT',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': file.type || 'application/octet-stream',
'Content-Type': resolvedMimeType,
},
body: buffer,
},