fix(security): cap KB document download size to prevent memory-exhaustion DoS (#5240)

Knowledge-base ingestion downloaded an attacker-controlled external fileUrl
with no byte cap: downloadFileFromUrl defaults maxBytes to MAX_SAFE_INTEGER,
so the streaming reader buffered the entire response into memory uncapped.
An authenticated user could OOM the processing worker by pointing fileUrl at
a server that streams an unbounded body.

Wire the documented 100MB file-size limit (MAX_FILE_SIZE) into the ingestion
download helper. The existing stream limiter aborts the read once the cap is
exceeded and rejects up front on an oversized Content-Length, so the body is
never fully buffered.
This commit is contained in:
Waleed
2026-06-27 12:22:52 -07:00
committed by GitHub
parent 845a6276d9
commit b8a197b99c
@@ -22,6 +22,7 @@ import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
import { StorageService } from '@/lib/uploads'
import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
import { mistralParserTool } from '@/tools/mistral/parser'
const logger = createLogger('DocumentProcessor')
@@ -380,8 +381,18 @@ async function handleFileForOCR(
}
}
/**
* Downloads an ingestion source file, enforcing the {@link MAX_FILE_SIZE} document
* limit. `maxBytes` aborts the streaming read once the cap is exceeded (and rejects
* up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl`
* pointing at an unbounded body cannot exhaust the processing worker's memory.
*/
async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise<Buffer> {
return downloadFileFromUrl(fileUrl, { timeoutMs: TIMEOUTS.FILE_DOWNLOAD, userId })
return downloadFileFromUrl(fileUrl, {
timeoutMs: TIMEOUTS.FILE_DOWNLOAD,
maxBytes: MAX_FILE_SIZE,
userId,
})
}
async function downloadFileForBase64(fileUrl: string, userId?: string): Promise<Buffer> {