mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
fix(security): cap KB document download size to prevent memory-exhaustion DoS (#5240)
Knowledge-base ingestion downloaded an attacker-controlled external fileUrl with no byte cap: downloadFileFromUrl defaults maxBytes to MAX_SAFE_INTEGER, so the streaming reader buffered the entire response into memory uncapped. An authenticated user could OOM the processing worker by pointing fileUrl at a server that streams an unbounded body. Wire the documented 100MB file-size limit (MAX_FILE_SIZE) into the ingestion download helper. The existing stream limiter aborts the read once the cap is exceeded and rejects up front on an oversized Content-Length, so the body is never fully buffered.
This commit is contained in:
@@ -22,6 +22,7 @@ import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
|
||||
import { StorageService } from '@/lib/uploads'
|
||||
import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
|
||||
import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
|
||||
import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
|
||||
import { mistralParserTool } from '@/tools/mistral/parser'
|
||||
|
||||
const logger = createLogger('DocumentProcessor')
|
||||
@@ -380,8 +381,18 @@ async function handleFileForOCR(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads an ingestion source file, enforcing the {@link MAX_FILE_SIZE} document
|
||||
* limit. `maxBytes` aborts the streaming read once the cap is exceeded (and rejects
|
||||
* up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl`
|
||||
* pointing at an unbounded body cannot exhaust the processing worker's memory.
|
||||
*/
|
||||
async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise<Buffer> {
|
||||
return downloadFileFromUrl(fileUrl, { timeoutMs: TIMEOUTS.FILE_DOWNLOAD, userId })
|
||||
return downloadFileFromUrl(fileUrl, {
|
||||
timeoutMs: TIMEOUTS.FILE_DOWNLOAD,
|
||||
maxBytes: MAX_FILE_SIZE,
|
||||
userId,
|
||||
})
|
||||
}
|
||||
|
||||
async function downloadFileForBase64(fileUrl: string, userId?: string): Promise<Buffer> {
|
||||
|
||||
Reference in New Issue
Block a user