mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
feat(zip): add support for zip files (#5788)
* zip checkpoint * fix(zip-uploads): harden extract path from review findings - Replace the whole-buffer central-directory signature scan with an EOCD-anchored walk (shared with zip-guard) so zips containing STORED nested archives are no longer falsely rejected, and report the accurate cap (new 'central_dir_too_large' reason) instead of 'Maximum is 1000' - Make extraction all-or-nothing: a discarding validation pass inflates every entry against the caps before anything is uploaded, so lying headers or corrupt entries can't leave partial trees (corrupt DEFLATE streams now surface as ArchiveError instead of raw zlib errors) - Single-source ArchiveError messages; callers surface err.message and map only status/reason (removes the three divergent message maps) - Guard save/import against archives (saving stranded the contents), extend the workspace ownership check to all three operations, dedupe fileNames, and refuse re-extracting into a non-empty folder instead of duplicating the tree with ' (1)' copies - Harden the extraction folder name (dot segments, control chars, separators) and compute the destination before extracting - Sniff small '.zip'-named uploads so mislabeled text files stay readable instead of dead-ending between read and extract - Scope zip acceptance to the mothership flow only (attachment list, accept attribute, upload/presigned gates) so execution/workspace/chat surfaces keep rejecting zips up front - Don't count skipped noise entries toward the 1000-file cap; restore per-entry zip-slip forensics via skippedUnsafePaths logging - Teach materialize_file(fileNames: [...]) in the extract guidance to match the declared schema * fix(zip-uploads): address review follow-ups on the extract path - Roll back already-uploaded files when an upload fails mid-extraction (storage/DB error, quota crossed), so callers and retries never observe a partial tree - Fold reserved system folder names (.changelogs, .plans) into the 'archive' fallback so extraction can't write into alias-backing namespaces or bypass the already-extracted lookup that hides them - Align the archive byte-sniff budget with the read path's inline text cap (5MB), so any mislabeled '.zip' small enough to be read inline is sniffed and read instead of dead-ending * fix(zip-uploads): detect prior nested-only extractions in the re-extract guard The already-extracted check only looked for files directly inside the archive root folder, so a zip whose entries are all nested (src/index.ts) left only subfolders there and a second extract slipped past the guard, duplicating the tree with ' (1)' suffixes. Extraction roots its whole tree at that folder, so a prior run always leaves a direct file OR a direct subfolder — check both.
This commit is contained in:
Siddharth Ganesan
@@ -588,7 +588,9 @@ describe('/api/files/presigned', () => {
|
||||
|
||||
const response = await POST(request)
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockValidateAttachmentFileType).toHaveBeenCalledWith('screenshot.png')
|
||||
expect(mockValidateAttachmentFileType).toHaveBeenCalledWith('screenshot.png', {
|
||||
allowArchives: true,
|
||||
})
|
||||
expect(mockValidateFileType).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const fileValidationError = validateAttachmentFileType(fileName)
|
||||
const fileValidationError = validateAttachmentFileType(fileName, { allowArchives: true })
|
||||
if (fileValidationError) {
|
||||
throw new ValidationError(fileValidationError.message)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import type { StorageContext } from '@/lib/uploads/config'
|
||||
import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager'
|
||||
import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { MAX_WORKSPACE_FORMDATA_FILE_SIZE } from '@/lib/uploads/shared/types'
|
||||
import { isImageFileType, resolveFileType } from '@/lib/uploads/utils/file-utils'
|
||||
import { isArchiveFileName, isImageFileType, resolveFileType } from '@/lib/uploads/utils/file-utils'
|
||||
import {
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS,
|
||||
SUPPORTED_IMAGE_EXTENSIONS,
|
||||
@@ -34,9 +34,12 @@ import { createErrorResponse, InvalidRequestError } from '@/app/api/files/utils'
|
||||
|
||||
const ALLOWED_EXTENSIONS = new Set<string>(SUPPORTED_ATTACHMENT_EXTENSIONS)
|
||||
|
||||
function validateFileExtension(filename: string): boolean {
|
||||
function validateFileExtension(filename: string, context: StorageContext): boolean {
|
||||
const extension = filename.split('.').pop()?.toLowerCase()
|
||||
if (!extension) return false
|
||||
// Archives are only extractable in the mothership copilot flow; every other
|
||||
// context keeps rejecting them up front instead of failing downstream.
|
||||
if (context === 'mothership' && isArchiveFileName(filename)) return true
|
||||
return ALLOWED_EXTENSIONS.has(extension)
|
||||
}
|
||||
|
||||
@@ -150,7 +153,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
for (const file of files) {
|
||||
const originalName = file.name || 'untitled.md'
|
||||
|
||||
if (!validateFileExtension(originalName)) {
|
||||
if (!validateFileExtension(originalName, context)) {
|
||||
const extension = originalName.split('.').pop()?.toLowerCase() || 'unknown'
|
||||
throw new InvalidRequestError(
|
||||
`File type '${extension}' is not allowed. Allowed types: ${Array.from(ALLOWED_EXTENSIONS).join(', ')}`
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { Buffer, isUtf8 } from 'buffer'
|
||||
import type { Readable } from 'stream'
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
@@ -21,6 +20,12 @@ import {
|
||||
ShareValidationError,
|
||||
upsertFileShare,
|
||||
} from '@/lib/public-shares/share-manager'
|
||||
import {
|
||||
ArchiveError,
|
||||
type DecompressResult,
|
||||
decompressArchiveBufferToWorkspaceFiles,
|
||||
MAX_ARCHIVE_BYTES,
|
||||
} from '@/lib/uploads/archive'
|
||||
import { ensureWorkspaceFileFolderPath } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
|
||||
import {
|
||||
fetchWorkspaceFileBuffer,
|
||||
@@ -203,102 +208,6 @@ const uniqueZipEntryName = (name: string, usedNames: Set<string>): string => {
|
||||
return candidate
|
||||
}
|
||||
|
||||
/** Input archive download cap for the decompress operation. */
|
||||
const MAX_DECOMPRESS_ARCHIVE_BYTES = 100 * 1024 * 1024
|
||||
/** Maximum number of entries extracted from a single archive. */
|
||||
const MAX_DECOMPRESS_ENTRIES = 1000
|
||||
/** Maximum uncompressed size for any single archive entry. */
|
||||
const MAX_DECOMPRESS_ENTRY_BYTES = 100 * 1024 * 1024
|
||||
/** Maximum total uncompressed size across all entries, to bound zip-bomb expansion. */
|
||||
const MAX_DECOMPRESS_TOTAL_BYTES = 200 * 1024 * 1024
|
||||
|
||||
const S_IFMT = 0o170000
|
||||
const S_IFLNK = 0o120000
|
||||
|
||||
/**
|
||||
* Read a zip entry's declared uncompressed size without materializing it. This
|
||||
* value comes straight from the (attacker-controlled) ZIP metadata, so it is only
|
||||
* usable as a cheap fast-reject for honestly-declared archives — never as the
|
||||
* authoritative cap. {@link inflateEntryWithinCaps} enforces the real limit on the
|
||||
* inflated byte stream.
|
||||
*/
|
||||
const readEntryUncompressedSize = (entry: JSZip.JSZipObject): number | undefined => {
|
||||
const data = (entry as JSZip.JSZipObject & { _data?: { uncompressedSize?: number } })._data
|
||||
const size = data?.uncompressedSize
|
||||
return typeof size === 'number' && Number.isFinite(size) ? size : undefined
|
||||
}
|
||||
|
||||
type InflateResult = { ok: true; buffer: Buffer } | { ok: false; reason: 'entry' | 'total' }
|
||||
|
||||
/**
|
||||
* Inflate a single zip entry through a streaming counting sink, tearing the
|
||||
* stream down the moment cumulative output would exceed the per-entry cap or the
|
||||
* remaining total budget. The declared uncompressed size in the ZIP header is
|
||||
* attacker-controlled and is NOT trusted here: a forged-small or absent size
|
||||
* cannot cause the full (potentially gigabyte-scale) entry to be materialized in
|
||||
* memory, because enforcement happens on the actual inflated bytes as they
|
||||
* arrive. Peak memory is bounded by the cap plus one DEFLATE chunk.
|
||||
*/
|
||||
const inflateEntryWithinCaps = (
|
||||
entry: JSZip.JSZipObject,
|
||||
remainingTotalBudget: number
|
||||
): Promise<InflateResult> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = []
|
||||
let size = 0
|
||||
let settled = false
|
||||
const stream = entry.nodeStream() as Readable
|
||||
|
||||
const settle = (result: InflateResult) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
stream.destroy()
|
||||
resolve(result)
|
||||
}
|
||||
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length
|
||||
if (size > MAX_DECOMPRESS_ENTRY_BYTES) {
|
||||
settle({ ok: false, reason: 'entry' })
|
||||
return
|
||||
}
|
||||
if (size > remainingTotalBudget) {
|
||||
settle({ ok: false, reason: 'total' })
|
||||
return
|
||||
}
|
||||
chunks.push(chunk)
|
||||
})
|
||||
stream.on('end', () => settle({ ok: true, buffer: Buffer.concat(chunks, size) }))
|
||||
stream.on('error', (error) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
stream.destroy()
|
||||
reject(error)
|
||||
})
|
||||
})
|
||||
|
||||
/** True when a zip entry's unix mode marks it as a symlink (never extracted). */
|
||||
const isSymlinkEntry = (entry: JSZip.JSZipObject): boolean => {
|
||||
const mode = (entry as JSZip.JSZipObject & { unixPermissions?: number | null }).unixPermissions
|
||||
return typeof mode === 'number' && (mode & S_IFMT) === S_IFLNK
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a zip entry path into safe workspace folder segments, guarding against
|
||||
* zip-slip. Returns null for traversal (`..`), so the entry is skipped rather than
|
||||
* written outside its intended location.
|
||||
*/
|
||||
const sanitizeArchiveEntryPath = (rawPath: string): string[] | null => {
|
||||
const segments = rawPath
|
||||
.replace(/\\/g, '/')
|
||||
.split('/')
|
||||
.map((segment) => segment.trim())
|
||||
.filter((segment) => segment.length > 0 && segment !== '.')
|
||||
|
||||
if (segments.length === 0 || segments.includes('..')) return null
|
||||
return segments
|
||||
}
|
||||
|
||||
const isLikelyTextBuffer = (buffer: Buffer): boolean => isUtf8(buffer) && !buffer.includes(0)
|
||||
|
||||
/**
|
||||
@@ -896,135 +805,53 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
if (denied) return denied
|
||||
|
||||
const archiveBuffer = await downloadFileFromStorage(archive, requestId, logger, {
|
||||
maxBytes: MAX_DECOMPRESS_ARCHIVE_BYTES,
|
||||
maxBytes: MAX_ARCHIVE_BYTES,
|
||||
})
|
||||
|
||||
let zip: JSZip
|
||||
let result: DecompressResult
|
||||
try {
|
||||
zip = await JSZip.loadAsync(archiveBuffer)
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: `"${archive.name}" is not a valid .zip archive` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const entries = Object.values(zip.files).filter(
|
||||
(entry) => !entry.dir && !isSymlinkEntry(entry)
|
||||
)
|
||||
if (entries.length > MAX_DECOMPRESS_ENTRIES) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `Archive has too many entries to extract. Maximum is ${MAX_DECOMPRESS_ENTRIES}.`,
|
||||
},
|
||||
{ status: 413 }
|
||||
)
|
||||
}
|
||||
|
||||
const entryTooLargeResponse = (name: string) =>
|
||||
NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `Archive entry "${name}" is too large to extract. Maximum is ${
|
||||
MAX_DECOMPRESS_ENTRY_BYTES / (1024 * 1024)
|
||||
} MB per file.`,
|
||||
},
|
||||
{ status: 413 }
|
||||
)
|
||||
const totalTooLargeResponse = () =>
|
||||
NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `Archive expands to more than the ${
|
||||
MAX_DECOMPRESS_TOTAL_BYTES / (1024 * 1024)
|
||||
} MB extraction limit.`,
|
||||
},
|
||||
{ status: 413 }
|
||||
)
|
||||
|
||||
// Resolve which entries are safe to extract first, so unsafe entries
|
||||
// (skipped below) never count toward the size caps.
|
||||
const safeEntries: Array<{ entry: JSZip.JSZipObject; segments: string[] }> = []
|
||||
let skippedCount = 0
|
||||
for (const entry of entries) {
|
||||
const segments = sanitizeArchiveEntryPath(entry.name)
|
||||
if (!segments) {
|
||||
skippedCount += 1
|
||||
logger.warn('Skipping unsafe archive entry', { name: entry.name })
|
||||
continue
|
||||
result = await decompressArchiveBufferToWorkspaceFiles(archiveBuffer, {
|
||||
workspaceId,
|
||||
userId,
|
||||
})
|
||||
} catch (archiveError) {
|
||||
if (archiveError instanceof ArchiveError) {
|
||||
// The error message is single-sourced in ArchiveError (caps included);
|
||||
// only the HTTP status is mapped here.
|
||||
const status = archiveError.reason === 'invalid' ? 400 : 413
|
||||
return NextResponse.json(
|
||||
{ success: false, error: `"${archive.name}": ${archiveError.message}` },
|
||||
{ status }
|
||||
)
|
||||
}
|
||||
safeEntries.push({ entry, segments })
|
||||
throw archiveError
|
||||
}
|
||||
|
||||
let declaredTotal = 0
|
||||
for (const { entry } of safeEntries) {
|
||||
const declaredSize = readEntryUncompressedSize(entry)
|
||||
if (declaredSize === undefined) continue
|
||||
if (declaredSize > MAX_DECOMPRESS_ENTRY_BYTES) return entryTooLargeResponse(entry.name)
|
||||
declaredTotal += declaredSize
|
||||
if (declaredTotal > MAX_DECOMPRESS_TOTAL_BYTES) return totalTooLargeResponse()
|
||||
}
|
||||
|
||||
const pending: Array<{ segments: string[]; buffer: Buffer }> = []
|
||||
let totalBytes = 0
|
||||
for (const { entry, segments } of safeEntries) {
|
||||
const result = await inflateEntryWithinCaps(
|
||||
entry,
|
||||
MAX_DECOMPRESS_TOTAL_BYTES - totalBytes
|
||||
)
|
||||
if (!result.ok) {
|
||||
return result.reason === 'entry'
|
||||
? entryTooLargeResponse(entry.name)
|
||||
: totalTooLargeResponse()
|
||||
}
|
||||
totalBytes += result.buffer.length
|
||||
pending.push({ segments, buffer: result.buffer })
|
||||
}
|
||||
|
||||
if (pending.length === 0) {
|
||||
if (result.extracted.length === 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `No files could be extracted from "${archive.name}".`,
|
||||
},
|
||||
{ success: false, error: `No files could be extracted from "${archive.name}".` },
|
||||
{ status: 422 }
|
||||
)
|
||||
}
|
||||
|
||||
const folderIdCache = new Map<string, string | null>()
|
||||
const extractedFiles: UserFile[] = []
|
||||
for (const { segments, buffer } of pending) {
|
||||
const leafName = segments[segments.length - 1]
|
||||
const folderSegments = segments.slice(0, -1)
|
||||
const folderKey = folderSegments.join('/')
|
||||
let folderId = folderIdCache.get(folderKey)
|
||||
if (folderId === undefined) {
|
||||
folderId = await ensureWorkspaceFileFolderPath({
|
||||
workspaceId,
|
||||
userId,
|
||||
pathSegments: folderSegments,
|
||||
})
|
||||
folderIdCache.set(folderKey, folderId)
|
||||
}
|
||||
const extractedFiles = result.extracted.map((file) => ({
|
||||
...file,
|
||||
url: ensureAbsoluteUrl(file.url),
|
||||
}))
|
||||
|
||||
const mimeType = getMimeTypeFromExtension(getFileExtension(leafName))
|
||||
const uploaded = await uploadWorkspaceFile(
|
||||
workspaceId,
|
||||
userId,
|
||||
buffer,
|
||||
leafName,
|
||||
mimeType,
|
||||
{ folderId }
|
||||
)
|
||||
extractedFiles.push({ ...uploaded, url: ensureAbsoluteUrl(uploaded.url) })
|
||||
if (result.skippedUnsafePaths.length > 0) {
|
||||
logger.warn('Skipped unsafe archive entries', {
|
||||
fileId: archive.id,
|
||||
name: archive.name,
|
||||
entryNames: result.skippedUnsafePaths,
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Archive decompressed', {
|
||||
fileId: archive.id,
|
||||
name: archive.name,
|
||||
extractedCount: extractedFiles.length,
|
||||
skippedCount,
|
||||
skippedCount: result.skipped,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
|
||||
@@ -15,7 +15,7 @@ import { createLogger } from '@sim/logger'
|
||||
import { useParams } from 'next/navigation'
|
||||
import { getMothershipAttachmentPreviewUrl } from '@/lib/copilot/chat/attachment-preview'
|
||||
import { SIM_RESOURCE_DRAG_TYPE, SIM_RESOURCES_DRAG_TYPE } from '@/lib/copilot/resource-types'
|
||||
import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation'
|
||||
import { MOTHERSHIP_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation'
|
||||
import { useChatSurface } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context'
|
||||
import {
|
||||
AnimatedPlaceholderEffect,
|
||||
@@ -597,7 +597,7 @@ const UserInputImpl = forwardRef<UserInputHandle, UserInputProps>(function UserI
|
||||
type='file'
|
||||
onChange={handleFileChange}
|
||||
className='hidden'
|
||||
accept={CHAT_ACCEPT_ATTRIBUTE}
|
||||
accept={MOTHERSHIP_ACCEPT_ATTRIBUTE}
|
||||
multiple
|
||||
/>
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
|
||||
import type { BlockVisibilityState } from '@/lib/core/config/block-visibility'
|
||||
import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags'
|
||||
import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { buildArchiveExtractGuidance, isArchiveFileName } from '@/lib/uploads/utils/file-utils'
|
||||
import { stripVersionSuffix } from '@/tools/utils'
|
||||
|
||||
const logger = createLogger('CopilotChatPayload')
|
||||
@@ -343,15 +344,25 @@ export async function buildCopilotRequestPayload(
|
||||
} catch {
|
||||
encodedUploadName = displayName
|
||||
}
|
||||
const lines = [
|
||||
`File "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
|
||||
`Read with: read("uploads/${encodedUploadName}")`,
|
||||
`To save permanently: materialize_file(fileName: "${displayName}")`,
|
||||
]
|
||||
if (displayName.endsWith('.json')) {
|
||||
lines.push(
|
||||
`To import as a workflow: materialize_file(fileName: "${displayName}", operation: "import")`
|
||||
)
|
||||
let lines: string[]
|
||||
if (isArchiveFileName(displayName)) {
|
||||
// A .zip is stored in uploads/ but its contents aren't readable until
|
||||
// the agent extracts it once into workspace files/ (explicit step).
|
||||
lines = [
|
||||
`Archive "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
|
||||
buildArchiveExtractGuidance(displayName),
|
||||
]
|
||||
} else {
|
||||
lines = [
|
||||
`File "${displayName}" (${mediaType}, ${f.size} bytes) uploaded.`,
|
||||
`Read with: read("uploads/${encodedUploadName}")`,
|
||||
`To save permanently: materialize_file(fileName: "${displayName}")`,
|
||||
]
|
||||
if (displayName.endsWith('.json')) {
|
||||
lines.push(
|
||||
`To import as a workflow: materialize_file(fileName: "${displayName}", operation: "import")`
|
||||
)
|
||||
}
|
||||
}
|
||||
uploadContexts.push({
|
||||
type: 'uploaded_file',
|
||||
|
||||
@@ -2843,8 +2843,8 @@ export const MaterializeFile: ToolCatalogEntry = {
|
||||
operation: {
|
||||
type: 'string',
|
||||
description:
|
||||
'What to do with the file. "save" promotes it to a permanent files/ path. "import" imports a workflow JSON as a workspace workflow. Defaults to "save".',
|
||||
enum: ['save', 'import'],
|
||||
'What to do with the file. "save" promotes it to a permanent files/ path. "import" imports a workflow JSON as a workspace workflow. "extract" decompresses a .zip upload into files/<archive>/. Defaults to "save".',
|
||||
enum: ['save', 'import', 'extract'],
|
||||
default: 'save',
|
||||
},
|
||||
},
|
||||
@@ -4720,6 +4720,7 @@ export const ManageSkillOperationValues = [
|
||||
export const MaterializeFileOperation = {
|
||||
save: 'save',
|
||||
import: 'import',
|
||||
extract: 'extract',
|
||||
} as const
|
||||
|
||||
export type MaterializeFileOperation =
|
||||
@@ -4728,6 +4729,7 @@ export type MaterializeFileOperation =
|
||||
export const MaterializeFileOperationValues = [
|
||||
MaterializeFileOperation.save,
|
||||
MaterializeFileOperation.import,
|
||||
MaterializeFileOperation.extract,
|
||||
] as const
|
||||
|
||||
export const QueryUserTableOperation = {
|
||||
|
||||
@@ -2656,8 +2656,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
operation: {
|
||||
type: 'string',
|
||||
description:
|
||||
'What to do with the file. "save" promotes it to a permanent files/ path. "import" imports a workflow JSON as a workspace workflow. Defaults to "save".',
|
||||
enum: ['save', 'import'],
|
||||
'What to do with the file. "save" promotes it to a permanent files/ path. "import" imports a workflow JSON as a workspace workflow. "extract" decompresses a .zip upload into files/<archive>/. Defaults to "save".',
|
||||
enum: ['save', 'import', 'extract'],
|
||||
default: 'save',
|
||||
},
|
||||
},
|
||||
|
||||
@@ -6,6 +6,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockCheckStorageQuotaForBillingContext,
|
||||
mockDecompress,
|
||||
mockFetchBuffer,
|
||||
mockFindFolder,
|
||||
mockFindUpload,
|
||||
mockHasCloudStorage,
|
||||
mockHeadObject,
|
||||
@@ -14,6 +17,9 @@ const {
|
||||
mockResolveStorageBillingContext,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckStorageQuotaForBillingContext: vi.fn(),
|
||||
mockDecompress: vi.fn(),
|
||||
mockFetchBuffer: vi.fn(),
|
||||
mockFindFolder: vi.fn(),
|
||||
mockFindUpload: vi.fn(),
|
||||
mockHasCloudStorage: vi.fn(),
|
||||
mockHeadObject: vi.fn(),
|
||||
@@ -33,7 +39,26 @@ vi.mock('@/lib/uploads', () => ({
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
fetchWorkspaceFileBuffer: vi.fn(),
|
||||
fetchWorkspaceFileBuffer: mockFetchBuffer,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => ({
|
||||
findWorkspaceFileFolderIdByPath: mockFindFolder,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/archive', () => ({
|
||||
decompressArchiveBufferToWorkspaceFiles: mockDecompress,
|
||||
ArchiveError: class ArchiveError extends Error {
|
||||
reason: string
|
||||
entryName?: string
|
||||
constructor(reason: string, message: string, entryName?: string) {
|
||||
super(message)
|
||||
this.name = 'ArchiveError'
|
||||
this.reason = reason
|
||||
this.entryName = entryName
|
||||
}
|
||||
},
|
||||
MAX_ARCHIVE_BYTES: 100 * 1024 * 1024,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => ({
|
||||
@@ -50,6 +75,8 @@ vi.mock('@/lib/billing/storage', () => ({
|
||||
|
||||
vi.mock('@/lib/copilot/vfs/path-utils', () => ({
|
||||
canonicalWorkspaceFilePath: vi.fn(() => 'files/report.txt'),
|
||||
encodeVfsPathSegments: (segments: string[]) =>
|
||||
segments.map((s) => encodeURIComponent(s)).join('/'),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/workflows/operations/import-export', () => ({ parseWorkflowJson: vi.fn() }))
|
||||
@@ -293,3 +320,210 @@ describe('executeMaterializeFile - save storage transition', () => {
|
||||
expect(mockMaybeNotifyStorageLimitForBillingContext).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('executeMaterializeFile - extract operation', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockFindFolder.mockResolvedValue(null)
|
||||
})
|
||||
|
||||
function zipRow(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 'wf_zip',
|
||||
key: 'mothership/abc/bundle.zip',
|
||||
userId: 'user-1',
|
||||
workspaceId: 'ws-1',
|
||||
context: 'mothership',
|
||||
chatId: 'chat-1',
|
||||
originalName: 'bundle.zip',
|
||||
displayName: 'bundle.zip',
|
||||
contentType: 'application/zip',
|
||||
size: 2048,
|
||||
deletedAt: null,
|
||||
uploadedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
it('dispatches to the archive extractor and returns the unpacked files', async () => {
|
||||
mockFindUpload.mockResolvedValue(zipRow())
|
||||
mockFetchBuffer.mockResolvedValue(Buffer.from('zip-bytes'))
|
||||
mockDecompress.mockResolvedValue({
|
||||
extracted: [
|
||||
{ id: 'f1', name: 'a.txt', url: '/x', size: 1, type: 'text/plain', key: 'k1' },
|
||||
{ id: 'f2', name: 'b.txt', url: '/y', size: 2, type: 'text/plain', key: 'k2' },
|
||||
],
|
||||
skipped: 0,
|
||||
skippedUnsafePaths: [],
|
||||
})
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['bundle.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockDecompress).toHaveBeenCalledTimes(1)
|
||||
expect(mockDecompress).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
expect.objectContaining({
|
||||
workspaceId: 'ws-1',
|
||||
userId: 'user-1',
|
||||
rootFolderSegments: ['bundle'],
|
||||
skipNoiseEntries: true,
|
||||
})
|
||||
)
|
||||
expect(result.output).toMatchObject({ succeeded: ['bundle.zip'], failed: [] })
|
||||
expect(result.resources).toEqual([
|
||||
{ type: 'file', id: 'f1', title: 'a.txt' },
|
||||
{ type: 'file', id: 'f2', title: 'b.txt' },
|
||||
])
|
||||
})
|
||||
|
||||
it('refuses to extract an upload that belongs to a different workspace', async () => {
|
||||
mockFindUpload.mockResolvedValue(zipRow({ workspaceId: 'other-ws' }))
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['bundle.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
const output = result.output as { failed: Array<{ fileName: string; error: string }> }
|
||||
expect(output.failed[0].error).toContain('does not belong to this workspace')
|
||||
expect(mockDecompress).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports an already-extracted archive instead of duplicating the tree', async () => {
|
||||
mockFindUpload.mockResolvedValue(zipRow())
|
||||
mockFindFolder.mockResolvedValue('folder-existing')
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'f-old' }])
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['bundle.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
const output = result.output as { failed: Array<{ fileName: string; error: string }> }
|
||||
expect(output.failed[0].error).toContain('already extracted')
|
||||
expect(mockDecompress).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('detects a prior nested-only extraction via subfolders, not just direct files', async () => {
|
||||
// A zip containing only nested entries (src/index.ts) leaves NO direct files
|
||||
// under the archive root — only subfolders. The guard must still refuse.
|
||||
mockFindUpload.mockResolvedValue(zipRow())
|
||||
mockFindFolder.mockResolvedValue('folder-existing')
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([]) // no direct files
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'subfolder-1' }]) // but a subfolder tree
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['bundle.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
const output = result.output as { failed: Array<{ fileName: string; error: string }> }
|
||||
expect(output.failed[0].error).toContain('already extracted')
|
||||
expect(mockDecompress).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('dedupes repeated fileNames so one call cannot double-extract', async () => {
|
||||
mockFindUpload.mockResolvedValue(zipRow())
|
||||
mockFetchBuffer.mockResolvedValue(Buffer.from('zip-bytes'))
|
||||
mockDecompress.mockResolvedValue({
|
||||
extracted: [{ id: 'f1', name: 'a.txt', url: '/x', size: 1, type: 'text/plain', key: 'k1' }],
|
||||
skipped: 0,
|
||||
skippedUnsafePaths: [],
|
||||
})
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['bundle.zip', 'bundle.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockDecompress).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('folds reserved system folder names into the "archive" fallback folder', async () => {
|
||||
// '.changelogs' / '.plans' back workflow changelog/plan aliases; extraction
|
||||
// must never write into them (and the already-extracted lookup hides them,
|
||||
// so a second extract would silently duplicate).
|
||||
mockFindUpload.mockResolvedValue(
|
||||
zipRow({ displayName: '.changelogs.zip', originalName: '.changelogs.zip' })
|
||||
)
|
||||
mockFetchBuffer.mockResolvedValue(Buffer.from('zip-bytes'))
|
||||
mockDecompress.mockResolvedValue({
|
||||
extracted: [{ id: 'f1', name: 'a.txt', url: '/x', size: 1, type: 'text/plain', key: 'k1' }],
|
||||
skipped: 0,
|
||||
skippedUnsafePaths: [],
|
||||
})
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['.changelogs.zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockDecompress).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
expect.objectContaining({ rootFolderSegments: ['archive'] })
|
||||
)
|
||||
})
|
||||
|
||||
it('folds degenerate archive names into the "archive" fallback folder', async () => {
|
||||
mockFindUpload.mockResolvedValue(zipRow({ displayName: '..zip', originalName: '..zip' }))
|
||||
mockFetchBuffer.mockResolvedValue(Buffer.from('zip-bytes'))
|
||||
mockDecompress.mockResolvedValue({
|
||||
extracted: [{ id: 'f1', name: 'a.txt', url: '/x', size: 1, type: 'text/plain', key: 'k1' }],
|
||||
skipped: 0,
|
||||
skippedUnsafePaths: [],
|
||||
})
|
||||
|
||||
const result = await executeMaterializeFile(
|
||||
{ fileNames: ['..zip'], operation: 'extract' },
|
||||
context
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(mockDecompress).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
expect.objectContaining({ rootFolderSegments: ['archive'] })
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('executeMaterializeFile - save operation on archives', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockFindFolder.mockResolvedValue(null)
|
||||
})
|
||||
|
||||
it('refuses to save a .zip upload and points at extract instead', async () => {
|
||||
mockFindUpload.mockResolvedValue({
|
||||
id: 'wf_zip',
|
||||
key: 'mothership/abc/bundle.zip',
|
||||
userId: 'user-1',
|
||||
workspaceId: 'ws-1',
|
||||
context: 'mothership',
|
||||
chatId: 'chat-1',
|
||||
originalName: 'bundle.zip',
|
||||
displayName: 'bundle.zip',
|
||||
contentType: 'application/zip',
|
||||
size: 2048,
|
||||
deletedAt: null,
|
||||
uploadedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
|
||||
const result = await executeMaterializeFile({ fileNames: ['bundle.zip'] }, context)
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
const output = result.output as { failed: Array<{ fileName: string; error: string }> }
|
||||
expect(output.failed[0].error).toContain('operation: "extract"')
|
||||
expect(mockDecompress).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { db } from '@sim/db'
|
||||
import { workflow, workspaceFiles } from '@sim/db/schema'
|
||||
import { workflow, workspaceFileFolder, workspaceFiles } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
@@ -13,10 +13,19 @@ import {
|
||||
} from '@/lib/billing/storage'
|
||||
import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types'
|
||||
import { findMothershipUploadRowByChatAndName } from '@/lib/copilot/tools/handlers/upload-file-reader'
|
||||
import { canonicalWorkspaceFilePath } from '@/lib/copilot/vfs/path-utils'
|
||||
import { canonicalWorkspaceFilePath, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils'
|
||||
import { isReservedWorkflowAliasBackingDisplayPath } from '@/lib/copilot/vfs/workflow-aliases'
|
||||
import { getServePathPrefix } from '@/lib/uploads'
|
||||
import {
|
||||
ArchiveError,
|
||||
type DecompressResult,
|
||||
decompressArchiveBufferToWorkspaceFiles,
|
||||
MAX_ARCHIVE_BYTES,
|
||||
} from '@/lib/uploads/archive'
|
||||
import { findWorkspaceFileFolderIdByPath } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
|
||||
import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { hasCloudStorage, headObject } from '@/lib/uploads/core/storage-service'
|
||||
import { isArchiveFileName } from '@/lib/uploads/utils/file-utils'
|
||||
import { parseWorkflowJson } from '@/lib/workflows/operations/import-export'
|
||||
import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils'
|
||||
import { deduplicateWorkflowName } from '@/lib/workflows/utils'
|
||||
@@ -42,6 +51,20 @@ function toFileRecord(row: typeof workspaceFiles.$inferSelect) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cross-workspace ownership guard shared by every operation. The resolver is
|
||||
* chat-scoped and current write paths always stamp matching workspaceIds, so
|
||||
* this is defense in depth — but it must hold uniformly: without it, save would
|
||||
* flip a foreign-workspace row into this workspace and import would read its
|
||||
* bytes, the exact leak extract blocks.
|
||||
*/
|
||||
function uploadBelongsToWorkspace(
|
||||
row: { workspaceId: string | null },
|
||||
workspaceId: string
|
||||
): boolean {
|
||||
return row.workspaceId === workspaceId
|
||||
}
|
||||
|
||||
async function executeSave(
|
||||
fileName: string,
|
||||
chatId: string,
|
||||
@@ -54,10 +77,18 @@ async function executeSave(
|
||||
error: `Upload not found: "${fileName}". Use glob("uploads/*") to list available uploads.`,
|
||||
}
|
||||
}
|
||||
if (row.workspaceId !== workspaceId) {
|
||||
if (!uploadBelongsToWorkspace(row, workspaceId)) {
|
||||
return { success: false, error: `Upload not found: "${fileName}".` }
|
||||
}
|
||||
|
||||
const displayName = row.displayName ?? row.originalName
|
||||
if (isArchiveFileName(displayName)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `"${fileName}" is a .zip archive — save it by extracting instead: materialize_file(fileNames: ["${fileName}"], operation: "extract") unpacks it into files/ where the contents stay readable. The raw .zip remains in uploads/ for this chat.`,
|
||||
}
|
||||
}
|
||||
|
||||
const head = await headObject(row.key, 'mothership')
|
||||
if (!head && hasCloudStorage()) {
|
||||
return { success: false, error: `Upload object not found: "${fileName}".` }
|
||||
@@ -158,6 +189,18 @@ async function executeImport(
|
||||
error: `Upload not found: "${fileName}". Use glob("uploads/*") to list available uploads.`,
|
||||
}
|
||||
}
|
||||
if (!uploadBelongsToWorkspace(row, workspaceId)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Upload "${fileName}" does not belong to this workspace.`,
|
||||
}
|
||||
}
|
||||
if (isArchiveFileName(row.displayName ?? row.originalName)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `"${fileName}" is a .zip archive, not a workflow JSON. Extract it first: materialize_file(fileNames: ["${fileName}"], operation: "extract").`,
|
||||
}
|
||||
}
|
||||
|
||||
const buffer = await fetchWorkspaceFileBuffer(toFileRecord(row))
|
||||
const content = buffer.toString('utf-8')
|
||||
@@ -254,13 +297,198 @@ async function executeImport(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a zip display name into a safe extraction folder name. Mirrors the VFS
|
||||
* segment normalization (NFC, control-char strip) and rejects the degenerate
|
||||
* names the folder layer throws plain Errors for (dot segments, separators,
|
||||
* empty), so a hostile upload name like `..zip` or `\x01.zip` lands in the
|
||||
* `archive` fallback instead of surfacing a raw internal error — and so the
|
||||
* VFS-encoded destination path can be computed before anything is extracted.
|
||||
* Reserved system backing folders (`.changelogs`, `.plans`) also fall back:
|
||||
* extraction must never write into — or hide behind — those namespaces (the
|
||||
* already-extracted lookup skips them, so they'd also duplicate silently).
|
||||
*/
|
||||
function archiveFolderBaseName(displayName: string): string {
|
||||
const stripped = displayName
|
||||
.replace(/\.zip$/i, '')
|
||||
.normalize('NFC')
|
||||
.replace(/[\x00-\x1f\x7f]/g, '')
|
||||
.replace(/[/\\]/g, '-')
|
||||
.trim()
|
||||
if (
|
||||
!stripped ||
|
||||
stripped === '.' ||
|
||||
stripped === '..' ||
|
||||
isReservedWorkflowAliasBackingDisplayPath(stripped)
|
||||
) {
|
||||
return 'archive'
|
||||
}
|
||||
return stripped
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress an uploaded `.zip` into the workspace `files/<archive>/` folder tree
|
||||
* (reusing the shared, capped, zip-slip/bomb-safe extractor). The raw archive
|
||||
* stays in uploads/; the extracted files persist in the workspace so the agent
|
||||
* can read them with the normal files/ tooling. This is the explicit "extract
|
||||
* before reading a zip" step.
|
||||
*/
|
||||
async function executeExtract(
|
||||
fileName: string,
|
||||
chatId: string,
|
||||
workspaceId: string,
|
||||
userId: string
|
||||
): Promise<ToolCallResult> {
|
||||
const row = await findMothershipUploadRowByChatAndName(chatId, fileName)
|
||||
if (!row) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Upload not found: "${fileName}". Use glob("uploads/*") to list available uploads.`,
|
||||
}
|
||||
}
|
||||
|
||||
if (!uploadBelongsToWorkspace(row, workspaceId)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Upload "${fileName}" does not belong to this workspace.`,
|
||||
}
|
||||
}
|
||||
|
||||
const displayName = row.displayName ?? row.originalName
|
||||
if (!isArchiveFileName(displayName)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `"${fileName}" is not a .zip archive — only .zip uploads can be extracted. Read it directly with read("uploads/${fileName}").`,
|
||||
}
|
||||
}
|
||||
|
||||
const record = toFileRecord(row)
|
||||
if (record.size > MAX_ARCHIVE_BYTES) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Archive too large to extract: "${fileName}" (${Math.round(
|
||||
record.size / 1024 / 1024
|
||||
)}MB, limit ${MAX_ARCHIVE_BYTES / 1024 / 1024}MB).`,
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the destination up front (the encoded path is a pure function of the
|
||||
// hardened base name), so nothing can throw after files have been written.
|
||||
const baseName = archiveFolderBaseName(displayName)
|
||||
const folderPath = `files/${encodeVfsPathSegments([baseName])}`
|
||||
|
||||
// Re-running extract must not silently duplicate the tree with " (1)"-suffixed
|
||||
// copies: when the destination folder already holds content, report it as
|
||||
// already extracted instead of extracting beside the previous run. Direct
|
||||
// files AND direct subfolders both count — extraction roots its whole tree
|
||||
// here, so a prior run of a nested-only zip (e.g. src/index.ts) leaves a
|
||||
// subfolder even when no file sits at the top level.
|
||||
const existingFolderId = await findWorkspaceFileFolderIdByPath(workspaceId, [baseName])
|
||||
if (existingFolderId) {
|
||||
const [[existingFile], [existingSubfolder]] = await Promise.all([
|
||||
db
|
||||
.select({ id: workspaceFiles.id })
|
||||
.from(workspaceFiles)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceFiles.folderId, existingFolderId),
|
||||
eq(workspaceFiles.context, 'workspace'),
|
||||
isNull(workspaceFiles.deletedAt)
|
||||
)
|
||||
)
|
||||
.limit(1),
|
||||
db
|
||||
.select({ id: workspaceFileFolder.id })
|
||||
.from(workspaceFileFolder)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceFileFolder.parentId, existingFolderId),
|
||||
isNull(workspaceFileFolder.deletedAt)
|
||||
)
|
||||
)
|
||||
.limit(1),
|
||||
])
|
||||
if (existingFile || existingSubfolder) {
|
||||
return {
|
||||
success: false,
|
||||
error: `"${fileName}" appears to be already extracted — ${folderPath}/ exists and contains content. List it with glob("${folderPath}/**"). To re-extract, delete that folder first.`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let result: DecompressResult
|
||||
try {
|
||||
const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_ARCHIVE_BYTES })
|
||||
result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId,
|
||||
userId,
|
||||
rootFolderSegments: [baseName],
|
||||
// The agent-facing extract drops macOS/Windows filesystem cruft so the
|
||||
// unpacked files/ tree only contains meaningful entries.
|
||||
skipNoiseEntries: true,
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof ArchiveError) {
|
||||
// Reads sniff small uploads' magic bytes, so a mislabeled ".zip" that
|
||||
// fails to parse here is genuinely readable via read() — say so instead
|
||||
// of bouncing the model between extract and read forever.
|
||||
const mislabeledHint =
|
||||
err.reason === 'invalid'
|
||||
? ` If the file is not actually a zip archive, read it directly with read("uploads/${fileName}").`
|
||||
: ''
|
||||
return {
|
||||
success: false,
|
||||
error: `Cannot extract "${fileName}": ${err.message}${mislabeledHint}`,
|
||||
}
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
if (result.extracted.length === 0) {
|
||||
return { success: false, error: `No files could be extracted from "${fileName}".` }
|
||||
}
|
||||
|
||||
const count = result.extracted.length
|
||||
|
||||
if (result.skippedUnsafePaths.length > 0) {
|
||||
logger.warn('Skipped unsafe archive entries during extract', {
|
||||
fileName,
|
||||
chatId,
|
||||
entryNames: result.skippedUnsafePaths,
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Extracted archive into workspace files', {
|
||||
fileName,
|
||||
chatId,
|
||||
folder: baseName,
|
||||
extractedCount: count,
|
||||
skipped: result.skipped,
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
message: `Extracted ${count} file${count === 1 ? '' : 's'} from "${fileName}" into ${folderPath}/. They now persist in the workspace — list them with glob("${folderPath}/**") and read one with read("${folderPath}/<path>/content").`,
|
||||
fileCount: count,
|
||||
path: folderPath,
|
||||
},
|
||||
resources: result.extracted.map((f) => ({ type: 'file' as const, id: f.id, title: f.name })),
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeMaterializeFile(
|
||||
params: Record<string, unknown>,
|
||||
context: ExecutionContext
|
||||
): Promise<ToolCallResult> {
|
||||
const fileNames: string[] =
|
||||
(params.fileNames as string[] | undefined) ??
|
||||
([params.fileName as string | undefined].filter(Boolean) as string[])
|
||||
// Dedupe: a repeated name in one call would re-run the operation against the
|
||||
// same upload (for extract, duplicating the unpacked tree with " (1)" copies).
|
||||
const fileNames: string[] = Array.from(
|
||||
new Set(
|
||||
(params.fileNames as string[] | undefined) ??
|
||||
([params.fileName as string | undefined].filter(Boolean) as string[])
|
||||
)
|
||||
)
|
||||
|
||||
if (fileNames.length === 0) {
|
||||
return { success: false, error: "Missing required parameter 'fileNames'" }
|
||||
@@ -275,12 +503,13 @@ export async function executeMaterializeFile(
|
||||
}
|
||||
|
||||
const operation = (params.operation as string | undefined) || 'save'
|
||||
// Only save/import are implemented. Reject anything else with guidance instead of
|
||||
// silently falling back to save (table/knowledge_base are handled by their subagents).
|
||||
if (operation !== 'save' && operation !== 'import') {
|
||||
// save (promote upload → workspace file), import (JSON → workflow), and extract
|
||||
// (decompress a .zip upload → workspace files/) are implemented. Reject anything
|
||||
// else with guidance instead of silently falling back to save.
|
||||
if (operation !== 'save' && operation !== 'import' && operation !== 'extract') {
|
||||
return {
|
||||
success: false,
|
||||
error: `Unsupported materialize_file operation "${operation}". Use "save" or "import". For CSV/TSV/JSON → use the table subagent; for documents → use the knowledge subagent.`,
|
||||
error: `Unsupported materialize_file operation "${operation}". Use "save", "import", or "extract". For CSV/TSV/JSON → use the table subagent; for documents → use the knowledge subagent.`,
|
||||
}
|
||||
}
|
||||
const succeeded: string[] = []
|
||||
@@ -292,6 +521,8 @@ export async function executeMaterializeFile(
|
||||
let result: ToolCallResult
|
||||
if (operation === 'import') {
|
||||
result = await executeImport(fileName, context.chatId, context.workspaceId, context.userId)
|
||||
} else if (operation === 'extract') {
|
||||
result = await executeExtract(fileName, context.chatId, context.workspaceId, context.userId)
|
||||
} else {
|
||||
result = await executeSave(fileName, context.chatId, context.workspaceId)
|
||||
}
|
||||
|
||||
@@ -7,16 +7,27 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('@sim/db', () => dbChainMock)
|
||||
|
||||
const { mockReadFileRecord } = vi.hoisted(() => ({
|
||||
const { mockReadFileRecord, mockFetchBuffer } = vi.hoisted(() => ({
|
||||
mockReadFileRecord: vi.fn(),
|
||||
mockFetchBuffer: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/copilot/vfs/file-reader', () => ({
|
||||
readFileRecord: mockReadFileRecord,
|
||||
MAX_TEXT_READ_BYTES: 5 * 1024 * 1024,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
fetchWorkspaceFileBuffer: mockFetchBuffer,
|
||||
}))
|
||||
|
||||
/** A buffer beginning with the ZIP local-file-header magic (PK\x03\x04). */
|
||||
const ZIP_SHAPED = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00])
|
||||
|
||||
import { WorkspaceFileGrepError } from '@/lib/copilot/vfs/operations'
|
||||
import {
|
||||
findMothershipUploadRowByChatAndName,
|
||||
grepChatUpload,
|
||||
listChatUploads,
|
||||
readChatUpload,
|
||||
} from './upload-file-reader'
|
||||
@@ -167,6 +178,46 @@ describe('readChatUpload', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('returns extract-first guidance for a .zip upload instead of reading bytes', async () => {
|
||||
const row = makeRow({ id: 'wf_z', displayName: 'bundle.zip', contentType: 'application/zip' })
|
||||
mockOrderByThenLimit([row])
|
||||
mockFetchBuffer.mockResolvedValueOnce(ZIP_SHAPED)
|
||||
|
||||
const result = await readChatUpload('bundle.zip', CHAT_ID)
|
||||
|
||||
expect(result?.content).toContain('materialize_file')
|
||||
expect(result?.content).toContain('extract')
|
||||
expect(mockReadFileRecord).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns extract-first guidance for a large .zip without downloading it', async () => {
|
||||
const row = makeRow({
|
||||
id: 'wf_z',
|
||||
displayName: 'huge.zip',
|
||||
contentType: 'application/zip',
|
||||
size: 50 * 1024 * 1024,
|
||||
})
|
||||
mockOrderByThenLimit([row])
|
||||
|
||||
const result = await readChatUpload('huge.zip', CHAT_ID)
|
||||
|
||||
expect(result?.content).toContain('materialize_file')
|
||||
expect(mockFetchBuffer).not.toHaveBeenCalled()
|
||||
expect(mockReadFileRecord).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reads a small mislabeled ".zip" (non-zip bytes) normally instead of dead-ending it', async () => {
|
||||
const row = makeRow({ id: 'wf_m', displayName: 'data.zip', contentType: 'text/csv' })
|
||||
mockOrderByThenLimit([row])
|
||||
mockFetchBuffer.mockResolvedValueOnce(Buffer.from('a,b,c\n1,2,3\n'))
|
||||
mockReadFileRecord.mockResolvedValueOnce({ content: 'a,b,c\n1,2,3', totalLines: 2 })
|
||||
|
||||
const result = await readChatUpload('data.zip', CHAT_ID)
|
||||
|
||||
expect(result).toEqual({ content: 'a,b,c\n1,2,3', totalLines: 2 })
|
||||
expect(mockReadFileRecord).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns null when no row matches', async () => {
|
||||
mockOrderByThenLimit([])
|
||||
dbChainMockFns.orderBy.mockResolvedValueOnce([] as never)
|
||||
@@ -177,3 +228,24 @@ describe('readChatUpload', () => {
|
||||
expect(mockReadFileRecord).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('grepChatUpload', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
mockReadFileRecord.mockReset()
|
||||
})
|
||||
|
||||
it('throws WorkspaceFileGrepError with extract-first guidance for a .zip upload', async () => {
|
||||
const row = makeRow({ id: 'wf_z', displayName: 'bundle.zip', contentType: 'application/zip' })
|
||||
mockOrderByThenLimit([row])
|
||||
mockFetchBuffer.mockResolvedValueOnce(ZIP_SHAPED)
|
||||
|
||||
const error = await grepChatUpload('bundle.zip', CHAT_ID, 'foo').catch((e) => e)
|
||||
|
||||
expect(error).toBeInstanceOf(WorkspaceFileGrepError)
|
||||
expect(error.message).toContain('materialize_file')
|
||||
expect(error.message).toContain('extract')
|
||||
expect(mockReadFileRecord).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,7 +3,11 @@ import { workspaceFiles } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { and, asc, desc, eq, isNull, or } from 'drizzle-orm'
|
||||
import { type FileReadResult, readFileRecord } from '@/lib/copilot/vfs/file-reader'
|
||||
import {
|
||||
type FileReadResult,
|
||||
MAX_TEXT_READ_BYTES,
|
||||
readFileRecord,
|
||||
} from '@/lib/copilot/vfs/file-reader'
|
||||
import {
|
||||
type GrepCountEntry,
|
||||
type GrepMatch,
|
||||
@@ -12,11 +16,42 @@ import {
|
||||
WorkspaceFileGrepError,
|
||||
} from '@/lib/copilot/vfs/operations'
|
||||
import { decodeVfsSegment, encodeVfsSegment } from '@/lib/copilot/vfs/path-utils'
|
||||
import { isZipShaped } from '@/lib/file-parsers/zip-guard'
|
||||
import { getServePathPrefix } from '@/lib/uploads'
|
||||
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import {
|
||||
fetchWorkspaceFileBuffer,
|
||||
type WorkspaceFileRecord,
|
||||
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { buildArchiveExtractGuidance, isArchiveFileName } from '@/lib/uploads/utils/file-utils'
|
||||
|
||||
const logger = createLogger('UploadFileReader')
|
||||
|
||||
/**
|
||||
* Sniff budget for uploads whose NAME says archive: below this size the actual
|
||||
* bytes decide (a mislabeled text file named `data.zip` stays readable instead
|
||||
* of being trapped between read-says-extract and extract-says-invalid); above
|
||||
* it the extension is trusted so a real 100MB zip is never downloaded just to
|
||||
* refuse it. Aligned with the read path's inline text cap — any mislabeled
|
||||
* file too big to sniff would be rejected by read() as too large anyway, so
|
||||
* nothing readable is ever dead-ended.
|
||||
*/
|
||||
const ARCHIVE_SNIFF_MAX_BYTES = MAX_TEXT_READ_BYTES
|
||||
|
||||
/**
|
||||
* True when the upload should get extract-first guidance: named like an archive
|
||||
* and — for small files — actually shaped like one.
|
||||
*/
|
||||
async function isActualArchiveUpload(record: WorkspaceFileRecord): Promise<boolean> {
|
||||
if (!isArchiveFileName(record.name)) return false
|
||||
if (record.size > ARCHIVE_SNIFF_MAX_BYTES) return true
|
||||
try {
|
||||
const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: ARCHIVE_SNIFF_MAX_BYTES })
|
||||
return isZipShaped(buffer)
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical comparison key for an upload's VFS name. Accepts both the raw display
|
||||
* name and a percent-encoded segment (decode first — a no-op for raw names —
|
||||
@@ -142,7 +177,8 @@ export async function listChatUploads(chatId: string): Promise<WorkspaceFileReco
|
||||
/**
|
||||
* Read a specific uploaded file by display name within a chat session.
|
||||
* Resolves names with `normalizeVfsSegment` so macOS screenshot spacing (e.g. U+202F)
|
||||
* matches when the model passes a visually equivalent path.
|
||||
* matches when the model passes a visually equivalent path. A `.zip` upload is not
|
||||
* read directly — it returns extract-first guidance instead of binary bytes.
|
||||
*/
|
||||
export async function readChatUpload(
|
||||
filename: string,
|
||||
@@ -151,7 +187,11 @@ export async function readChatUpload(
|
||||
try {
|
||||
const row = await findMothershipUploadRowByChatAndName(chatId, filename)
|
||||
if (!row) return null
|
||||
return readFileRecord(toWorkspaceFileRecord(row))
|
||||
const record = toWorkspaceFileRecord(row)
|
||||
if (await isActualArchiveUpload(record)) {
|
||||
return { content: `[${buildArchiveExtractGuidance(record.name)}]`, totalLines: 1 }
|
||||
}
|
||||
return readFileRecord(record)
|
||||
} catch (err) {
|
||||
logger.warn('Failed to read chat upload', {
|
||||
filename,
|
||||
@@ -183,6 +223,9 @@ export async function grepChatUpload(
|
||||
)
|
||||
}
|
||||
const record = toWorkspaceFileRecord(row)
|
||||
if (await isActualArchiveUpload(record)) {
|
||||
throw new WorkspaceFileGrepError(buildArchiveExtractGuidance(record.name))
|
||||
}
|
||||
const result = await readFileRecord(record)
|
||||
if (!result) {
|
||||
throw new WorkspaceFileGrepError(`Upload content not found for "${filename}".`)
|
||||
|
||||
@@ -26,7 +26,8 @@ function recordSpanError(span: Span, err: unknown) {
|
||||
|
||||
const logger = createLogger('FileReader')
|
||||
|
||||
const MAX_TEXT_READ_BYTES = 5 * 1024 * 1024 // 5 MB
|
||||
/** Inline text-read cap — exported so callers can align their own byte-sniff budgets with what read() can actually display. */
|
||||
export const MAX_TEXT_READ_BYTES = 5 * 1024 * 1024 // 5 MB
|
||||
const MAX_IMAGE_READ_BYTES = 5 * 1024 * 1024 // 5 MB
|
||||
// Parseable-document byte cap. Large office/PDF files can still
|
||||
// produce huge extracted text; reject up front to avoid wasting a
|
||||
|
||||
@@ -66,7 +66,7 @@ export class ZipBombError extends Error {
|
||||
* closed: a ZIP-shaped buffer the guard cannot parse must be rejected rather
|
||||
* than handed to a decompression library.
|
||||
*/
|
||||
function isZipShaped(buffer: Buffer): boolean {
|
||||
export function isZipShaped(buffer: Buffer): boolean {
|
||||
if (buffer.length < 4) {
|
||||
return false
|
||||
}
|
||||
@@ -217,6 +217,64 @@ function sumDeclaredUncompressedSize(buffer: Buffer, abortAboveBytes: number): n
|
||||
return total
|
||||
}
|
||||
|
||||
/** Parse-time shape of a ZIP central directory, read without decompressing anything. */
|
||||
export interface ZipCentralDirectoryStats {
|
||||
/** Records in the contiguous central-directory run — what a per-signature parser allocates. */
|
||||
entryCount: number
|
||||
/** Summed declared extra-field bytes across those records. */
|
||||
totalExtraFieldBytes: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the real central directory (EOCD-anchored, decoy-resistant, ZIP64-aware —
|
||||
* the same anchoring as {@link assertOoxmlArchiveWithinLimits}) and report its
|
||||
* record count and summed extra-field bytes, so callers can bound a parser's
|
||||
* object graph before handing it the buffer. The walk covers the CONTIGUOUS run
|
||||
* of records at the central-directory offset rather than trusting the EOCD's
|
||||
* declared count, because that run is what JSZip actually allocates one entry
|
||||
* per — a lied count can neither hide records nor inflate the tally. Unlike a
|
||||
* raw whole-buffer signature scan, STORED entry payloads (e.g. a nested `.zip`
|
||||
* archived without recompression) are never miscounted as records. Returns
|
||||
* `null` when the buffer is not a parseable ZIP, so callers can fail closed.
|
||||
*/
|
||||
export function readZipCentralDirectoryStats(buffer: Buffer): ZipCentralDirectoryStats | null {
|
||||
if (buffer.length < EOCD_MIN_SIZE) {
|
||||
return null
|
||||
}
|
||||
|
||||
const eocdOffset = findEocdOffset(buffer)
|
||||
if (eocdOffset < 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
const location = locateCentralDirectory(buffer, eocdOffset)
|
||||
if (!location) {
|
||||
return null
|
||||
}
|
||||
|
||||
let entryCount = 0
|
||||
let totalExtraFieldBytes = 0
|
||||
let cursor = location.offset
|
||||
while (
|
||||
cursor + CENTRAL_DIRECTORY_HEADER_MIN_SIZE <= buffer.length &&
|
||||
buffer.readUInt32LE(cursor) === CENTRAL_DIRECTORY_HEADER_SIGNATURE
|
||||
) {
|
||||
const fileNameLength = buffer.readUInt16LE(cursor + 28)
|
||||
const extraFieldLength = buffer.readUInt16LE(cursor + 30)
|
||||
const commentLength = buffer.readUInt16LE(cursor + 32)
|
||||
|
||||
entryCount += 1
|
||||
totalExtraFieldBytes += extraFieldLength
|
||||
cursor += CENTRAL_DIRECTORY_HEADER_MIN_SIZE + fileNameLength + extraFieldLength + commentLength
|
||||
}
|
||||
|
||||
if (entryCount < location.entryCount) {
|
||||
return null
|
||||
}
|
||||
|
||||
return { entryCount, totalExtraFieldBytes }
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject an OOXML archive whose declared expanded size or compression ratio
|
||||
* exceeds safe bounds, before any decompression library materializes it.
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { Buffer } from 'buffer'
|
||||
import JSZip from 'jszip'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockEnsureFolder, mockUpload, mockDelete } = vi.hoisted(() => ({
|
||||
mockEnsureFolder: vi.fn(),
|
||||
mockUpload: vi.fn(),
|
||||
mockDelete: vi.fn(),
|
||||
}))
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => ({
|
||||
ensureWorkspaceFileFolderPath: mockEnsureFolder,
|
||||
}))
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
uploadWorkspaceFile: mockUpload,
|
||||
deleteWorkspaceFile: mockDelete,
|
||||
}))
|
||||
|
||||
import {
|
||||
decompressArchiveBufferToWorkspaceFiles,
|
||||
MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES,
|
||||
MAX_ARCHIVE_CENTRAL_DIR_RECORDS,
|
||||
MAX_ARCHIVE_ENTRY_BYTES,
|
||||
} from '@/lib/uploads/archive'
|
||||
|
||||
async function buildZip(
|
||||
files: Record<string, string | Buffer>,
|
||||
opts?: { symlinks?: string[] }
|
||||
): Promise<Buffer> {
|
||||
const zip = new JSZip()
|
||||
for (const [name, content] of Object.entries(files)) {
|
||||
const isLink = opts?.symlinks?.includes(name)
|
||||
zip.file(name, content, isLink ? { unixPermissions: 0o120777 } : undefined)
|
||||
}
|
||||
// platform: 'UNIX' so unixPermissions (incl. the symlink mode) round-trip,
|
||||
// mirroring how macOS/Linux `zip` authors archives.
|
||||
return Buffer.from(await zip.generateAsync({ type: 'uint8array', platform: 'UNIX' }))
|
||||
}
|
||||
|
||||
const CD_HEADER_SIZE = 46
|
||||
const EOCD_SIZE = 22
|
||||
|
||||
/**
|
||||
* Hand-craft a structurally valid ZIP central directory + EOCD: `records`
|
||||
* zero-name CD headers each declaring `extraPerRecord` extra-field bytes
|
||||
* (with the bytes actually present, so the walk advances correctly), and an
|
||||
* EOCD at the tail pointing at offset 0. There are no local file entries —
|
||||
* the pre-parse guard must reject before JSZip ever needs them.
|
||||
*/
|
||||
function craftCentralDirectory(records: number, extraPerRecord: number): Buffer {
|
||||
const recordSize = CD_HEADER_SIZE + extraPerRecord
|
||||
const buffer = Buffer.alloc(records * recordSize + EOCD_SIZE)
|
||||
for (let r = 0; r < records; r++) {
|
||||
const base = r * recordSize
|
||||
buffer.writeUInt32LE(0x02014b50, base) // central-directory file header signature
|
||||
buffer.writeUInt16LE(0, base + 28) // file name length
|
||||
buffer.writeUInt16LE(extraPerRecord, base + 30) // extra field length
|
||||
buffer.writeUInt16LE(0, base + 32) // comment length
|
||||
}
|
||||
const eocd = records * recordSize
|
||||
buffer.writeUInt32LE(0x06054b50, eocd) // EOCD signature
|
||||
buffer.writeUInt16LE(Math.min(records, 0xffff), eocd + 8) // entries on this disk
|
||||
buffer.writeUInt16LE(Math.min(records, 0xffff), eocd + 10) // total entries
|
||||
buffer.writeUInt32LE(records * recordSize, eocd + 12) // central directory size
|
||||
buffer.writeUInt32LE(0, eocd + 16) // central directory offset
|
||||
buffer.writeUInt16LE(0, eocd + 20) // comment length
|
||||
return buffer
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockEnsureFolder.mockResolvedValue('folder_1')
|
||||
mockDelete.mockResolvedValue(undefined)
|
||||
mockUpload.mockImplementation(async (_ws: string, _uid: string, buf: Buffer, name: string) => ({
|
||||
id: `f_${name}`,
|
||||
name,
|
||||
url: `/api/files/serve/${name}`,
|
||||
key: `workspace/ws/${name}`,
|
||||
size: buf.length,
|
||||
type: 'text/plain',
|
||||
}))
|
||||
})
|
||||
|
||||
describe('decompressArchiveBufferToWorkspaceFiles', () => {
|
||||
it('extracts entries as workspace files under the root folder', async () => {
|
||||
const buffer = await buildZip({ 'report.txt': 'hi', 'data/sheet.csv': 'a,b' })
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
rootFolderSegments: ['bundle'],
|
||||
})
|
||||
|
||||
expect(result.extracted).toHaveLength(2)
|
||||
expect(result.skippedUnsafePaths).toEqual([])
|
||||
expect(mockUpload).toHaveBeenCalledTimes(2)
|
||||
const leafNames = mockUpload.mock.calls.map((c) => c[3]).sort()
|
||||
expect(leafNames).toEqual(['report.txt', 'sheet.csv'])
|
||||
// Entries are rooted under the archive's folder; nested paths are preserved.
|
||||
expect(mockEnsureFolder).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ pathSegments: ['bundle'] })
|
||||
)
|
||||
expect(mockEnsureFolder).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ pathSegments: ['bundle', 'data'] })
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an archive with more central-directory records than the cap, before parsing', async () => {
|
||||
// A structurally valid central directory (EOCD-anchored) with one record more
|
||||
// than the parse-graph cap. JSZip would build one entry per record in the
|
||||
// contiguous run, so the pre-parse guard must reject before loadAsync runs —
|
||||
// and with the accurate central-directory message, not the file-count one.
|
||||
const buffer = craftCentralDirectory(MAX_ARCHIVE_CENTRAL_DIR_RECORDS + 1, 0)
|
||||
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(buffer, { workspaceId: 'ws', userId: 'u' })
|
||||
).rejects.toMatchObject({
|
||||
name: 'ArchiveError',
|
||||
reason: 'central_dir_too_large',
|
||||
message: expect.stringContaining(String(MAX_ARCHIVE_CENTRAL_DIR_RECORDS)),
|
||||
})
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects an archive whose central-directory extra fields exceed the byte cap, before parsing', async () => {
|
||||
// A handful of records — far below the record cap — each declaring (and
|
||||
// carrying) the maximum 0xFFFF extra-field bytes, so their sum crosses
|
||||
// MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES. JSZip retains one object per declared
|
||||
// extra field during loadAsync, so the guard must reject on summed extra
|
||||
// bytes, not just on record count.
|
||||
const EXTRA_PER_RECORD = 0xffff
|
||||
const records = Math.ceil(MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES / EXTRA_PER_RECORD) + 5
|
||||
expect(records).toBeLessThan(MAX_ARCHIVE_CENTRAL_DIR_RECORDS)
|
||||
const buffer = craftCentralDirectory(records, EXTRA_PER_RECORD)
|
||||
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(buffer, { workspaceId: 'ws', userId: 'u' })
|
||||
).rejects.toMatchObject({ name: 'ArchiveError', reason: 'central_dir_too_large' })
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('extracts a zip whose STORED entry contains foreign central-directory signatures', async () => {
|
||||
// Regression: a whole-buffer signature scan would count the PK\x01\x02
|
||||
// signatures inside this stored payload (a nested archive's central
|
||||
// directory travels verbatim inside STORED entries) and falsely reject the
|
||||
// archive. The EOCD-anchored walk must ignore entry payloads entirely.
|
||||
const signatures = Buffer.alloc((MAX_ARCHIVE_CENTRAL_DIR_RECORDS + 1) * 4)
|
||||
for (let r = 0; r <= MAX_ARCHIVE_CENTRAL_DIR_RECORDS; r++) {
|
||||
signatures.writeUInt32LE(0x02014b50, r * 4)
|
||||
}
|
||||
const zip = new JSZip()
|
||||
zip.file('inner.zip', signatures, { compression: 'STORE' })
|
||||
const buffer = Buffer.from(await zip.generateAsync({ type: 'uint8array' }))
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
})
|
||||
|
||||
expect(result.extracted).toHaveLength(1)
|
||||
expect(mockUpload).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('uploads nothing when an entry is corrupted mid-archive (all-or-nothing)', async () => {
|
||||
// Corrupt one entry's DEFLATE bytes AFTER the central directory is built, so
|
||||
// loadAsync parses fine and only streaming inflation fails. The validation
|
||||
// pass must catch it before ANY entry is uploaded, and the raw zlib error
|
||||
// must surface as the module's ArchiveError, not leak through.
|
||||
const zip = new JSZip()
|
||||
zip.file('fine.txt', 'this entry is intact')
|
||||
// Incompressible payload so the DEFLATE stream is large enough to stomp
|
||||
// without touching the following records.
|
||||
zip.file('bad.bin', Buffer.from(Array.from({ length: 20000 }, (_, i) => (i * 137) % 251)))
|
||||
const buffer = Buffer.from(
|
||||
await zip.generateAsync({ type: 'uint8array', compression: 'DEFLATE' })
|
||||
)
|
||||
const nameOffset = buffer.indexOf(Buffer.from('bad.bin'))
|
||||
// Local file header: name follows the 30-byte fixed header; data follows the
|
||||
// name (+ extra field, empty here). Stomp a chunk of the DEFLATE stream.
|
||||
buffer.fill(0xff, nameOffset + 'bad.bin'.length, nameOffset + 'bad.bin'.length + 256)
|
||||
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(buffer, { workspaceId: 'ws', userId: 'u' })
|
||||
).rejects.toMatchObject({ name: 'ArchiveError', reason: 'invalid' })
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rolls back already-uploaded files when an upload fails mid-extraction', async () => {
|
||||
// Pass 1 validates caps, but an upload itself can still fail mid-loop
|
||||
// (storage/DB error, quota crossed). Every file written before the failure
|
||||
// must be deleted so callers and retries never observe a partial tree.
|
||||
const buffer = await buildZip({ 'a.txt': 'first', 'b.txt': 'second', 'c.txt': 'third' })
|
||||
mockUpload
|
||||
.mockResolvedValueOnce({ id: 'f_a', name: 'a.txt', url: '/a', key: 'k/a', size: 5 })
|
||||
.mockResolvedValueOnce({ id: 'f_b', name: 'b.txt', url: '/b', key: 'k/b', size: 6 })
|
||||
.mockRejectedValueOnce(new Error('storage quota exceeded'))
|
||||
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(buffer, { workspaceId: 'ws', userId: 'u' })
|
||||
).rejects.toThrow('storage quota exceeded')
|
||||
|
||||
expect(mockDelete).toHaveBeenCalledTimes(2)
|
||||
expect(mockDelete).toHaveBeenCalledWith('ws', 'f_a')
|
||||
expect(mockDelete).toHaveBeenCalledWith('ws', 'f_b')
|
||||
})
|
||||
|
||||
it('does not count noise entries toward the extraction cap when they are being skipped', async () => {
|
||||
// macOS Finder zips carry a __MACOSX/._* shadow per file, doubling the raw
|
||||
// entry count. 501 files + 501 shadows = 1002 raw entries — over the
|
||||
// 1000-file cap — but with skipNoiseEntries set only the 501 real files are
|
||||
// extracted, so the archive must be accepted.
|
||||
const files: Record<string, string> = {}
|
||||
for (let i = 0; i < 501; i++) {
|
||||
files[`f${i}.txt`] = 'x'
|
||||
files[`__MACOSX/._f${i}.txt`] = 'shadow'
|
||||
}
|
||||
const buffer = await buildZip(files)
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
skipNoiseEntries: true,
|
||||
})
|
||||
|
||||
expect(result.extracted).toHaveLength(501)
|
||||
expect(result.skipped).toBe(501)
|
||||
})
|
||||
|
||||
it('throws ArchiveError invalid for a non-zip buffer (no files written)', async () => {
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(Buffer.from('not a zip at all'), {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
})
|
||||
).rejects.toMatchObject({ name: 'ArchiveError', reason: 'invalid' })
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('excludes symlinks (uncounted) and skips zip-slip traversal (counted in skipped)', async () => {
|
||||
const buffer = await buildZip(
|
||||
{
|
||||
'safe.txt': 'ok',
|
||||
'..\\evil.txt': 'evil',
|
||||
'link.txt': '/etc/passwd',
|
||||
},
|
||||
{ symlinks: ['link.txt'] }
|
||||
)
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
})
|
||||
|
||||
// Only the traversal entry counts toward `skipped`; the symlink is filtered
|
||||
// out before the skip tally (it never becomes a candidate entry). The
|
||||
// traversal entry's raw name is preserved for the callers' forensic logs.
|
||||
expect(result.extracted).toHaveLength(1)
|
||||
expect(result.skipped).toBe(1)
|
||||
expect(result.skippedUnsafePaths).toEqual(['..\\evil.txt'])
|
||||
expect(mockUpload).toHaveBeenCalledTimes(1)
|
||||
expect(mockUpload.mock.calls[0][3]).toBe('safe.txt')
|
||||
})
|
||||
|
||||
it('extracts macOS/Windows filesystem-noise entries by default (skipNoiseEntries unset)', async () => {
|
||||
const buffer = await buildZip({ '__MACOSX/a.txt': 'x', '.DS_Store': 'y' })
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
})
|
||||
|
||||
// Parity with the HTTP decompress route, which extracts these verbatim.
|
||||
expect(result.extracted).toHaveLength(2)
|
||||
expect(result.skipped).toBe(0)
|
||||
expect(mockUpload).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('drops filesystem-noise entries when skipNoiseEntries is set', async () => {
|
||||
const buffer = await buildZip({ '__MACOSX/a.txt': 'x', '.DS_Store': 'y' })
|
||||
|
||||
const result = await decompressArchiveBufferToWorkspaceFiles(buffer, {
|
||||
workspaceId: 'ws',
|
||||
userId: 'u',
|
||||
skipNoiseEntries: true,
|
||||
})
|
||||
|
||||
expect(result.extracted).toEqual([])
|
||||
expect(result.skipped).toBe(2)
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects an entry whose declared size exceeds the per-entry cap', async () => {
|
||||
const zip = new JSZip()
|
||||
// Highly compressible zeros keep the archive tiny on disk while the declared
|
||||
// uncompressed size blows past the per-entry cap.
|
||||
zip.file('big.bin', Buffer.alloc(MAX_ARCHIVE_ENTRY_BYTES + 1024))
|
||||
const buffer = Buffer.from(
|
||||
await zip.generateAsync({ type: 'uint8array', compression: 'DEFLATE' })
|
||||
)
|
||||
|
||||
await expect(
|
||||
decompressArchiveBufferToWorkspaceFiles(buffer, { workspaceId: 'ws', userId: 'u' })
|
||||
).rejects.toMatchObject({ name: 'ArchiveError', reason: 'entry_too_large' })
|
||||
expect(mockUpload).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,386 @@
|
||||
import { Buffer } from 'buffer'
|
||||
import type { Readable } from 'stream'
|
||||
import JSZip from 'jszip'
|
||||
import { readZipCentralDirectoryStats } from '@/lib/file-parsers/zip-guard'
|
||||
import { ensureWorkspaceFileFolderPath } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
|
||||
import {
|
||||
deleteWorkspaceFile,
|
||||
uploadWorkspaceFile,
|
||||
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils'
|
||||
import type { UserFile } from '@/executor/types'
|
||||
|
||||
/**
|
||||
* Shared, zip-bomb / zip-slip safe archive primitives plus the one-shot
|
||||
* "decompress into workspace files/" extractor.
|
||||
*
|
||||
* The declared sizes in a ZIP header are attacker-controlled, so the real caps
|
||||
* are always enforced on the inflated byte stream — never on metadata. The
|
||||
* parser's object graph is additionally bounded BEFORE parsing via
|
||||
* {@link readZipCentralDirectoryStats} (EOCD-anchored central-directory walk),
|
||||
* bounding both the record count (JSZip builds one entry per record in the
|
||||
* contiguous CD run) and the summed declared extra-field bytes (JSZip retains
|
||||
* one object per CD extra field), so a crafted archive cannot balloon the
|
||||
* parser's heap. Extraction is all-or-nothing: every entry is first inflated
|
||||
* through a counting sink (discarded, nothing persisted) to prove the archive
|
||||
* fits the caps, and only then inflated again and uploaded — so a cap violation
|
||||
* or lying header can never leave a partial tree behind. Peak memory stays ~one
|
||||
* entry in both passes.
|
||||
*/
|
||||
|
||||
/** Input archive download/size cap. */
|
||||
export const MAX_ARCHIVE_BYTES = 100 * 1024 * 1024
|
||||
/** Maximum number of entries extracted from a single archive. */
|
||||
export const MAX_ARCHIVE_ENTRIES = 1000
|
||||
/** Maximum uncompressed size for any single archive entry. */
|
||||
export const MAX_ARCHIVE_ENTRY_BYTES = 100 * 1024 * 1024
|
||||
/** Maximum total uncompressed size across all entries, to bound zip-bomb expansion. */
|
||||
export const MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
|
||||
|
||||
const S_IFMT = 0o170000
|
||||
const S_IFLNK = 0o120000
|
||||
|
||||
/** Memory bound for the parse-time object graph (distinct from the file-extraction cap). */
|
||||
export const MAX_ARCHIVE_CENTRAL_DIR_RECORDS = 10_000
|
||||
/**
|
||||
* Cap on the summed declared extra-field bytes across all central-directory
|
||||
* records. JSZip allocates one object per CD extra field (>= 4 bytes each), so
|
||||
* bounding the summed extra-field bytes bounds the parse-time object graph even
|
||||
* when the record count is tiny (one record may declare up to 65535 extra bytes
|
||||
* ≈ 16k tiny fields). Legit archives use only tens of bytes/entry, so 4MB is
|
||||
* generous headroom.
|
||||
*/
|
||||
export const MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES = 4 * 1024 * 1024
|
||||
|
||||
/** Reason a {@link ArchiveError} was raised, for mapping to a caller response/status. */
|
||||
export type ArchiveErrorReason =
|
||||
| 'invalid'
|
||||
| 'too_many_entries'
|
||||
| 'central_dir_too_large'
|
||||
| 'entry_too_large'
|
||||
| 'total_too_large'
|
||||
|
||||
/**
|
||||
* Raised for malformed archives and cap violations. `message` is the single
|
||||
* user-facing source of truth (caps included) — callers surface it verbatim and
|
||||
* use `reason` only for response mapping (e.g. HTTP status), never to rebuild
|
||||
* the text.
|
||||
*/
|
||||
export class ArchiveError extends Error {
|
||||
readonly reason: ArchiveErrorReason
|
||||
readonly entryName?: string
|
||||
|
||||
constructor(reason: ArchiveErrorReason, message: string, entryName?: string) {
|
||||
super(message)
|
||||
this.name = 'ArchiveError'
|
||||
this.reason = reason
|
||||
this.entryName = entryName
|
||||
}
|
||||
}
|
||||
|
||||
const MB = 1024 * 1024
|
||||
|
||||
/**
|
||||
* Bound JSZip's parse-time object graph before handing it the buffer, using the
|
||||
* real (EOCD-anchored) central directory: record count bounds the entry graph,
|
||||
* and summed declared extra-field bytes bound the per-field objects. Throws with
|
||||
* the accurate cap in the message — these caps are parse-graph bounds, distinct
|
||||
* from {@link MAX_ARCHIVE_ENTRIES} which limits extracted files after parsing.
|
||||
*/
|
||||
function assertCentralDirWithinCaps(buffer: Buffer): void {
|
||||
const stats = readZipCentralDirectoryStats(buffer)
|
||||
if (!stats) {
|
||||
throw new ArchiveError(
|
||||
'invalid',
|
||||
'Not a valid .zip archive — its central directory could not be parsed.'
|
||||
)
|
||||
}
|
||||
if (stats.entryCount > MAX_ARCHIVE_CENTRAL_DIR_RECORDS) {
|
||||
throw new ArchiveError(
|
||||
'central_dir_too_large',
|
||||
`Archive central directory has ${stats.entryCount} records; the maximum that can be parsed safely is ${MAX_ARCHIVE_CENTRAL_DIR_RECORDS}.`
|
||||
)
|
||||
}
|
||||
if (stats.totalExtraFieldBytes > MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES) {
|
||||
throw new ArchiveError(
|
||||
'central_dir_too_large',
|
||||
`Archive central-directory metadata is too large to parse safely (over ${MAX_ARCHIVE_CENTRAL_DIR_EXTRA_BYTES / MB} MB of extra fields).`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a zip entry's declared uncompressed size without materializing it. Comes
|
||||
* straight from (attacker-controlled) ZIP metadata, so it is only a cheap
|
||||
* fast-reject for honestly-declared archives — never the authoritative cap.
|
||||
*/
|
||||
const readEntryUncompressedSize = (entry: JSZip.JSZipObject): number | undefined => {
|
||||
const data = (entry as JSZip.JSZipObject & { _data?: { uncompressedSize?: number } })._data
|
||||
const size = data?.uncompressedSize
|
||||
return typeof size === 'number' && Number.isFinite(size) ? size : undefined
|
||||
}
|
||||
|
||||
type InflateResult =
|
||||
| { ok: true; size: number; buffer: Buffer | null }
|
||||
| { ok: false; reason: 'entry' | 'total' }
|
||||
|
||||
/**
|
||||
* Inflate a single zip entry through a streaming counting sink, tearing the
|
||||
* stream down the moment cumulative output would exceed the per-entry cap or the
|
||||
* remaining total budget. The declared uncompressed size is NOT trusted:
|
||||
* enforcement happens on the actual inflated bytes as they arrive, so peak memory
|
||||
* is bounded by the cap plus one DEFLATE chunk. With `retain: false` the bytes
|
||||
* are counted and discarded (the validation pass), so peak memory is ~one chunk.
|
||||
*/
|
||||
const inflateEntryWithinCaps = (
|
||||
entry: JSZip.JSZipObject,
|
||||
remainingTotalBudget: number,
|
||||
retain: boolean
|
||||
): Promise<InflateResult> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = []
|
||||
let size = 0
|
||||
let settled = false
|
||||
const stream = entry.nodeStream() as Readable
|
||||
|
||||
const settle = (result: InflateResult) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
stream.destroy()
|
||||
resolve(result)
|
||||
}
|
||||
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length
|
||||
if (size > MAX_ARCHIVE_ENTRY_BYTES) {
|
||||
settle({ ok: false, reason: 'entry' })
|
||||
return
|
||||
}
|
||||
if (size > remainingTotalBudget) {
|
||||
settle({ ok: false, reason: 'total' })
|
||||
return
|
||||
}
|
||||
if (retain) chunks.push(chunk)
|
||||
})
|
||||
stream.on('end', () =>
|
||||
settle({ ok: true, size, buffer: retain ? Buffer.concat(chunks, size) : null })
|
||||
)
|
||||
stream.on('error', () => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
stream.destroy()
|
||||
// A stream error here means the entry's compressed data is corrupt or
|
||||
// truncated (it passed the central-directory parse) — surface it under the
|
||||
// module's ArchiveError contract, not as a raw zlib error.
|
||||
reject(
|
||||
new ArchiveError(
|
||||
'invalid',
|
||||
`Archive entry "${entry.name}" could not be decompressed — the archive may be corrupted or truncated.`,
|
||||
entry.name
|
||||
)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
/** True when a zip entry's unix mode marks it as a symlink (never extracted). */
|
||||
const isSymlinkEntry = (entry: JSZip.JSZipObject): boolean => {
|
||||
const mode = (entry as JSZip.JSZipObject & { unixPermissions?: number | null }).unixPermissions
|
||||
return typeof mode === 'number' && (mode & S_IFMT) === S_IFLNK
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a zip entry path into safe segments, guarding against zip-slip.
|
||||
* Returns null for traversal (`..`) and empty paths so the entry is skipped.
|
||||
*/
|
||||
const sanitizeArchiveEntryPath = (rawPath: string): string[] | null => {
|
||||
const segments = rawPath
|
||||
.replace(/\\/g, '/')
|
||||
.split('/')
|
||||
.map((segment) => segment.trim())
|
||||
.filter((segment) => segment.length > 0 && segment !== '.')
|
||||
|
||||
if (segments.length === 0 || segments.includes('..')) return null
|
||||
return segments
|
||||
}
|
||||
|
||||
/** Filesystem cruft that should never surface as a readable archive entry. */
|
||||
const isArchiveNoiseEntry = (segments: string[]): boolean => {
|
||||
if (segments[0] === '__MACOSX') return true
|
||||
const leaf = segments[segments.length - 1]
|
||||
return leaf === '.DS_Store' || leaf === 'Thumbs.db'
|
||||
}
|
||||
|
||||
export interface DecompressResult {
|
||||
/** Workspace files created under the target folder, in extraction order. */
|
||||
extracted: UserFile[]
|
||||
/** Count of entries skipped as unsafe (zip-slip) or filesystem noise. */
|
||||
skipped: number
|
||||
/**
|
||||
* Raw entry paths rejected by the zip-slip sanitizer (traversal or empty),
|
||||
* so callers can log the attempted names for forensics. Noise entries
|
||||
* (`__MACOSX/`, `.DS_Store`) are counted in `skipped` but never listed here.
|
||||
*/
|
||||
skippedUnsafePaths: string[]
|
||||
}
|
||||
|
||||
/** Throw the (single-sourced) ArchiveError for a streaming cap violation. */
|
||||
function throwInflateCapError(reason: 'entry' | 'total', entryName: string): never {
|
||||
if (reason === 'entry') {
|
||||
throw new ArchiveError(
|
||||
'entry_too_large',
|
||||
`Archive entry "${entryName}" is too large to extract. Maximum is ${MAX_ARCHIVE_ENTRY_BYTES / MB} MB per file.`,
|
||||
entryName
|
||||
)
|
||||
}
|
||||
throw new ArchiveError(
|
||||
'total_too_large',
|
||||
`Archive expands to more than the ${MAX_ARCHIVE_TOTAL_BYTES / MB} MB extraction limit.`,
|
||||
entryName
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress an archive buffer into workspace files under `rootFolderSegments`
|
||||
* (default: the workspace root). Reuses the same caps and zip-slip / zip-bomb /
|
||||
* symlink guards everywhere. Throws {@link ArchiveError} for an invalid archive
|
||||
* or a cap violation; returns `{ extracted: [] }` when every entry was skipped.
|
||||
*
|
||||
* All-or-nothing: pass 1 inflates every entry through a discarding counting sink
|
||||
* to enforce the per-entry and total caps on REAL inflated bytes (declared sizes
|
||||
* can lie), and nothing is uploaded until the whole archive has passed — so a
|
||||
* mid-archive violation never leaves a partial tree in the workspace. Pass 2
|
||||
* re-inflates and uploads one entry at a time. Peak memory stays ~one entry in
|
||||
* both passes; the cost is inflating twice (CPU only, bounded by the caps).
|
||||
*
|
||||
* Filesystem-noise entries (`__MACOSX/`, `.DS_Store`, `Thumbs.db`) are extracted
|
||||
* verbatim unless `skipNoiseEntries` is set — the HTTP decompress route preserves
|
||||
* them; the agent-facing extract path drops them.
|
||||
*/
|
||||
export async function decompressArchiveBufferToWorkspaceFiles(
|
||||
buffer: Buffer,
|
||||
opts: {
|
||||
workspaceId: string
|
||||
userId: string
|
||||
rootFolderSegments?: string[]
|
||||
skipNoiseEntries?: boolean
|
||||
}
|
||||
): Promise<DecompressResult> {
|
||||
const { workspaceId, userId, rootFolderSegments = [], skipNoiseEntries = false } = opts
|
||||
|
||||
assertCentralDirWithinCaps(buffer)
|
||||
|
||||
let zip: JSZip
|
||||
try {
|
||||
zip = await JSZip.loadAsync(buffer)
|
||||
} catch {
|
||||
throw new ArchiveError('invalid', 'Not a valid .zip archive.')
|
||||
}
|
||||
|
||||
const realEntries = Object.values(zip.files).filter(
|
||||
(entry) => !entry.dir && !isSymlinkEntry(entry)
|
||||
)
|
||||
|
||||
// Resolve safe entries first so unsafe ones never count toward the size caps.
|
||||
const safeEntries: Array<{ entry: JSZip.JSZipObject; segments: string[] }> = []
|
||||
const skippedUnsafePaths: string[] = []
|
||||
let skipped = 0
|
||||
for (const entry of realEntries) {
|
||||
const segments = sanitizeArchiveEntryPath(entry.name)
|
||||
if (!segments) {
|
||||
skippedUnsafePaths.push(entry.name)
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
if (skipNoiseEntries && isArchiveNoiseEntry(segments)) {
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
safeEntries.push({ entry, segments })
|
||||
}
|
||||
|
||||
// The entry cap applies to what will actually be extracted — a macOS zip whose
|
||||
// __MACOSX/ shadows are about to be dropped must not be rejected for them.
|
||||
if (safeEntries.length > MAX_ARCHIVE_ENTRIES) {
|
||||
throw new ArchiveError(
|
||||
'too_many_entries',
|
||||
`Archive has ${safeEntries.length} files; the maximum is ${MAX_ARCHIVE_ENTRIES}.`
|
||||
)
|
||||
}
|
||||
|
||||
// Cheap declared-size fast-reject for honestly-declared archives.
|
||||
let declaredTotal = 0
|
||||
for (const { entry } of safeEntries) {
|
||||
const declaredSize = readEntryUncompressedSize(entry)
|
||||
if (declaredSize === undefined) continue
|
||||
if (declaredSize > MAX_ARCHIVE_ENTRY_BYTES) throwInflateCapError('entry', entry.name)
|
||||
declaredTotal += declaredSize
|
||||
if (declaredTotal > MAX_ARCHIVE_TOTAL_BYTES) throwInflateCapError('total', entry.name)
|
||||
}
|
||||
|
||||
// Pass 1 — validate: inflate every entry against the caps without retaining or
|
||||
// persisting anything, so a lying header aborts before any upload happens.
|
||||
let validatedTotal = 0
|
||||
for (const { entry } of safeEntries) {
|
||||
const result = await inflateEntryWithinCaps(
|
||||
entry,
|
||||
MAX_ARCHIVE_TOTAL_BYTES - validatedTotal,
|
||||
false
|
||||
)
|
||||
if (!result.ok) throwInflateCapError(result.reason, entry.name)
|
||||
validatedTotal += result.size
|
||||
}
|
||||
|
||||
// Pass 2 — extract: the archive is proven within caps; inflate again and upload.
|
||||
// Uploads themselves can still fail mid-loop (storage/DB errors, quota crossed
|
||||
// by another writer), so a failure rolls back every file written so far —
|
||||
// callers and their retries must never observe a partial tree.
|
||||
const folderIdCache = new Map<string, string | null>()
|
||||
const extracted: UserFile[] = []
|
||||
let totalBytes = 0
|
||||
try {
|
||||
for (const { entry, segments } of safeEntries) {
|
||||
const result = await inflateEntryWithinCaps(entry, MAX_ARCHIVE_TOTAL_BYTES - totalBytes, true)
|
||||
if (!result.ok) throwInflateCapError(result.reason, entry.name)
|
||||
totalBytes += result.size
|
||||
const entryBuffer = result.buffer as Buffer
|
||||
|
||||
const leafName = segments[segments.length - 1]
|
||||
const folderSegments = [...rootFolderSegments, ...segments.slice(0, -1)]
|
||||
const folderKey = folderSegments.join('/')
|
||||
let folderId = folderIdCache.get(folderKey)
|
||||
if (folderId === undefined) {
|
||||
folderId = await ensureWorkspaceFileFolderPath({
|
||||
workspaceId,
|
||||
userId,
|
||||
pathSegments: folderSegments,
|
||||
})
|
||||
folderIdCache.set(folderKey, folderId)
|
||||
}
|
||||
|
||||
const mimeType = getMimeTypeFromExtension(getFileExtension(leafName))
|
||||
const uploaded = await uploadWorkspaceFile(
|
||||
workspaceId,
|
||||
userId,
|
||||
entryBuffer,
|
||||
leafName,
|
||||
mimeType,
|
||||
{
|
||||
folderId,
|
||||
}
|
||||
)
|
||||
extracted.push(uploaded)
|
||||
}
|
||||
} catch (error) {
|
||||
for (const file of extracted) {
|
||||
try {
|
||||
await deleteWorkspaceFile(workspaceId, file.id)
|
||||
} catch {
|
||||
// Best-effort: a file whose cleanup fails is still soft-deletable by hand;
|
||||
// the original error is what the caller needs to see.
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
return { extracted, skipped, skippedUnsafePaths }
|
||||
}
|
||||
@@ -970,13 +970,17 @@ export async function getWorkspaceFile(
|
||||
/**
|
||||
* Download workspace file content
|
||||
*/
|
||||
export async function fetchWorkspaceFileBuffer(fileRecord: WorkspaceFileRecord): Promise<Buffer> {
|
||||
export async function fetchWorkspaceFileBuffer(
|
||||
fileRecord: WorkspaceFileRecord,
|
||||
options: { maxBytes?: number } = {}
|
||||
): Promise<Buffer> {
|
||||
logger.info(`Downloading workspace file: ${fileRecord.name}`)
|
||||
|
||||
try {
|
||||
const buffer = await downloadFile({
|
||||
key: fileRecord.key,
|
||||
context: fileRecord.storageContext ?? 'workspace',
|
||||
maxBytes: options.maxBytes,
|
||||
})
|
||||
logger.info(
|
||||
`Successfully downloaded workspace file: ${fileRecord.name} (${buffer.length} bytes)`
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { omit } from '@sim/utils/object'
|
||||
import type { StorageContext } from '@/lib/uploads'
|
||||
import { ACCEPTED_FILE_TYPES, SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation'
|
||||
import {
|
||||
ACCEPTED_FILE_TYPES,
|
||||
SUPPORTED_ARCHIVE_EXTENSIONS,
|
||||
SUPPORTED_DOCUMENT_EXTENSIONS,
|
||||
} from '@/lib/uploads/utils/validation'
|
||||
import { isUuid } from '@/executor/constants'
|
||||
import type { UserFile } from '@/executor/types'
|
||||
|
||||
@@ -206,6 +210,26 @@ export function getFileExtension(filename: string): string {
|
||||
return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : ''
|
||||
}
|
||||
|
||||
const ARCHIVE_EXTENSIONS = new Set<string>(SUPPORTED_ARCHIVE_EXTENSIONS)
|
||||
|
||||
/**
|
||||
* True when a file name is a supported archive (zip). Detection is by extension
|
||||
* so it is robust to the varied/empty MIME types browsers assign to archives.
|
||||
*/
|
||||
export function isArchiveFileName(filename: string): boolean {
|
||||
return ARCHIVE_EXTENSIONS.has(getFileExtension(filename))
|
||||
}
|
||||
|
||||
/**
|
||||
* Single source of truth for the "extract a .zip first" guidance shown wherever
|
||||
* the agent tries to read/grep a raw archive (upload reader, chat payload). A
|
||||
* `.zip`'s contents aren't readable until it is decompressed into workspace
|
||||
* `files/`, so this points at the explicit one-time extract step.
|
||||
*/
|
||||
export function buildArchiveExtractGuidance(name: string): string {
|
||||
return `"${name}" is a .zip archive — its contents can't be read directly. Extract it once with materialize_file(fileNames: ["${name}"], operation: "extract"), then read the unpacked files under files/ (e.g. glob("files/<archive>/**") then read("files/<archive>/<path>/content")).`
|
||||
}
|
||||
|
||||
const EXTENSION_TO_MIME: Record<string, string> = {
|
||||
// Images
|
||||
jpg: 'image/jpeg',
|
||||
|
||||
@@ -95,6 +95,13 @@ export const SUPPORTED_AUDIO_EXTENSIONS = [
|
||||
|
||||
export const SUPPORTED_VIDEO_EXTENSIONS = ['mp4', 'mov', 'avi', 'mkv', 'webm'] as const
|
||||
|
||||
/**
|
||||
* Archive formats accepted as chat attachments. A `.zip` is stored once in
|
||||
* uploads/; the agent must extract it (materialize_file operation "extract") to
|
||||
* decompress it into workspace files/ before reading its contents.
|
||||
*/
|
||||
export const SUPPORTED_ARCHIVE_EXTENSIONS = ['zip'] as const
|
||||
|
||||
export const SUPPORTED_IMAGE_EXTENSIONS = [
|
||||
'png',
|
||||
'jpg',
|
||||
@@ -207,12 +214,30 @@ const SUPPORTED_IMAGE_MIME_TYPES = [
|
||||
'image/vnd.microsoft.icon',
|
||||
]
|
||||
|
||||
const SUPPORTED_ARCHIVE_MIME_TYPES = [
|
||||
'application/zip',
|
||||
'application/x-zip-compressed',
|
||||
'application/x-zip',
|
||||
]
|
||||
|
||||
export const CHAT_ACCEPT_ATTRIBUTE = [
|
||||
ACCEPT_ATTRIBUTE,
|
||||
...SUPPORTED_IMAGE_MIME_TYPES,
|
||||
...SUPPORTED_IMAGE_EXTENSIONS.map((ext) => `.${ext}`),
|
||||
].join(',')
|
||||
|
||||
/**
|
||||
* Accept attribute for the mothership copilot input only. Archives are scoped
|
||||
* here — NOT in {@link CHAT_ACCEPT_ATTRIBUTE} — because only the copilot flow
|
||||
* has zip handling (materialize_file "extract"); a zip picked in a workflow or
|
||||
* deployed chat would flow into execution, where no parser exists.
|
||||
*/
|
||||
export const MOTHERSHIP_ACCEPT_ATTRIBUTE = [
|
||||
CHAT_ACCEPT_ATTRIBUTE,
|
||||
...SUPPORTED_ARCHIVE_MIME_TYPES,
|
||||
...SUPPORTED_ARCHIVE_EXTENSIONS.map((ext) => `.${ext}`),
|
||||
].join(',')
|
||||
|
||||
export interface FileValidationError {
|
||||
code: 'UNSUPPORTED_FILE_TYPE' | 'MIME_TYPE_MISMATCH'
|
||||
message: string
|
||||
@@ -234,15 +259,27 @@ export const SUPPORTED_ATTACHMENT_EXTENSIONS = Array.from(
|
||||
*
|
||||
* Permits documents, code, images, audio, and video — anything users would
|
||||
* reasonably attach to a chat message. Rejects executables and unknown types.
|
||||
* Archives (`.zip`) are accepted only with `allowArchives` — the mothership
|
||||
* copilot flow, which alone can extract them; every other upload surface keeps
|
||||
* rejecting them up front rather than failing mid-run downstream.
|
||||
*/
|
||||
export function validateAttachmentFileType(fileName: string): FileValidationError | null {
|
||||
export function validateAttachmentFileType(
|
||||
fileName: string,
|
||||
options?: { allowArchives?: boolean }
|
||||
): FileValidationError | null {
|
||||
const raw = extractExtension(fileName)
|
||||
const extension = isAlphanumericExtension(raw) ? raw : ''
|
||||
|
||||
if (!SUPPORTED_ATTACHMENT_EXTENSIONS.includes(extension)) {
|
||||
const allowed =
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS.includes(extension) ||
|
||||
(options?.allowArchives === true &&
|
||||
(SUPPORTED_ARCHIVE_EXTENSIONS as readonly string[]).includes(extension))
|
||||
|
||||
if (!allowed) {
|
||||
const archiveNote = options?.allowArchives ? ', and zip archives' : ''
|
||||
return {
|
||||
code: 'UNSUPPORTED_FILE_TYPE',
|
||||
message: `Unsupported file type${extension ? `: ${extension}` : ` for "${fileName}"`}. Supported types include documents, code, images, audio, and video.`,
|
||||
message: `Unsupported file type${extension ? `: ${extension}` : ` for "${fileName}"`}. Supported types include documents, code, images, audio, video${archiveNote}.`,
|
||||
supportedTypes: [...SUPPORTED_ATTACHMENT_EXTENSIONS],
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user