opensource/sim
3
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-09-24 15:45:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(tables): route large CSV imports to the background job instead of 413 (#4927)

Browse Source 
* fix(tables): route large CSV imports to the background job instead of 413

* fix(tables): drop duplicate error toast on async import failure

* fix(tables): guard importId on async cancel and drop mutation objects from deps
This commit is contained in:
Theodore Li
2026-06-09 14:35:41 -04:00
committed by GitHub
parent 37f1141fc0
commit 2f70188091
2 changed files with 147 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx
+78 -40
View File
@@ -23,14 +23,26 @@ import {
TableRow,
toast,
} from '@/components/emcn'
import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
import type { TableDefinition } from '@/lib/table/types'
import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables'
import {
type CsvImportMode,
cancelTableImport,
useImportCsvIntoTable,
useImportCsvIntoTableAsync,
} from '@/hooks/queries/tables'
import { useImportTrayStore } from '@/stores/table/import-tray/store'
const logger = createLogger('ImportCsvDialog')
const MAX_SAMPLE_ROWS = 5
const MAX_EXAMPLES_IN_ERROR = 3
/**
* Bytes read for the preview/mapping. We never parse the whole file client-side — the importer
* streams it server-side and the DB row-count trigger enforces the row limit.
*/
const CSV_PREVIEW_BYTES = 512 * 1024
/**
* Sentinel value for the "Do not import" option in the mapping combobox. The
* whitespace is intentional: valid column names must match `NAME_PATTERN`
@@ -92,7 +104,18 @@ interface ParsedCsv {
file: File
headers: string[]
sampleRows: Record<string, unknown>[]
totalRows: number
}
/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
const sliced = file.size > CSV_PREVIEW_BYTES
const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
let bytes = new Uint8Array(await blob.arrayBuffer())
if (sliced) {
const lastNewline = bytes.lastIndexOf(0x0a)
if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
}
return parseCsvBuffer(bytes, delimiter)
}
export function ImportCsvDialog({
@@ -110,6 +133,7 @@ export function ImportCsvDialog({
const [createHeaders, setCreateHeaders] = useState<Set<string>>(new Set())
const [mode, setMode] = useState<CsvImportMode>('append')
const importMutation = useImportCsvIntoTable()
const importAsyncMutation = useImportCsvIntoTableAsync()
function resetState() {
setParsed(null)
@@ -155,15 +179,13 @@ export function ImportCsvDialog({
setParsing(true)
setParseError(null)
try {
const arrayBuffer = await file.arrayBuffer()
const delimiter = ext === 'tsv' ? '\t' : ','
const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter)
const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
const { headers, rows } = await parseCsvPreview(file, delimiter)
const autoMapping = buildAutoMapping(headers, table.schema)
setParsed({
file,
headers,
sampleRows: rows.slice(0, MAX_SAMPLE_ROWS),
totalRows: rows.length,
})
setMapping(autoMapping)
} catch (err) {
@@ -256,28 +278,62 @@ export function ImportCsvDialog({
}
}, [mapping, parsed?.headers, table.schema.columns, createHeaders])
const appendCapacityDeficit =
parsed && mode === 'append' && table.rowCount + parsed.totalRows > table.maxRows
? table.rowCount + parsed.totalRows - table.maxRows
: 0
const replaceCapacityDeficit =
parsed && mode === 'replace' && parsed.totalRows > table.maxRows
? parsed.totalRows - table.maxRows
: 0
const canSubmit =
parsed !== null &&
!importMutation.isPending &&
!importAsyncMutation.isPending &&
missingRequired.length === 0 &&
duplicateTargets.length === 0 &&
mappedCount + createCount > 0 &&
appendCapacityDeficit === 0 &&
replaceCapacityDeficit === 0
mappedCount + createCount > 0
async function handleSubmit() {
if (!parsed || !canSubmit) return
setSubmitError(null)
const createColumns = createHeaders.size > 0 ? [...createHeaders] : undefined
// Large files can't be POSTed through the server (request-body cap) — upload them
// straight to storage and import in the background instead. Seed the header tray and
// close the dialog immediately so the indicator is visible during the upload, then run
// the upload + kickoff in the background (don't block the dialog on it).
if (parsed.file.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES) {
useImportTrayStore.getState().startUpload({
uploadId: table.id,
workspaceId,
title: parsed.file.name,
})
onOpenChange(false)
toast.success(`Importing "${parsed.file.name}" into "${table.name}" in the background`)
importAsyncMutation.mutate(
{
workspaceId,
tableId: table.id,
file: parsed.file,
mode,
mapping,
createColumns,
onProgress: (percent) => {
useImportTrayStore.getState().setUploadPercent(table.id, percent)
},
},
{
onSuccess: (data) => {
useImportTrayStore.getState().endUpload(table.id)
// The server row drives the tray once the list refetches. If canceled mid-upload, flag
// the id so it's not shown and cancel the worker server-side.
if (useImportTrayStore.getState().consumeCanceled(table.id) && data?.importId) {
useImportTrayStore.getState().cancel(table.id)
void cancelTableImport(workspaceId, table.id, data.importId).catch(() => {})
}
},
onError: () => {
// The hook's onError surfaces the toast; just clear the tray indicator here.
useImportTrayStore.getState().endUpload(table.id)
},
}
)
return
}
try {
const result = await importMutation.mutateAsync({
workspaceId,
@@ -285,7 +341,7 @@ export function ImportCsvDialog({
file: parsed.file,
mode,
mapping,
createColumns: createHeaders.size > 0 ? [...createHeaders] : undefined,
createColumns,
})
const data = result.data
if (mode === 'append') {
@@ -307,11 +363,7 @@ export function ImportCsvDialog({
}
}
const hasWarning =
missingRequired.length > 0 ||
duplicateTargets.length > 0 ||
appendCapacityDeficit > 0 ||
replaceCapacityDeficit > 0
const hasWarning = missingRequired.length > 0 || duplicateTargets.length > 0
return (
<ChipModal
@@ -344,7 +396,7 @@ export function ImportCsvDialog({
{parsed.file.name}
</span>
<span className='text-[var(--text-tertiary)] text-xs'>
{parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns
{parsed.headers.length} columns
</span>
</div>
<Button variant='ghost' size='sm' onClick={resetState}>
@@ -442,20 +494,6 @@ export function ImportCsvDialog({
Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one)
</p>
)}
{appendCapacityDeficit > 0 && (
<p className='text-[var(--text-error)] text-caption leading-tight'>
Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '}
{appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to
Replace.
</p>
)}
{replaceCapacityDeficit > 0 && (
<p className='text-[var(--text-error)] text-caption leading-tight'>
CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table limit
of {table.maxRows.toLocaleString()} by {replaceCapacityDeficit.toLocaleString()}
.
</p>
)}
</div>
)}
apps/sim/app/workspace/[workspaceId]/tables/tables.tsx
+69 -18
View File
@@ -2,12 +2,13 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { createLogger } from '@sim/logger'
import { generateId } from '@sim/utils/id'
import { useParams, useRouter } from 'next/navigation'
import type { ComboboxOption } from '@/components/emcn'
import { ChipCombobox, ChipConfirmModal, toast, Upload } from '@/components/emcn'
import { Columns3, Rows3, Table as TableIcon } from '@/components/emcn/icons'
import type { TableDefinition } from '@/lib/table'
import { generateUniqueTableName } from '@/lib/table/constants'
import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES, generateUniqueTableName } from '@/lib/table/constants'
import type {
FilterTag,
ResourceColumn,
@@ -24,14 +25,17 @@ import {
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
import {
ImportCsvDialog,
ImportProgressMenu,
TablesListContextMenu,
} from '@/app/workspace/[workspaceId]/tables/components'
import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu'
import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
import {
cancelTableImport,
downloadTableExport,
useCreateTable,
useDeleteTable,
useImportCsvAsync,
useRenameTable,
useTablesList,
useUploadCsvToTable,
@@ -40,6 +44,7 @@ import { useWorkspaceMembersQuery } from '@/hooks/queries/workspace'
import { useDebounce } from '@/hooks/use-debounce'
import { useInlineRename } from '@/hooks/use-inline-rename'
import { usePermissionConfig } from '@/hooks/use-permission-config'
import { useImportTrayStore } from '@/stores/table/import-tray/store'
const logger = createLogger('Tables')
@@ -76,6 +81,7 @@ export function Tables() {
const renameTable = useRenameTable(workspaceId)
const createTable = useCreateTable(workspaceId)
const uploadCsv = useUploadCsvToTable()
const importCsvAsync = useImportCsvAsync()
const tableRename = useInlineRename({
onSave: (tableId, name) => renameTable.mutate({ tableId, name }),
@@ -407,37 +413,80 @@ export function Tables() {
const list = e.target.files
if (!list || list.length === 0 || !workspaceId) return
const csvFiles = Array.from(list).filter((f) => {
const ext = f.name.split('.').pop()?.toLowerCase()
return ext === 'csv' || ext === 'tsv'
})
if (csvFiles.length === 0) {
toast.error('No CSV or TSV files selected')
if (csvInputRef.current) csvInputRef.current.value = ''
return
}
// Large files can't be POSTed through the server (request-body cap) — upload them
// straight to storage and import in the background. These are tracked by the import
// tray, never the header upload button, so don't touch uploading/uploadProgress here.
const asyncFiles = csvFiles.filter((f) => f.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
const syncFiles = csvFiles.filter((f) => f.size < CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
try {
setUploading(true)
const csvFiles = Array.from(list).filter((f) => {
const ext = f.name.split('.').pop()?.toLowerCase()
return ext === 'csv' || ext === 'tsv'
})
if (csvFiles.length === 0) {
toast.error('No CSV or TSV files selected')
return
for (const file of asyncFiles) {
// Show the indicator immediately under a temporary id (the real table id doesn't
// exist until kickoff returns), then let the tray track it. Don't redirect — the
// table is still empty/importing, so stay on the list.
const pendingId = `pending_${generateId()}`
useImportTrayStore
.getState()
.startUpload({ uploadId: pendingId, workspaceId, title: file.name })
toast.success(`Importing "${file.name}" in the background`)
try {
const result = await importCsvAsync.mutateAsync({
workspaceId,
file,
onProgress: (percent) => {
useImportTrayStore.getState().setUploadPercent(pendingId, percent)
},
})
useImportTrayStore.getState().endUpload(pendingId)
// The server row drives the tray once the list refetches (mutation invalidates it).
// If canceled mid-upload, flag the real id so it's not shown and cancel server-side.
if (
result?.tableId &&
result.importId &&
useImportTrayStore.getState().consumeCanceled(pendingId)
) {
useImportTrayStore.getState().cancel(result.tableId)
void cancelTableImport(workspaceId, result.tableId, result.importId).catch(() => {})
}
} catch {
// The hook's onError surfaces the toast; just clear the tray indicator here.
useImportTrayStore.getState().endUpload(pendingId)
}
}
setUploadProgress({ completed: 0, total: csvFiles.length })
if (syncFiles.length === 0) return
setUploading(true)
setUploadProgress({ completed: 0, total: syncFiles.length })
const failed: string[] = []
for (let i = 0; i < csvFiles.length; i++) {
for (let i = 0; i < syncFiles.length; i++) {
const file = syncFiles[i]
try {
const result = await uploadCsv.mutateAsync({ workspaceId, file: csvFiles[i] })
const result = await uploadCsv.mutateAsync({ workspaceId, file })
if (csvFiles.length === 1) {
if (syncFiles.length === 1 && asyncFiles.length === 0) {
const tableId = result?.data?.table?.id
if (tableId) {
router.push(`/workspace/${workspaceId}/tables/${tableId}`)
}
}
} catch (err) {
failed.push(csvFiles[i].name)
failed.push(file.name)
logger.error('Error uploading CSV:', err)
} finally {
setUploadProgress({ completed: i + 1, total: csvFiles.length })
setUploadProgress({ completed: i + 1, total: syncFiles.length })
}
}
@@ -459,7 +508,8 @@ export function Tables() {
}
}
},
[workspaceId, router, uploadCsv]
// eslint-disable-next-line react-hooks/exhaustive-deps -- mutation objects are unstable; mutateAsync is stable in v5
[workspaceId, router]
)
const handleListUploadCsv = useCallback(() => {
@@ -508,6 +558,7 @@ export function Tables() {
sort={sortConfig}
filter={filterContent}
filterTags={filterTags}
leadingActions={<ImportProgressMenu workspaceId={workspaceId} />}
headerActions={[
{
label: uploadButtonLabel,