mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
feat(tables): import csv into existing tables (#4199)
* feat(tables): import csv into existing tables * update types * address comments * address comment
This commit is contained in:
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const {
|
||||
mockCheckSessionOrInternalAuth,
|
||||
mockCheckAccess,
|
||||
mockBatchInsertRows,
|
||||
mockReplaceTableRows,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckSessionOrInternalAuth: vi.fn(),
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockBatchInsertRows: vi.fn(),
|
||||
mockReplaceTableRows: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/auth/hybrid', () => ({
|
||||
AuthType: { SESSION: 'session', API_KEY: 'api_key', INTERNAL_JWT: 'internal_jwt' },
|
||||
checkSessionOrInternalAuth: mockCheckSessionOrInternalAuth,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/utils/request', () => ({
|
||||
generateRequestId: vi.fn().mockReturnValue('req-test-123'),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/utils/uuid', () => ({
|
||||
generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) => {
|
||||
const message = result.status === 404 ? 'Table not found' : 'Access denied'
|
||||
return NextResponse.json({ error: message }, { status: result.status })
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* The route imports `batchInsertRows` and `replaceTableRows` from the barrel,
|
||||
* which forwards them from `./service`. Mocking the service module replaces
|
||||
* both without having to touch the other real helpers (`parseCsvBuffer`,
|
||||
* `coerceRowsForTable`, etc.) exported through the barrel.
|
||||
*/
|
||||
vi.mock('@/lib/table/service', () => ({
|
||||
batchInsertRows: mockBatchInsertRows,
|
||||
replaceTableRows: mockReplaceTableRows,
|
||||
}))
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/import-csv/route'
|
||||
|
||||
function createCsvFile(contents: string, name = 'data.csv', type = 'text/csv'): File {
|
||||
return new File([contents], name, { type })
|
||||
}
|
||||
|
||||
function createFormData(
|
||||
file: File,
|
||||
options?: {
|
||||
workspaceId?: string | null
|
||||
mode?: string | null
|
||||
mapping?: unknown
|
||||
}
|
||||
): FormData {
|
||||
const form = new FormData()
|
||||
form.append('file', file)
|
||||
if (options?.workspaceId !== null) {
|
||||
form.append('workspaceId', options?.workspaceId ?? 'workspace-1')
|
||||
}
|
||||
if (options?.mode !== null) {
|
||||
form.append('mode', options?.mode ?? 'append')
|
||||
}
|
||||
if (options?.mapping !== undefined) {
|
||||
form.append(
|
||||
'mapping',
|
||||
typeof options.mapping === 'string' ? options.mapping : JSON.stringify(options.mapping)
|
||||
)
|
||||
}
|
||||
return form
|
||||
}
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string', required: true },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
},
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
async function callPost(form: FormData, { tableId }: { tableId: string } = { tableId: 'tbl_1' }) {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import-csv`, {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
describe('POST /api/table/[tableId]/import-csv', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockBatchInsertRows.mockImplementation(async (data: { rows: unknown[] }) =>
|
||||
data.rows.map((_, i) => ({ id: `row_${i}` }))
|
||||
)
|
||||
mockReplaceTableRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 })
|
||||
})
|
||||
|
||||
it('returns 401 when the user is not authenticated', async () => {
|
||||
mockCheckSessionOrInternalAuth.mockResolvedValueOnce({
|
||||
success: false,
|
||||
error: 'Authentication required',
|
||||
})
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('returns 400 when the mode is invalid', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'bogus' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/Invalid mode/)
|
||||
})
|
||||
|
||||
it('returns 403 when the user lacks workspace write access', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({ ok: false, status: 403 })
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(403)
|
||||
})
|
||||
|
||||
it('returns 400 when the target table is archived', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
table: buildTable({ archivedAt: new Date('2024-01-02') }),
|
||||
})
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/archived/i)
|
||||
})
|
||||
|
||||
it('returns 400 when the CSV is missing a required column', async () => {
|
||||
const response = await callPost(createFormData(createCsvFile('age\n30')))
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/missing required columns/i)
|
||||
expect(data.details?.missingRequired).toEqual(['name'])
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('appends rows via batchInsertRows', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mode).toBe('append')
|
||||
expect(data.data.insertedCount).toBe(2)
|
||||
expect(mockBatchInsertRows).toHaveBeenCalledTimes(1)
|
||||
const callArgs = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] }
|
||||
expect(callArgs.rows).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
expect(mockReplaceTableRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects append when it would exceed maxRows', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
table: buildTable({ rowCount: 99, maxRows: 100 }),
|
||||
})
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/exceed table row limit/)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('replaces rows via replaceTableRows', async () => {
|
||||
mockReplaceTableRows.mockResolvedValueOnce({ deletedCount: 5, insertedCount: 2 })
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'replace' })
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mode).toBe('replace')
|
||||
expect(data.data.deletedCount).toBe(5)
|
||||
expect(data.data.insertedCount).toBe(2)
|
||||
expect(mockReplaceTableRows).toHaveBeenCalledTimes(1)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses an explicit mapping when provided', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('First Name,Years\nAlice,30\nBob,40', 'people.csv'), {
|
||||
mode: 'append',
|
||||
mapping: { 'First Name': 'name', Years: 'age' },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mappedColumns).toEqual(['First Name', 'Years'])
|
||||
const callArgs = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] }
|
||||
expect(callArgs.rows).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
})
|
||||
|
||||
it('returns 400 when the mapping targets a non-existent column', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('a\nAlice'), {
|
||||
mode: 'append',
|
||||
mapping: { a: 'nonexistent' },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/do not exist on the table/)
|
||||
})
|
||||
|
||||
it('returns 400 when a mapping value is not a string or null', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), {
|
||||
mode: 'append',
|
||||
mapping: { name: 42 },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/Mapping values must be/)
|
||||
})
|
||||
|
||||
it('surfaces unique violations from batchInsertRows as 400', async () => {
|
||||
mockBatchInsertRows.mockRejectedValueOnce(
|
||||
new Error('Row 1: Column "name" must be unique. Value "Alice" already exists in row row_xxx')
|
||||
)
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/must be unique/)
|
||||
expect(data.data?.insertedCount).toBe(0)
|
||||
})
|
||||
|
||||
it('accepts TSV files', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(
|
||||
createCsvFile('name\tage\nAlice\t30', 'data.tsv', 'text/tab-separated-values'),
|
||||
{ mode: 'append' }
|
||||
)
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockBatchInsertRows).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns 400 for unsupported file extensions', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age', 'data.json', 'application/json'))
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/CSV and TSV/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,267 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { generateId } from '@/lib/core/utils/uuid'
|
||||
import {
|
||||
batchInsertRows,
|
||||
buildAutoMapping,
|
||||
CSV_MAX_BATCH_SIZE,
|
||||
CSV_MAX_FILE_SIZE_BYTES,
|
||||
type CsvHeaderMapping,
|
||||
CsvImportValidationError,
|
||||
coerceRowsForTable,
|
||||
parseCsvBuffer,
|
||||
replaceTableRows,
|
||||
validateMapping,
|
||||
} from '@/lib/table'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableImportCSVExisting')
|
||||
|
||||
const IMPORT_MODES = new Set(['append', 'replace'])
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest, { params }: RouteParams) {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const formData = await request.formData()
|
||||
const file = formData.get('file')
|
||||
const workspaceId = formData.get('workspaceId') as string | null
|
||||
const rawMode = (formData.get('mode') as string | null) ?? 'append'
|
||||
const rawMapping = formData.get('mapping') as string | null
|
||||
|
||||
if (!file || !(file instanceof File)) {
|
||||
return NextResponse.json({ error: 'CSV file is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (file.size > CSV_MAX_FILE_SIZE_BYTES) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
if (!workspaceId) {
|
||||
return NextResponse.json({ error: 'Workspace ID is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (!IMPORT_MODES.has(rawMode)) {
|
||||
return NextResponse.json(
|
||||
{ error: `Invalid mode "${rawMode}". Must be "append" or "replace".` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const mode = rawMode as 'append' | 'replace'
|
||||
|
||||
const ext = file.name.split('.').pop()?.toLowerCase()
|
||||
if (ext !== 'csv' && ext !== 'tsv') {
|
||||
return NextResponse.json({ error: 'Only CSV and TSV files are supported' }, { status: 400 })
|
||||
}
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (table.workspaceId !== workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (table.archivedAt) {
|
||||
return NextResponse.json({ error: 'Cannot import into an archived table' }, { status: 400 })
|
||||
}
|
||||
|
||||
let mapping: CsvHeaderMapping | undefined
|
||||
if (rawMapping) {
|
||||
try {
|
||||
const parsed = JSON.parse(rawMapping)
|
||||
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'mapping must be a JSON object mapping CSV headers to column names' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
mapping = parsed as CsvHeaderMapping
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'mapping must be valid JSON' }, { status: 400 })
|
||||
}
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await file.arrayBuffer())
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
const { headers, rows } = await parseCsvBuffer(buffer, delimiter)
|
||||
|
||||
const effectiveMapping = mapping ?? buildAutoMapping(headers, table.schema)
|
||||
|
||||
let validation: ReturnType<typeof validateMapping>
|
||||
try {
|
||||
validation = validateMapping({
|
||||
csvHeaders: headers,
|
||||
mapping: effectiveMapping,
|
||||
tableSchema: table.schema,
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof CsvImportValidationError) {
|
||||
return NextResponse.json({ error: err.message, details: err.details }, { status: 400 })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
if (validation.mappedHeaders.length === 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `No CSV headers map to columns on the table. CSV headers: ${headers.join(', ')}. Table columns: ${table.schema.columns.map((c) => c.name).join(', ')}`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const coerced = coerceRowsForTable(rows, table.schema, validation.effectiveMap)
|
||||
|
||||
if (mode === 'append') {
|
||||
if (table.rowCount + coerced.length > table.maxRows) {
|
||||
const deficit = table.rowCount + coerced.length - table.maxRows
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Append would exceed table row limit (${table.maxRows}). Currently ${table.rowCount} rows, ${coerced.length} new rows, ${deficit} over.`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
let inserted = 0
|
||||
try {
|
||||
for (let i = 0; i < coerced.length; i += CSV_MAX_BATCH_SIZE) {
|
||||
const batch = coerced.slice(i, i + CSV_MAX_BATCH_SIZE)
|
||||
const batchRequestId = generateId().slice(0, 8)
|
||||
const result = await batchInsertRows(
|
||||
{
|
||||
tableId: table.id,
|
||||
rows: batch,
|
||||
workspaceId,
|
||||
userId: authResult.userId,
|
||||
},
|
||||
table,
|
||||
batchRequestId
|
||||
)
|
||||
inserted += result.length
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
logger.warn(`[${requestId}] Append failed mid-import for table ${tableId}`, {
|
||||
inserted,
|
||||
total: coerced.length,
|
||||
error: message,
|
||||
})
|
||||
const isClientError =
|
||||
message.includes('row limit') ||
|
||||
message.includes('Insufficient capacity') ||
|
||||
message.includes('Schema validation') ||
|
||||
message.includes('must be unique') ||
|
||||
message.includes('Row size exceeds') ||
|
||||
/^Row \d+:/.test(message)
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: isClientError ? message : 'Failed to import CSV',
|
||||
data: { insertedCount: inserted },
|
||||
},
|
||||
{ status: isClientError ? 400 : 500 }
|
||||
)
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Append CSV imported`, {
|
||||
tableId: table.id,
|
||||
fileName: file.name,
|
||||
mode,
|
||||
inserted,
|
||||
mappedColumns: validation.mappedHeaders.length,
|
||||
skippedHeaders: validation.skippedHeaders.length,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
mode,
|
||||
insertedCount: inserted,
|
||||
mappedColumns: validation.mappedHeaders,
|
||||
skippedHeaders: validation.skippedHeaders,
|
||||
unmappedColumns: validation.unmappedColumns,
|
||||
sourceFile: file.name,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await replaceTableRows(
|
||||
{ tableId: table.id, rows: coerced, workspaceId, userId: authResult.userId },
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
logger.info(`[${requestId}] Replace CSV imported`, {
|
||||
tableId: table.id,
|
||||
fileName: file.name,
|
||||
mode,
|
||||
deleted: result.deletedCount,
|
||||
inserted: result.insertedCount,
|
||||
mappedColumns: validation.mappedHeaders.length,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
mode,
|
||||
deletedCount: result.deletedCount,
|
||||
insertedCount: result.insertedCount,
|
||||
mappedColumns: validation.mappedHeaders,
|
||||
skippedHeaders: validation.skippedHeaders,
|
||||
unmappedColumns: validation.unmappedColumns,
|
||||
sourceFile: file.name,
|
||||
},
|
||||
})
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
const isClientError =
|
||||
message.includes('row limit') ||
|
||||
message.includes('Schema validation') ||
|
||||
message.includes('must be unique') ||
|
||||
message.includes('Row size exceeds') ||
|
||||
/^Row \d+:/.test(message)
|
||||
if (isClientError) {
|
||||
return NextResponse.json({ error: message }, { status: 400 })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
logger.error(`[${requestId}] CSV import into existing table failed:`, error)
|
||||
|
||||
const isClientError =
|
||||
message.includes('CSV file has no') ||
|
||||
message.includes('already exists') ||
|
||||
message.includes('Invalid column name')
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: isClientError ? message : 'Failed to import CSV' },
|
||||
{ status: isClientError ? 400 : 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -5,157 +5,22 @@ import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { generateId } from '@/lib/core/utils/uuid'
|
||||
import {
|
||||
batchInsertRows,
|
||||
CSV_MAX_BATCH_SIZE,
|
||||
CSV_MAX_FILE_SIZE_BYTES,
|
||||
coerceRowsForTable,
|
||||
createTable,
|
||||
deleteTable,
|
||||
getWorkspaceTableLimits,
|
||||
inferSchemaFromCsv,
|
||||
parseCsvBuffer,
|
||||
sanitizeName,
|
||||
type TableSchema,
|
||||
} from '@/lib/table'
|
||||
import type { ColumnDefinition, RowData } from '@/lib/table/types'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
import { normalizeColumn } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableImportCSV')
|
||||
|
||||
const MAX_CSV_FILE_SIZE = 50 * 1024 * 1024
|
||||
const MAX_BATCH_SIZE = 1000
|
||||
const SCHEMA_SAMPLE_SIZE = 100
|
||||
|
||||
type ColumnType = 'string' | 'number' | 'boolean' | 'date'
|
||||
|
||||
async function parseCsvBuffer(
|
||||
buffer: Buffer,
|
||||
delimiter = ','
|
||||
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
|
||||
const { parse } = await import('csv-parse/sync')
|
||||
const parsed = parse(buffer.toString('utf-8'), {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
trim: true,
|
||||
relax_column_count: true,
|
||||
relax_quotes: true,
|
||||
skip_records_with_error: true,
|
||||
cast: false,
|
||||
delimiter,
|
||||
}) as Record<string, unknown>[]
|
||||
|
||||
if (parsed.length === 0) {
|
||||
throw new Error('CSV file has no data rows')
|
||||
}
|
||||
|
||||
const headers = Object.keys(parsed[0])
|
||||
if (headers.length === 0) {
|
||||
throw new Error('CSV file has no headers')
|
||||
}
|
||||
|
||||
return { headers, rows: parsed }
|
||||
}
|
||||
|
||||
function inferColumnType(values: unknown[]): ColumnType {
|
||||
const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
|
||||
if (nonEmpty.length === 0) return 'string'
|
||||
|
||||
const allNumber = nonEmpty.every((v) => {
|
||||
const n = Number(v)
|
||||
return !Number.isNaN(n) && String(v).trim() !== ''
|
||||
})
|
||||
if (allNumber) return 'number'
|
||||
|
||||
const allBoolean = nonEmpty.every((v) => {
|
||||
const s = String(v).toLowerCase()
|
||||
return s === 'true' || s === 'false'
|
||||
})
|
||||
if (allBoolean) return 'boolean'
|
||||
|
||||
const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/
|
||||
const allDate = nonEmpty.every((v) => {
|
||||
const s = String(v)
|
||||
return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s))
|
||||
})
|
||||
if (allDate) return 'date'
|
||||
|
||||
return 'string'
|
||||
}
|
||||
|
||||
function inferSchema(headers: string[], rows: Record<string, unknown>[]): ColumnDefinition[] {
|
||||
const sample = rows.slice(0, SCHEMA_SAMPLE_SIZE)
|
||||
const seen = new Set<string>()
|
||||
|
||||
return headers.map((name) => {
|
||||
let colName = sanitizeName(name)
|
||||
let suffix = 2
|
||||
while (seen.has(colName.toLowerCase())) {
|
||||
colName = `${sanitizeName(name)}_${suffix}`
|
||||
suffix++
|
||||
}
|
||||
seen.add(colName.toLowerCase())
|
||||
|
||||
return {
|
||||
name: colName,
|
||||
type: inferColumnType(sample.map((r) => r[name])),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips non-alphanumeric characters (except underscore), collapses runs of
|
||||
* underscores, and ensures the name starts with a letter or underscore.
|
||||
* Used for both table names and column names to satisfy NAME_PATTERN.
|
||||
*/
|
||||
function sanitizeName(raw: string, fallbackPrefix = 'col'): string {
|
||||
let name = raw
|
||||
.trim()
|
||||
.replace(/[^a-zA-Z0-9_]/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_+|_+$/g, '')
|
||||
|
||||
if (!name || /^\d/.test(name)) {
|
||||
name = `${fallbackPrefix}_${name}`
|
||||
}
|
||||
|
||||
return name
|
||||
}
|
||||
|
||||
function coerceValue(value: unknown, colType: ColumnType): string | number | boolean | null {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
switch (colType) {
|
||||
case 'number': {
|
||||
const n = Number(value)
|
||||
return Number.isNaN(n) ? null : n
|
||||
}
|
||||
case 'boolean': {
|
||||
const s = String(value).toLowerCase()
|
||||
if (s === 'true') return true
|
||||
if (s === 'false') return false
|
||||
return null
|
||||
}
|
||||
case 'date': {
|
||||
const d = new Date(String(value))
|
||||
return Number.isNaN(d.getTime()) ? String(value) : d.toISOString()
|
||||
}
|
||||
default:
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
|
||||
function coerceRows(
|
||||
rows: Record<string, unknown>[],
|
||||
columns: ColumnDefinition[],
|
||||
headerToColumn: Map<string, string>
|
||||
): RowData[] {
|
||||
const colTypeMap = new Map(columns.map((c) => [c.name, c.type as ColumnType]))
|
||||
|
||||
return rows.map((row) => {
|
||||
const coerced: RowData = {}
|
||||
for (const [header, value] of Object.entries(row)) {
|
||||
const colName = headerToColumn.get(header)
|
||||
if (colName) {
|
||||
coerced[colName] = coerceValue(value, colTypeMap.get(colName) ?? 'string')
|
||||
}
|
||||
}
|
||||
return coerced
|
||||
})
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
@@ -173,9 +38,11 @@ export async function POST(request: NextRequest) {
|
||||
return NextResponse.json({ error: 'CSV file is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (file.size > MAX_CSV_FILE_SIZE) {
|
||||
if (file.size > CSV_MAX_FILE_SIZE_BYTES) {
|
||||
return NextResponse.json(
|
||||
{ error: `File exceeds maximum allowed size of ${MAX_CSV_FILE_SIZE / (1024 * 1024)} MB` },
|
||||
{
|
||||
error: `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
@@ -198,9 +65,7 @@ export async function POST(request: NextRequest) {
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
const { headers, rows } = await parseCsvBuffer(buffer, delimiter)
|
||||
|
||||
const columns = inferSchema(headers, rows)
|
||||
const headerToColumn = new Map(headers.map((h, i) => [h, columns[i].name]))
|
||||
|
||||
const { columns, headerToColumn } = inferSchemaFromCsv(headers, rows)
|
||||
const tableName = sanitizeName(file.name.replace(/\.[^.]+$/, ''), 'imported_table')
|
||||
const planLimits = await getWorkspaceTableLimits(workspaceId)
|
||||
|
||||
@@ -222,10 +87,10 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
|
||||
try {
|
||||
const coerced = coerceRows(rows, columns, headerToColumn)
|
||||
const coerced = coerceRowsForTable(rows, normalizedSchema, headerToColumn)
|
||||
let inserted = 0
|
||||
for (let i = 0; i < coerced.length; i += MAX_BATCH_SIZE) {
|
||||
const batch = coerced.slice(i, i + MAX_BATCH_SIZE)
|
||||
for (let i = 0; i < coerced.length; i += CSV_MAX_BATCH_SIZE) {
|
||||
const batch = coerced.slice(i, i + CSV_MAX_BATCH_SIZE)
|
||||
const batchRequestId = generateId().slice(0, 8)
|
||||
const result = await batchInsertRows(
|
||||
{ tableId: table.id, rows: batch, workspaceId, userId: authResult.userId },
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
ModalFooter,
|
||||
ModalHeader,
|
||||
Skeleton,
|
||||
Upload,
|
||||
} from '@/components/emcn'
|
||||
import {
|
||||
ArrowLeft,
|
||||
@@ -45,6 +46,7 @@ import type { ColumnDefinition, Filter, SortDirection, TableRow as TableRowType
|
||||
import type { ColumnOption, SortConfig } from '@/app/workspace/[workspaceId]/components'
|
||||
import { ResourceHeader, ResourceOptionsBar } from '@/app/workspace/[workspaceId]/components'
|
||||
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
||||
import { ImportCsvDialog } from '@/app/workspace/[workspaceId]/tables/components/import-csv-dialog'
|
||||
import {
|
||||
useAddTableColumn,
|
||||
useBatchCreateTableRows,
|
||||
@@ -200,6 +202,7 @@ export function Table({
|
||||
const lastCheckboxRowRef = useRef<number | null>(null)
|
||||
const [showDeleteTableConfirm, setShowDeleteTableConfirm] = useState(false)
|
||||
const [deletingColumn, setDeletingColumn] = useState<string | null>(null)
|
||||
const [isImportCsvOpen, setIsImportCsvOpen] = useState(false)
|
||||
|
||||
const [columnWidths, setColumnWidths] = useState<Record<string, number>>({})
|
||||
const columnWidthsRef = useRef(columnWidths)
|
||||
@@ -1555,6 +1558,21 @@ export function Table({
|
||||
[handleAddColumn, addColumnMutation.isPending]
|
||||
)
|
||||
|
||||
const headerActions = useMemo(
|
||||
() =>
|
||||
tableData
|
||||
? [
|
||||
{
|
||||
label: 'Import CSV',
|
||||
icon: Upload,
|
||||
onClick: () => setIsImportCsvOpen(true),
|
||||
disabled: userPermissions.canEdit !== true,
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
[tableData, userPermissions.canEdit]
|
||||
)
|
||||
|
||||
const activeSortState = useMemo(() => {
|
||||
if (!queryOptions.sort) return null
|
||||
const entries = Object.entries(queryOptions.sort)
|
||||
@@ -1619,7 +1637,12 @@ export function Table({
|
||||
<div ref={containerRef} className='flex h-full flex-col overflow-hidden'>
|
||||
{!embedded && (
|
||||
<>
|
||||
<ResourceHeader icon={TableIcon} breadcrumbs={breadcrumbs} create={createAction} />
|
||||
<ResourceHeader
|
||||
icon={TableIcon}
|
||||
breadcrumbs={breadcrumbs}
|
||||
create={createAction}
|
||||
actions={headerActions}
|
||||
/>
|
||||
|
||||
<ResourceOptionsBar
|
||||
sort={sortConfig}
|
||||
@@ -1875,6 +1898,15 @@ export function Table({
|
||||
</Modal>
|
||||
)}
|
||||
|
||||
{tableData && (
|
||||
<ImportCsvDialog
|
||||
open={isImportCsvOpen}
|
||||
onOpenChange={setIsImportCsvOpen}
|
||||
workspaceId={workspaceId}
|
||||
table={tableData}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Modal
|
||||
open={deletingColumn !== null}
|
||||
onOpenChange={(open) => {
|
||||
|
||||
+513
@@ -0,0 +1,513 @@
|
||||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import {
|
||||
Button,
|
||||
ButtonGroup,
|
||||
ButtonGroupItem,
|
||||
Combobox,
|
||||
type ComboboxOption,
|
||||
Label,
|
||||
Modal,
|
||||
ModalBody,
|
||||
ModalContent,
|
||||
ModalFooter,
|
||||
ModalHeader,
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
toast,
|
||||
} from '@/components/emcn'
|
||||
import { cn } from '@/lib/core/utils/cn'
|
||||
import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/csv-import'
|
||||
import type { TableDefinition } from '@/lib/table/types'
|
||||
import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables'
|
||||
|
||||
const logger = createLogger('ImportCsvDialog')
|
||||
|
||||
const MAX_SAMPLE_ROWS = 5
|
||||
const MAX_EXAMPLES_IN_ERROR = 3
|
||||
/**
|
||||
* Sentinel value for the "Do not import" option in the mapping combobox. The
|
||||
* whitespace is intentional: valid column names must match `NAME_PATTERN`
|
||||
* (`/^[a-z_][a-z0-9_]*$/i`), so no real column can share this value.
|
||||
*/
|
||||
const SKIP_VALUE = '__ skip __'
|
||||
|
||||
/**
|
||||
* Converts the verbose backend error messages into a short, human-friendly
|
||||
* summary suitable for the modal footer. Specifically collapses repeated
|
||||
* `Row N: Column "X" must be unique. Value "Y" already exists in row row_abc`
|
||||
* segments into a single concise summary without internal row IDs.
|
||||
*/
|
||||
function summarizeImportError(message: string): string {
|
||||
const uniqueMatches = [
|
||||
...message.matchAll(/Column\s+"([^"]+)"\s+must be unique\.\s+Value\s+"([^"]+)"/g),
|
||||
]
|
||||
if (uniqueMatches.length > 0) {
|
||||
const column = uniqueMatches[0][1]
|
||||
const values = Array.from(new Set(uniqueMatches.map((m) => m[2])))
|
||||
const preview = values
|
||||
.slice(0, MAX_EXAMPLES_IN_ERROR)
|
||||
.map((v) => `"${v}"`)
|
||||
.join(', ')
|
||||
const extra = values.length - MAX_EXAMPLES_IN_ERROR
|
||||
const suffix = extra > 0 ? `, +${extra} more` : ''
|
||||
return `${values.length} row${values.length === 1 ? '' : 's'} conflict on unique column "${column}" (${preview}${suffix})`
|
||||
}
|
||||
|
||||
const requiredMatch = message.match(/missing required columns?:\s*(.+)/i)
|
||||
if (requiredMatch) {
|
||||
return `Missing required column(s): ${requiredMatch[1].replace(/[.;]+$/, '')}`
|
||||
}
|
||||
|
||||
const rowLimitMatch = message.match(/row limit[^.;]*/i)
|
||||
if (rowLimitMatch) {
|
||||
return rowLimitMatch[0].trim()
|
||||
}
|
||||
|
||||
const stripped = message.replace(/\s+in row\s+row_[a-f0-9]+/gi, '').trim()
|
||||
if (stripped.length > 180) return `${stripped.slice(0, 177)}...`
|
||||
return stripped
|
||||
}
|
||||
|
||||
interface ImportCsvDialogProps {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
workspaceId: string
|
||||
table: TableDefinition
|
||||
onImported?: (result: { insertedCount?: number; deletedCount?: number }) => void
|
||||
}
|
||||
|
||||
interface ParsedCsv {
|
||||
file: File
|
||||
headers: string[]
|
||||
sampleRows: Record<string, unknown>[]
|
||||
totalRows: number
|
||||
}
|
||||
|
||||
export function ImportCsvDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
workspaceId,
|
||||
table,
|
||||
onImported,
|
||||
}: ImportCsvDialogProps) {
|
||||
const [parsed, setParsed] = useState<ParsedCsv | null>(null)
|
||||
const [parseError, setParseError] = useState<string | null>(null)
|
||||
const [submitError, setSubmitError] = useState<string | null>(null)
|
||||
const [parsing, setParsing] = useState(false)
|
||||
const [mapping, setMapping] = useState<Record<string, string | null>>({})
|
||||
const [mode, setMode] = useState<CsvImportMode>('append')
|
||||
const [isDragging, setIsDragging] = useState(false)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const importMutation = useImportCsvIntoTable()
|
||||
|
||||
const resetState = useCallback(() => {
|
||||
setParsed(null)
|
||||
setParseError(null)
|
||||
setSubmitError(null)
|
||||
setMapping({})
|
||||
setMode('append')
|
||||
setIsDragging(false)
|
||||
setParsing(false)
|
||||
if (fileInputRef.current) fileInputRef.current.value = ''
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) resetState()
|
||||
}, [open, resetState])
|
||||
|
||||
useEffect(() => {
|
||||
resetState()
|
||||
}, [table.id, resetState])
|
||||
|
||||
const columnOptions: ComboboxOption[] = useMemo(() => {
|
||||
const options: ComboboxOption[] = [{ label: 'Do not import', value: SKIP_VALUE }]
|
||||
for (const col of table.schema.columns) {
|
||||
options.push({
|
||||
label: col.required ? `${col.name} (required)` : col.name,
|
||||
value: col.name,
|
||||
})
|
||||
}
|
||||
return options
|
||||
}, [table.schema.columns])
|
||||
|
||||
const handleFileSelected = useCallback(
|
||||
async (file: File) => {
|
||||
const ext = file.name.split('.').pop()?.toLowerCase()
|
||||
if (ext !== 'csv' && ext !== 'tsv') {
|
||||
setParseError('Only CSV and TSV files are supported')
|
||||
return
|
||||
}
|
||||
setParsing(true)
|
||||
setParseError(null)
|
||||
try {
|
||||
const arrayBuffer = await file.arrayBuffer()
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter)
|
||||
const autoMapping = buildAutoMapping(headers, table.schema)
|
||||
setParsed({
|
||||
file,
|
||||
headers,
|
||||
sampleRows: rows.slice(0, MAX_SAMPLE_ROWS),
|
||||
totalRows: rows.length,
|
||||
})
|
||||
setMapping(autoMapping)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to parse CSV'
|
||||
logger.error('CSV parse failed', err)
|
||||
setParseError(message)
|
||||
} finally {
|
||||
setParsing(false)
|
||||
}
|
||||
},
|
||||
[table.schema]
|
||||
)
|
||||
|
||||
const handleFileInputChange = useCallback(
|
||||
(e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = e.target.files?.[0]
|
||||
if (file) void handleFileSelected(file)
|
||||
},
|
||||
[handleFileSelected]
|
||||
)
|
||||
|
||||
const handleDragEnter = useCallback((e: React.DragEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault()
|
||||
setIsDragging(true)
|
||||
}, [])
|
||||
|
||||
const handleDragOver = useCallback((e: React.DragEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault()
|
||||
}, [])
|
||||
|
||||
const handleDragLeave = useCallback((e: React.DragEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault()
|
||||
setIsDragging(false)
|
||||
}, [])
|
||||
|
||||
const handleDrop = useCallback(
|
||||
(e: React.DragEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault()
|
||||
setIsDragging(false)
|
||||
const file = e.dataTransfer.files?.[0]
|
||||
if (file) void handleFileSelected(file)
|
||||
},
|
||||
[handleFileSelected]
|
||||
)
|
||||
|
||||
const handleMappingChange = useCallback((header: string, value: string) => {
|
||||
setSubmitError(null)
|
||||
setMapping((prev) => ({
|
||||
...prev,
|
||||
[header]: value === SKIP_VALUE ? null : value,
|
||||
}))
|
||||
}, [])
|
||||
|
||||
const handleModeChange = useCallback((value: string) => {
|
||||
setSubmitError(null)
|
||||
setMode(value as CsvImportMode)
|
||||
}, [])
|
||||
|
||||
const { missingRequired, duplicateTargets, mappedCount, skipCount } = useMemo(() => {
|
||||
const mappedTargets = new Map<string, string[]>()
|
||||
let mapped = 0
|
||||
let skipped = 0
|
||||
for (const header of parsed?.headers ?? []) {
|
||||
const target = mapping[header]
|
||||
if (!target) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
mapped++
|
||||
const existing = mappedTargets.get(target) ?? []
|
||||
existing.push(header)
|
||||
mappedTargets.set(target, existing)
|
||||
}
|
||||
const dupes = [...mappedTargets.entries()]
|
||||
.filter(([, headers]) => headers.length > 1)
|
||||
.map(([col]) => col)
|
||||
const mappedSet = new Set(mappedTargets.keys())
|
||||
const missing = table.schema.columns
|
||||
.filter((c) => c.required && !mappedSet.has(c.name))
|
||||
.map((c) => c.name)
|
||||
return {
|
||||
missingRequired: missing,
|
||||
duplicateTargets: dupes,
|
||||
mappedCount: mapped,
|
||||
skipCount: skipped,
|
||||
}
|
||||
}, [mapping, parsed?.headers, table.schema.columns])
|
||||
|
||||
const appendCapacityDeficit = useMemo(() => {
|
||||
if (!parsed || mode !== 'append') return 0
|
||||
const projected = table.rowCount + parsed.totalRows
|
||||
return projected > table.maxRows ? projected - table.maxRows : 0
|
||||
}, [mode, parsed, table.maxRows, table.rowCount])
|
||||
|
||||
const replaceCapacityDeficit = useMemo(() => {
|
||||
if (!parsed || mode !== 'replace') return 0
|
||||
return parsed.totalRows > table.maxRows ? parsed.totalRows - table.maxRows : 0
|
||||
}, [mode, parsed, table.maxRows])
|
||||
|
||||
const canSubmit =
|
||||
parsed !== null &&
|
||||
!importMutation.isPending &&
|
||||
missingRequired.length === 0 &&
|
||||
duplicateTargets.length === 0 &&
|
||||
mappedCount > 0 &&
|
||||
appendCapacityDeficit === 0 &&
|
||||
replaceCapacityDeficit === 0
|
||||
|
||||
const importCsv = importMutation.mutateAsync
|
||||
const handleSubmit = useCallback(async () => {
|
||||
if (!parsed || !canSubmit) return
|
||||
setSubmitError(null)
|
||||
try {
|
||||
const result = await importCsv({
|
||||
workspaceId,
|
||||
tableId: table.id,
|
||||
file: parsed.file,
|
||||
mode,
|
||||
mapping,
|
||||
})
|
||||
const data = result.data
|
||||
if (mode === 'append') {
|
||||
toast.success(`Imported ${data?.insertedCount ?? 0} rows into "${table.name}"`)
|
||||
} else {
|
||||
toast.success(
|
||||
`Replaced rows in "${table.name}": deleted ${data?.deletedCount ?? 0}, inserted ${data?.insertedCount ?? 0}`
|
||||
)
|
||||
}
|
||||
onImported?.({
|
||||
insertedCount: data?.insertedCount,
|
||||
deletedCount: data?.deletedCount,
|
||||
})
|
||||
onOpenChange(false)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to import CSV'
|
||||
setSubmitError(summarizeImportError(message))
|
||||
logger.error('CSV import into existing table failed', err)
|
||||
}
|
||||
}, [
|
||||
canSubmit,
|
||||
importCsv,
|
||||
mapping,
|
||||
mode,
|
||||
onImported,
|
||||
onOpenChange,
|
||||
parsed,
|
||||
table.id,
|
||||
table.name,
|
||||
workspaceId,
|
||||
])
|
||||
|
||||
const hasWarning =
|
||||
missingRequired.length > 0 ||
|
||||
duplicateTargets.length > 0 ||
|
||||
appendCapacityDeficit > 0 ||
|
||||
replaceCapacityDeficit > 0
|
||||
|
||||
return (
|
||||
<Modal open={open} onOpenChange={onOpenChange}>
|
||||
<ModalContent size='lg'>
|
||||
<ModalHeader>Import CSV into {table.name}</ModalHeader>
|
||||
<ModalBody>
|
||||
{!parsed ? (
|
||||
<div className='flex flex-col gap-2'>
|
||||
<Label>Upload CSV</Label>
|
||||
<Button
|
||||
type='button'
|
||||
variant='default'
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
onDragEnter={handleDragEnter}
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
disabled={parsing}
|
||||
className={cn(
|
||||
'!bg-[var(--surface-1)] hover-hover:!bg-[var(--surface-4)] w-full justify-center border border-[var(--border-1)] border-dashed py-2.5',
|
||||
isDragging && 'border-[var(--surface-7)]'
|
||||
)}
|
||||
>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type='file'
|
||||
accept='.csv,.tsv'
|
||||
onChange={handleFileInputChange}
|
||||
className='hidden'
|
||||
/>
|
||||
<div className='flex flex-col gap-0.5 text-center'>
|
||||
<span className='text-[var(--text-primary)]'>
|
||||
{parsing
|
||||
? 'Parsing...'
|
||||
: isDragging
|
||||
? 'Drop file here'
|
||||
: 'Drop CSV or TSV here or click to browse'}
|
||||
</span>
|
||||
<span className='text-[var(--text-tertiary)] text-xs'>
|
||||
Map columns to append or replace rows in this table
|
||||
</span>
|
||||
</div>
|
||||
</Button>
|
||||
{parseError && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>{parseError}</p>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className='flex flex-col gap-4'>
|
||||
<div className='flex items-center justify-between gap-3 rounded-sm border border-[var(--border)] p-2'>
|
||||
<div className='flex min-w-0 flex-col'>
|
||||
<span className='truncate text-caption text-[var(--text-primary)]'>
|
||||
{parsed.file.name}
|
||||
</span>
|
||||
<span className='text-[var(--text-tertiary)] text-xs'>
|
||||
{parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns
|
||||
</span>
|
||||
</div>
|
||||
<Button variant='ghost' size='sm' onClick={resetState}>
|
||||
Change file
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className='flex flex-col gap-2'>
|
||||
<Label>Mode</Label>
|
||||
<ButtonGroup value={mode} onValueChange={handleModeChange}>
|
||||
<ButtonGroupItem value='append'>Append</ButtonGroupItem>
|
||||
<ButtonGroupItem value='replace'>Replace all rows</ButtonGroupItem>
|
||||
</ButtonGroup>
|
||||
</div>
|
||||
|
||||
<div className='flex flex-col gap-2'>
|
||||
<Label>Column mapping</Label>
|
||||
<div className='overflow-hidden rounded-sm border border-[var(--border)]'>
|
||||
<div className='max-h-[320px] overflow-auto'>
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>CSV column</TableHead>
|
||||
<TableHead>Target column</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{parsed.headers.map((header) => {
|
||||
const sample = parsed.sampleRows
|
||||
.map((r) =>
|
||||
r[header] === '' || r[header] == null ? '' : String(r[header])
|
||||
)
|
||||
.filter(Boolean)
|
||||
.slice(0, 2)
|
||||
.join(', ')
|
||||
return (
|
||||
<TableRow key={header}>
|
||||
<TableCell>
|
||||
<div className='flex min-w-0 flex-col'>
|
||||
<span className='truncate text-[var(--text-primary)]'>
|
||||
{header}
|
||||
</span>
|
||||
{sample && (
|
||||
<span className='truncate text-[var(--text-tertiary)] text-xs'>
|
||||
{sample}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Combobox
|
||||
options={columnOptions}
|
||||
value={mapping[header] ?? SKIP_VALUE}
|
||||
onChange={(value) => handleMappingChange(header, value)}
|
||||
size='sm'
|
||||
className='w-full'
|
||||
/>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)
|
||||
})}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
</div>
|
||||
<span className='text-[var(--text-tertiary)] text-xs'>
|
||||
{mappedCount} mapped · {skipCount} skipped
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{hasWarning && (
|
||||
<div className='flex flex-col gap-1'>
|
||||
{missingRequired.length > 0 && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>
|
||||
Missing required column(s): {missingRequired.join(', ')}
|
||||
</p>
|
||||
)}
|
||||
{duplicateTargets.length > 0 && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>
|
||||
Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one)
|
||||
</p>
|
||||
)}
|
||||
{appendCapacityDeficit > 0 && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>
|
||||
Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '}
|
||||
{appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to
|
||||
Replace.
|
||||
</p>
|
||||
)}
|
||||
{replaceCapacityDeficit > 0 && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>
|
||||
CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table
|
||||
limit of {table.maxRows.toLocaleString()} by{' '}
|
||||
{replaceCapacityDeficit.toLocaleString()}.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{mode === 'replace' && !hasWarning && (
|
||||
<p className='text-[var(--text-error)] text-caption leading-tight'>
|
||||
Replace will permanently delete the {table.rowCount.toLocaleString()} existing
|
||||
row(s) before inserting the new rows.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{submitError && (
|
||||
<p
|
||||
className='text-[var(--text-error)] text-caption leading-tight'
|
||||
title={submitError}
|
||||
>
|
||||
{submitError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button
|
||||
variant='default'
|
||||
onClick={() => onOpenChange(false)}
|
||||
disabled={importMutation.isPending}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
variant={mode === 'replace' ? 'destructive' : 'primary'}
|
||||
onClick={handleSubmit}
|
||||
disabled={!canSubmit}
|
||||
>
|
||||
{importMutation.isPending
|
||||
? mode === 'replace'
|
||||
? 'Replacing...'
|
||||
: 'Importing...'
|
||||
: mode === 'replace'
|
||||
? 'Replace rows'
|
||||
: 'Append rows'}
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from './import-csv-dialog'
|
||||
@@ -1,2 +1,3 @@
|
||||
export * from './import-csv-dialog'
|
||||
export * from './table-context-menu'
|
||||
export * from './tables-list-context-menu'
|
||||
|
||||
+14
-1
@@ -6,6 +6,7 @@ import {
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
Upload,
|
||||
} from '@/components/emcn'
|
||||
import { Copy, Database, Pencil, Trash } from '@/components/emcn/icons'
|
||||
|
||||
@@ -17,8 +18,10 @@ interface TableContextMenuProps {
|
||||
onDelete?: () => void
|
||||
onViewSchema?: () => void
|
||||
onRename?: () => void
|
||||
onImportCsv?: () => void
|
||||
disableDelete?: boolean
|
||||
disableRename?: boolean
|
||||
disableImport?: boolean
|
||||
menuRef?: React.RefObject<HTMLDivElement | null>
|
||||
}
|
||||
|
||||
@@ -30,8 +33,10 @@ export function TableContextMenu({
|
||||
onDelete,
|
||||
onViewSchema,
|
||||
onRename,
|
||||
onImportCsv,
|
||||
disableDelete = false,
|
||||
disableRename = false,
|
||||
disableImport = false,
|
||||
}: TableContextMenuProps) {
|
||||
return (
|
||||
<DropdownMenu open={isOpen} onOpenChange={(open) => !open && onClose()} modal={false}>
|
||||
@@ -67,7 +72,15 @@ export function TableContextMenu({
|
||||
Rename
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{(onViewSchema || onRename) && (onCopyId || onDelete) && <DropdownMenuSeparator />}
|
||||
{onImportCsv && (
|
||||
<DropdownMenuItem disabled={disableImport} onSelect={onImportCsv}>
|
||||
<Upload />
|
||||
Import CSV...
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{(onViewSchema || onRename || onImportCsv) && (onCopyId || onDelete) && (
|
||||
<DropdownMenuSeparator />
|
||||
)}
|
||||
{onCopyId && (
|
||||
<DropdownMenuItem onSelect={onCopyId}>
|
||||
<Copy />
|
||||
|
||||
@@ -27,7 +27,10 @@ import type {
|
||||
} from '@/app/workspace/[workspaceId]/components'
|
||||
import { ownerCell, Resource, timeCell } from '@/app/workspace/[workspaceId]/components'
|
||||
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
|
||||
import { TablesListContextMenu } from '@/app/workspace/[workspaceId]/tables/components'
|
||||
import {
|
||||
ImportCsvDialog,
|
||||
TablesListContextMenu,
|
||||
} from '@/app/workspace/[workspaceId]/tables/components'
|
||||
import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu'
|
||||
import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
|
||||
import {
|
||||
@@ -76,6 +79,7 @@ export function Tables() {
|
||||
const uploadCsv = useUploadCsvToTable()
|
||||
|
||||
const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false)
|
||||
const [isImportDialogOpen, setIsImportDialogOpen] = useState(false)
|
||||
const [activeTable, setActiveTable] = useState<TableDefinition | null>(null)
|
||||
const [searchTerm, setSearchTerm] = useState('')
|
||||
const debouncedSearchTerm = useDebounce(searchTerm, 300)
|
||||
@@ -525,10 +529,24 @@ export function Tables() {
|
||||
if (activeTable) navigator.clipboard.writeText(activeTable.id)
|
||||
}}
|
||||
onDelete={() => setIsDeleteDialogOpen(true)}
|
||||
onImportCsv={() => setIsImportDialogOpen(true)}
|
||||
disableDelete={userPermissions.canEdit !== true}
|
||||
disableRename={userPermissions.canEdit !== true}
|
||||
disableImport={userPermissions.canEdit !== true}
|
||||
/>
|
||||
|
||||
{activeTable && (
|
||||
<ImportCsvDialog
|
||||
open={isImportDialogOpen}
|
||||
onOpenChange={(open) => {
|
||||
setIsImportDialogOpen(open)
|
||||
if (!open) setActiveTable(null)
|
||||
}}
|
||||
workspaceId={workspaceId}
|
||||
table={activeTable}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Modal open={isDeleteDialogOpen} onOpenChange={setIsDeleteDialogOpen}>
|
||||
<ModalContent size='sm'>
|
||||
<ModalHeader>Delete Table</ModalHeader>
|
||||
|
||||
@@ -780,6 +780,77 @@ export function useUploadCsvToTable() {
|
||||
})
|
||||
}
|
||||
|
||||
export type CsvHeaderMapping = Record<string, string | null>
|
||||
export type CsvImportMode = 'append' | 'replace'
|
||||
|
||||
interface ImportCsvIntoTableParams {
|
||||
workspaceId: string
|
||||
tableId: string
|
||||
file: File
|
||||
mode: CsvImportMode
|
||||
mapping?: CsvHeaderMapping
|
||||
}
|
||||
|
||||
interface ImportCsvIntoTableResponse {
|
||||
success: boolean
|
||||
data?: {
|
||||
tableId: string
|
||||
mode: CsvImportMode
|
||||
insertedCount?: number
|
||||
deletedCount?: number
|
||||
mappedColumns?: string[]
|
||||
skippedHeaders?: string[]
|
||||
unmappedColumns?: string[]
|
||||
sourceFile?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a CSV file to an existing table in append or replace mode. Supports
|
||||
* an optional explicit header-to-column mapping; when omitted the server
|
||||
* auto-maps headers by sanitized name.
|
||||
*/
|
||||
export function useImportCsvIntoTable() {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
return useMutation({
|
||||
mutationFn: async ({
|
||||
workspaceId,
|
||||
tableId,
|
||||
file,
|
||||
mode,
|
||||
mapping,
|
||||
}: ImportCsvIntoTableParams): Promise<ImportCsvIntoTableResponse> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
formData.append('workspaceId', workspaceId)
|
||||
formData.append('mode', mode)
|
||||
if (mapping) {
|
||||
formData.append('mapping', JSON.stringify(mapping))
|
||||
}
|
||||
|
||||
const response = await fetch(`/api/table/${tableId}/import-csv`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => ({}))
|
||||
throw new Error(data.error || 'CSV import failed')
|
||||
}
|
||||
|
||||
return response.json()
|
||||
},
|
||||
onSettled: (_data, _error, variables) => {
|
||||
if (!variables) return
|
||||
invalidateRowCount(queryClient, variables.workspaceId, variables.tableId)
|
||||
},
|
||||
onError: (error) => {
|
||||
logger.error('Failed to import CSV into table:', error)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useDeleteColumn({ workspaceId, tableId }: RowMutationContext) {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
|
||||
@@ -2830,6 +2830,22 @@ export const UserTable: ToolCatalogEntry = {
|
||||
type: 'number',
|
||||
description: 'Maximum rows to return or affect (optional, default 100)',
|
||||
},
|
||||
mapping: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.',
|
||||
additionalProperties: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Target column name on the table. Use null to skip this CSV header instead of a column name.',
|
||||
},
|
||||
},
|
||||
mode: {
|
||||
type: 'string',
|
||||
description:
|
||||
"Import mode for import_file. 'append' (default) adds rows; 'replace' truncates existing rows in a transaction before inserting the new rows.",
|
||||
enum: ['append', 'replace'],
|
||||
},
|
||||
name: { type: 'string', description: "Table name (required for 'create')" },
|
||||
newName: { type: 'string', description: 'New column name (required for rename_column)' },
|
||||
newType: {
|
||||
|
||||
@@ -10,7 +10,7 @@ export interface ToolRuntimeSchemaEntry {
|
||||
}
|
||||
|
||||
export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
agent: {
|
||||
['agent']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -23,7 +23,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
auth: {
|
||||
['auth']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -36,7 +36,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
check_deployment_status: {
|
||||
['check_deployment_status']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -48,7 +48,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
complete_job: {
|
||||
['complete_job']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -61,7 +61,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
context_write: {
|
||||
['context_write']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -78,7 +78,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
crawl_website: {
|
||||
['crawl_website']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -113,7 +113,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
create_file: {
|
||||
['create_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -149,7 +149,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
create_folder: {
|
||||
['create_folder']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -170,7 +170,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
create_job: {
|
||||
['create_job']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -220,7 +220,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
create_workflow: {
|
||||
['create_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -245,7 +245,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
create_workspace_mcp_server: {
|
||||
['create_workspace_mcp_server']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -266,7 +266,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
debug: {
|
||||
['debug']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
context: {
|
||||
@@ -285,7 +285,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
delete_file: {
|
||||
['delete_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -314,7 +314,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
delete_folder: {
|
||||
['delete_folder']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -330,7 +330,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
delete_workflow: {
|
||||
['delete_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -346,7 +346,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
delete_workspace_mcp_server: {
|
||||
['delete_workspace_mcp_server']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -359,7 +359,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
deploy: {
|
||||
['deploy']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -373,7 +373,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
deploy_api: {
|
||||
['deploy_api']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -447,7 +447,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
],
|
||||
},
|
||||
},
|
||||
deploy_chat: {
|
||||
['deploy_chat']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -595,7 +595,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
],
|
||||
},
|
||||
},
|
||||
deploy_mcp: {
|
||||
['deploy_mcp']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -711,7 +711,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['deploymentType', 'deploymentStatus'],
|
||||
},
|
||||
},
|
||||
download_to_workspace_file: {
|
||||
['download_to_workspace_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -730,7 +730,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
edit_content: {
|
||||
['edit_content']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -762,7 +762,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
edit_workflow: {
|
||||
['edit_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -801,13 +801,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
file: {
|
||||
['file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
function_execute: {
|
||||
['function_execute']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -868,7 +868,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
generate_api_key: {
|
||||
['generate_api_key']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -886,7 +886,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
generate_image: {
|
||||
['generate_image']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -923,7 +923,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
generate_visualization: {
|
||||
['generate_visualization']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -963,7 +963,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_block_outputs: {
|
||||
['get_block_outputs']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -984,7 +984,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_block_upstream_references: {
|
||||
['get_block_upstream_references']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1006,7 +1006,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_deployed_workflow_state: {
|
||||
['get_deployed_workflow_state']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1019,7 +1019,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_deployment_version: {
|
||||
['get_deployment_version']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1036,7 +1036,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_execution_summary: {
|
||||
['get_execution_summary']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1063,7 +1063,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_job_logs: {
|
||||
['get_job_logs']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1088,7 +1088,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_page_contents: {
|
||||
['get_page_contents']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1116,14 +1116,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_platform_actions: {
|
||||
['get_platform_actions']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_workflow_data: {
|
||||
['get_workflow_data']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1142,7 +1142,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
get_workflow_logs: {
|
||||
['get_workflow_logs']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1168,7 +1168,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
glob: {
|
||||
['glob']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1187,7 +1187,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
grep: {
|
||||
['grep']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1234,7 +1234,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
job: {
|
||||
['job']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -1247,7 +1247,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
knowledge: {
|
||||
['knowledge']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -1260,7 +1260,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
knowledge_base: {
|
||||
['knowledge_base']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1452,7 +1452,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
list_folders: {
|
||||
['list_folders']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1464,14 +1464,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
list_user_workspaces: {
|
||||
['list_user_workspaces']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
list_workspace_mcp_servers: {
|
||||
['list_workspace_mcp_servers']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1483,7 +1483,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
manage_credential: {
|
||||
['manage_credential']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1512,7 +1512,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
manage_custom_tool: {
|
||||
['manage_custom_tool']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1591,7 +1591,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
manage_job: {
|
||||
['manage_job']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1661,7 +1661,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
manage_mcp_tool: {
|
||||
['manage_mcp_tool']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1712,7 +1712,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
manage_skill: {
|
||||
['manage_skill']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1744,7 +1744,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
materialize_file: {
|
||||
['materialize_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1778,7 +1778,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
move_folder: {
|
||||
['move_folder']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1796,7 +1796,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
move_workflow: {
|
||||
['move_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1816,7 +1816,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
oauth_get_auth_link: {
|
||||
['oauth_get_auth_link']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1830,7 +1830,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
oauth_request_access: {
|
||||
['oauth_request_access']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1844,7 +1844,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
open_resource: {
|
||||
['open_resource']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1872,7 +1872,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
read: {
|
||||
['read']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1899,7 +1899,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
redeploy: {
|
||||
['redeploy']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1967,7 +1967,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
],
|
||||
},
|
||||
},
|
||||
rename_file: {
|
||||
['rename_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2002,7 +2002,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
rename_workflow: {
|
||||
['rename_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2019,7 +2019,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
research: {
|
||||
['research']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
topic: {
|
||||
@@ -2032,7 +2032,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
respond: {
|
||||
['respond']: {
|
||||
parameters: {
|
||||
additionalProperties: true,
|
||||
properties: {
|
||||
@@ -2055,7 +2055,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
restore_resource: {
|
||||
['restore_resource']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2073,7 +2073,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
revert_to_version: {
|
||||
['revert_to_version']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2090,7 +2090,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
run: {
|
||||
['run']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
context: {
|
||||
@@ -2107,7 +2107,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
run_block: {
|
||||
['run_block']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2139,7 +2139,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
run_from_block: {
|
||||
['run_from_block']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2171,7 +2171,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
run_workflow: {
|
||||
['run_workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2199,7 +2199,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
run_workflow_until_block: {
|
||||
['run_workflow_until_block']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2231,7 +2231,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
scrape_page: {
|
||||
['scrape_page']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2252,7 +2252,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
search_documentation: {
|
||||
['search_documentation']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2269,7 +2269,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
search_library_docs: {
|
||||
['search_library_docs']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2290,7 +2290,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
search_online: {
|
||||
['search_online']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2331,7 +2331,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
search_patterns: {
|
||||
['search_patterns']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2353,7 +2353,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
set_block_enabled: {
|
||||
['set_block_enabled']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2375,7 +2375,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
set_environment_variables: {
|
||||
['set_environment_variables']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2409,7 +2409,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
set_global_workflow_variables: {
|
||||
['set_global_workflow_variables']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2447,7 +2447,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
superagent: {
|
||||
['superagent']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
task: {
|
||||
@@ -2461,7 +2461,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
table: {
|
||||
['table']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
request: {
|
||||
@@ -2474,7 +2474,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
tool_search_tool_regex: {
|
||||
['tool_search_tool_regex']: {
|
||||
parameters: {
|
||||
properties: {
|
||||
case_insensitive: {
|
||||
@@ -2495,7 +2495,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
update_job_history: {
|
||||
['update_job_history']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2513,7 +2513,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
update_workspace_mcp_server: {
|
||||
['update_workspace_mcp_server']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2538,7 +2538,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
user_memory: {
|
||||
['user_memory']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2586,7 +2586,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
user_table: {
|
||||
['user_table']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -2635,6 +2635,22 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
type: 'number',
|
||||
description: 'Maximum rows to return or affect (optional, default 100)',
|
||||
},
|
||||
mapping: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Optional explicit CSV-header → table-column mapping for import_file, as { "csvHeader": "columnName" | null }. When omitted, headers are auto-matched by sanitized name (case-insensitive fallback). Use null to skip a CSV column.',
|
||||
additionalProperties: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Target column name on the table. Use null to skip this CSV header instead of a column name.',
|
||||
},
|
||||
},
|
||||
mode: {
|
||||
type: 'string',
|
||||
description:
|
||||
"Import mode for import_file. 'append' (default) adds rows; 'replace' truncates existing rows in a transaction before inserting the new rows.",
|
||||
enum: ['append', 'replace'],
|
||||
},
|
||||
name: {
|
||||
type: 'string',
|
||||
description: "Table name (required for 'create')",
|
||||
@@ -2761,13 +2777,13 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
required: ['success', 'message'],
|
||||
},
|
||||
},
|
||||
workflow: {
|
||||
['workflow']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
},
|
||||
resultSchema: undefined,
|
||||
},
|
||||
workspace_file: {
|
||||
['workspace_file']: {
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
extractTabularData,
|
||||
serializeOutputForFile,
|
||||
unwrapFunctionExecuteOutput,
|
||||
} from '@/lib/copilot/request/tools/files'
|
||||
|
||||
describe('unwrapFunctionExecuteOutput', () => {
|
||||
it('unwraps the function_execute envelope { result, stdout }', () => {
|
||||
expect(unwrapFunctionExecuteOutput({ result: 'name,age\nAlice,30', stdout: '' })).toBe(
|
||||
'name,age\nAlice,30'
|
||||
)
|
||||
})
|
||||
|
||||
it('passes through objects that do not have both result + stdout', () => {
|
||||
const output = { data: { rows: [], totalCount: 0 } }
|
||||
expect(unwrapFunctionExecuteOutput(output)).toBe(output)
|
||||
})
|
||||
|
||||
it('passes through strings and arrays untouched', () => {
|
||||
expect(unwrapFunctionExecuteOutput('hello')).toBe('hello')
|
||||
const arr: unknown[] = [{ a: 1 }]
|
||||
expect(unwrapFunctionExecuteOutput(arr)).toBe(arr)
|
||||
})
|
||||
})
|
||||
|
||||
describe('serializeOutputForFile (csv)', () => {
|
||||
it('returns raw CSV text when function_execute result is already a CSV string', () => {
|
||||
const output = {
|
||||
result: 'name,age\nAlice,30\nBob,40',
|
||||
stdout: '(2 rows)',
|
||||
}
|
||||
expect(serializeOutputForFile(output, 'csv')).toBe('name,age\nAlice,30\nBob,40')
|
||||
})
|
||||
|
||||
it('converts a result array of objects into CSV', () => {
|
||||
const output = {
|
||||
result: [
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
],
|
||||
stdout: '',
|
||||
}
|
||||
expect(serializeOutputForFile(output, 'csv')).toBe('name,age\nAlice,30\nBob,40')
|
||||
})
|
||||
|
||||
it('returns the raw string when the non-envelope output is already a CSV string', () => {
|
||||
expect(serializeOutputForFile('a,b\n1,2', 'csv')).toBe('a,b\n1,2')
|
||||
})
|
||||
|
||||
it('falls back to JSON.stringify when the payload is not tabular and not a string', () => {
|
||||
const output = { result: { foo: 'bar' }, stdout: '' }
|
||||
expect(serializeOutputForFile(output, 'csv')).toBe('{\n "foo": "bar"\n}')
|
||||
})
|
||||
})
|
||||
|
||||
describe('serializeOutputForFile (json / txt / md)', () => {
|
||||
it('unwraps the envelope for json format so the file contains only result', () => {
|
||||
const output = { result: { hello: 'world' }, stdout: 'log' }
|
||||
expect(serializeOutputForFile(output, 'json')).toBe('{\n "hello": "world"\n}')
|
||||
})
|
||||
|
||||
it('returns the string payload as-is for txt/md/html formats', () => {
|
||||
const output = { result: '# Report\n\nHello', stdout: '' }
|
||||
expect(serializeOutputForFile(output, 'md')).toBe('# Report\n\nHello')
|
||||
expect(serializeOutputForFile(output, 'txt')).toBe('# Report\n\nHello')
|
||||
expect(serializeOutputForFile(output, 'html')).toBe('# Report\n\nHello')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractTabularData', () => {
|
||||
it('extracts rows directly from an array input', () => {
|
||||
expect(extractTabularData([{ a: 1 }, { a: 2 }])).toEqual([{ a: 1 }, { a: 2 }])
|
||||
})
|
||||
|
||||
it('does NOT unwrap function_execute envelopes on its own (callers must pre-unwrap)', () => {
|
||||
// Caller is responsible for unwrapping { result, stdout } envelopes first.
|
||||
// Keeping that concern out of this function prevents a double unwrap when
|
||||
// the user's payload itself happens to have matching keys.
|
||||
expect(extractTabularData({ result: [{ a: 1 }], stdout: '' })).toBeNull()
|
||||
})
|
||||
|
||||
it('extracts rows from the user_table query_rows shape', () => {
|
||||
const rows = extractTabularData({
|
||||
data: {
|
||||
rows: [
|
||||
{ id: 'row_1', data: { name: 'Alice' } },
|
||||
{ id: 'row_2', data: { name: 'Bob' } },
|
||||
],
|
||||
totalCount: 2,
|
||||
},
|
||||
})
|
||||
expect(rows).toEqual([{ name: 'Alice' }, { name: 'Bob' }])
|
||||
})
|
||||
|
||||
it('returns null for non-tabular inputs', () => {
|
||||
expect(extractTabularData('plain string')).toBeNull()
|
||||
expect(extractTabularData(null)).toBeNull()
|
||||
expect(extractTabularData({ foo: 'bar' })).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -26,8 +26,29 @@ export const FORMAT_TO_CONTENT_TYPE: Record<OutputFormat, string> = {
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to pull a flat array of row-objects out of the various shapes that
|
||||
* `function_execute` and `user_table` can return.
|
||||
* Unwraps the `function_execute` response envelope `{ result, stdout }` so the
|
||||
* rest of the serialization code works on the user's actual payload (a string,
|
||||
* array, object, etc.) instead of JSON-stringifying the envelope itself.
|
||||
*
|
||||
* Only unwraps when both keys are present — that's the unique shape of
|
||||
* `function_execute` (see `apps/sim/tools/function/types.ts` `CodeExecutionOutput`).
|
||||
* `user_table` returns `{ data, message, success }` which is left alone.
|
||||
*/
|
||||
export function unwrapFunctionExecuteOutput(output: unknown): unknown {
|
||||
if (!output || typeof output !== 'object' || Array.isArray(output)) return output
|
||||
const obj = output as Record<string, unknown>
|
||||
if ('result' in obj && 'stdout' in obj) {
|
||||
return obj.result
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to pull a flat array of row-objects out of an already-unwrapped tool
|
||||
* payload. Callers are responsible for stripping any `function_execute`
|
||||
* envelope first (via {@link unwrapFunctionExecuteOutput}) — this function
|
||||
* does not re-unwrap, so a user payload that coincidentally has `result` and
|
||||
* `stdout` keys is not mistaken for another envelope.
|
||||
*/
|
||||
export function extractTabularData(output: unknown): Record<string, unknown>[] | null {
|
||||
if (!output || typeof output !== 'object') return null
|
||||
@@ -41,14 +62,6 @@ export function extractTabularData(output: unknown): Record<string, unknown>[] |
|
||||
|
||||
const obj = output as Record<string, unknown>
|
||||
|
||||
// function_execute shape: { result: [...], stdout: "..." }
|
||||
if (Array.isArray(obj.result)) {
|
||||
const rows = obj.result
|
||||
if (rows.length > 0 && typeof rows[0] === 'object' && rows[0] !== null) {
|
||||
return rows as Record<string, unknown>[]
|
||||
}
|
||||
}
|
||||
|
||||
// user_table query_rows shape: { data: { rows: [{ data: {...} }], totalCount } }
|
||||
if (obj.data && typeof obj.data === 'object' && !Array.isArray(obj.data)) {
|
||||
const data = obj.data as Record<string, unknown>
|
||||
@@ -112,16 +125,18 @@ export function resolveOutputFormat(fileName: string, explicit?: string): Output
|
||||
}
|
||||
|
||||
export function serializeOutputForFile(output: unknown, format: OutputFormat): string {
|
||||
if (typeof output === 'string') return output
|
||||
const unwrapped = unwrapFunctionExecuteOutput(output)
|
||||
|
||||
if (typeof unwrapped === 'string') return unwrapped
|
||||
|
||||
if (format === 'csv') {
|
||||
const rows = extractTabularData(output)
|
||||
const rows = extractTabularData(unwrapped)
|
||||
if (rows && rows.length > 0) {
|
||||
return convertRowsToCsv(rows)
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.stringify(output, null, 2)
|
||||
return JSON.stringify(unwrapped, null, 2)
|
||||
}
|
||||
|
||||
export async function maybeWriteOutputToFile(
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const {
|
||||
mockResolveWorkspaceFileReference,
|
||||
mockDownloadWorkspaceFile,
|
||||
mockGetTableById,
|
||||
mockBatchInsertRows,
|
||||
mockReplaceTableRows,
|
||||
} = vi.hoisted(() => ({
|
||||
mockResolveWorkspaceFileReference: vi.fn(),
|
||||
mockDownloadWorkspaceFile: vi.fn(),
|
||||
mockGetTableById: vi.fn(),
|
||||
mockBatchInsertRows: vi.fn(),
|
||||
mockReplaceTableRows: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/logger', () => ({
|
||||
createLogger: () => ({
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/utils/uuid', () => ({
|
||||
generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
resolveWorkspaceFileReference: mockResolveWorkspaceFileReference,
|
||||
downloadWorkspaceFile: mockDownloadWorkspaceFile,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/service', () => ({
|
||||
addTableColumn: vi.fn(),
|
||||
batchInsertRows: mockBatchInsertRows,
|
||||
batchUpdateRows: vi.fn(),
|
||||
createTable: vi.fn(),
|
||||
deleteColumn: vi.fn(),
|
||||
deleteColumns: vi.fn(),
|
||||
deleteRow: vi.fn(),
|
||||
deleteRowsByFilter: vi.fn(),
|
||||
deleteRowsByIds: vi.fn(),
|
||||
deleteTable: vi.fn(),
|
||||
getRowById: vi.fn(),
|
||||
getTableById: mockGetTableById,
|
||||
insertRow: vi.fn(),
|
||||
queryRows: vi.fn(),
|
||||
renameColumn: vi.fn(),
|
||||
renameTable: vi.fn(),
|
||||
replaceTableRows: mockReplaceTableRows,
|
||||
updateColumnConstraints: vi.fn(),
|
||||
updateColumnType: vi.fn(),
|
||||
updateRow: vi.fn(),
|
||||
updateRowsByFilter: vi.fn(),
|
||||
}))
|
||||
|
||||
import { userTableServerTool } from '@/lib/copilot/tools/server/table/user-table'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string', required: true },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
},
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('userTableServerTool.import_file', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockResolveWorkspaceFileReference.mockResolvedValue({
|
||||
name: 'people.csv',
|
||||
type: 'text/csv',
|
||||
})
|
||||
mockDownloadWorkspaceFile.mockResolvedValue(Buffer.from('name,age\nAlice,30\nBob,40'))
|
||||
mockGetTableById.mockResolvedValue(buildTable())
|
||||
mockBatchInsertRows.mockImplementation(async (data: { rows: unknown[] }) =>
|
||||
data.rows.map((_, i) => ({ id: `row_${i}` }))
|
||||
)
|
||||
mockReplaceTableRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 })
|
||||
})
|
||||
|
||||
it('appends rows using auto-mapping by default', async () => {
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(result.data?.mode).toBe('append')
|
||||
expect(result.data?.rowCount).toBe(2)
|
||||
expect(mockBatchInsertRows).toHaveBeenCalledTimes(1)
|
||||
expect(mockReplaceTableRows).not.toHaveBeenCalled()
|
||||
const call = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] }
|
||||
expect(call.rows).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
})
|
||||
|
||||
it('replaces rows in replace mode', async () => {
|
||||
mockReplaceTableRows.mockResolvedValueOnce({ deletedCount: 3, insertedCount: 2 })
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1', mode: 'replace' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(result.data?.mode).toBe('replace')
|
||||
expect(result.data?.deletedCount).toBe(3)
|
||||
expect(result.data?.insertedCount).toBe(2)
|
||||
expect(mockReplaceTableRows).toHaveBeenCalledTimes(1)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses the caller-provided mapping', async () => {
|
||||
mockDownloadWorkspaceFile.mockResolvedValueOnce(
|
||||
Buffer.from('Full Name,Years\nAlice,30\nBob,40')
|
||||
)
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: {
|
||||
tableId: 'tbl_1',
|
||||
fileId: 'file-1',
|
||||
mapping: { 'Full Name': 'name', Years: 'age' },
|
||||
},
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
const call = mockBatchInsertRows.mock.calls[0][0] as { rows: unknown[] }
|
||||
expect(call.rows).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects unknown modes', async () => {
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1', mode: 'merge' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.message).toMatch(/Invalid mode/)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses to import into an archived table', async () => {
|
||||
mockGetTableById.mockResolvedValueOnce(buildTable({ archivedAt: new Date('2024-02-01') }))
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.message).toMatch(/archived/i)
|
||||
})
|
||||
|
||||
it('refuses to import when the table belongs to a different workspace', async () => {
|
||||
mockGetTableById.mockResolvedValueOnce(buildTable({ workspaceId: 'workspace-other' }))
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.message).toMatch(/not found/i)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports missing required columns instead of inserting', async () => {
|
||||
mockDownloadWorkspaceFile.mockResolvedValueOnce(Buffer.from('age\n30'))
|
||||
const result = await userTableServerTool.execute(
|
||||
{
|
||||
operation: 'import_file',
|
||||
args: { tableId: 'tbl_1', fileId: 'file-1' },
|
||||
},
|
||||
{ userId: 'user-1', workspaceId: 'workspace-1' }
|
||||
)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.message).toMatch(/missing required columns/i)
|
||||
expect(mockBatchInsertRows).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -7,6 +7,17 @@ import {
|
||||
} from '@/lib/copilot/tools/server/base-tool'
|
||||
import { generateId } from '@/lib/core/utils/uuid'
|
||||
import { COLUMN_TYPES } from '@/lib/table/constants'
|
||||
import {
|
||||
buildAutoMapping,
|
||||
CSV_MAX_BATCH_SIZE,
|
||||
type CsvHeaderMapping,
|
||||
CsvImportValidationError,
|
||||
coerceRowsForTable,
|
||||
inferSchemaFromCsv,
|
||||
parseCsvBuffer,
|
||||
sanitizeName,
|
||||
validateMapping,
|
||||
} from '@/lib/table/csv-import'
|
||||
import {
|
||||
addTableColumn,
|
||||
batchInsertRows,
|
||||
@@ -24,12 +35,13 @@ import {
|
||||
queryRows,
|
||||
renameColumn,
|
||||
renameTable,
|
||||
replaceTableRows,
|
||||
updateColumnConstraints,
|
||||
updateColumnType,
|
||||
updateRow,
|
||||
updateRowsByFilter,
|
||||
} from '@/lib/table/service'
|
||||
import type { ColumnDefinition, RowData, TableDefinition } from '@/lib/table/types'
|
||||
import type { RowData, TableDefinition } from '@/lib/table/types'
|
||||
import {
|
||||
downloadWorkspaceFile,
|
||||
resolveWorkspaceFileReference,
|
||||
@@ -48,22 +60,27 @@ type UserTableResult = {
|
||||
data?: any
|
||||
}
|
||||
|
||||
const MAX_BATCH_SIZE = 1000
|
||||
const SCHEMA_SAMPLE_SIZE = 100
|
||||
const MAX_BATCH_SIZE = CSV_MAX_BATCH_SIZE
|
||||
|
||||
type ColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json'
|
||||
|
||||
function sanitizeColumnName(raw: string): string {
|
||||
let name = raw
|
||||
.trim()
|
||||
.replace(/[^a-zA-Z0-9_]/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_|_$/g, '')
|
||||
if (!name || /^\d/.test(name)) name = `col_${name}`
|
||||
return name
|
||||
async function resolveWorkspaceFile(
|
||||
fileReference: string,
|
||||
workspaceId: string
|
||||
): Promise<{ buffer: Buffer; name: string; type: string }> {
|
||||
const record = await resolveWorkspaceFileReference(workspaceId, fileReference)
|
||||
if (!record) {
|
||||
throw new Error(
|
||||
`File not found: "${fileReference}". Use glob("files/by-id/*/meta.json") to list canonical file IDs.`
|
||||
)
|
||||
}
|
||||
const buffer = await downloadWorkspaceFile(record)
|
||||
return { buffer, name: record.name, type: record.type }
|
||||
}
|
||||
|
||||
function sanitizeHeaders(
|
||||
/**
|
||||
* Sanitizes raw JSON headers/rows so they conform to the same rules as CSV
|
||||
* imports (so `inferSchemaFromCsv` and friends can be reused).
|
||||
*/
|
||||
function sanitizeJsonHeaders(
|
||||
headers: string[],
|
||||
rows: Record<string, unknown>[]
|
||||
): { headers: string[]; rows: Record<string, unknown>[] } {
|
||||
@@ -71,7 +88,7 @@ function sanitizeHeaders(
|
||||
const seen = new Set<string>()
|
||||
|
||||
for (const raw of headers) {
|
||||
let safe = sanitizeColumnName(raw)
|
||||
let safe = sanitizeName(raw)
|
||||
while (seen.has(safe)) safe = `${safe}_`
|
||||
seen.add(safe)
|
||||
renamed.set(raw, safe)
|
||||
@@ -92,35 +109,6 @@ function sanitizeHeaders(
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveWorkspaceFile(
|
||||
fileReference: string,
|
||||
workspaceId: string
|
||||
): Promise<{ buffer: Buffer; name: string; type: string }> {
|
||||
const record = await resolveWorkspaceFileReference(workspaceId, fileReference)
|
||||
if (!record) {
|
||||
throw new Error(
|
||||
`File not found: "${fileReference}". Use glob("files/by-id/*/meta.json") to list canonical file IDs.`
|
||||
)
|
||||
}
|
||||
const buffer = await downloadWorkspaceFile(record)
|
||||
return { buffer, name: record.name, type: record.type }
|
||||
}
|
||||
|
||||
function parseFileRows(
|
||||
buffer: Buffer,
|
||||
fileName: string,
|
||||
contentType: string
|
||||
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
|
||||
const ext = fileName.split('.').pop()?.toLowerCase()
|
||||
if (ext === 'json' || contentType === 'application/json') {
|
||||
return parseJsonRows(buffer)
|
||||
}
|
||||
if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') {
|
||||
return parseCsvRows(buffer)
|
||||
}
|
||||
throw new Error(`Unsupported file format: "${ext}". Supported: csv, tsv, json`)
|
||||
}
|
||||
|
||||
async function parseJsonRows(
|
||||
buffer: Buffer
|
||||
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
|
||||
@@ -138,98 +126,23 @@ async function parseJsonRows(
|
||||
}
|
||||
for (const key of Object.keys(row)) headerSet.add(key)
|
||||
}
|
||||
return sanitizeHeaders([...headerSet], parsed)
|
||||
return sanitizeJsonHeaders([...headerSet], parsed)
|
||||
}
|
||||
|
||||
async function parseCsvRows(
|
||||
buffer: Buffer
|
||||
async function parseFileRows(
|
||||
buffer: Buffer,
|
||||
fileName: string,
|
||||
contentType: string
|
||||
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
|
||||
const { parse } = await import('csv-parse/sync')
|
||||
const parsed = parse(buffer.toString('utf-8'), {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
trim: true,
|
||||
relax_column_count: true,
|
||||
relax_quotes: true,
|
||||
skip_records_with_error: true,
|
||||
cast: false,
|
||||
}) as Record<string, unknown>[]
|
||||
if (parsed.length === 0) {
|
||||
throw new Error('CSV file has no data rows')
|
||||
const ext = fileName.split('.').pop()?.toLowerCase()
|
||||
if (ext === 'json' || contentType === 'application/json') {
|
||||
return parseJsonRows(buffer)
|
||||
}
|
||||
const headers = Object.keys(parsed[0])
|
||||
if (headers.length === 0) {
|
||||
throw new Error('CSV file has no headers')
|
||||
if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') {
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
return parseCsvBuffer(buffer, delimiter)
|
||||
}
|
||||
return sanitizeHeaders(headers, parsed)
|
||||
}
|
||||
|
||||
function inferColumnType(values: unknown[]): ColumnType {
|
||||
const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
|
||||
if (nonEmpty.length === 0) return 'string'
|
||||
|
||||
const allNumber = nonEmpty.every((v) => {
|
||||
const n = Number(v)
|
||||
return !Number.isNaN(n) && String(v).trim() !== ''
|
||||
})
|
||||
if (allNumber) return 'number'
|
||||
|
||||
const allBoolean = nonEmpty.every((v) => {
|
||||
const s = String(v).toLowerCase()
|
||||
return s === 'true' || s === 'false'
|
||||
})
|
||||
if (allBoolean) return 'boolean'
|
||||
|
||||
const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/
|
||||
const allDate = nonEmpty.every((v) => {
|
||||
const s = String(v)
|
||||
return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s))
|
||||
})
|
||||
if (allDate) return 'date'
|
||||
|
||||
return 'string'
|
||||
}
|
||||
|
||||
function inferSchema(headers: string[], rows: Record<string, unknown>[]): ColumnDefinition[] {
|
||||
const sample = rows.slice(0, SCHEMA_SAMPLE_SIZE)
|
||||
return headers.map((name) => ({
|
||||
name,
|
||||
type: inferColumnType(sample.map((r) => r[name])),
|
||||
}))
|
||||
}
|
||||
|
||||
function coerceValue(value: unknown, colType: ColumnType): string | number | boolean | null {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
switch (colType) {
|
||||
case 'number': {
|
||||
const n = Number(value)
|
||||
return Number.isNaN(n) ? null : n
|
||||
}
|
||||
case 'boolean': {
|
||||
const s = String(value).toLowerCase()
|
||||
return s === 'true'
|
||||
}
|
||||
case 'date':
|
||||
return new Date(String(value)).toISOString()
|
||||
default:
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
|
||||
function coerceRows(
|
||||
rows: Record<string, unknown>[],
|
||||
columns: ColumnDefinition[],
|
||||
columnMap: Map<string, ColumnDefinition>
|
||||
): RowData[] {
|
||||
return rows.map((row) => {
|
||||
const coerced: RowData = {}
|
||||
for (const col of columns) {
|
||||
if (row[col.name] !== undefined) {
|
||||
coerced[col.name] = coerceValue(row[col.name], col.type as ColumnType)
|
||||
}
|
||||
}
|
||||
return coerced
|
||||
})
|
||||
throw new Error(`Unsupported file format: "${ext}". Supported: csv, tsv, json`)
|
||||
}
|
||||
|
||||
async function batchInsertAll(
|
||||
@@ -240,11 +153,16 @@ async function batchInsertAll(
|
||||
context?: ServerToolContext
|
||||
): Promise<number> {
|
||||
let inserted = 0
|
||||
const userId = context?.userId
|
||||
for (let i = 0; i < rows.length; i += MAX_BATCH_SIZE) {
|
||||
assertServerToolNotAborted(context, 'Request aborted before table mutation could be applied.')
|
||||
const batch = rows.slice(i, i + MAX_BATCH_SIZE)
|
||||
const requestId = generateId().slice(0, 8)
|
||||
const result = await batchInsertRows({ tableId, rows: batch, workspaceId }, table, requestId)
|
||||
const result = await batchInsertRows(
|
||||
{ tableId, rows: batch, workspaceId, userId },
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
inserted += result.length
|
||||
}
|
||||
return inserted
|
||||
@@ -384,7 +302,7 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
const requestId = generateId().slice(0, 8)
|
||||
assertNotAborted()
|
||||
const row = await insertRow(
|
||||
{ tableId: args.tableId, data: args.data, workspaceId },
|
||||
{ tableId: args.tableId, data: args.data, workspaceId, userId: context.userId },
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
@@ -415,7 +333,7 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
const requestId = generateId().slice(0, 8)
|
||||
assertNotAborted()
|
||||
const rows = await batchInsertRows(
|
||||
{ tableId: args.tableId, rows: args.rows, workspaceId },
|
||||
{ tableId: args.tableId, rows: args.rows, workspaceId, userId: context.userId },
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
@@ -724,7 +642,7 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
return { success: false, message: 'File contains no data rows' }
|
||||
}
|
||||
|
||||
const columns = inferSchema(headers, rows)
|
||||
const { columns, headerToColumn } = inferSchemaFromCsv(headers, rows)
|
||||
const tableName = args.name || file.name.replace(/\.[^.]+$/, '')
|
||||
const requestId = generateId().slice(0, 8)
|
||||
assertNotAborted()
|
||||
@@ -739,8 +657,7 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
requestId
|
||||
)
|
||||
|
||||
const columnMap = new Map(columns.map((c) => [c.name, c]))
|
||||
const coerced = coerceRows(rows, columns, columnMap)
|
||||
const coerced = coerceRowsForTable(rows, { columns }, headerToColumn)
|
||||
const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context)
|
||||
|
||||
logger.info('Table created from file', {
|
||||
@@ -769,6 +686,10 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
const filePath = (args as Record<string, unknown>).filePath as string | undefined
|
||||
const tableId = (args as Record<string, unknown>).tableId as string | undefined
|
||||
const fileReference = fileId || filePath
|
||||
const rawMode = (args as Record<string, unknown>).mode as string | undefined
|
||||
const rawMapping = (args as Record<string, unknown>).mapping as
|
||||
| CsvHeaderMapping
|
||||
| undefined
|
||||
if (!fileReference) {
|
||||
return {
|
||||
success: false,
|
||||
@@ -782,11 +703,21 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
if (!workspaceId) {
|
||||
return { success: false, message: 'Workspace ID is required' }
|
||||
}
|
||||
if (rawMode && rawMode !== 'append' && rawMode !== 'replace') {
|
||||
return {
|
||||
success: false,
|
||||
message: `Invalid mode "${rawMode}". Must be "append" or "replace".`,
|
||||
}
|
||||
}
|
||||
const mode: 'append' | 'replace' = rawMode === 'replace' ? 'replace' : 'append'
|
||||
|
||||
const table = await getTableById(tableId)
|
||||
if (!table) {
|
||||
if (!table || table.workspaceId !== workspaceId) {
|
||||
return { success: false, message: `Table not found: ${tableId}` }
|
||||
}
|
||||
if (table.archivedAt) {
|
||||
return { success: false, message: `Table is archived: ${tableId}` }
|
||||
}
|
||||
|
||||
const file = await resolveWorkspaceFile(fileReference, workspaceId)
|
||||
const { headers, rows } = await parseFileRows(file.buffer, file.name, file.type)
|
||||
@@ -794,47 +725,86 @@ export const userTableServerTool: BaseServerTool<UserTableArgs, UserTableResult>
|
||||
return { success: false, message: 'File contains no data rows' }
|
||||
}
|
||||
|
||||
const tableColumns = table.schema.columns as ColumnDefinition[]
|
||||
const tableColNames = new Set(tableColumns.map((c) => c.name))
|
||||
const mappedHeaders = headers.filter((h) => tableColNames.has(h))
|
||||
if (mappedHeaders.length === 0) {
|
||||
const mapping: CsvHeaderMapping = rawMapping ?? buildAutoMapping(headers, table.schema)
|
||||
|
||||
let validation: ReturnType<typeof validateMapping>
|
||||
try {
|
||||
validation = validateMapping({
|
||||
csvHeaders: headers,
|
||||
mapping,
|
||||
tableSchema: table.schema,
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof CsvImportValidationError) {
|
||||
return { success: false, message: err.message }
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
if (validation.mappedHeaders.length === 0) {
|
||||
return {
|
||||
success: false,
|
||||
message: `No matching columns between file (${headers.join(', ')}) and table (${tableColumns.map((c) => c.name).join(', ')})`,
|
||||
message: `No matching columns between file (${headers.join(', ')}) and table (${table.schema.columns.map((c) => c.name).join(', ')})`,
|
||||
}
|
||||
}
|
||||
|
||||
const requiredMissing = tableColumns
|
||||
.filter((c) => c.required && !headers.includes(c.name))
|
||||
.map((c) => c.name)
|
||||
if (requiredMissing.length > 0) {
|
||||
const coerced = coerceRowsForTable(rows, table.schema, validation.effectiveMap)
|
||||
|
||||
if (mode === 'replace') {
|
||||
assertNotAborted()
|
||||
const requestId = generateId().slice(0, 8)
|
||||
const result = await replaceTableRows(
|
||||
{ tableId: table.id, rows: coerced, workspaceId, userId: context.userId },
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
logger.info('Rows replaced from file', {
|
||||
tableId: table.id,
|
||||
fileName: file.name,
|
||||
mode,
|
||||
matchedColumns: validation.mappedHeaders.length,
|
||||
deleted: result.deletedCount,
|
||||
inserted: result.insertedCount,
|
||||
userId: context.userId,
|
||||
})
|
||||
|
||||
return {
|
||||
success: false,
|
||||
message: `File is missing required columns: ${requiredMissing.join(', ')}`,
|
||||
success: true,
|
||||
message: `Replaced rows in "${table.name}" from "${file.name}": deleted ${result.deletedCount}, inserted ${result.insertedCount}`,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
tableName: table.name,
|
||||
mode,
|
||||
matchedColumns: validation.mappedHeaders,
|
||||
skippedColumns: validation.skippedHeaders,
|
||||
deletedCount: result.deletedCount,
|
||||
insertedCount: result.insertedCount,
|
||||
sourceFile: file.name,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const columnMap = new Map(tableColumns.map((c) => [c.name, c]))
|
||||
const matchedColumns = tableColumns.filter((c) => headers.includes(c.name))
|
||||
const coerced = coerceRows(rows, matchedColumns, columnMap)
|
||||
const inserted = await batchInsertAll(table.id, coerced, table, workspaceId, context)
|
||||
|
||||
logger.info('Rows imported from file', {
|
||||
tableId: table.id,
|
||||
fileName: file.name,
|
||||
matchedColumns: mappedHeaders.length,
|
||||
mode,
|
||||
matchedColumns: validation.mappedHeaders.length,
|
||||
rows: inserted,
|
||||
userId: context.userId,
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Imported ${inserted} rows into "${table.name}" from "${file.name}" (${mappedHeaders.length} columns matched)`,
|
||||
message: `Imported ${inserted} rows into "${table.name}" from "${file.name}" (${validation.mappedHeaders.length} columns matched)`,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
tableName: table.name,
|
||||
matchedColumns: mappedHeaders,
|
||||
skippedColumns: headers.filter((h) => !tableColNames.has(h)),
|
||||
mode,
|
||||
matchedColumns: validation.mappedHeaders,
|
||||
skippedColumns: validation.skippedHeaders,
|
||||
rowCount: inserted,
|
||||
sourceFile: file.name,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
buildAutoMapping,
|
||||
CsvImportValidationError,
|
||||
coerceRowsForTable,
|
||||
coerceValue,
|
||||
inferColumnType,
|
||||
inferSchemaFromCsv,
|
||||
parseCsvBuffer,
|
||||
sanitizeName,
|
||||
validateMapping,
|
||||
} from '@/lib/table/csv-import'
|
||||
import type { TableSchema } from '@/lib/table/types'
|
||||
|
||||
describe('csv-import', () => {
|
||||
describe('parseCsvBuffer', () => {
|
||||
it('parses a CSV string and extracts headers', async () => {
|
||||
const { headers, rows } = await parseCsvBuffer('a,b\n1,2\n3,4')
|
||||
expect(headers).toEqual(['a', 'b'])
|
||||
expect(rows).toEqual([
|
||||
{ a: '1', b: '2' },
|
||||
{ a: '3', b: '4' },
|
||||
])
|
||||
})
|
||||
|
||||
it('strips a UTF-8 BOM from the first header', async () => {
|
||||
const text = `\uFEFFname,age\nAlice,30`
|
||||
const { headers } = await parseCsvBuffer(text)
|
||||
expect(headers).toEqual(['name', 'age'])
|
||||
})
|
||||
|
||||
it('parses a Uint8Array input in browser-like environments', async () => {
|
||||
const bytes = new TextEncoder().encode('a,b\n1,2')
|
||||
const { headers, rows } = await parseCsvBuffer(bytes)
|
||||
expect(headers).toEqual(['a', 'b'])
|
||||
expect(rows).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('parses TSV when delimiter is tab', async () => {
|
||||
const { headers, rows } = await parseCsvBuffer('a\tb\n1\t2', '\t')
|
||||
expect(headers).toEqual(['a', 'b'])
|
||||
expect(rows).toEqual([{ a: '1', b: '2' }])
|
||||
})
|
||||
|
||||
it('throws when the file has no data rows', async () => {
|
||||
await expect(parseCsvBuffer('a,b')).rejects.toThrow(/no data rows/i)
|
||||
})
|
||||
})
|
||||
|
||||
describe('inferColumnType', () => {
|
||||
it('returns "string" for empty samples', () => {
|
||||
expect(inferColumnType([])).toBe('string')
|
||||
expect(inferColumnType([null, undefined, ''])).toBe('string')
|
||||
})
|
||||
|
||||
it('detects numeric columns', () => {
|
||||
expect(inferColumnType(['1', '2', '3.14'])).toBe('number')
|
||||
})
|
||||
|
||||
it('detects boolean columns (case-insensitive)', () => {
|
||||
expect(inferColumnType(['true', 'FALSE', 'True'])).toBe('boolean')
|
||||
})
|
||||
|
||||
it('detects ISO date columns', () => {
|
||||
expect(inferColumnType(['2024-01-01', '2024-02-01T12:00:00'])).toBe('date')
|
||||
})
|
||||
|
||||
it('falls back to "string"', () => {
|
||||
expect(inferColumnType(['abc', 'def'])).toBe('string')
|
||||
expect(inferColumnType(['1', 'abc'])).toBe('string')
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeName', () => {
|
||||
it('strips unsupported chars and collapses underscores', () => {
|
||||
expect(sanitizeName('Hello World!')).toBe('Hello_World')
|
||||
expect(sanitizeName(' foo-bar ')).toBe('foo_bar')
|
||||
})
|
||||
|
||||
it('prefixes names that start with a digit', () => {
|
||||
expect(sanitizeName('123abc')).toBe('col_123abc')
|
||||
})
|
||||
|
||||
it('fills in an empty name with the prefix', () => {
|
||||
expect(sanitizeName('$$$')).toBe('col_')
|
||||
})
|
||||
})
|
||||
|
||||
describe('inferSchemaFromCsv', () => {
|
||||
it('produces sanitized column names and inferred types', () => {
|
||||
const { columns, headerToColumn } = inferSchemaFromCsv(
|
||||
['First Name', 'Age', 'Active'],
|
||||
[
|
||||
{ 'First Name': 'Alice', Age: '30', Active: 'true' },
|
||||
{ 'First Name': 'Bob', Age: '40', Active: 'false' },
|
||||
]
|
||||
)
|
||||
expect(columns).toEqual([
|
||||
{ name: 'First_Name', type: 'string' },
|
||||
{ name: 'Age', type: 'number' },
|
||||
{ name: 'Active', type: 'boolean' },
|
||||
])
|
||||
expect(headerToColumn.get('First Name')).toBe('First_Name')
|
||||
expect(headerToColumn.get('Age')).toBe('Age')
|
||||
})
|
||||
|
||||
it('disambiguates duplicate sanitized headers', () => {
|
||||
const { columns } = inferSchemaFromCsv(
|
||||
['a b', 'a-b', 'a.b'],
|
||||
[{ 'a b': '1', 'a-b': '2', 'a.b': '3' }]
|
||||
)
|
||||
expect(columns.map((c) => c.name)).toEqual(['a_b', 'a_b_2', 'a_b_3'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('coerceValue', () => {
|
||||
it('returns null for empty values', () => {
|
||||
expect(coerceValue(null, 'string')).toBeNull()
|
||||
expect(coerceValue(undefined, 'number')).toBeNull()
|
||||
expect(coerceValue('', 'boolean')).toBeNull()
|
||||
})
|
||||
|
||||
it('coerces numbers', () => {
|
||||
expect(coerceValue('42', 'number')).toBe(42)
|
||||
expect(coerceValue('not a number', 'number')).toBeNull()
|
||||
})
|
||||
|
||||
it('coerces booleans strictly', () => {
|
||||
expect(coerceValue('true', 'boolean')).toBe(true)
|
||||
expect(coerceValue('FALSE', 'boolean')).toBe(false)
|
||||
expect(coerceValue('yes', 'boolean')).toBeNull()
|
||||
})
|
||||
|
||||
it('coerces dates to ISO strings and falls back to the original string', () => {
|
||||
expect(coerceValue('2024-01-01', 'date')).toBe(new Date('2024-01-01').toISOString())
|
||||
expect(coerceValue('not-a-date', 'date')).toBe('not-a-date')
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildAutoMapping', () => {
|
||||
const schema: TableSchema = {
|
||||
columns: [
|
||||
{ name: 'First_Name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
}
|
||||
|
||||
it('maps by exact sanitized name', () => {
|
||||
const mapping = buildAutoMapping(['First_Name', 'age'], schema)
|
||||
expect(mapping).toEqual({ First_Name: 'First_Name', age: 'age' })
|
||||
})
|
||||
|
||||
it('falls back to a case/punctuation-insensitive match', () => {
|
||||
const mapping = buildAutoMapping(['first name', 'AGE'], schema)
|
||||
expect(mapping).toEqual({ 'first name': 'First_Name', AGE: 'age' })
|
||||
})
|
||||
|
||||
it('returns null for headers without a match', () => {
|
||||
const mapping = buildAutoMapping(['unmatched'], schema)
|
||||
expect(mapping).toEqual({ unmatched: null })
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateMapping', () => {
|
||||
const schema: TableSchema = {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string', required: true },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
}
|
||||
|
||||
it('accepts a valid mapping and lists skipped/unmapped', () => {
|
||||
const result = validateMapping({
|
||||
csvHeaders: ['name', 'age', 'extra'],
|
||||
mapping: { name: 'name', age: 'age', extra: null },
|
||||
tableSchema: schema,
|
||||
})
|
||||
expect(result.mappedHeaders).toEqual(['name', 'age'])
|
||||
expect(result.skippedHeaders).toEqual(['extra'])
|
||||
expect(result.unmappedColumns).toEqual([])
|
||||
expect(result.effectiveMap.get('name')).toBe('name')
|
||||
expect(result.effectiveMap.has('extra')).toBe(false)
|
||||
})
|
||||
|
||||
it('throws when a required column is missing', () => {
|
||||
expect(() =>
|
||||
validateMapping({
|
||||
csvHeaders: ['age'],
|
||||
mapping: { age: 'age' },
|
||||
tableSchema: schema,
|
||||
})
|
||||
).toThrow(CsvImportValidationError)
|
||||
})
|
||||
|
||||
it('throws when a mapping targets a non-existent column', () => {
|
||||
expect(() =>
|
||||
validateMapping({
|
||||
csvHeaders: ['name'],
|
||||
mapping: { name: 'nonexistent' },
|
||||
tableSchema: schema,
|
||||
})
|
||||
).toThrow(/do not exist on the table/)
|
||||
})
|
||||
|
||||
it('throws when multiple headers map to the same column', () => {
|
||||
expect(() =>
|
||||
validateMapping({
|
||||
csvHeaders: ['a', 'b'],
|
||||
mapping: { a: 'name', b: 'name' },
|
||||
tableSchema: schema,
|
||||
})
|
||||
).toThrow(/same column/)
|
||||
})
|
||||
|
||||
it('throws when mapping references an unknown CSV header', () => {
|
||||
expect(() =>
|
||||
validateMapping({
|
||||
csvHeaders: ['name'],
|
||||
mapping: { name: 'name', bogus: 'age' },
|
||||
tableSchema: schema,
|
||||
})
|
||||
).toThrow(/unknown CSV headers/)
|
||||
})
|
||||
|
||||
it('throws when a mapping value is neither a string nor null', () => {
|
||||
expect(() =>
|
||||
validateMapping({
|
||||
csvHeaders: ['name'],
|
||||
mapping: { name: 42 as unknown as string },
|
||||
tableSchema: schema,
|
||||
})
|
||||
).toThrow(/Mapping values must be/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('coerceRowsForTable', () => {
|
||||
const schema: TableSchema = {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'active', type: 'boolean' },
|
||||
],
|
||||
}
|
||||
|
||||
it('applies the table column type using the effective mapping', () => {
|
||||
const rows = coerceRowsForTable(
|
||||
[
|
||||
{ Name: 'Alice', Age: '30', Active: 'true' },
|
||||
{ Name: 'Bob', Age: 'oops', Active: 'false' },
|
||||
],
|
||||
schema,
|
||||
new Map([
|
||||
['Name', 'name'],
|
||||
['Age', 'age'],
|
||||
['Active', 'active'],
|
||||
])
|
||||
)
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ name: 'Alice', age: 30, active: true },
|
||||
{ name: 'Bob', age: null, active: false },
|
||||
])
|
||||
})
|
||||
|
||||
it('drops CSV headers absent from the mapping', () => {
|
||||
const rows = coerceRowsForTable(
|
||||
[{ name: 'Alice', extra: 'keep me out' }],
|
||||
schema,
|
||||
new Map([['name', 'name']])
|
||||
)
|
||||
expect(rows).toEqual([{ name: 'Alice' }])
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,391 @@
|
||||
/**
|
||||
* Shared CSV import helpers for user-defined tables.
|
||||
*
|
||||
* Used by:
|
||||
* - `POST /api/table/import-csv` (create new table from CSV)
|
||||
* - `POST /api/table/[tableId]/import-csv` (append/replace into existing table)
|
||||
* - Copilot `user-table` tool (`create_from_file`, `import_file`)
|
||||
*
|
||||
* Keeping a single implementation avoids drift between HTTP and agent code paths.
|
||||
*/
|
||||
|
||||
import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types'
|
||||
|
||||
/** Narrower type than `COLUMN_TYPES` used internally for coercion. */
|
||||
export type CsvColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json'
|
||||
|
||||
/** Number of CSV rows sampled when inferring column types for a new table. */
|
||||
export const CSV_SCHEMA_SAMPLE_SIZE = 100
|
||||
|
||||
/** Maximum rows inserted per `batchInsertRows` call during import. */
|
||||
export const CSV_MAX_BATCH_SIZE = 1000
|
||||
|
||||
/** Maximum CSV/TSV file size accepted by import routes (50 MB). */
|
||||
export const CSV_MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Error thrown when the user-supplied mapping or CSV does not line up with the
|
||||
* target table. Callers should translate this into a 400 response.
|
||||
*/
|
||||
export class CsvImportValidationError extends Error {
|
||||
readonly code = 'CSV_IMPORT_VALIDATION' as const
|
||||
readonly details: {
|
||||
missingRequired?: string[]
|
||||
duplicateTargets?: string[]
|
||||
unknownColumns?: string[]
|
||||
unknownHeaders?: string[]
|
||||
}
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
details: {
|
||||
missingRequired?: string[]
|
||||
duplicateTargets?: string[]
|
||||
unknownColumns?: string[]
|
||||
unknownHeaders?: string[]
|
||||
} = {}
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'CsvImportValidationError'
|
||||
this.details = details
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a CSV/TSV payload using `csv-parse/sync`. Accepts a Node `Buffer`,
|
||||
* browser-friendly `Uint8Array`, or already-decoded string. Strips a leading
|
||||
* UTF-8 BOM so headers are not silently prefixed with `\uFEFF`.
|
||||
*/
|
||||
export async function parseCsvBuffer(
|
||||
input: Buffer | Uint8Array | string,
|
||||
delimiter = ','
|
||||
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
|
||||
const { parse } = await import('csv-parse/sync')
|
||||
|
||||
let text: string
|
||||
if (typeof input === 'string') {
|
||||
text = input
|
||||
} else if (typeof Buffer !== 'undefined' && Buffer.isBuffer(input)) {
|
||||
text = input.toString('utf-8')
|
||||
} else {
|
||||
text = new TextDecoder('utf-8').decode(input as Uint8Array)
|
||||
}
|
||||
text = text.replace(/^\uFEFF/, '')
|
||||
|
||||
const parsed = parse(text, {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
trim: true,
|
||||
relax_column_count: true,
|
||||
relax_quotes: true,
|
||||
skip_records_with_error: true,
|
||||
cast: false,
|
||||
delimiter,
|
||||
}) as Record<string, unknown>[]
|
||||
|
||||
if (parsed.length === 0) {
|
||||
throw new Error('CSV file has no data rows')
|
||||
}
|
||||
|
||||
const headers = Object.keys(parsed[0])
|
||||
if (headers.length === 0) {
|
||||
throw new Error('CSV file has no headers')
|
||||
}
|
||||
|
||||
return { headers, rows: parsed }
|
||||
}
|
||||
|
||||
/**
|
||||
* Infers a column type from a sample of non-empty values. Order matters: we
|
||||
* prefer narrower types (number > boolean > ISO date) and fall back to string.
|
||||
* JSON is never inferred automatically.
|
||||
*/
|
||||
export function inferColumnType(values: unknown[]): Exclude<CsvColumnType, 'json'> {
|
||||
const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
|
||||
if (nonEmpty.length === 0) return 'string'
|
||||
|
||||
const allNumber = nonEmpty.every((v) => {
|
||||
const n = Number(v)
|
||||
return !Number.isNaN(n) && String(v).trim() !== ''
|
||||
})
|
||||
if (allNumber) return 'number'
|
||||
|
||||
const allBoolean = nonEmpty.every((v) => {
|
||||
const s = String(v).toLowerCase()
|
||||
return s === 'true' || s === 'false'
|
||||
})
|
||||
if (allBoolean) return 'boolean'
|
||||
|
||||
const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/
|
||||
const allDate = nonEmpty.every((v) => {
|
||||
const s = String(v)
|
||||
return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s))
|
||||
})
|
||||
if (allDate) return 'date'
|
||||
|
||||
return 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes a raw header into a valid column/table name. Strips disallowed
|
||||
* characters, collapses runs of underscores, and ensures the first character
|
||||
* is a letter or underscore (prefixing with `fallbackPrefix` otherwise).
|
||||
*/
|
||||
export function sanitizeName(raw: string, fallbackPrefix = 'col'): string {
|
||||
let name = raw
|
||||
.trim()
|
||||
.replace(/[^a-zA-Z0-9_]/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_+|_+$/g, '')
|
||||
|
||||
if (!name || /^\d/.test(name)) {
|
||||
name = `${fallbackPrefix}_${name}`
|
||||
}
|
||||
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns column definitions inferred from CSV headers + sample rows. Duplicate
|
||||
* sanitized names are suffixed with `_2`, `_3`, etc. Also returns the header ->
|
||||
* column-name mapping used when coercing row values.
|
||||
*/
|
||||
export function inferSchemaFromCsv(
|
||||
headers: string[],
|
||||
rows: Record<string, unknown>[]
|
||||
): { columns: ColumnDefinition[]; headerToColumn: Map<string, string> } {
|
||||
const sample = rows.slice(0, CSV_SCHEMA_SAMPLE_SIZE)
|
||||
const seen = new Set<string>()
|
||||
const headerToColumn = new Map<string, string>()
|
||||
|
||||
const columns = headers.map((header) => {
|
||||
const base = sanitizeName(header)
|
||||
let colName = base
|
||||
let suffix = 2
|
||||
while (seen.has(colName.toLowerCase())) {
|
||||
colName = `${base}_${suffix}`
|
||||
suffix++
|
||||
}
|
||||
seen.add(colName.toLowerCase())
|
||||
headerToColumn.set(header, colName)
|
||||
|
||||
return {
|
||||
name: colName,
|
||||
type: inferColumnType(sample.map((r) => r[header])),
|
||||
} satisfies ColumnDefinition
|
||||
})
|
||||
|
||||
return { columns, headerToColumn }
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerces a single value to the requested column type. Returns `null` for
|
||||
* empty inputs or values that cannot be parsed (numbers/booleans). Dates fall
|
||||
* back to the original string when unparseable so that schema validation can
|
||||
* reject it with context rather than silently inserting `null`.
|
||||
*/
|
||||
export function coerceValue(
|
||||
value: unknown,
|
||||
colType: CsvColumnType
|
||||
): string | number | boolean | null | Record<string, unknown> | unknown[] {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
switch (colType) {
|
||||
case 'number': {
|
||||
const n = Number(value)
|
||||
return Number.isNaN(n) ? null : n
|
||||
}
|
||||
case 'boolean': {
|
||||
const s = String(value).toLowerCase()
|
||||
if (s === 'true') return true
|
||||
if (s === 'false') return false
|
||||
return null
|
||||
}
|
||||
case 'date': {
|
||||
const d = new Date(String(value))
|
||||
return Number.isNaN(d.getTime()) ? String(value) : d.toISOString()
|
||||
}
|
||||
case 'json': {
|
||||
if (typeof value === 'object') return value as Record<string, unknown> | unknown[]
|
||||
try {
|
||||
return JSON.parse(String(value))
|
||||
} catch {
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mapping from raw CSV header to target column name, with `null` indicating
|
||||
* "do not import this column".
|
||||
*/
|
||||
export type CsvHeaderMapping = Record<string, string | null>
|
||||
|
||||
export interface CsvMappingValidationResult {
|
||||
/** Columns present in the CSV that landed on a real table column. */
|
||||
mappedHeaders: string[]
|
||||
/** Columns in the CSV that the user/client chose to skip. */
|
||||
skippedHeaders: string[]
|
||||
/** Target column names that ended up unmapped (resolved from the mapping). */
|
||||
unmappedColumns: string[]
|
||||
/** Effective header -> column map (after dropping unknown / null targets). */
|
||||
effectiveMap: Map<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates a user-supplied mapping against the target table schema. Rejects
|
||||
* unknown target columns, duplicate targets, and required table columns that
|
||||
* are not covered by the CSV. Returns the normalized header -> column map.
|
||||
*/
|
||||
export function validateMapping(params: {
|
||||
csvHeaders: string[]
|
||||
mapping: CsvHeaderMapping
|
||||
tableSchema: TableSchema
|
||||
}): CsvMappingValidationResult {
|
||||
const { csvHeaders, mapping, tableSchema } = params
|
||||
const columnByName = new Map(tableSchema.columns.map((c) => [c.name, c]))
|
||||
|
||||
const unknownHeaders = Object.keys(mapping).filter((h) => !csvHeaders.includes(h))
|
||||
if (unknownHeaders.length > 0) {
|
||||
throw new CsvImportValidationError(
|
||||
`Mapping references unknown CSV headers: ${unknownHeaders.join(', ')}`,
|
||||
{ unknownHeaders }
|
||||
)
|
||||
}
|
||||
|
||||
const invalidTargets = Object.entries(mapping).filter(
|
||||
([, target]) => target !== null && typeof target !== 'string'
|
||||
)
|
||||
if (invalidTargets.length > 0) {
|
||||
throw new CsvImportValidationError(
|
||||
`Mapping values must be a column name (string) or null, got: ${invalidTargets
|
||||
.map(([header]) => header)
|
||||
.join(', ')}`
|
||||
)
|
||||
}
|
||||
|
||||
const targetsSeen = new Map<string, string[]>()
|
||||
const unknownColumns: string[] = []
|
||||
const effectiveMap = new Map<string, string>()
|
||||
const skippedHeaders: string[] = []
|
||||
|
||||
for (const header of csvHeaders) {
|
||||
const target = header in mapping ? mapping[header] : undefined
|
||||
if (target === null || target === undefined) {
|
||||
skippedHeaders.push(header)
|
||||
continue
|
||||
}
|
||||
if (!columnByName.has(target)) {
|
||||
unknownColumns.push(target)
|
||||
continue
|
||||
}
|
||||
const existing = targetsSeen.get(target) ?? []
|
||||
existing.push(header)
|
||||
targetsSeen.set(target, existing)
|
||||
effectiveMap.set(header, target)
|
||||
}
|
||||
|
||||
if (unknownColumns.length > 0) {
|
||||
throw new CsvImportValidationError(
|
||||
`Mapping references columns that do not exist on the table: ${unknownColumns.join(', ')}`,
|
||||
{ unknownColumns }
|
||||
)
|
||||
}
|
||||
|
||||
const duplicateTargets = [...targetsSeen.entries()]
|
||||
.filter(([, headers]) => headers.length > 1)
|
||||
.map(([col]) => col)
|
||||
if (duplicateTargets.length > 0) {
|
||||
throw new CsvImportValidationError(
|
||||
`Multiple CSV headers map to the same column(s): ${duplicateTargets.join(', ')}`,
|
||||
{ duplicateTargets }
|
||||
)
|
||||
}
|
||||
|
||||
const mappedTargets = new Set(effectiveMap.values())
|
||||
const unmappedColumns = tableSchema.columns
|
||||
.filter((c) => !mappedTargets.has(c.name))
|
||||
.map((c) => c.name)
|
||||
|
||||
const missingRequired = tableSchema.columns
|
||||
.filter((c) => c.required && !mappedTargets.has(c.name))
|
||||
.map((c) => c.name)
|
||||
if (missingRequired.length > 0) {
|
||||
throw new CsvImportValidationError(
|
||||
`CSV is missing required columns: ${missingRequired.join(', ')}`,
|
||||
{ missingRequired }
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
mappedHeaders: [...effectiveMap.keys()],
|
||||
skippedHeaders,
|
||||
unmappedColumns,
|
||||
effectiveMap,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an auto-mapping from CSV headers to table columns: prefers exact
|
||||
* sanitized-name matches and falls back to a case- and punctuation-insensitive
|
||||
* comparison. Unmapped headers are set to `null`.
|
||||
*/
|
||||
export function buildAutoMapping(csvHeaders: string[], tableSchema: TableSchema): CsvHeaderMapping {
|
||||
const mapping: CsvHeaderMapping = {}
|
||||
const columns = tableSchema.columns
|
||||
|
||||
const exactByName = new Map(columns.map((c) => [c.name, c.name]))
|
||||
const loose = new Map<string, string>()
|
||||
for (const col of columns) {
|
||||
loose.set(col.name.toLowerCase().replace(/[^a-z0-9]/g, ''), col.name)
|
||||
}
|
||||
|
||||
const usedTargets = new Set<string>()
|
||||
|
||||
for (const header of csvHeaders) {
|
||||
const sanitized = sanitizeName(header)
|
||||
const exact = exactByName.get(sanitized)
|
||||
if (exact && !usedTargets.has(exact)) {
|
||||
mapping[header] = exact
|
||||
usedTargets.add(exact)
|
||||
continue
|
||||
}
|
||||
const key = header.toLowerCase().replace(/[^a-z0-9]/g, '')
|
||||
const fuzzy = loose.get(key)
|
||||
if (fuzzy && !usedTargets.has(fuzzy)) {
|
||||
mapping[header] = fuzzy
|
||||
usedTargets.add(fuzzy)
|
||||
continue
|
||||
}
|
||||
mapping[header] = null
|
||||
}
|
||||
|
||||
return mapping
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerces parsed CSV rows into `RowData` objects keyed by target column name,
|
||||
* applying the column types declared in `tableSchema`. Headers not present in
|
||||
* `headerToColumn` are dropped. Missing table columns remain unset (schema
|
||||
* validation decides whether that's acceptable).
|
||||
*/
|
||||
export function coerceRowsForTable(
|
||||
rows: Record<string, unknown>[],
|
||||
tableSchema: TableSchema,
|
||||
headerToColumn: Map<string, string>
|
||||
): RowData[] {
|
||||
const typeByName = new Map(tableSchema.columns.map((c) => [c.name, c.type as CsvColumnType]))
|
||||
|
||||
return rows.map((row) => {
|
||||
const coerced: RowData = {}
|
||||
for (const [header, value] of Object.entries(row)) {
|
||||
const colName = headerToColumn.get(header)
|
||||
if (!colName) continue
|
||||
const colType = typeByName.get(colName) ?? 'string'
|
||||
coerced[colName] = coerceValue(value, colType) as RowData[string]
|
||||
}
|
||||
return coerced
|
||||
})
|
||||
}
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
export * from './billing'
|
||||
export * from './constants'
|
||||
export * from './csv-import'
|
||||
export * from './llm'
|
||||
export * from './query-builder'
|
||||
export * from './service'
|
||||
|
||||
@@ -30,6 +30,8 @@ import type {
|
||||
QueryOptions,
|
||||
QueryResult,
|
||||
RenameColumnData,
|
||||
ReplaceRowsData,
|
||||
ReplaceRowsResult,
|
||||
RowData,
|
||||
TableDefinition,
|
||||
TableMetadata,
|
||||
@@ -776,6 +778,120 @@ export async function batchInsertRows(
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces all rows in a table with a new set of rows. Deletes existing rows
|
||||
* and inserts the provided rows inside a single transaction so the table is
|
||||
* never observed in an empty intermediate state by other readers.
|
||||
*
|
||||
* Validates each row against the schema, enforces unique constraints within the
|
||||
* new rows (existing rows are deleted, so DB-side checks are unnecessary), and
|
||||
* enforces `maxRows` before the replace executes.
|
||||
*
|
||||
* @param data - Replace data (rows to install)
|
||||
* @param table - Table definition
|
||||
* @param requestId - Request ID for logging
|
||||
* @returns Count of rows deleted and inserted
|
||||
* @throws Error if validation fails or capacity exceeded
|
||||
*/
|
||||
export async function replaceTableRows(
|
||||
data: ReplaceRowsData,
|
||||
table: TableDefinition,
|
||||
requestId: string
|
||||
): Promise<ReplaceRowsResult> {
|
||||
if (data.tableId !== table.id) {
|
||||
throw new Error(`Table ID mismatch: ${data.tableId} vs ${table.id}`)
|
||||
}
|
||||
if (data.workspaceId !== table.workspaceId) {
|
||||
throw new Error(`Workspace ID mismatch: ${data.workspaceId} does not own table ${data.tableId}`)
|
||||
}
|
||||
if (data.rows.length > table.maxRows) {
|
||||
throw new Error(
|
||||
`Cannot replace: ${data.rows.length} rows exceeds table row limit (${table.maxRows})`
|
||||
)
|
||||
}
|
||||
|
||||
for (let i = 0; i < data.rows.length; i++) {
|
||||
const row = data.rows[i]
|
||||
|
||||
const sizeValidation = validateRowSize(row)
|
||||
if (!sizeValidation.valid) {
|
||||
throw new Error(`Row ${i + 1}: ${sizeValidation.errors.join(', ')}`)
|
||||
}
|
||||
|
||||
const schemaValidation = validateRowAgainstSchema(row, table.schema)
|
||||
if (!schemaValidation.valid) {
|
||||
throw new Error(`Row ${i + 1}: ${schemaValidation.errors.join(', ')}`)
|
||||
}
|
||||
}
|
||||
|
||||
const uniqueColumns = getUniqueColumns(table.schema)
|
||||
if (uniqueColumns.length > 0 && data.rows.length > 0) {
|
||||
const seen = new Map<string, Map<string, number>>()
|
||||
for (const col of uniqueColumns) {
|
||||
seen.set(col.name, new Map())
|
||||
}
|
||||
for (let i = 0; i < data.rows.length; i++) {
|
||||
const row = data.rows[i]
|
||||
for (const col of uniqueColumns) {
|
||||
const value = row[col.name]
|
||||
if (value === null || value === undefined) continue
|
||||
const normalized = typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value)
|
||||
const map = seen.get(col.name)!
|
||||
if (map.has(normalized)) {
|
||||
throw new Error(
|
||||
`Row ${i + 1}: Column "${col.name}" must be unique. Value "${String(value)}" duplicates row ${map.get(normalized)! + 1} in batch`
|
||||
)
|
||||
}
|
||||
map.set(normalized, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date()
|
||||
|
||||
const result = await db.transaction(async (trx) => {
|
||||
await trx.execute(
|
||||
sql`SELECT 1 FROM user_table_definitions WHERE id = ${data.tableId} FOR UPDATE`
|
||||
)
|
||||
|
||||
const deletedRows = await trx
|
||||
.delete(userTableRows)
|
||||
.where(eq(userTableRows.tableId, data.tableId))
|
||||
.returning({ id: userTableRows.id })
|
||||
|
||||
let insertedCount = 0
|
||||
if (data.rows.length > 0) {
|
||||
const rowsToInsert = data.rows.map((rowData, i) => ({
|
||||
id: `row_${generateId().replace(/-/g, '')}`,
|
||||
tableId: data.tableId,
|
||||
workspaceId: data.workspaceId,
|
||||
data: rowData,
|
||||
position: i,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
...(data.userId ? { createdBy: data.userId } : {}),
|
||||
}))
|
||||
|
||||
const batchSize = TABLE_LIMITS.MAX_BATCH_INSERT_SIZE
|
||||
for (let i = 0; i < rowsToInsert.length; i += batchSize) {
|
||||
const chunk = rowsToInsert.slice(i, i + batchSize)
|
||||
const inserted = await trx.insert(userTableRows).values(chunk).returning({
|
||||
id: userTableRows.id,
|
||||
})
|
||||
insertedCount += inserted.length
|
||||
}
|
||||
}
|
||||
|
||||
return { deletedCount: deletedRows.length, insertedCount }
|
||||
})
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Replaced rows in table ${data.tableId}: deleted ${result.deletedCount}, inserted ${result.insertedCount}`
|
||||
)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Upserts a row: updates an existing row if a match is found on the conflict target
|
||||
* column, otherwise inserts a new row.
|
||||
|
||||
@@ -241,6 +241,18 @@ export interface BulkDeleteByIdsResult {
|
||||
missingRowIds: string[]
|
||||
}
|
||||
|
||||
export interface ReplaceRowsData {
|
||||
tableId: string
|
||||
rows: RowData[]
|
||||
workspaceId: string
|
||||
userId?: string
|
||||
}
|
||||
|
||||
export interface ReplaceRowsResult {
|
||||
deletedCount: number
|
||||
insertedCount: number
|
||||
}
|
||||
|
||||
export interface RenameColumnData {
|
||||
tableId: string
|
||||
oldName: string
|
||||
|
||||
Reference in New Issue
Block a user