mirror of
https://github.com/simstudioai/sim.git
synced 2026-08-30 17:05:18 +08:00
feat(tables): add row TTL expiration
This commit is contained in:
@@ -24,6 +24,7 @@ Every column has a type, which decides how its values are stored and validated.
|
||||
| **Currency** | An amount in a currency you pick per column | `$1,234.56` |
|
||||
| **Boolean** | `true` or `false` | `true` |
|
||||
| **Date** | A date | `2026-03-16` |
|
||||
| **TTL** | A row expiration date, stored as Unix epoch seconds | `2026-03-16 2:30 PM` |
|
||||
| **JSON** | An object or array | `{ "tier": "pro" }` |
|
||||
| **Select** | One of a fixed set of options, or several | `Pro` |
|
||||
|
||||
@@ -31,6 +32,8 @@ Types are enforced as you enter values, so a Number column only takes numbers.
|
||||
|
||||
A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts.
|
||||
|
||||
A table can have one TTL column. Adding it enables row expiration; rows with a non-empty TTL value are deleted after that time passes. Deleting the TTL column disables expiration for the table. TTL cells use the date editor, while APIs and workflows read and write integer Unix epoch seconds, matching DynamoDB TTL.
|
||||
|
||||
## Editing a table
|
||||
|
||||
Open the **Tables** section in the sidebar and click **New table** to create one. Add columns from the column header, type into a cell to edit it, and paste rows from a spreadsheet to bulk-load. Filter and sort from the toolbar without changing the underlying data. The editor has full keyboard support; see [keyboard shortcuts](/keyboard-shortcuts).
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { createMockRequest } from '@sim/testing'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockEnqueue, mockGetJobQueue, mockVerifyCronAuth } = vi.hoisted(() => ({
|
||||
mockEnqueue: vi.fn(),
|
||||
mockGetJobQueue: vi.fn(),
|
||||
mockVerifyCronAuth: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mockVerifyCronAuth }))
|
||||
vi.mock('@/lib/core/async-jobs', () => ({ getJobQueue: mockGetJobQueue }))
|
||||
|
||||
import { GET } from '@/app/api/cron/cleanup-table-row-ttl/route'
|
||||
|
||||
describe('table row TTL cleanup route', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(new Date('2026-08-22T17:12:00Z'))
|
||||
mockVerifyCronAuth.mockReturnValue(null)
|
||||
mockEnqueue.mockResolvedValue('job-ttl-1')
|
||||
mockGetJobQueue.mockResolvedValue({ enqueue: mockEnqueue })
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('enqueues one serialized cleanup job', async () => {
|
||||
const response = await GET(
|
||||
createMockRequest(
|
||||
'GET',
|
||||
undefined,
|
||||
{},
|
||||
'http://localhost:3000/api/cron/cleanup-table-row-ttl'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
await expect(response.json()).resolves.toEqual({ triggered: true, jobId: 'job-ttl-1' })
|
||||
expect(mockEnqueue).toHaveBeenCalledWith(
|
||||
'cleanup-table-row-ttl',
|
||||
{},
|
||||
expect.objectContaining({
|
||||
maxAttempts: 1,
|
||||
jobId: 'cleanup-table-row-ttl:5958062',
|
||||
concurrencyKey: 'cleanup:table-row-ttl',
|
||||
concurrencyLimit: 1,
|
||||
runner: expect.any(Function),
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('deduplicates retries within the same five-minute schedule window', async () => {
|
||||
const request = () =>
|
||||
createMockRequest(
|
||||
'GET',
|
||||
undefined,
|
||||
{},
|
||||
'http://localhost:3000/api/cron/cleanup-table-row-ttl'
|
||||
)
|
||||
|
||||
await GET(request())
|
||||
vi.advanceTimersByTime(2 * 60 * 1000)
|
||||
await GET(request())
|
||||
|
||||
expect(mockEnqueue.mock.calls[0]?.[2]?.jobId).toBe(mockEnqueue.mock.calls[1]?.[2]?.jobId)
|
||||
})
|
||||
|
||||
it('returns the cron auth refusal without touching the queue', async () => {
|
||||
mockVerifyCronAuth.mockReturnValue(new Response(null, { status: 401 }))
|
||||
|
||||
const response = await GET(
|
||||
createMockRequest(
|
||||
'GET',
|
||||
undefined,
|
||||
{},
|
||||
'http://localhost:3000/api/cron/cleanup-table-row-ttl'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockGetJobQueue).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { verifyCronAuth } from '@/lib/auth/internal'
|
||||
import { getJobQueue } from '@/lib/core/async-jobs'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('CleanupTableRowTtlApi')
|
||||
const TTL_CLEANUP_INTERVAL_MS = 5 * 60 * 1000
|
||||
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const authError = verifyCronAuth(request, 'table row TTL cleanup')
|
||||
if (authError) return authError
|
||||
|
||||
const queue = await getJobQueue()
|
||||
const scheduleWindow = Math.floor(Date.now() / TTL_CLEANUP_INTERVAL_MS)
|
||||
const jobId = await queue.enqueue(
|
||||
'cleanup-table-row-ttl',
|
||||
{},
|
||||
{
|
||||
maxAttempts: 1,
|
||||
jobId: `cleanup-table-row-ttl:${scheduleWindow}`,
|
||||
name: 'Table row TTL cleanup',
|
||||
concurrencyKey: 'cleanup:table-row-ttl',
|
||||
concurrencyLimit: 1,
|
||||
runner: async (_payload, signal) => {
|
||||
const { runCleanupTableRowTtl } = await import('@/background/cleanup-table-row-ttl')
|
||||
return runCleanupTableRowTtl(signal)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
logger.info('Table row TTL cleanup dispatched', { jobId })
|
||||
return NextResponse.json({ triggered: true, jobId })
|
||||
} catch (error) {
|
||||
logger.error('Failed to dispatch table row TTL cleanup', { error })
|
||||
return NextResponse.json({ error: 'Failed to dispatch table row TTL cleanup' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { ColumnDefinition } from '@/lib/table'
|
||||
import { columnTypeOptionsForTable } from './column-types'
|
||||
|
||||
describe('columnTypeOptionsForTable', () => {
|
||||
const ttlColumn: ColumnDefinition = { name: 'expires_at', type: 'ttl' }
|
||||
|
||||
it('disables TTL with an explanation when the table already has one', () => {
|
||||
const availableTtl = columnTypeOptionsForTable([{ name: 'name', type: 'string' }]).find(
|
||||
(option) => option.type === 'ttl'
|
||||
)
|
||||
const unavailableTtl = columnTypeOptionsForTable([ttlColumn]).find(
|
||||
(option) => option.type === 'ttl'
|
||||
)
|
||||
|
||||
expect(availableTtl?.disabledReason).toBeUndefined()
|
||||
expect(unavailableTtl?.disabledReason).toBe('Only one TTL column allowed per table')
|
||||
})
|
||||
|
||||
it('keeps TTL enabled while editing the existing TTL column', () => {
|
||||
const ttlOption = columnTypeOptionsForTable([ttlColumn], ttlColumn).find(
|
||||
(option) => option.type === 'ttl'
|
||||
)
|
||||
|
||||
expect(ttlOption?.disabledReason).toBeUndefined()
|
||||
})
|
||||
})
|
||||
+3
-3
@@ -251,7 +251,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) {
|
||||
required={column.required}
|
||||
hint={hint}
|
||||
mono
|
||||
value={formatValueForInput(value, column.type)}
|
||||
value={formatValueForInput(value, column.type, timeZone)}
|
||||
onChange={onChange}
|
||||
placeholder='{"key": "value"}'
|
||||
rows={4}
|
||||
@@ -260,7 +260,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) {
|
||||
}
|
||||
|
||||
if (definition.editor === 'date') {
|
||||
const parts = dateValueToLocalParts(formatValueForInput(value, 'date'))
|
||||
const parts = dateValueToLocalParts(formatValueForInput(value, column.type, timeZone))
|
||||
return (
|
||||
<ChipModalField type='custom' title={title} required={column.required} hint={hint}>
|
||||
<div className='flex items-center gap-2'>
|
||||
@@ -306,7 +306,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) {
|
||||
inputType={
|
||||
definition.inputMode === 'decimal' && !definition.acceptsFormattedInput ? 'number' : 'text'
|
||||
}
|
||||
value={formatValueForInput(value, column.type)}
|
||||
value={formatValueForInput(value, column.type, timeZone)}
|
||||
onChange={onChange}
|
||||
placeholder={`Enter ${column.name}`}
|
||||
/>
|
||||
|
||||
+3
@@ -13,6 +13,7 @@ interface CellContentProps {
|
||||
/** Current workspace id — lets string cells holding an in-workspace resource
|
||||
* URL render as a tagged-resource chip instead of a plain external link. */
|
||||
workspaceId: string
|
||||
timeZone: string
|
||||
isEditing: boolean
|
||||
initialCharacter?: string | null
|
||||
onSave: (value: unknown, reason: SaveReason) => void
|
||||
@@ -38,6 +39,7 @@ export function CellContent({
|
||||
exec,
|
||||
column,
|
||||
workspaceId,
|
||||
timeZone,
|
||||
isEditing,
|
||||
initialCharacter,
|
||||
onSave,
|
||||
@@ -52,6 +54,7 @@ export function CellContent({
|
||||
waitingOnLabels,
|
||||
isEnrichmentOutput,
|
||||
currentWorkspaceId: workspaceId,
|
||||
timeZone,
|
||||
})
|
||||
|
||||
return (
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { resolveCellRender } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render'
|
||||
import type { DisplayColumn } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/types'
|
||||
|
||||
function column(type: DisplayColumn['type']): DisplayColumn {
|
||||
return {
|
||||
key: 'expires_at',
|
||||
name: 'expires_at',
|
||||
type,
|
||||
groupSize: 1,
|
||||
groupStartColIndex: 0,
|
||||
headerLabel: 'expires_at',
|
||||
isGroupStart: true,
|
||||
}
|
||||
}
|
||||
|
||||
describe('resolveCellRender', () => {
|
||||
it('renders TTL epoch seconds through the date presentation', () => {
|
||||
expect(
|
||||
resolveCellRender({
|
||||
value: 1_700_000_000,
|
||||
exec: undefined,
|
||||
column: column('ttl'),
|
||||
waitingOnLabels: undefined,
|
||||
timeZone: 'America/New_York',
|
||||
})
|
||||
).toEqual({ kind: 'date', text: '2023-11-14T17:13:20-05:00' })
|
||||
})
|
||||
})
|
||||
+7
-1
@@ -53,6 +53,8 @@ interface ResolveCellRenderInput {
|
||||
/** Current workspace id — a URL pointing to a resource in this workspace
|
||||
* renders as a tagged-resource chip rather than a plain external link. */
|
||||
currentWorkspaceId?: string
|
||||
/** Effective viewer timezone for instant-like column presentations. */
|
||||
timeZone?: string
|
||||
}
|
||||
|
||||
export function resolveCellRender({
|
||||
@@ -62,6 +64,7 @@ export function resolveCellRender({
|
||||
waitingOnLabels,
|
||||
isEnrichmentOutput,
|
||||
currentWorkspaceId,
|
||||
timeZone,
|
||||
}: ResolveCellRenderInput): CellRenderKind {
|
||||
const isNull = value === null || value === undefined
|
||||
const isEmpty = isNull || value === ''
|
||||
@@ -137,7 +140,10 @@ export function resolveCellRender({
|
||||
return { kind: 'text', text: columnTypeOf(column).formatForDisplay(value, column) }
|
||||
}
|
||||
if (column.type === 'json') return { kind: 'json', text: JSON.stringify(value) }
|
||||
if (column.type === 'date') return { kind: 'date', text: String(value) }
|
||||
const definition = columnTypeOf(column)
|
||||
if (definition.editor === 'date') {
|
||||
return { kind: 'date', text: definition.formatForInput(value, column, { timezone: timeZone }) }
|
||||
}
|
||||
if (column.type === 'string') {
|
||||
const text = stringifyValue(value)
|
||||
return resolveLinkKind(text, currentWorkspaceId) ?? { kind: 'text', text }
|
||||
|
||||
+4
-4
@@ -70,7 +70,7 @@ function InlineDateEditor({
|
||||
const popoverPointerAtRef = useRef(0)
|
||||
const timeZone = useTimezone()
|
||||
|
||||
const storedValue = formatValueForInput(value, column.type)
|
||||
const storedValue = formatValueForInput(value, column.type, timeZone)
|
||||
const initialDraft =
|
||||
initialCharacter !== undefined
|
||||
? initialCharacter
|
||||
@@ -115,7 +115,7 @@ function InlineDateEditor({
|
||||
// silently shifting the instant of a value someone else wrote.
|
||||
if (storageVal === undefined && initialCharacter === undefined && current === initialDraft) {
|
||||
doneRef.current = true
|
||||
onSave(storedValue || null, reason)
|
||||
onSave(storedValue ? cleanCellValue(storedValue, column, timeZone) : null, reason)
|
||||
return
|
||||
}
|
||||
const raw = storageVal ?? displayToStorage(current, timeZone) ?? current
|
||||
@@ -132,9 +132,9 @@ function InlineDateEditor({
|
||||
return
|
||||
}
|
||||
doneRef.current = true
|
||||
onSave(raw || null, reason)
|
||||
onSave(raw ? cleanCellValue(raw, column, timeZone) : null, reason)
|
||||
},
|
||||
[invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue]
|
||||
[invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue, column]
|
||||
)
|
||||
|
||||
const handleKeyDown = useCallback(
|
||||
|
||||
+5
@@ -26,6 +26,8 @@ export interface DataRowProps {
|
||||
/** Current workspace id — forwarded to cells so in-workspace resource URLs
|
||||
* render as tagged-resource chips. */
|
||||
workspaceId: string
|
||||
/** Effective viewer timezone used to render TTL instants. */
|
||||
timeZone: string
|
||||
rowIndex: number
|
||||
isFirstRow: boolean
|
||||
editingColumnName: string | null
|
||||
@@ -114,6 +116,7 @@ function dataRowPropsAreEqual(prev: DataRowProps, next: DataRowProps): boolean {
|
||||
prev.row !== next.row ||
|
||||
prev.columns !== next.columns ||
|
||||
prev.workspaceId !== next.workspaceId ||
|
||||
prev.timeZone !== next.timeZone ||
|
||||
prev.rowIndex !== next.rowIndex ||
|
||||
prev.isFirstRow !== next.isFirstRow ||
|
||||
prev.editingColumnName !== next.editingColumnName ||
|
||||
@@ -161,6 +164,7 @@ export const DataRow = React.memo(function DataRow({
|
||||
row,
|
||||
columns,
|
||||
workspaceId,
|
||||
timeZone,
|
||||
rowIndex,
|
||||
isFirstRow,
|
||||
editingColumnName,
|
||||
@@ -396,6 +400,7 @@ export const DataRow = React.memo(function DataRow({
|
||||
<div className={CELL_CONTENT}>
|
||||
<CellContent
|
||||
workspaceId={workspaceId}
|
||||
timeZone={timeZone}
|
||||
value={
|
||||
pendingCellValue && column.key in pendingCellValue
|
||||
? pendingCellValue[column.key]
|
||||
|
||||
+1
@@ -4904,6 +4904,7 @@ export function TableGrid({
|
||||
row={row}
|
||||
columns={displayColumns}
|
||||
workspaceId={workspaceId}
|
||||
timeZone={timeZone}
|
||||
rowIndex={index}
|
||||
isFirstRow={index === 0}
|
||||
editingColumnName={
|
||||
|
||||
@@ -194,4 +194,16 @@ describe('formatValueForInput', () => {
|
||||
)
|
||||
expect(formatValueForInput('2026-07-06', 'date')).toBe('2026-07-06')
|
||||
})
|
||||
|
||||
it('renders TTL instants in the editor timezone without changing the instant', () => {
|
||||
expect(formatValueForInput(1_700_000_000, 'ttl', 'America/New_York')).toBe(
|
||||
'2023-11-14T17:13:20-05:00'
|
||||
)
|
||||
expect(
|
||||
cleanCellValue('2023-11-14 17:13:20', { name: 'expires_at', type: 'ttl' }, 'America/New_York')
|
||||
).toBe(1_700_000_000)
|
||||
expect(
|
||||
cleanCellValue('2023-11-14', { name: 'expires_at', type: 'ttl' }, 'America/New_York')
|
||||
).toBe(1_699_938_000)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -55,7 +55,7 @@ export function cleanCellValue(
|
||||
// Everything else runs the SAME coercion the server will run, so the
|
||||
// optimistic cache holds exactly the value that gets persisted.
|
||||
const columnType = columnTypeOf(column)
|
||||
const coerced = columnType.coerce(value as JsonValue, column)
|
||||
const coerced = columnType.coerce(value as JsonValue, column, { timezone: timeZone })
|
||||
if (coerced.ok) return coerced.value
|
||||
const salvaged = columnType.salvage?.(value as JsonValue, column)
|
||||
return salvaged?.ok ? salvaged.value : null
|
||||
@@ -68,7 +68,7 @@ export function cleanCellValue(
|
||||
* row data already has the new mapping's value) would otherwise render
|
||||
* `[object Object]` via `String(value)`.
|
||||
*/
|
||||
export function formatValueForInput(value: unknown, type: string): string {
|
||||
export function formatValueForInput(value: unknown, type: string, timeZone?: string): string {
|
||||
if (value === null || value === undefined) return ''
|
||||
const definition = columnTypeById(type)
|
||||
// Shape-drift guard, kept ahead of the registry: a column whose declared type
|
||||
@@ -78,7 +78,11 @@ export function formatValueForInput(value: unknown, type: string): string {
|
||||
if (typeof value === 'object' && !definition.storesOpaqueIds && type !== 'json') {
|
||||
return JSON.stringify(value)
|
||||
}
|
||||
return definition.formatForInput(value, { name: '', type: type as ColumnType })
|
||||
return definition.formatForInput(
|
||||
value,
|
||||
{ name: '', type: type as ColumnType },
|
||||
{ timezone: timeZone }
|
||||
)
|
||||
}
|
||||
|
||||
/** A canonical date-cell value split into its wall-clock editing parts. */
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockDeleteExecute,
|
||||
mockListExecute,
|
||||
mockSignalTableRowsChanged,
|
||||
mockTask,
|
||||
mockWithLockedTable,
|
||||
} = vi.hoisted(() => ({
|
||||
mockDeleteExecute: vi.fn(),
|
||||
mockListExecute: vi.fn(),
|
||||
mockSignalTableRowsChanged: vi.fn(),
|
||||
mockTask: vi.fn((config: unknown) => config),
|
||||
mockWithLockedTable: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/db', () => ({
|
||||
dbFor: vi.fn(() => ({ execute: mockListExecute })),
|
||||
}))
|
||||
|
||||
vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
|
||||
vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: mockSignalTableRowsChanged }))
|
||||
vi.mock('@/lib/table/service', () => ({ withLockedTable: mockWithLockedTable }))
|
||||
|
||||
import { cleanupTableRowTtlTask, runCleanupTableRowTtl } from '@/background/cleanup-table-row-ttl'
|
||||
|
||||
const table = {
|
||||
id: 'table-1',
|
||||
workspaceId: 'workspace-1',
|
||||
schema: { columns: [{ id: 'col-ttl', name: 'expires_at', type: 'ttl' }] },
|
||||
locks: { insertLocked: false, updateLocked: false, deleteLocked: false, schemaLocked: false },
|
||||
}
|
||||
|
||||
describe('table row TTL cleanup', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockListExecute.mockResolvedValue([{ id: table.id, workspaceId: table.workspaceId }])
|
||||
mockWithLockedTable.mockImplementation(
|
||||
async (
|
||||
_tableId: string,
|
||||
mutate: (
|
||||
fresh: typeof table,
|
||||
trx: { execute: typeof mockDeleteExecute }
|
||||
) => Promise<unknown>
|
||||
) => mutate(table, { execute: mockDeleteExecute })
|
||||
)
|
||||
})
|
||||
|
||||
it('deletes expired rows in locked, keyset batches and signals the table', async () => {
|
||||
mockDeleteExecute
|
||||
.mockResolvedValueOnce([{ count: 500, lastId: 'row-500' }])
|
||||
.mockResolvedValueOnce([{ count: 12, lastId: 'row-512' }])
|
||||
|
||||
await expect(runCleanupTableRowTtl()).resolves.toEqual({
|
||||
batches: 2,
|
||||
deleted: 512,
|
||||
limitReached: false,
|
||||
})
|
||||
expect(mockWithLockedTable).toHaveBeenCalledTimes(2)
|
||||
expect(mockDeleteExecute).toHaveBeenCalledTimes(2)
|
||||
expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id)
|
||||
})
|
||||
|
||||
it('compares TTL values with whole Date.now epoch seconds', async () => {
|
||||
const nowEpochMilliseconds = 1_700_000_000_123
|
||||
const nowEpochSeconds = 1_700_000_000
|
||||
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(nowEpochMilliseconds)
|
||||
mockDeleteExecute.mockResolvedValue([{ count: 0, lastId: null }])
|
||||
|
||||
try {
|
||||
await runCleanupTableRowTtl()
|
||||
} finally {
|
||||
nowSpy.mockRestore()
|
||||
}
|
||||
|
||||
expect(mockListExecute.mock.calls[0][0]).toMatchObject({
|
||||
values: expect.arrayContaining([nowEpochSeconds]),
|
||||
})
|
||||
expect(mockDeleteExecute.mock.calls[0][0]).toMatchObject({
|
||||
values: expect.arrayContaining([nowEpochSeconds]),
|
||||
})
|
||||
})
|
||||
|
||||
it('does no work when already aborted', async () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await expect(runCleanupTableRowTtl(controller.signal)).resolves.toEqual({
|
||||
batches: 0,
|
||||
deleted: 0,
|
||||
limitReached: false,
|
||||
})
|
||||
expect(mockListExecute).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('honors a delete lock re-read inside the table advisory lock', async () => {
|
||||
mockWithLockedTable.mockImplementationOnce(async (_tableId, mutate) =>
|
||||
mutate(
|
||||
{ ...table, locks: { ...table.locks, deleteLocked: true } },
|
||||
{ execute: mockDeleteExecute }
|
||||
)
|
||||
)
|
||||
|
||||
await expect(runCleanupTableRowTtl()).resolves.toEqual({
|
||||
batches: 0,
|
||||
deleted: 0,
|
||||
limitReached: false,
|
||||
})
|
||||
expect(mockDeleteExecute).not.toHaveBeenCalled()
|
||||
expect(mockSignalTableRowsChanged).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops after one hundred full batches', async () => {
|
||||
mockDeleteExecute.mockResolvedValue([{ count: 500, lastId: 'row-cursor' }])
|
||||
|
||||
await expect(runCleanupTableRowTtl()).resolves.toEqual({
|
||||
batches: 100,
|
||||
deleted: 50_000,
|
||||
limitReached: true,
|
||||
})
|
||||
expect(mockDeleteExecute).toHaveBeenCalledTimes(100)
|
||||
expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('registers one serialized Trigger.dev task', () => {
|
||||
expect(cleanupTableRowTtlTask).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 'cleanup-table-row-ttl',
|
||||
queue: { concurrencyLimit: 1 },
|
||||
})
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,207 @@
|
||||
import { dbFor } from '@sim/db'
|
||||
import { userTableDefinitions, userTableRows } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { task } from '@trigger.dev/sdk'
|
||||
import { sql } from 'drizzle-orm'
|
||||
import { asOrchestrationError } from '@/lib/core/orchestration/types'
|
||||
import { getColumnId } from '@/lib/table/column-keys'
|
||||
import { signalTableRowsChanged } from '@/lib/table/events'
|
||||
import { assertRowDelete, TableLockedError } from '@/lib/table/mutation-locks'
|
||||
import type { DbTransaction } from '@/lib/table/planner'
|
||||
import { withLockedTable } from '@/lib/table/service'
|
||||
|
||||
const logger = createLogger('CleanupTableRowTtl')
|
||||
const cleanupDb = dbFor('cleanup')
|
||||
|
||||
const TTL_CLEANUP_BATCH_SIZE = 500
|
||||
const TTL_CLEANUP_MAX_BATCHES = 100
|
||||
|
||||
interface ExpiredTtlTableRef {
|
||||
[key: string]: unknown
|
||||
id: string
|
||||
workspaceId: string
|
||||
}
|
||||
|
||||
interface DeletedTtlBatch {
|
||||
attempted: boolean
|
||||
deleted: number
|
||||
lastId: string | null
|
||||
}
|
||||
|
||||
export interface TableRowTtlCleanupResult {
|
||||
batches: number
|
||||
deleted: number
|
||||
limitReached: boolean
|
||||
}
|
||||
|
||||
async function listExpiredTtlTables(nowEpochSeconds: number): Promise<ExpiredTtlTableRef[]> {
|
||||
const rows = await cleanupDb.execute<ExpiredTtlTableRef>(sql`
|
||||
SELECT
|
||||
${userTableDefinitions.id} AS id,
|
||||
${userTableDefinitions.workspaceId} AS "workspaceId"
|
||||
FROM ${userTableDefinitions}
|
||||
WHERE ${userTableDefinitions.archivedAt} IS NULL
|
||||
AND ${userTableDefinitions.deleteLocked} = false
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM jsonb_array_elements(
|
||||
COALESCE(${userTableDefinitions.schema}->'columns', '[]'::jsonb)
|
||||
) AS ttl_column(column_definition)
|
||||
JOIN ${userTableRows} AS table_row
|
||||
ON table_row.table_id = ${userTableDefinitions.id}
|
||||
AND table_row.workspace_id = ${userTableDefinitions.workspaceId}
|
||||
WHERE ttl_column.column_definition->>'type' = 'ttl'
|
||||
AND jsonb_typeof(
|
||||
table_row.data->COALESCE(
|
||||
ttl_column.column_definition->>'id',
|
||||
ttl_column.column_definition->>'name'
|
||||
)
|
||||
) = 'number'
|
||||
AND (
|
||||
table_row.data->>COALESCE(
|
||||
ttl_column.column_definition->>'id',
|
||||
ttl_column.column_definition->>'name'
|
||||
)
|
||||
)::numeric <= ${nowEpochSeconds}
|
||||
)
|
||||
ORDER BY ${userTableDefinitions.id}
|
||||
LIMIT ${TTL_CLEANUP_MAX_BATCHES}
|
||||
`)
|
||||
return Array.isArray(rows) ? rows : []
|
||||
}
|
||||
|
||||
function parseDeletedBatch(rows: unknown): Omit<DeletedTtlBatch, 'attempted'> {
|
||||
const [row] = Array.isArray(rows)
|
||||
? (rows as Array<{ count?: number | string; lastId?: string | null }>)
|
||||
: []
|
||||
if (!row) throw new Error('Table row TTL cleanup did not return a deleted count')
|
||||
|
||||
const deleted = Number(row.count)
|
||||
if (!Number.isSafeInteger(deleted) || deleted < 0 || deleted > TTL_CLEANUP_BATCH_SIZE) {
|
||||
throw new Error('Table row TTL cleanup returned an invalid deleted count')
|
||||
}
|
||||
if (deleted > 0 && typeof row.lastId !== 'string') {
|
||||
throw new Error('Table row TTL cleanup did not return a row cursor')
|
||||
}
|
||||
return { deleted, lastId: row.lastId ?? null }
|
||||
}
|
||||
|
||||
async function deleteExpiredTableRowBatch(
|
||||
trx: DbTransaction,
|
||||
tableId: string,
|
||||
workspaceId: string,
|
||||
columnKey: string,
|
||||
nowEpochSeconds: number,
|
||||
afterId?: string
|
||||
): Promise<Omit<DeletedTtlBatch, 'attempted'>> {
|
||||
const rows = await trx.execute<{ count: number | string; lastId: string | null }>(sql`
|
||||
WITH candidates AS MATERIALIZED (
|
||||
SELECT table_row.id
|
||||
FROM ${userTableRows} AS table_row
|
||||
WHERE table_row.table_id = ${tableId}
|
||||
AND table_row.workspace_id = ${workspaceId}
|
||||
${afterId ? sql`AND table_row.id > ${afterId}` : sql``}
|
||||
AND jsonb_typeof(table_row.data->${columnKey}) = 'number'
|
||||
AND (table_row.data->>${columnKey})::numeric <= ${nowEpochSeconds}
|
||||
ORDER BY table_row.id
|
||||
LIMIT ${TTL_CLEANUP_BATCH_SIZE}
|
||||
FOR UPDATE OF table_row SKIP LOCKED
|
||||
), deleted AS (
|
||||
DELETE FROM ${userTableRows} AS table_row
|
||||
USING candidates
|
||||
WHERE table_row.id = candidates.id
|
||||
RETURNING table_row.id
|
||||
)
|
||||
SELECT
|
||||
count(*)::integer AS count,
|
||||
max(id) AS "lastId"
|
||||
FROM deleted
|
||||
`)
|
||||
return parseDeletedBatch(rows)
|
||||
}
|
||||
|
||||
async function deleteExpiredRowsForTable(
|
||||
ref: ExpiredTtlTableRef,
|
||||
nowEpochSeconds: number,
|
||||
afterId?: string
|
||||
): Promise<DeletedTtlBatch> {
|
||||
try {
|
||||
return await withLockedTable(
|
||||
ref.id,
|
||||
async (table, trx) => {
|
||||
try {
|
||||
assertRowDelete(table)
|
||||
} catch (error) {
|
||||
if (error instanceof TableLockedError) {
|
||||
return { attempted: false, deleted: 0, lastId: null }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const ttlColumn = table.schema.columns.find((column) => column.type === 'ttl')
|
||||
if (!ttlColumn) return { attempted: false, deleted: 0, lastId: null }
|
||||
|
||||
const batch = await deleteExpiredTableRowBatch(
|
||||
trx,
|
||||
table.id,
|
||||
table.workspaceId,
|
||||
getColumnId(ttlColumn),
|
||||
nowEpochSeconds,
|
||||
afterId
|
||||
)
|
||||
return { attempted: true, ...batch }
|
||||
},
|
||||
{ expectedWorkspaceId: ref.workspaceId }
|
||||
)
|
||||
} catch (error) {
|
||||
if (asOrchestrationError(error)?.code === 'not_found') {
|
||||
return { attempted: false, deleted: 0, lastId: null }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Deletes rows whose table TTL cell is at or before the current Unix epoch second. */
|
||||
export async function runCleanupTableRowTtl(
|
||||
signal?: AbortSignal
|
||||
): Promise<TableRowTtlCleanupResult> {
|
||||
if (signal?.aborted) return { batches: 0, deleted: 0, limitReached: false }
|
||||
|
||||
const nowEpochSeconds = Math.floor(Date.now() / 1000)
|
||||
const tableRefs = await listExpiredTtlTables(nowEpochSeconds)
|
||||
let deleted = 0
|
||||
let batches = 0
|
||||
let lastBatchDeleted = 0
|
||||
|
||||
for (const ref of tableRefs) {
|
||||
let afterId: string | undefined
|
||||
let tableDeleted = 0
|
||||
|
||||
while (batches < TTL_CLEANUP_MAX_BATCHES && !signal?.aborted) {
|
||||
const batch = await deleteExpiredRowsForTable(ref, nowEpochSeconds, afterId)
|
||||
if (!batch.attempted) break
|
||||
|
||||
batches++
|
||||
deleted += batch.deleted
|
||||
tableDeleted += batch.deleted
|
||||
lastBatchDeleted = batch.deleted
|
||||
afterId = batch.lastId ?? undefined
|
||||
if (batch.deleted < TTL_CLEANUP_BATCH_SIZE) break
|
||||
}
|
||||
|
||||
if (tableDeleted > 0) signalTableRowsChanged(ref.id)
|
||||
if (batches === TTL_CLEANUP_MAX_BATCHES || signal?.aborted) break
|
||||
}
|
||||
|
||||
const limitReached =
|
||||
batches === TTL_CLEANUP_MAX_BATCHES &&
|
||||
(lastBatchDeleted === TTL_CLEANUP_BATCH_SIZE || tableRefs.length === TTL_CLEANUP_MAX_BATCHES)
|
||||
logger.info('Table row TTL cleanup completed', { batches, deleted, limitReached })
|
||||
return { batches, deleted, limitReached }
|
||||
}
|
||||
|
||||
export const cleanupTableRowTtlTask = task({
|
||||
id: 'cleanup-table-row-ttl',
|
||||
queue: { concurrencyLimit: 1 },
|
||||
run: () => runCleanupTableRowTtl(),
|
||||
})
|
||||
@@ -4191,7 +4191,7 @@ export const QueryUserTable: ToolCatalogEntry = {
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.',
|
||||
'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
@@ -5453,7 +5453,7 @@ export const TableColumns: ToolCatalogEntry = {
|
||||
column: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Column definition for add_column: { name, type, unique?, position? }; select (enum) columns also take { options: [names], multiple?: true } — options is required for select.',
|
||||
'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
columnName: {
|
||||
type: 'string',
|
||||
@@ -5474,7 +5474,7 @@ export const TableColumns: ToolCatalogEntry = {
|
||||
newType: {
|
||||
type: 'string',
|
||||
description:
|
||||
'New column type for update_column: string, number, boolean, date, json, select. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell.',
|
||||
'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.',
|
||||
},
|
||||
options: {
|
||||
type: 'array',
|
||||
@@ -5653,7 +5653,7 @@ export const TableManage: ToolCatalogEntry = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Table schema with a columns array (required for create). Each column: { name, type, unique? }; a select (enum) column also requires options (display names) and takes multiple?.',
|
||||
'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
tableId: {
|
||||
type: 'string',
|
||||
@@ -5699,12 +5699,12 @@ export const TableRows: ToolCatalogEntry = {
|
||||
data: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME.',
|
||||
'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.',
|
||||
},
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES.',
|
||||
'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
@@ -5729,7 +5729,8 @@ export const TableRows: ToolCatalogEntry = {
|
||||
},
|
||||
rows: {
|
||||
type: 'array',
|
||||
description: 'Array of row data objects (required for batch_insert_rows)',
|
||||
description:
|
||||
'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.',
|
||||
},
|
||||
tableId: { type: 'string', description: 'Table ID (required for every operation)' },
|
||||
updates: {
|
||||
@@ -6058,7 +6059,7 @@ export const UserTable: ToolCatalogEntry = {
|
||||
column: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Column definition for add_column: { name, type, unique?, position? }. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.',
|
||||
'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
columnName: {
|
||||
type: 'string',
|
||||
@@ -6077,7 +6078,8 @@ export const UserTable: ToolCatalogEntry = {
|
||||
},
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Row data as key-value pairs (required for insert_row, update_row)',
|
||||
description:
|
||||
'Row data as key-value pairs (required for insert_row, update_row). TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.',
|
||||
},
|
||||
dependencies: {
|
||||
type: 'object',
|
||||
@@ -6112,7 +6114,7 @@ export const UserTable: ToolCatalogEntry = {
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.',
|
||||
'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.',
|
||||
},
|
||||
groupId: {
|
||||
type: 'string',
|
||||
@@ -6201,7 +6203,7 @@ export const UserTable: ToolCatalogEntry = {
|
||||
newType: {
|
||||
type: 'string',
|
||||
description:
|
||||
'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.',
|
||||
'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.',
|
||||
},
|
||||
options: {
|
||||
type: 'array',
|
||||
@@ -6279,7 +6281,8 @@ export const UserTable: ToolCatalogEntry = {
|
||||
},
|
||||
rows: {
|
||||
type: 'array',
|
||||
description: 'Array of row data objects (required for batch_insert_rows)',
|
||||
description:
|
||||
'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.',
|
||||
},
|
||||
runMode: {
|
||||
type: 'string',
|
||||
@@ -6290,7 +6293,7 @@ export const UserTable: ToolCatalogEntry = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.',
|
||||
'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
scope: {
|
||||
type: 'string',
|
||||
|
||||
@@ -4080,7 +4080,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.',
|
||||
'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
@@ -5339,7 +5339,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
column: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Column definition for add_column: { name, type, unique?, position? }; select (enum) columns also take { options: [names], multiple?: true } — options is required for select.',
|
||||
'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
columnName: {
|
||||
type: 'string',
|
||||
@@ -5363,7 +5363,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
newType: {
|
||||
type: 'string',
|
||||
description:
|
||||
'New column type for update_column: string, number, boolean, date, json, select. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell.',
|
||||
'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.',
|
||||
},
|
||||
options: {
|
||||
type: 'array',
|
||||
@@ -5569,7 +5569,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Table schema with a columns array (required for create). Each column: { name, type, unique? }; a select (enum) column also requires options (display names) and takes multiple?.',
|
||||
'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
tableId: {
|
||||
type: 'string',
|
||||
@@ -5619,12 +5619,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
data: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME.',
|
||||
'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.',
|
||||
},
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES.',
|
||||
'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
@@ -5654,7 +5654,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
rows: {
|
||||
type: 'array',
|
||||
description: 'Array of row data objects (required for batch_insert_rows)',
|
||||
description:
|
||||
'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.',
|
||||
},
|
||||
tableId: {
|
||||
type: 'string',
|
||||
@@ -5998,7 +5999,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
column: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Column definition for add_column: { name, type, unique?, position? }. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.',
|
||||
'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
columnName: {
|
||||
type: 'string',
|
||||
@@ -6017,7 +6018,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Row data as key-value pairs (required for insert_row, update_row)',
|
||||
description:
|
||||
'Row data as key-value pairs (required for insert_row, update_row). TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.',
|
||||
},
|
||||
dependencies: {
|
||||
type: 'object',
|
||||
@@ -6057,7 +6059,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
filter: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.',
|
||||
'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.',
|
||||
},
|
||||
groupId: {
|
||||
type: 'string',
|
||||
@@ -6154,7 +6156,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
newType: {
|
||||
type: 'string',
|
||||
description:
|
||||
'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.',
|
||||
'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.',
|
||||
},
|
||||
options: {
|
||||
type: 'array',
|
||||
@@ -6242,7 +6244,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
},
|
||||
rows: {
|
||||
type: 'array',
|
||||
description: 'Array of row data objects (required for batch_insert_rows)',
|
||||
description:
|
||||
'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.',
|
||||
},
|
||||
runMode: {
|
||||
type: 'string',
|
||||
@@ -6253,7 +6256,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
|
||||
schema: {
|
||||
type: 'object',
|
||||
description:
|
||||
'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.',
|
||||
'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.',
|
||||
},
|
||||
scope: {
|
||||
type: 'string',
|
||||
|
||||
@@ -181,6 +181,7 @@ const JOB_TYPE_TO_TASK_ID: Record<JobType, string> = {
|
||||
'workflow-group-cell': 'workflow-group-cell',
|
||||
'cleanup-logs': 'cleanup-logs',
|
||||
'cleanup-soft-deletes': 'cleanup-soft-deletes',
|
||||
'cleanup-table-row-ttl': 'cleanup-table-row-ttl',
|
||||
'cleanup-tasks': 'cleanup-tasks',
|
||||
'run-data-drain': 'run-data-drain',
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ export type JobType =
|
||||
| 'workflow-group-cell'
|
||||
| 'cleanup-logs'
|
||||
| 'cleanup-soft-deletes'
|
||||
| 'cleanup-table-row-ttl'
|
||||
| 'cleanup-tasks'
|
||||
| 'run-data-drain'
|
||||
|
||||
|
||||
@@ -121,6 +121,62 @@ describe('conversion write-back', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('ttl columns', () => {
|
||||
const column = { name: 'expires_at', type: 'ttl' } as ColumnDefinition
|
||||
|
||||
it('stores integer epoch seconds while accepting date-shaped input', () => {
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20Z', column)).toEqual({
|
||||
ok: true,
|
||||
value: 1_700_000_000,
|
||||
})
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000, column)).toEqual({
|
||||
ok: true,
|
||||
value: 1_700_000_000,
|
||||
})
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce('1700000000', column)).toEqual({
|
||||
ok: true,
|
||||
value: 1_700_000_000,
|
||||
})
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20.123Z', column)).toEqual({
|
||||
ok: true,
|
||||
value: 1_700_000_000,
|
||||
})
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce('not-a-date', column)).toEqual({ ok: false })
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000.5, column)).toEqual({ ok: false })
|
||||
})
|
||||
|
||||
it('renders and edits epoch seconds as a date', () => {
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.formatForDisplay(1_700_000_000, column)).toBe(
|
||||
'11/14/2023 10:13:20 PM'
|
||||
)
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column)).toBe(
|
||||
'2023-11-14T22:13:20Z'
|
||||
)
|
||||
expect(
|
||||
COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column, {
|
||||
timezone: 'America/New_York',
|
||||
})
|
||||
).toBe('2023-11-14T17:13:20-05:00')
|
||||
})
|
||||
|
||||
it('preserves the exact instant across both sides of a daylight-saving fold', () => {
|
||||
expect(
|
||||
COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_162_200, column, {
|
||||
timezone: 'America/New_York',
|
||||
})
|
||||
).toBe('2023-11-05T01:30:00-04:00')
|
||||
expect(
|
||||
COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_165_800, column, {
|
||||
timezone: 'America/New_York',
|
||||
})
|
||||
).toBe('2023-11-05T01:30:00-05:00')
|
||||
})
|
||||
|
||||
it('limits a table to one ttl column', () => {
|
||||
expect(COLUMN_TYPE_REGISTRY.ttl.maxPerTable).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('intentional divergences from the pre-registry behavior', () => {
|
||||
// A differential run of the registry against the pre-refactor implementations
|
||||
// (55 values x 7 column shapes) found ZERO coercion differences and exactly
|
||||
|
||||
@@ -195,6 +195,18 @@ describe('Validation', () => {
|
||||
expect(result.errors).toContain('Duplicate column names found')
|
||||
})
|
||||
|
||||
it('rejects more than one TTL column', () => {
|
||||
const result = validateTableSchema({
|
||||
columns: [
|
||||
{ name: 'expires_at', type: 'ttl' },
|
||||
{ name: 'delete_at', type: 'ttl' },
|
||||
],
|
||||
} as TableSchema)
|
||||
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.errors).toContain('A table can have at most 1 TTL column')
|
||||
})
|
||||
|
||||
it('should reject null schema', () => {
|
||||
const result = validateTableSchema(null as unknown as TableSchema)
|
||||
expect(result.valid).toBe(false)
|
||||
|
||||
@@ -273,6 +273,7 @@ export const COLUMN_TYPE_SERVER_REGISTRY: Record<ColumnType, ColumnTypeServerEnt
|
||||
number: COLUMN_TYPE_REGISTRY.number,
|
||||
boolean: COLUMN_TYPE_REGISTRY.boolean,
|
||||
date: COLUMN_TYPE_REGISTRY.date,
|
||||
ttl: COLUMN_TYPE_REGISTRY.ttl,
|
||||
json: COLUMN_TYPE_REGISTRY.json,
|
||||
select: {
|
||||
...COLUMN_TYPE_REGISTRY.select,
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
selectColumnType,
|
||||
} from '@/lib/table/column-types/select'
|
||||
import { stringColumnType } from '@/lib/table/column-types/string'
|
||||
import { ttlColumnType } from '@/lib/table/column-types/ttl'
|
||||
import type { ColumnType, ColumnTypeDefinition } from '@/lib/table/column-types/types'
|
||||
import { COLUMN_TYPES, TYPE_SPECIFIC_COLUMN_KEYS } from '@/lib/table/column-types/types'
|
||||
import type { ColumnDefinition, JsonValue } from '@/lib/table/types'
|
||||
@@ -46,6 +47,7 @@ export const COLUMN_TYPE_REGISTRY: Record<ColumnType, ColumnTypeDefinition> = {
|
||||
number: numberColumnType,
|
||||
boolean: booleanColumnType,
|
||||
date: dateColumnType,
|
||||
ttl: ttlColumnType,
|
||||
json: jsonColumnType,
|
||||
select: selectColumnType,
|
||||
currency: currencyColumnType,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { retypeCellRewrite } from '@/lib/table/columns/service'
|
||||
import type { ColumnDefinition } from '@/lib/table/types'
|
||||
|
||||
const column = (over: Partial<ColumnDefinition>): ColumnDefinition =>
|
||||
({ name: 'col', type: 'string', ...over }) as ColumnDefinition
|
||||
|
||||
describe('TTL column type', () => {
|
||||
it('converts epoch seconds to an ISO date before retyping', () => {
|
||||
expect(
|
||||
retypeCellRewrite(1_700_000_000, column({ type: 'date' }), column({ type: 'ttl' }))
|
||||
).toEqual({ value: '2023-11-14T22:13:20Z' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,114 @@
|
||||
import { TypeTtl } from '@sim/emcn/icons'
|
||||
import type { ColumnTypeDefinition } from '@/lib/table/column-types/types'
|
||||
import {
|
||||
formatDateCellDisplay,
|
||||
formatInstantInTimeZone,
|
||||
type NormalizeDateCellOptions,
|
||||
normalizeDateCellValue,
|
||||
} from '@/lib/table/dates'
|
||||
import type { ColumnDefinition } from '@/lib/table/types'
|
||||
|
||||
const NUMERIC_VALUE_PATTERN = /^-?\d+(?:\.\d+)?$/
|
||||
const EXPLICIT_OFFSET_PATTERN = /(?:Z|[+-]\d{2}:?\d{2})$/i
|
||||
|
||||
function isRepresentableEpochSeconds(value: number): boolean {
|
||||
return Number.isSafeInteger(value) && !Number.isNaN(new Date(value * 1000).getTime())
|
||||
}
|
||||
|
||||
/** Converts a TTL cell input to integer Unix epoch seconds. */
|
||||
export function parseTtlEpochSeconds(
|
||||
value: unknown,
|
||||
options?: NormalizeDateCellOptions
|
||||
): number | null {
|
||||
if (typeof value === 'number') return isRepresentableEpochSeconds(value) ? value : null
|
||||
|
||||
if (value instanceof Date) {
|
||||
const milliseconds = value.getTime()
|
||||
return Number.isNaN(milliseconds) ? null : Math.floor(milliseconds / 1000)
|
||||
}
|
||||
|
||||
if (typeof value !== 'string') return null
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return null
|
||||
|
||||
if (NUMERIC_VALUE_PATTERN.test(trimmed)) {
|
||||
const numeric = Number(trimmed)
|
||||
return isRepresentableEpochSeconds(numeric) ? numeric : null
|
||||
}
|
||||
|
||||
if (EXPLICIT_OFFSET_PATTERN.test(trimmed)) {
|
||||
const milliseconds = Date.parse(trimmed)
|
||||
if (Number.isNaN(milliseconds)) return null
|
||||
const seconds = Math.floor(milliseconds / 1000)
|
||||
return isRepresentableEpochSeconds(seconds) ? seconds : null
|
||||
}
|
||||
|
||||
const normalized = normalizeDateCellValue(trimmed, options)
|
||||
if (normalized === null) return null
|
||||
const instant = /^\d{4}-\d{2}-\d{2}$/.test(normalized)
|
||||
? normalizeDateCellValue(`${normalized}T00:00:00`, options)
|
||||
: normalized
|
||||
if (instant === null) return null
|
||||
const milliseconds = Date.parse(instant)
|
||||
if (Number.isNaN(milliseconds)) return null
|
||||
const seconds = Math.floor(milliseconds / 1000)
|
||||
return isRepresentableEpochSeconds(seconds) ? seconds : null
|
||||
}
|
||||
|
||||
function epochSecondsToIso(value: unknown): string | null {
|
||||
const seconds = typeof value === 'number' ? value : Number(value)
|
||||
if (!isRepresentableEpochSeconds(seconds)) return null
|
||||
return new Date(seconds * 1000).toISOString().replace('.000Z', 'Z')
|
||||
}
|
||||
|
||||
function epochSecondsToEditable(value: unknown, timeZone?: string): string | null {
|
||||
const iso = epochSecondsToIso(value)
|
||||
if (!iso || !timeZone) return iso
|
||||
return formatInstantInTimeZone(new Date(iso), timeZone)
|
||||
}
|
||||
|
||||
export const ttlColumnType: ColumnTypeDefinition = {
|
||||
id: 'ttl',
|
||||
label: 'TTL',
|
||||
maxPerTable: 1,
|
||||
icon: TypeTtl,
|
||||
jsonbCast: 'numeric',
|
||||
storesOpaqueIds: false,
|
||||
supportsUnique: true,
|
||||
sampleValue: 1_706_659_200,
|
||||
ownedMetadata: [],
|
||||
workflowInputType: 'number',
|
||||
editor: 'date',
|
||||
expandable: false,
|
||||
typeaheadPattern: /[\d\-/]/,
|
||||
parseErrorMessage: 'Invalid expiration date',
|
||||
|
||||
coerce(value, _column, context) {
|
||||
const seconds = parseTtlEpochSeconds(value, context)
|
||||
return seconds === null ? { ok: false } : { ok: true, value: seconds }
|
||||
},
|
||||
|
||||
coerceImport(value, options) {
|
||||
return parseTtlEpochSeconds(value, options) ?? String(value)
|
||||
},
|
||||
|
||||
valueForConversion(value, target: ColumnDefinition) {
|
||||
if (target.type !== 'date') return value
|
||||
return epochSecondsToIso(value) ?? value
|
||||
},
|
||||
|
||||
validateCell(value, column) {
|
||||
return typeof value === 'number' && isRepresentableEpochSeconds(value)
|
||||
? null
|
||||
: `${column.name} must be valid epoch seconds`
|
||||
},
|
||||
|
||||
formatForDisplay(value) {
|
||||
const iso = epochSecondsToIso(value)
|
||||
return iso === null ? String(value) : formatDateCellDisplay(iso, { seconds: true })
|
||||
},
|
||||
|
||||
formatForInput(value, _column, context) {
|
||||
return epochSecondsToEditable(value, context?.timezone) ?? String(value)
|
||||
},
|
||||
}
|
||||
@@ -37,6 +37,7 @@ export const COLUMN_TYPES = [
|
||||
'currency',
|
||||
'boolean',
|
||||
'date',
|
||||
'ttl',
|
||||
'json',
|
||||
'select',
|
||||
] as const
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition, TableLocks } from '@/lib/table/types'
|
||||
|
||||
const { mockTimeoutExecute, mockWithLockedTable } = vi.hoisted(() => ({
|
||||
mockTimeoutExecute: vi.fn(),
|
||||
mockWithLockedTable: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/service', () => ({ withLockedTable: mockWithLockedTable }))
|
||||
|
||||
import { addTableColumn, updateColumnType } from '@/lib/table/columns/service'
|
||||
|
||||
const UNLOCKED: TableLocks = {
|
||||
schemaLocked: false,
|
||||
insertLocked: false,
|
||||
updateLocked: false,
|
||||
deleteLocked: false,
|
||||
}
|
||||
|
||||
function makeTable(): TableDefinition {
|
||||
return {
|
||||
id: 'table-1',
|
||||
name: 'Tasks',
|
||||
schema: {
|
||||
columns: [
|
||||
{ id: 'col-name', name: 'name', type: 'string' },
|
||||
{ id: 'col-ttl', name: 'expires_at', type: 'ttl' },
|
||||
],
|
||||
},
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
locks: UNLOCKED,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
const transaction = new Proxy(
|
||||
{ execute: mockTimeoutExecute },
|
||||
{
|
||||
get(target, property) {
|
||||
if (property in target) return target[property as keyof typeof target]
|
||||
throw new Error(`Unexpected transaction method: ${String(property)}`)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
describe('TTL column mutation limit', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockTimeoutExecute.mockResolvedValue([])
|
||||
mockWithLockedTable.mockImplementation(async (_tableId, mutate) =>
|
||||
mutate(makeTable(), transaction)
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects adding a second TTL column before persistence', async () => {
|
||||
await expect(
|
||||
addTableColumn('table-1', { name: 'another_expiry', type: 'ttl' }, 'request-1')
|
||||
).rejects.toThrow('A table can have at most 1 TTL column')
|
||||
})
|
||||
|
||||
it('rejects retyping another column to TTL before scanning cells', async () => {
|
||||
await expect(
|
||||
updateColumnType({ tableId: 'table-1', columnName: 'name', newType: 'ttl' }, 'request-1')
|
||||
).rejects.toThrow('A table can have at most 1 TTL column')
|
||||
expect(mockTimeoutExecute).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -164,6 +164,21 @@ function formatOffsetSuffix(offsetMinutes: number): string {
|
||||
return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}`
|
||||
}
|
||||
|
||||
/** Formats an instant as canonical wall time in an IANA timezone. */
|
||||
export function formatInstantInTimeZone(date: Date, timeZone: string): string {
|
||||
const wall = getWallClockParts(date, timeZone)
|
||||
const wallAsUtc = Date.UTC(
|
||||
wall.year,
|
||||
wall.month - 1,
|
||||
wall.day,
|
||||
wall.hour,
|
||||
wall.minute,
|
||||
wall.second
|
||||
)
|
||||
const offsetMinutes = Math.round((wallAsUtc - date.getTime()) / 60_000)
|
||||
return `${wall.year}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}${formatOffsetSuffix(offsetMinutes)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Trailing offset (minutes east of UTC) of a datetime string, or null when
|
||||
* naive. Recognizes exactly what `Date.parse` recognizes: numeric offsets,
|
||||
|
||||
@@ -172,6 +172,15 @@ describe('import', () => {
|
||||
)
|
||||
expect(coerceValue('not-a-date', 'date')).toBe('not-a-date')
|
||||
})
|
||||
|
||||
it('coerces TTL imports to epoch seconds and preserves invalid input for row validation', () => {
|
||||
expect(coerceValue('2023-11-14T22:13:20Z', 'ttl')).toBe(1_700_000_000)
|
||||
expect(coerceValue('1700000000', 'ttl')).toBe(1_700_000_000)
|
||||
expect(coerceValue('2023-11-14 17:13:20', 'ttl', { timezone: 'America/New_York' })).toBe(
|
||||
1_700_000_000
|
||||
)
|
||||
expect(coerceValue('not-a-date', 'ttl')).toBe('not-a-date')
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildAutoMapping', () => {
|
||||
|
||||
@@ -39,6 +39,9 @@ SHELL=/bin/sh
|
||||
# Enterprise data drains
|
||||
0 * * * * curl -fsS -m 300 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/run-data-drains"
|
||||
|
||||
# Deletes table rows whose TTL column has expired
|
||||
*/5 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-table-row-ttl"
|
||||
|
||||
# Microsoft Graph subscription renewal (Teams chat triggers expire after ~3 days)
|
||||
0 */12 * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/renew-subscriptions"
|
||||
|
||||
|
||||
@@ -1454,6 +1454,16 @@ cronjobs:
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 1
|
||||
|
||||
# Deletes table rows whose TTL column contains an expired Unix timestamp.
|
||||
cleanupTableRowTtl:
|
||||
enabled: true
|
||||
name: cleanup-table-row-ttl
|
||||
schedule: "*/5 * * * *"
|
||||
path: "/api/cron/cleanup-table-row-ttl"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 1
|
||||
|
||||
# Deletes prebuilt sandbox images that no workspace sandbox references and that
|
||||
# have gone unused past the retention window, from the provider and locally.
|
||||
# A no-op on deployments whose sandbox provider installs at run time.
|
||||
|
||||
@@ -160,6 +160,7 @@ export { TypeCurrency } from './type-currency'
|
||||
export { TypeJson } from './type-json'
|
||||
export { TypeNumber } from './type-number'
|
||||
export { TypeText } from './type-text'
|
||||
export { TypeTtl } from './type-ttl'
|
||||
export { Undo } from './undo'
|
||||
export { Unlink } from './unlink'
|
||||
export { Unlock } from './unlock'
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { SVGProps } from 'react'
|
||||
|
||||
export function TypeTtl(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg
|
||||
width='24'
|
||||
height='24'
|
||||
viewBox='-1.75 -1.5 24 24'
|
||||
fill='none'
|
||||
stroke='currentColor'
|
||||
strokeWidth='1.55'
|
||||
strokeLinecap='round'
|
||||
strokeLinejoin='round'
|
||||
xmlns='http://www.w3.org/2000/svg'
|
||||
aria-hidden='true'
|
||||
{...props}
|
||||
>
|
||||
<circle cx='10.25' cy='10.5' r='7.75' />
|
||||
<path d='M10.25 6V10.5L13.5 12.5' />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
@@ -86,6 +86,7 @@ const INDIRECT_ZOD_ROUTES = new Set([
|
||||
'apps/sim/app/api/settings/allowed-mcp-domains/route.ts',
|
||||
'apps/sim/app/api/cron/cleanup-tasks/route.ts',
|
||||
'apps/sim/app/api/cron/cleanup-soft-deletes/route.ts',
|
||||
'apps/sim/app/api/cron/cleanup-table-row-ttl/route.ts',
|
||||
'apps/sim/app/api/cron/cleanup-stale-executions/route.ts',
|
||||
'apps/sim/app/api/cron/cleanup-sandbox-images/route.ts',
|
||||
'apps/sim/app/api/cron/renew-subscriptions/route.ts',
|
||||
|
||||
Reference in New Issue
Block a user