fix(sanitize): repair malformed subblock states (#4490)

* fix(sanitize): repair malformed subblock states

* address comments

* address comment
This commit is contained in:
Vikhyath Mondreti
2026-05-07 00:54:38 -07:00
committed by GitHub
parent 76d602fed5
commit 09425553b3
8 changed files with 423 additions and 76 deletions
@@ -133,6 +133,69 @@ describe('migrateSubblockIds', () => {
expect(blocks.b1.subBlocks.code.value).toBe('console.log("hi")')
})
it('should repair malformed subBlocks for every block type without deleting values', () => {
const input: Record<string, BlockState> = {
b1: makeBlock({
type: 'function',
subBlocks: {
code: { id: 'code', type: 'unknown', value: 'console.log("hi")' },
language: { value: 'javascript' },
undefined: { type: 'unknown', value: null },
noId: { type: 'short-input', value: 'stale' },
noType: { id: 'noType', value: 'stale' },
unknownType: { id: 'unknownType', type: 'unknown', value: 'preserved' },
notRecord: 'stale',
arrayValue: ['a', 'b'],
} as unknown as BlockState['subBlocks'],
}),
}
const { blocks, migrated } = migrateSubblockIds(input)
expect(migrated).toBe(true)
expect(blocks.b1.subBlocks.code).toEqual({
id: 'code',
type: 'code',
value: 'console.log("hi")',
})
expect(blocks.b1.subBlocks.language).toEqual({
id: 'language',
type: 'dropdown',
value: 'javascript',
})
expect(blocks.b1.subBlocks.undefined).toBeUndefined()
expect(blocks.b1.subBlocks.noId).toBeUndefined()
expect(blocks.b1.subBlocks.noType).toBeUndefined()
expect(blocks.b1.subBlocks.unknownType).toBeUndefined()
expect(blocks.b1.subBlocks.notRecord).toBeUndefined()
expect(blocks.b1.subBlocks.arrayValue).toBeUndefined()
})
it('should preserve malformed legacy subBlocks before renaming them', () => {
const input: Record<string, BlockState> = {
b1: makeBlock({
type: 'knowledge',
subBlocks: {
knowledgeBaseId: {
id: 'knowledgeBaseId',
type: 'unknown',
value: 'kb-uuid-123',
},
},
}),
}
const { blocks, migrated } = migrateSubblockIds(input)
expect(migrated).toBe(true)
expect(blocks.b1.subBlocks.knowledgeBaseId).toBeUndefined()
expect(blocks.b1.subBlocks.knowledgeBaseSelector).toEqual({
id: 'knowledgeBaseSelector',
type: 'knowledge-base-selector',
value: 'kb-uuid-123',
})
})
it('should migrate multiple blocks in one pass', () => {
const input: Record<string, BlockState> = {
b1: makeBlock({
@@ -1,4 +1,7 @@
import { createLogger } from '@sim/logger'
import { DEFAULT_SUBBLOCK_TYPE } from '@sim/workflow-persistence/subblocks'
import { isPlainRecord } from '@/lib/core/utils/records'
import { sanitizeMalformedSubBlocks } from '@/lib/workflows/sanitization/subblocks'
import {
buildCanonicalIndex,
buildSubBlockValues,
@@ -68,6 +71,7 @@ export const SUBBLOCK_ID_MIGRATIONS: Record<string, Record<string, string>> = {
* Returns a new subBlocks record if anything changed, or the original if not.
*/
function migrateBlockSubblockIds(
blockType: string,
subBlocks: Record<string, BlockState['subBlocks'][string]>,
renames: Record<string, string>
): { subBlocks: Record<string, BlockState['subBlocks'][string]>; migrated: boolean } {
@@ -83,6 +87,7 @@ function migrateBlockSubblockIds(
if (!migrated) return { subBlocks, migrated: false }
const result = { ...subBlocks }
const blockConfig = getBlock(blockType)
for (const [oldId, newId] of Object.entries(renames)) {
if (!(oldId in result)) continue
@@ -93,7 +98,24 @@ function migrateBlockSubblockIds(
}
const oldEntry = result[oldId]
result[newId] = { ...oldEntry, id: newId }
const configuredType = blockConfig?.subBlocks?.find((config) => config.id === newId)?.type
result[newId] = isPlainRecord(oldEntry)
? {
...oldEntry,
id: newId,
type:
configuredType ||
(typeof oldEntry.type === 'string' && oldEntry.type.length > 0
? oldEntry.type === 'unknown'
? DEFAULT_SUBBLOCK_TYPE
: oldEntry.type
: DEFAULT_SUBBLOCK_TYPE),
}
: ({
id: newId,
type: configuredType || DEFAULT_SUBBLOCK_TYPE,
value: oldEntry,
} as BlockState['subBlocks'][string])
delete result[oldId]
}
@@ -112,20 +134,28 @@ export function migrateSubblockIds(blocks: Record<string, BlockState>): {
const result: Record<string, BlockState> = {}
for (const [blockId, block] of Object.entries(blocks)) {
const renames = SUBBLOCK_ID_MIGRATIONS[block.type]
if (!renames || !block.subBlocks) {
if (!block.subBlocks) {
result[blockId] = block
continue
}
const { subBlocks, migrated } = migrateBlockSubblockIds(block.subBlocks, renames)
if (migrated) {
logger.info('Migrated legacy subblock IDs', {
blockId: block.id,
blockType: block.type,
})
const renames = SUBBLOCK_ID_MIGRATIONS[block.type]
const renamed = renames
? migrateBlockSubblockIds(block.type, block.subBlocks, renames)
: { subBlocks: block.subBlocks, migrated: false }
const renamedBlock = renamed.migrated ? { ...block, subBlocks: renamed.subBlocks } : block
const sanitized = sanitizeMalformedSubBlocks(renamedBlock)
const blockMigrated = renamed.migrated || sanitized.changed
if (blockMigrated) {
if (renamed.migrated) {
logger.info('Migrated legacy subblock IDs', {
blockId: block.id,
blockType: block.type,
})
}
anyMigrated = true
result[blockId] = { ...block, subBlocks }
result[blockId] = { ...renamedBlock, subBlocks: sanitized.subBlocks }
} else {
result[blockId] = block
}
@@ -1,8 +1,139 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { sanitizePathSegment } from '@/lib/workflows/operations/import-export'
import { describe, expect, it, vi } from 'vitest'
vi.unmock('@/blocks/registry')
import {
extractWorkflowName,
parseWorkflowJson,
sanitizePathSegment,
} from '@/lib/workflows/operations/import-export'
function createLegacyState() {
return {
blocks: {
'start-1': {
id: 'start-1',
type: 'start_trigger',
name: 'Start',
position: { x: 0, y: 0 },
enabled: true,
subBlocks: {
inputFormat: {
id: 'inputFormat',
type: 'input-format',
value: [],
},
undefined: {
type: 'unknown',
value: 'stale duplicate',
},
},
outputs: {},
data: {},
},
},
edges: [],
loops: {},
parallels: {},
variables: {},
metadata: {
name: 'Wrapped Workflow',
color: '#FFBF00',
},
}
}
describe('workflow import/export parsing', () => {
it('parses workflow exports wrapped in an API data envelope', () => {
const content = JSON.stringify({
data: {
version: '1.0',
exportedAt: '2026-05-07T06:45:06.892Z',
workflow: {
name: 'Wrapped Workflow',
},
state: createLegacyState(),
},
})
const result = parseWorkflowJson(content, false)
expect(result.errors).toEqual([])
expect(result.data?.blocks['start-1']).toBeDefined()
expect(result.data?.blocks['start-1'].subBlocks.inputFormat).toEqual({
id: 'inputFormat',
type: 'input-format',
value: [],
})
expect(result.data?.blocks['start-1'].subBlocks.undefined).toBeUndefined()
})
it('extracts workflow names from wrapped exports', () => {
const content = JSON.stringify({
data: {
workflow: {
name: 'Wrapped Workflow',
},
state: createLegacyState(),
},
})
expect(extractWorkflowName(content, 'wf.json')).toBe('Wrapped Workflow')
})
it('parses API envelopes that contain state without an export version', () => {
const content = JSON.stringify({
data: {
workflow: {
name: 'API Workflow',
},
state: createLegacyState(),
},
})
const result = parseWorkflowJson(content, false)
expect(result.errors).toEqual([])
expect(result.data?.blocks['start-1']).toBeDefined()
expect(result.data?.blocks['start-1'].subBlocks.undefined).toBeUndefined()
})
it('preserves malformed legacy renamed subBlocks during import parsing', () => {
const state = {
...createLegacyState(),
blocks: {
knowledge: {
id: 'knowledge',
type: 'knowledge',
name: 'Knowledge',
position: { x: 0, y: 0 },
enabled: true,
subBlocks: {
operation: { id: 'operation', type: 'dropdown', value: 'search' },
knowledgeBaseId: {
id: 'knowledgeBaseId',
type: 'unknown',
value: 'kb-uuid-123',
},
},
outputs: {},
data: {},
},
},
}
const content = JSON.stringify({ data: { workflow: { name: 'Knowledge Workflow' }, state } })
const result = parseWorkflowJson(content, false)
expect(result.errors).toEqual([])
expect(result.data?.blocks.knowledge.subBlocks.knowledgeBaseId).toBeUndefined()
expect(result.data?.blocks.knowledge.subBlocks.knowledgeBaseSelector).toEqual({
id: 'knowledgeBaseSelector',
type: 'knowledge-base-selector',
value: 'kb-uuid-123',
})
})
})
describe('sanitizePathSegment', () => {
it('should preserve ASCII alphanumeric characters', () => {
@@ -9,15 +9,37 @@ import {
type WorkflowStateContractInput,
workflowVariablesContract,
} from '@/lib/api/contracts/workflows'
import { migrateSubblockIds } from '@/lib/workflows/migrations/subblock-migrations'
import {
type ExportWorkflowState,
sanitizeForExport,
} from '@/lib/workflows/sanitization/json-sanitizer'
import { sanitizeMalformedSubBlocks } from '@/lib/workflows/sanitization/subblocks'
import { regenerateWorkflowIds } from '@/stores/workflows/utils'
import type { Variable, WorkflowState } from '@/stores/workflows/workflow/types'
const logger = createLogger('WorkflowImportExport')
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function unwrapWorkflowExportEnvelope(data: unknown): unknown {
if (!isRecord(data)) {
return data
}
const envelopeData = data.data
if (
isRecord(envelopeData) &&
(envelopeData.state || envelopeData.version || envelopeData.workflow)
) {
return envelopeData
}
return data
}
async function getJSZip() {
const { default: JSZip } = await import('jszip')
return JSZip
@@ -338,7 +360,7 @@ export interface WorkspaceImportMetadata {
function extractSortOrder(content: string): number | undefined {
try {
const parsed = JSON.parse(content)
const parsed = unwrapWorkflowExportEnvelope(JSON.parse(content)) as Record<string, any>
return parsed.state?.metadata?.sortOrder ?? parsed.metadata?.sortOrder
} catch {
return undefined
@@ -418,11 +440,15 @@ export async function extractWorkflowsFromFiles(files: File[]): Promise<Imported
export function extractWorkflowName(content: string, filename: string): string {
try {
const parsed = JSON.parse(content)
const parsed = unwrapWorkflowExportEnvelope(JSON.parse(content)) as Record<string, any>
if (parsed.state?.metadata?.name && typeof parsed.state.metadata.name === 'string') {
return parsed.state.metadata.name.trim()
}
if (parsed.workflow?.name && typeof parsed.workflow.name === 'string') {
return parsed.workflow.name.trim()
}
} catch {
// JSON parse failed, fall through to filename
}
@@ -441,63 +467,29 @@ export function extractWorkflowName(content: string, filename: string): string {
}
/**
* Normalize subblock values by converting empty strings to null and filtering out invalid subblocks.
* Normalize subblock values by converting empty strings to null and repairing invalid subblocks.
* This provides backwards compatibility for workflows exported before the null sanitization fix,
* preventing Zod validation errors like "Expected array, received string".
*
* Also filters out malformed subBlocks that may have been created by bugs in previous exports:
* - SubBlocks with key "undefined" (caused by assigning to undefined key)
* - SubBlocks missing required fields like `id`
* - SubBlocks with `type: "unknown"` (indicates malformed data)
* Also filters out subBlocks with the literal key "undefined", which cannot be associated
* with a stable block field.
*/
function normalizeSubblockValues(blocks: Record<string, any>): Record<string, any> {
const { blocks: migratedBlocks } = migrateSubblockIds(blocks)
const normalizedBlocks: Record<string, any> = {}
Object.entries(blocks).forEach(([blockId, block]) => {
Object.entries(migratedBlocks).forEach(([blockId, block]) => {
const normalizedBlock = { ...block }
if (block.subBlocks) {
const normalizedSubBlocks: Record<string, any> = {}
Object.entries(block.subBlocks).forEach(([subBlockId, subBlock]: [string, any]) => {
// Skip subBlocks with invalid keys (literal "undefined" string)
if (subBlockId === 'undefined') {
logger.warn(`Skipping malformed subBlock with key "undefined" in block ${blockId}`)
return
}
// Skip subBlocks that are null or not objects
if (!subBlock || typeof subBlock !== 'object') {
logger.warn(`Skipping invalid subBlock ${subBlockId} in block ${blockId}: not an object`)
return
}
// Skip subBlocks with type "unknown" (malformed data)
if (subBlock.type === 'unknown') {
logger.warn(
`Skipping malformed subBlock ${subBlockId} in block ${blockId}: type is "unknown"`
)
return
}
// Skip subBlocks missing required id field
if (!subBlock.id) {
logger.warn(
`Skipping malformed subBlock ${subBlockId} in block ${blockId}: missing id field`
)
return
}
const normalizedSubBlock = { ...subBlock }
// Convert empty strings to null for consistency
if (normalizedSubBlock.value === '') {
normalizedSubBlock.value = null
}
normalizedSubBlocks[subBlockId] = normalizedSubBlock
})
const { subBlocks: normalizedSubBlocks } = sanitizeMalformedSubBlocks(
{
id: typeof block.id === 'string' ? block.id : blockId,
type: typeof block.type === 'string' ? block.type : '',
subBlocks: block.subBlocks,
},
{ convertEmptyStringToNull: true }
)
normalizedBlock.subBlocks = normalizedSubBlocks
}
@@ -538,10 +530,12 @@ export function parseWorkflowJson(
return { data: null, errors }
}
data = unwrapWorkflowExportEnvelope(data)
// Handle new export format (version/exportedAt/state) or old format (blocks/edges at root)
let workflowData: any
if (data.version && data.state) {
// New format with versioning
if (isRecord(data.state)) {
// Export/API envelope format with workflow state nested under `state`
logger.info('Parsing workflow JSON with version', {
version: data.version,
exportedAt: data.exportedAt,
+8 -5
View File
@@ -158,13 +158,13 @@ const applyBlockMigrations = createMigrationPipeline([
blocks: migrateAgentBlocksToMessagesFormat(ctx.blocks),
}),
async (ctx) => {
const { blocks, migrated } = await migrateCredentialIds(ctx.blocks, ctx.workspaceId)
(ctx) => {
const { blocks, migrated } = migrateSubblockIds(ctx.blocks)
return { ...ctx, blocks, migrated: ctx.migrated || migrated }
},
(ctx) => {
const { blocks, migrated } = migrateSubblockIds(ctx.blocks)
async (ctx) => {
const { blocks, migrated } = await migrateCredentialIds(ctx.blocks, ctx.workspaceId)
return { ...ctx, blocks, migrated: ctx.migrated || migrated }
},
@@ -249,6 +249,7 @@ async function migrateCredentialIds(
for (const block of Object.values(blocks)) {
for (const [subBlockId, subBlock] of Object.entries(block.subBlocks || {})) {
if (!subBlock || typeof subBlock !== 'object') continue
const value = (subBlock as { value?: unknown }).value
if (
CREDENTIAL_SUBBLOCK_IDS.has(subBlockId) &&
@@ -350,7 +351,9 @@ export async function loadWorkflowFromNormalizedTables(
const { blocks: finalBlocks, migrated } = await applyBlockMigrations(raw.blocks, raw.workspaceId)
if (migrated) {
Promise.resolve().then(() => persistMigratedBlocks(workflowId, raw.blocks, finalBlocks))
Promise.resolve().then(() =>
persistMigratedBlocks(workflowId, raw.blocks, finalBlocks, raw.blockUpdatedAt)
)
}
const patchedLoops: Record<string, Loop> = { ...raw.loops }
@@ -0,0 +1,114 @@
import { createLogger } from '@sim/logger'
import { DEFAULT_SUBBLOCK_TYPE } from '@sim/workflow-persistence/subblocks'
import { isPlainRecord } from '@/lib/core/utils/records'
import { getBlock } from '@/blocks'
import type { BlockState } from '@/stores/workflows/workflow/types'
const logger = createLogger('WorkflowSubblockSanitization')
interface SanitizeMalformedSubBlocksOptions {
convertEmptyStringToNull?: boolean
}
interface SanitizableBlock {
id: string
type: string
subBlocks?: Record<string, unknown>
}
/**
* Repairs legacy subBlock metadata when the map key identifies a real field,
* and drops entries that cannot be associated with a stable subBlock.
*/
export function sanitizeMalformedSubBlocks(
block: SanitizableBlock,
options: SanitizeMalformedSubBlocksOptions = {}
): { subBlocks: Record<string, BlockState['subBlocks'][string]>; changed: boolean } {
let changed = false
const blockConfig = getBlock(block.type)
const result: Record<string, BlockState['subBlocks'][string]> = {}
for (const [subBlockId, subBlock] of Object.entries(block.subBlocks || {})) {
if (subBlockId === 'undefined') {
logger.warn('Skipping malformed subBlock with key "undefined"', { blockId: block.id })
changed = true
continue
}
const configuredType = blockConfig?.subBlocks?.find((config) => config.id === subBlockId)?.type
if (!isPlainRecord(subBlock)) {
if (!configuredType) {
logger.warn('Skipping malformed subBlock: unrecognized value entry', {
blockId: block.id,
subBlockId,
})
changed = true
continue
}
logger.warn('Repairing malformed subBlock value', { blockId: block.id, subBlockId })
result[subBlockId] = {
id: subBlockId,
type: configuredType || DEFAULT_SUBBLOCK_TYPE,
value: options.convertEmptyStringToNull && subBlock === '' ? null : subBlock,
} as BlockState['subBlocks'][string]
changed = true
continue
}
if (subBlock.type === 'unknown' && !configuredType) {
logger.warn('Skipping malformed subBlock: type is "unknown"', {
blockId: block.id,
subBlockId,
})
changed = true
continue
}
const id = typeof subBlock.id === 'string' && subBlock.id.length > 0 ? subBlock.id : subBlockId
const typeFromConfig =
configuredType || blockConfig?.subBlocks?.find((config) => config.id === id)?.type
const missingMetadata =
typeof subBlock.id !== 'string' ||
subBlock.id.length === 0 ||
typeof subBlock.type !== 'string' ||
subBlock.type.length === 0
if (missingMetadata && !typeFromConfig) {
logger.warn('Skipping malformed subBlock: unrecognized metadata entry', {
blockId: block.id,
subBlockId,
})
changed = true
continue
}
const type =
typeof subBlock.type === 'string' && subBlock.type.length > 0 && subBlock.type !== 'unknown'
? subBlock.type
: typeFromConfig || DEFAULT_SUBBLOCK_TYPE
const hasValue = Object.hasOwn(subBlock, 'value')
const value =
options.convertEmptyStringToNull && subBlock.value === ''
? null
: hasValue
? subBlock.value
: null
const repairedMetadata = id !== subBlock.id || type !== subBlock.type
const normalizedValue = hasValue && value !== subBlock.value
if (repairedMetadata) {
logger.warn('Repairing malformed subBlock metadata', { blockId: block.id, subBlockId })
changed = true
} else if (normalizedValue) {
logger.warn('Normalizing malformed subBlock value', { blockId: block.id, subBlockId })
changed = true
}
result[subBlockId] = { ...subBlock, id, type, value } as BlockState['subBlocks'][string]
}
return { subBlocks: changed ? result : (block.subBlocks as BlockState['subBlocks']), changed }
}
+1 -2
View File
@@ -1,6 +1,5 @@
{
"lockfileVersion": 1,
"configVersion": 0,
"workspaces": {
"": {
"name": "simstudio",
@@ -4049,7 +4048,7 @@
"ws": ["ws@8.20.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA=="],
"xlsx": ["xlsx@https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", { "bin": { "xlsx": "./bin/xlsx.njs" } }, "sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA=="],
"xlsx": ["xlsx@https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", { "bin": { "xlsx": "./bin/xlsx.njs" } }],
"xml": ["xml@1.0.1", "", {}, "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw=="],
+16 -3
View File
@@ -2,7 +2,7 @@ import { db, workflow, workflowBlocks, workflowEdges, workflowSubflows } from '@
import { createLogger } from '@sim/logger'
import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow'
import { SUBFLOW_TYPES } from '@sim/workflow-types/workflow'
import { eq } from 'drizzle-orm'
import { and, eq } from 'drizzle-orm'
import type { Edge } from 'reactflow'
import type { NormalizedWorkflowData } from './types'
@@ -10,6 +10,7 @@ const logger = createLogger('WorkflowPersistenceLoad')
export interface RawNormalizedWorkflow extends NormalizedWorkflowData {
workspaceId: string
blockUpdatedAt: Record<string, Date>
}
/**
@@ -50,6 +51,7 @@ export async function loadWorkflowFromNormalizedTablesRaw(
}
const blocksMap: Record<string, BlockState> = {}
const blockUpdatedAt: Record<string, Date> = {}
blocks.forEach((block) => {
const blockData = (block.data ?? {}) as BlockState['data']
@@ -73,6 +75,7 @@ export async function loadWorkflowFromNormalizedTablesRaw(
}
blocksMap[block.id] = assembled
blockUpdatedAt[block.id] = block.updatedAt
})
const edgesArray: Edge[] = edges.map((edge) => ({
@@ -151,6 +154,7 @@ export async function loadWorkflowFromNormalizedTablesRaw(
parallels,
isFromNormalizedTables: true,
workspaceId: workflowRow.workspaceId,
blockUpdatedAt,
}
} catch (error) {
logger.error(`Error loading workflow ${workflowId} from normalized tables:`, error)
@@ -161,7 +165,8 @@ export async function loadWorkflowFromNormalizedTablesRaw(
export async function persistMigratedBlocks(
workflowId: string,
originalBlocks: Record<string, BlockState>,
migratedBlocks: Record<string, BlockState>
migratedBlocks: Record<string, BlockState>,
originalBlockUpdatedAt: Record<string, Date> = {}
): Promise<void> {
try {
for (const [blockId, block] of Object.entries(migratedBlocks)) {
@@ -173,7 +178,15 @@ export async function persistMigratedBlocks(
data: block.data,
updatedAt: new Date(),
})
.where(eq(workflowBlocks.id, blockId))
.where(
originalBlockUpdatedAt[blockId]
? and(
eq(workflowBlocks.id, blockId),
eq(workflowBlocks.workflowId, workflowId),
eq(workflowBlocks.updatedAt, originalBlockUpdatedAt[blockId])
)
: and(eq(workflowBlocks.id, blockId), eq(workflowBlocks.workflowId, workflowId))
)
}
}
} catch (err) {