feat(pii): custom user-supplied regex patterns for redaction (#5732)

* feat(pii): custom user-supplied regex patterns for redaction

* fix(pii): enforce custom-regex syntax + safety at the boundary schema

* improvement(pii): always wrap custom-pattern redaction token in angle brackets

* chore(pii): register guardrails_validate in the dev minimal tool registry

* fix(pii): coerce empty guardrails entity-type checkbox (null) so the contract accepts it

* fix(pii): keep detect-all when a custom pattern is added; custom patterns win overlaps
This commit is contained in:
Theodore Li
2026-07-17 12:20:58 -07:00
committed by GitHub
parent caa454aeca
commit 61de0b5808
30 changed files with 971 additions and 80 deletions
+126 -12
View File
@@ -10,7 +10,8 @@ import logging
import time
from typing import Any
from fastapi import FastAPI
import regex as regex_module
from fastapi import FastAPI, HTTPException
from presidio_analyzer import (
AnalyzerEngine,
BatchAnalyzerEngine,
@@ -220,9 +221,11 @@ def _analyze_one(
entities: list[str] | None,
score_threshold: float | None,
return_decision_process: bool = False,
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
):
# Regex-only requests reuse a blank NlpArtifacts to skip the spaCy NLP pass;
# otherwise analyze() computes artifacts (runs spaCy) as usual.
# otherwise analyze() computes artifacts (runs spaCy) as usual. Custom-pattern
# recognizers are regex-based, so they run fine against the blank artifacts.
nlp_artifacts = (
_BLANK_ARTIFACTS.get(language) if _regex_only(entities, score_threshold) else None
)
@@ -233,6 +236,7 @@ def _analyze_one(
score_threshold=score_threshold,
return_decision_process=return_decision_process,
nlp_artifacts=nlp_artifacts,
ad_hoc_recognizers=ad_hoc_recognizers or None,
)
@@ -241,6 +245,7 @@ def _analyze_many(
language: str,
entities: list[str] | None,
score_threshold: float | None,
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
):
"""Analyze many texts, skipping the spaCy pass for regex-only requests."""
if _regex_only(entities, score_threshold):
@@ -252,6 +257,7 @@ def _analyze_many(
entities=entities,
score_threshold=score_threshold,
nlp_artifacts=blank,
ad_hoc_recognizers=ad_hoc_recognizers or None,
)
for text in texts
]
@@ -261,12 +267,92 @@ def _analyze_many(
language=language,
entities=entities or None,
score_threshold=score_threshold,
ad_hoc_recognizers=ad_hoc_recognizers or None,
)
)
app = FastAPI(title="Sim Presidio", docs_url=None, redoc_url=None)
# Internal entity id assigned to the i-th user-supplied custom pattern. Never
# surfaced: the anonymizer maps it back to the pattern's chosen `replacement`, and
# callers relabel any leftover CUSTOM_<i> span to the pattern's display name.
CUSTOM_ENTITY_PREFIX = "CUSTOM_"
class CustomPattern(BaseModel):
"""A user-supplied regex pattern. Matches are replaced with `replacement`,
wrapped in angle brackets (see `_wrap_token`)."""
regex: str
replacement: str = ""
name: str = ""
def _wrap_token(replacement: str) -> str:
"""Wrap the redaction token in angle brackets so custom matches read like the
built-in Presidio tokens (`<PERSON>`, `<EMAIL_ADDRESS>`). A value the user
already bracketed is left as-is so it never double-wraps to `<<X>>`."""
if len(replacement) >= 2 and replacement.startswith("<") and replacement.endswith(">"):
return replacement
return f"<{replacement}>"
def custom_operators(patterns: list[CustomPattern] | None) -> dict[str, dict[str, Any]]:
"""Raw replace-operator per custom pattern, keyed by its internal entity id."""
return {
f"{CUSTOM_ENTITY_PREFIX}{i}": {"type": "replace", "new_value": _wrap_token(p.replacement)}
for i, p in enumerate(patterns or [])
}
def build_custom_recognizers(
patterns: list[CustomPattern] | None, language: str
) -> tuple[list[PatternRecognizer], list[str]]:
"""Ad-hoc PatternRecognizers + their entity ids for the given custom patterns.
Each regex is precompiled so a malformed pattern fails fast as a 400 rather
than surfacing later as an opaque analyze-time 500."""
recognizers: list[PatternRecognizer] = []
entity_ids: list[str] = []
for i, p in enumerate(patterns or []):
try:
regex_module.compile(p.regex)
except regex_module.error as exc:
raise HTTPException(
status_code=400, detail=f"Invalid custom pattern regex: {exc}"
) from exc
entity = f"{CUSTOM_ENTITY_PREFIX}{i}"
recognizers.append(
PatternRecognizer(
supported_entity=entity,
# Score 1.0 so a user's explicit pattern wins any overlap with a
# built-in detector (e.g. spaCy tagging "EMP-123456" as ORGANIZATION
# under detect-all). Presidio resolves overlapping spans by score, so
# the custom replacement — not the built-in token — is applied.
patterns=[Pattern(name=p.name or entity, regex=p.regex, score=1.0)],
supported_language=language,
)
)
entity_ids.append(entity)
return recognizers, entity_ids
def resolve_entities(
req_entities: list[str] | None, custom_entity_ids: list[str]
) -> list[str] | None:
"""Effective entity filter.
`None` means detect-all built-ins (the guardrails "empty selection = detect
everything" convention); the ad-hoc custom recognizers still fire under `None`,
so adding a custom pattern augments detect-all rather than silently disabling
the built-in detectors. An explicit list — including the empty list, which is
the data-retention "only these custom patterns" shape — is used verbatim, with
the custom ids appended."""
if req_entities is None:
return None
return list(req_entities) + custom_entity_ids
class AnalyzeRequest(BaseModel):
text: str
@@ -274,6 +360,7 @@ class AnalyzeRequest(BaseModel):
entities: list[str] | None = None
score_threshold: float | None = None
return_decision_process: bool = False
patterns: list[CustomPattern] | None = None
class AnalyzeBatchRequest(BaseModel):
@@ -281,6 +368,7 @@ class AnalyzeBatchRequest(BaseModel):
language: str = "en"
entities: list[str] | None = None
score_threshold: float | None = None
patterns: list[CustomPattern] | None = None
class AnonymizeRequest(BaseModel):
@@ -288,6 +376,7 @@ class AnonymizeRequest(BaseModel):
analyzer_results: list[dict[str, Any]] = []
anonymizers: dict[str, dict[str, Any]] | None = None
operators: dict[str, dict[str, Any]] | None = None
patterns: list[CustomPattern] | None = None
class AnonymizeBatchItem(BaseModel):
@@ -299,6 +388,7 @@ class AnonymizeBatchRequest(BaseModel):
items: list[AnonymizeBatchItem] = []
anonymizers: dict[str, dict[str, Any]] | None = None
operators: dict[str, dict[str, Any]] | None = None
patterns: list[CustomPattern] | None = None
class RedactRequest(BaseModel):
@@ -308,6 +398,7 @@ class RedactRequest(BaseModel):
score_threshold: float | None = None
anonymizers: dict[str, dict[str, Any]] | None = None
operators: dict[str, dict[str, Any]] | None = None
patterns: list[CustomPattern] | None = None
class RedactBatchRequest(BaseModel):
@@ -317,6 +408,7 @@ class RedactBatchRequest(BaseModel):
score_threshold: float | None = None
anonymizers: dict[str, dict[str, Any]] | None = None
operators: dict[str, dict[str, Any]] | None = None
patterns: list[CustomPattern] | None = None
def build_operators(
@@ -332,6 +424,17 @@ def build_operators(
return operators
def resolve_operators(
anonymizers: dict[str, dict[str, Any]] | None,
operators: dict[str, dict[str, Any]] | None,
patterns: list[CustomPattern] | None,
) -> dict[str, OperatorConfig] | None:
"""Merge the caller's operators with the per-custom-pattern replace operators."""
raw = dict(anonymizers or operators or {})
raw.update(custom_operators(patterns))
return build_operators(raw)
def run_anonymize(
text: str,
raw_results: list[dict[str, Any]],
@@ -366,12 +469,15 @@ def supported_entities(language: str = "en") -> list[str]:
@app.post("/analyze")
def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]:
started = time.perf_counter()
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
entities = resolve_entities(req.entities, custom_ids)
results = _analyze_one(
req.text,
req.language,
req.entities,
entities,
req.score_threshold,
req.return_decision_process,
recognizers,
)
logger.info(
"analyze lang=%s chars=%d entities=%d duration_ms=%.1f",
@@ -387,14 +493,16 @@ def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]:
def analyze_batch(req: AnalyzeBatchRequest) -> list[list[dict[str, Any]]]:
"""Analyze many texts in one pass (spaCy nlp.pipe), returning one span list
per input in request order — the batched counterpart to /analyze."""
results = _analyze_many(req.texts, req.language, req.entities, req.score_threshold)
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
entities = resolve_entities(req.entities, custom_ids)
results = _analyze_many(req.texts, req.language, entities, req.score_threshold, recognizers)
return [[r.to_dict() for r in per_text] for per_text in results]
@app.post("/anonymize")
def anonymize(req: AnonymizeRequest) -> dict[str, Any]:
started = time.perf_counter()
operators = build_operators(req.anonymizers or req.operators)
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
result = run_anonymize(req.text, req.analyzer_results, operators)
logger.info(
"anonymize chars=%d spans=%d duration_ms=%.1f",
@@ -422,7 +530,7 @@ def anonymize_batch(req: AnonymizeBatchRequest) -> dict[str, list[str]]:
"""Mask many texts in one pass, returning masked text per item in request
order — the batched counterpart to /anonymize. Anonymization is pure string
work (no NLP), so callers should send only items with detected spans."""
operators = build_operators(req.anonymizers or req.operators)
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
return {
"texts": [
run_anonymize(item.text, item.analyzer_results, operators).text
@@ -438,8 +546,12 @@ def redact(req: RedactRequest) -> dict[str, str]:
with no detected PII passes through unchanged. The analyzer results feed the
anonymizer directly (no dict round-trip)."""
started = time.perf_counter()
operators = build_operators(req.anonymizers or req.operators)
results = _analyze_one(req.text, req.language, req.entities, req.score_threshold)
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
entities = resolve_entities(req.entities, custom_ids)
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
results = _analyze_one(
req.text, req.language, entities, req.score_threshold, ad_hoc_recognizers=recognizers
)
text = (
req.text
if not results
@@ -466,8 +578,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
the anonymizer directly (no dict round-trip), and anonymization runs only on
texts that actually matched."""
started = time.perf_counter()
operators = build_operators(req.anonymizers or req.operators)
analyzed = _analyze_many(req.texts, req.language, req.entities, req.score_threshold)
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
entities = resolve_entities(req.entities, custom_ids)
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
analyzed = _analyze_many(req.texts, req.language, entities, req.score_threshold, recognizers)
masked: list[str] = []
total_spans = 0
for text, per_text in zip(req.texts, analyzed):
@@ -484,8 +598,8 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
"redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
req.language,
len(req.texts),
len(req.entities) if req.entities else "all",
"skip" if _regex_only(req.entities, req.score_threshold) else "full",
len(entities) if entities else "all",
"skip" if _regex_only(entities, req.score_threshold) else "full",
total_spans,
(time.perf_counter() - started) * 1000,
)
@@ -52,7 +52,12 @@ describe('POST /api/guardrails/mask-batch', () => {
expect(res.status).toBe(200)
const json = await res.json()
expect(json.masked).toEqual(['M(a@b.com)', 'M(hello)'])
expect(mockMaskPIIBatch).toHaveBeenCalledWith(['a@b.com', 'hello'], ['EMAIL_ADDRESS'], 'en')
expect(mockMaskPIIBatch).toHaveBeenCalledWith(
['a@b.com', 'hello'],
['EMAIL_ADDRESS'],
'en',
undefined
)
})
it('rejects an invalid body with 400', async () => {
@@ -25,11 +25,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const parsed = await parseRequest(guardrailsMaskBatchContract, request, {})
if (!parsed.success) return parsed.response
const { texts, entityTypes, language } = parsed.data.body
const { texts, entityTypes, language, customPatterns } = parsed.data.body
try {
const startedAt = performance.now()
const masked = await maskPIIBatch(texts, entityTypes, language)
const masked = await maskPIIBatch(texts, entityTypes, language, customPatterns)
logger.info('Masked PII batch', {
count: texts.length,
durationMs: Math.round(performance.now() - startedAt),
@@ -16,6 +16,7 @@ import {
import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
import { validateHallucination } from '@/lib/guardrails/validate_hallucination'
import { validateJson } from '@/lib/guardrails/validate_json'
import { validatePII } from '@/lib/guardrails/validate_pii'
@@ -63,6 +64,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
piiEntityTypes,
piiMode,
piiLanguage,
piiCustomPatterns,
} = body
if (!validationType) {
@@ -280,6 +282,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
piiEntityTypes,
piiMode,
piiLanguage,
piiCustomPatterns,
authHeaders,
requestId
)
@@ -392,6 +395,7 @@ async function executeValidation(
piiEntityTypes: string[] | undefined,
piiMode: string | undefined,
piiLanguage: string | undefined,
piiCustomPatterns: CustomPiiPattern[] | undefined,
authHeaders: { cookie?: string; authorization?: string; billingAttribution?: string } | undefined,
requestId: string
): Promise<{
@@ -450,6 +454,7 @@ async function executeValidation(
entityTypes: piiEntityTypes || [], // Empty array = detect all PII types
mode: (piiMode as 'block' | 'mask') || 'block', // Default to block mode
language: piiLanguage || 'en',
customPatterns: piiCustomPatterns,
requestId,
})
}
+15
View File
@@ -214,6 +214,17 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
},
dependsOn: ['validationType'],
},
{
id: 'piiCustomPatterns',
title: 'Custom Patterns',
type: 'table',
columns: ['Name', 'Pattern', 'Replacement'],
condition: {
field: 'validationType',
value: ['pii'],
},
dependsOn: ['validationType'],
},
],
tools: {
access: ['guardrails_validate'],
@@ -260,6 +271,10 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
type: 'string',
description: 'Language for PII detection (default: en)',
},
piiCustomPatterns: {
type: 'json',
description: 'Custom regex patterns to detect and replace (name, pattern, replacement rows)',
},
},
outputs: {
input: {
@@ -0,0 +1,76 @@
/**
* @vitest-environment jsdom
*/
import { act } from 'react'
import { createRoot, type Root } from 'react-dom/client'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { CustomPatternsEditor } from '@/components/pii/custom-patterns-editor'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
let container: HTMLDivElement
let root: Root
beforeEach(() => {
container = document.createElement('div')
document.body.appendChild(container)
root = createRoot(container)
})
afterEach(() => {
act(() => root.unmount())
container.remove()
})
function row(regex: string): CustomPiiPattern {
return { name: 'X', regex, replacement: '<X>' }
}
function renderEditor(patterns: CustomPiiPattern[], onChange: (p: CustomPiiPattern[]) => void) {
act(() => root.render(<CustomPatternsEditor patterns={patterns} onChange={onChange} />))
}
function clickText(text: string) {
const button = [...container.querySelectorAll('button')].find((b) => b.textContent === text)
if (!button) throw new Error(`button "${text}" not found`)
act(() => button.dispatchEvent(new MouseEvent('click', { bubbles: true })))
}
describe('CustomPatternsEditor', () => {
it('renders one input row per pattern with no error for a valid regex', () => {
renderEditor([row('EMP-\\d{6}')], vi.fn())
const values = [...container.querySelectorAll('input')].map((i) => i.value)
expect(values).toContain('EMP-\\d{6}')
expect(container.textContent).not.toMatch(/Invalid regex/)
expect(container.textContent).not.toMatch(/potentially unsafe/)
})
it('shows an inline error for a syntactically invalid regex', () => {
renderEditor([row('(')], vi.fn())
expect(container.textContent).toMatch(/Invalid regex/)
})
it('shows an inline error for a catastrophic-backtracking pattern', () => {
renderEditor([row('(a+)+$')], vi.fn())
expect(container.textContent).toMatch(/potentially unsafe/)
})
it('appends an empty row when "Add pattern" is clicked', () => {
const onChange = vi.fn()
renderEditor([row('a+')], onChange)
clickText('Add pattern')
expect(onChange).toHaveBeenCalledWith([
{ name: 'X', regex: 'a+', replacement: '<X>' },
{ name: '', regex: '', replacement: '' },
])
})
it('removes a row when its remove button is clicked', () => {
const onChange = vi.fn()
renderEditor([row('a+'), row('b+')], onChange)
const remove = container.querySelector(
'button[aria-label="Remove pattern"]'
) as HTMLButtonElement
act(() => remove.dispatchEvent(new MouseEvent('click', { bubbles: true })))
expect(onChange).toHaveBeenCalledWith([{ name: 'X', regex: 'b+', replacement: '<X>' }])
})
})
@@ -0,0 +1,82 @@
'use client'
import { Chip, ChipInput } from '@sim/emcn'
import { Plus, Trash2 } from 'lucide-react'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
import { validateRegexPattern } from '@/lib/guardrails/validate_regex'
/** Matches the `.max(20)` bound on `customPatterns` in the boundary contract. */
const MAX_PATTERNS = 20
interface CustomPatternsEditorProps {
patterns: CustomPiiPattern[]
onChange: (patterns: CustomPiiPattern[]) => void
}
/**
* Editor for user-supplied custom regex patterns. Each row is a name label, the
* regex (validated inline for syntax + catastrophic-backtracking safety), and the
* verbatim replacement token that matches are redacted to. Shared by the Data
* Retention settings and any other PII-policy surface.
*/
export function CustomPatternsEditor({ patterns, onChange }: CustomPatternsEditorProps) {
function updateRow(index: number, patch: Partial<CustomPiiPattern>) {
onChange(patterns.map((pattern, i) => (i === index ? { ...pattern, ...patch } : pattern)))
}
function removeRow(index: number) {
onChange(patterns.filter((_, i) => i !== index))
}
function addRow() {
if (patterns.length >= MAX_PATTERNS) return
onChange([...patterns, { name: '', regex: '', replacement: '' }])
}
return (
<div className='flex flex-col gap-2'>
{patterns.map((pattern, index) => {
const validation = pattern.regex.length > 0 ? validateRegexPattern(pattern.regex) : null
const error = validation && !validation.valid ? validation.error : undefined
return (
<div key={index} className='flex flex-col gap-1'>
<div className='flex items-start gap-2'>
<ChipInput
placeholder='Name'
value={pattern.name}
onChange={(e) => updateRow(index, { name: e.target.value })}
className='w-[26%]'
/>
<ChipInput
placeholder='Pattern (regex)'
value={pattern.regex}
onChange={(e) => updateRow(index, { regex: e.target.value })}
inputClassName='font-mono'
error={Boolean(error)}
className='flex-1'
/>
<ChipInput
placeholder='EMPLOYEE_ID'
value={pattern.replacement}
onChange={(e) => updateRow(index, { replacement: e.target.value })}
className='w-[26%]'
/>
<button
type='button'
aria-label='Remove pattern'
onClick={() => removeRow(index)}
className='flex size-[30px] flex-shrink-0 items-center justify-center rounded-md text-[var(--text-icon)] transition-colors hover-hover:bg-[var(--surface-active)] hover-hover:text-[var(--text-error)]'
>
<Trash2 className='size-[14px]' />
</button>
</div>
{error && <span className='text-[var(--text-error)] text-small'>{error}</span>}
</div>
)
})}
<Chip leftIcon={Plus} onClick={addRow} disabled={patterns.length >= MAX_PATTERNS}>
Add pattern
</Chip>
</div>
)
}
@@ -19,10 +19,12 @@ import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { ArrowRight, Plus } from 'lucide-react'
import { CustomPatternsEditor } from '@/components/pii/custom-patterns-editor'
import type { UpdateOrganizationDataRetentionBody } from '@/lib/api/contracts/organization'
import type { RetentionOverride } from '@/lib/api/contracts/primitives'
import { isBillingEnabled } from '@/lib/core/config/env-flags'
import {
type CustomPiiPattern,
emptyPiiStages,
getEntityGroupsForLanguage,
isEntitySupportedForLanguage,
@@ -35,6 +37,7 @@ import {
type PiiStageKey,
type PiiStagePolicy,
type PiiStages,
sanitizeCustomPatterns,
stripNerEntities,
} from '@/lib/guardrails/pii-entities'
import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail'
@@ -137,15 +140,18 @@ function buildRetentionOverride(workspaceId: string, draft: PolicyDraft): Retent
}
/** Stable serialization of a stage set for dirty-detection. */
function serializeStages(stages: PiiStages): Array<[PiiStageKey, boolean, string[], PIILanguage]> {
function serializeStages(
stages: PiiStages
): Array<[PiiStageKey, boolean, string[], PIILanguage, CustomPiiPattern[]]> {
return PII_STAGES.map((key) => {
const policy = stages[key]
return [key, policy.enabled, [...policy.entityTypes].sort(), policy.language] as [
PiiStageKey,
boolean,
string[],
PIILanguage,
]
return [
key,
policy.enabled,
[...policy.entityTypes].sort(),
policy.language,
policy.customPatterns ?? [],
] as [PiiStageKey, boolean, string[], PIILanguage, CustomPiiPattern[]]
})
}
@@ -161,22 +167,29 @@ function normalizePolicyDraft(draft: PolicyDraft): string {
})
}
/** A stage is "on" iff it has at least one entity type selected. */
/** A stage is "on" iff it has at least one entity type or custom pattern. */
function stageHasContent(policy: PiiStagePolicy): boolean {
return policy.entityTypes.length > 0
return policy.entityTypes.length > 0 || (policy.customPatterns?.length ?? 0) > 0
}
function anyStageHasContent(stages: PiiStages): boolean {
return PII_STAGES.some((key) => stageHasContent(stages[key]))
}
/** Persist-time guarantee that `enabled` mirrors "has entity types" for every stage. */
/** Persist-time guarantee that `enabled` mirrors "has content" for every stage. */
function withSyncedEnabled(stages: PiiStages): PiiStages {
return PII_STAGES.reduce((acc, key) => {
// Block outputs are regex-only — strip any NER before persisting.
const entityTypes =
key === 'blockOutputs' ? stripNerEntities(stages[key].entityTypes) : stages[key].entityTypes
acc[key] = { ...stages[key], entityTypes, enabled: entityTypes.length > 0 }
// Drop half-typed rows (empty regex) so the boundary contract never rejects the save.
const customPatterns = sanitizeCustomPatterns(stages[key].customPatterns)
acc[key] = {
...stages[key],
entityTypes,
customPatterns,
enabled: entityTypes.length > 0 || customPatterns.length > 0,
}
return acc
}, {} as PiiStages)
}
@@ -195,7 +208,8 @@ function stageSummary(stages: PiiStages): string {
}
return PII_STAGES.map((key) => {
const policy = stages[key]
return `${short[key]} ${stageHasContent(policy) ? policy.entityTypes.length : 'off'}`
const count = policy.entityTypes.length + (policy.customPatterns?.length ?? 0)
return `${short[key]} ${stageHasContent(policy) ? count : 'off'}`
}).join(' · ')
}
@@ -343,8 +357,10 @@ function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanel
regexOnly: stageKey === 'blockOutputs',
})
function update(entityTypes: string[], language = value.language) {
onChange({ ...value, language, entityTypes, enabled: entityTypes.length > 0 })
function update(patch: Partial<PiiStagePolicy>) {
const merged = { ...value, ...patch }
const enabled = merged.entityTypes.length > 0 || (merged.customPatterns?.length ?? 0) > 0
onChange({ ...merged, enabled })
}
return (
@@ -363,20 +379,38 @@ function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanel
<EntityCheckboxGrid
groups={groups}
selected={value.entityTypes}
onChange={(entityTypes) => update(entityTypes)}
onChange={(entityTypes) => update({ entityTypes })}
belowSearch={
<div className='flex items-center justify-between gap-3'>
<span className='text-[var(--text-muted)] text-small'>Language</span>
<PiiLanguageSelect
value={value.language}
onChange={(language) =>
update(pruneEntitiesForLanguage(value.entityTypes, language), language)
update({
language,
entityTypes: pruneEntitiesForLanguage(value.entityTypes, language),
})
}
/>
</div>
}
/>
</div>
<div className='flex flex-col gap-2'>
<div className='flex items-center gap-1.5'>
<span className='text-[var(--text-muted)] text-small'>Custom patterns</span>
<Info side='top' align='start'>
Redact anything a regular expression can match (employee ids, internal urls, ticket
numbers). Each match is replaced with its replacement text, wrapped in angle brackets
(e.g. EMPLOYEE_ID &lt;EMPLOYEE_ID&gt;).
</Info>
</div>
<CustomPatternsEditor
patterns={value.customPatterns ?? []}
onChange={(customPatterns) => update({ customPatterns })}
/>
</div>
</div>
)
}
@@ -529,7 +563,10 @@ function PolicyDetail({
[effectiveStage]: {
...draft.piiStages[effectiveStage],
entityTypes: [],
enabled: false,
// Clearing entity types leaves any custom patterns intact,
// so the stage stays enabled while patterns remain.
enabled:
(draft.piiStages[effectiveStage].customPatterns?.length ?? 0) > 0,
},
},
})
@@ -236,6 +236,7 @@ export class BlockExecutor {
const redactionOptions = {
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
language: ctx.piiBlockOutputRedaction.language,
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
onFailure: 'throw' as const,
}
// Tools like the function executor offload large outputs to large-value
@@ -889,6 +890,7 @@ export class BlockExecutor {
fullContent = await redactObjectStrings(fullContent, {
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
language: ctx.piiBlockOutputRedaction.language,
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
onFailure: 'throw',
})
}
+3
View File
@@ -1,6 +1,7 @@
import type { Edge } from 'reactflow'
import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution'
import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
import type { NodeMetadata } from '@/executor/dag/types'
import type {
BlockLog,
@@ -159,6 +160,8 @@ export interface PiiBlockOutputRedaction {
entityTypes: string[]
/** Language whose Presidio recognizers apply. */
language: string
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
}
export interface ContextExtensions {
@@ -134,6 +134,7 @@ export class Memory {
content: await redactObjectStrings(message.content, {
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
language: ctx.piiBlockOutputRedaction.language,
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
onFailure: 'throw',
}),
}
+3 -1
View File
@@ -1,5 +1,5 @@
import { z } from 'zod'
import { unknownRecordSchema } from '@/lib/api/contracts/primitives'
import { customPatternSchema, unknownRecordSchema } from '@/lib/api/contracts/primitives'
import { defineRouteContract } from '@/lib/api/contracts/types'
import { DEFAULT_CODE_LANGUAGE } from '@/lib/execution/languages'
export const guardrailsValidateContract = defineRouteContract({
@@ -26,6 +26,7 @@ export const guardrailsValidateContract = defineRouteContract({
piiEntityTypes: z.array(z.string()).optional(),
piiMode: z.string().optional(),
piiLanguage: z.string().optional(),
piiCustomPatterns: z.array(customPatternSchema).max(20).optional(),
}),
response: {
mode: 'json',
@@ -49,6 +50,7 @@ const guardrailsMaskBatchBodySchema = z.object({
texts: z.array(z.string()).max(100_000),
entityTypes: z.array(z.string().min(1, 'Entity type cannot be empty')).max(200),
language: z.string().min(1).max(20).optional(),
customPatterns: z.array(customPatternSchema).max(20).optional(),
})
const guardrailsMaskBatchResponseSchema = z.object({
@@ -0,0 +1,92 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import {
customPatternSchema,
piiStagePolicySchema,
piiStagesSchema,
} from '@/lib/api/contracts/primitives'
describe('customPatternSchema', () => {
it('accepts a well-formed pattern', () => {
expect(
customPatternSchema.parse({ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' })
).toEqual({ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' })
})
it('rejects an empty regex', () => {
expect(customPatternSchema.safeParse({ name: 'x', regex: '', replacement: '' }).success).toBe(
false
)
})
it('rejects an over-long regex', () => {
expect(
customPatternSchema.safeParse({ name: '', regex: 'a'.repeat(513), replacement: '' }).success
).toBe(false)
})
it('rejects a syntactically invalid regex at the boundary (not just in the editor)', () => {
const parsed = customPatternSchema.safeParse({ name: 'bad', regex: '(', replacement: '' })
expect(parsed.success).toBe(false)
if (!parsed.success) {
expect(parsed.error.issues[0].path).toEqual(['regex'])
}
})
it('rejects a catastrophic-backtracking regex at the boundary', () => {
expect(
customPatternSchema.safeParse({ name: 'evil', regex: '(a+)+$', replacement: '' }).success
).toBe(false)
})
})
describe('piiStagePolicySchema', () => {
it('allows an enabled stage with only custom patterns (no entity types)', () => {
const parsed = piiStagePolicySchema.safeParse({
enabled: true,
entityTypes: [],
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
})
expect(parsed.success).toBe(true)
})
it('rejects an enabled stage with no entity types and no custom patterns', () => {
const parsed = piiStagePolicySchema.safeParse({ enabled: true, entityTypes: [] })
expect(parsed.success).toBe(false)
})
})
describe('piiStagesSchema', () => {
it('keeps custom patterns on blockOutputs while stripping NER, staying enabled', () => {
const parsed = piiStagesSchema.parse({
input: { enabled: false, entityTypes: [] },
blockOutputs: {
enabled: true,
entityTypes: ['PERSON', 'EMAIL_ADDRESS'],
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
},
logs: { enabled: false, entityTypes: [] },
})
expect(parsed.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS'])
expect(parsed.blockOutputs.customPatterns).toEqual([
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
])
expect(parsed.blockOutputs.enabled).toBe(true)
})
it('keeps blockOutputs enabled when only custom patterns survive the NER strip', () => {
const parsed = piiStagesSchema.parse({
input: { enabled: false, entityTypes: [] },
blockOutputs: {
enabled: true,
entityTypes: ['PERSON'],
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
},
logs: { enabled: false, entityTypes: [] },
})
expect(parsed.blockOutputs.entityTypes).toEqual([])
expect(parsed.blockOutputs.enabled).toBe(true)
})
})
+45 -5
View File
@@ -1,5 +1,6 @@
import { z } from 'zod'
import { PII_LANGUAGE_CODES, stripNerEntities } from '@/lib/guardrails/pii-entities'
import { validateRegexPattern } from '@/lib/guardrails/validate_regex'
export const unknownRecordSchema = z.record(z.string(), z.unknown())
@@ -119,6 +120,37 @@ export const userFileSchema = z
* expressible policy, so `enabled: true` with an empty list (which would resolve
* to off and silently skip masking) is rejected at the boundary.
*/
/**
* A user-supplied custom regex pattern. `name` is a label; `regex` is matched
* against text; matches are replaced with `replacement` wrapped in angle brackets
* (`EMPLOYEE_ID` → `<EMPLOYEE_ID>`). Bounds guard the Presidio boundary
* (ReDoS/oversized payloads).
*
* The `regex` is validated for both syntax and catastrophic-backtracking safety
* here at the write boundary — not just in the editor — so an invalid or unsafe
* pattern can never be persisted or reach Presidio (where it would abort the
* batch on a 400, or time out and silently fail open, leaving PII unredacted).
*/
export const customPatternSchema = z.object({
name: z.string().max(100, 'Pattern name is too long'),
regex: z
.string()
.min(1, 'Pattern cannot be empty')
.max(512, 'Pattern is too long')
.superRefine((regex, ctx) => {
const result = validateRegexPattern(regex)
if (!result.valid) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: result.error ?? 'Invalid regex pattern',
})
}
}),
replacement: z.string().max(100, 'Replacement is too long'),
})
export type CustomPiiPattern = z.output<typeof customPatternSchema>
export const piiStagePolicySchema = z
.object({
enabled: z.boolean(),
@@ -126,11 +158,17 @@ export const piiStagePolicySchema = z
entityTypes: z.array(z.string().min(1, 'Entity type cannot be empty')).max(100),
/** Language whose Presidio recognizers apply; defaults to English. */
language: z.enum(PII_LANGUAGE_CODES).optional(),
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns: z.array(customPatternSchema).max(20).optional(),
})
.refine((stage) => !stage.enabled || stage.entityTypes.length > 0, {
message: 'An enabled redaction stage must select at least one entity type.',
path: ['entityTypes'],
})
.refine(
(stage) =>
!stage.enabled || stage.entityTypes.length > 0 || (stage.customPatterns?.length ?? 0) > 0,
{
message: 'An enabled redaction stage must select at least one entity type or custom pattern.',
path: ['entityTypes'],
}
)
export type PiiStagePolicy = z.output<typeof piiStagePolicySchema>
@@ -150,12 +188,14 @@ export const piiStagesSchema = z
})
.transform((stages) => {
const entityTypes = stripNerEntities(stages.blockOutputs.entityTypes)
const customPatterns = stages.blockOutputs.customPatterns ?? []
return {
...stages,
blockOutputs: {
...stages.blockOutputs,
entityTypes,
enabled: stages.blockOutputs.enabled && entityTypes.length > 0,
enabled:
stages.blockOutputs.enabled && (entityTypes.length > 0 || customPatterns.length > 0),
},
}
})
+81 -10
View File
@@ -13,7 +13,7 @@ function settings(rules: PiiRedactionRule[]): DataRetentionSettings {
return { piiRedaction: { rules } }
}
const DISABLED = { enabled: false, entityTypes: [], language: 'en' }
const DISABLED = { enabled: false, entityTypes: [], language: 'en', customPatterns: [] }
describe('resolveEffectivePiiRedaction', () => {
const allRule: PiiRedactionRule = {
@@ -31,7 +31,12 @@ describe('resolveEffectivePiiRedaction', () => {
expect(result).toEqual({
input: DISABLED,
blockOutputs: DISABLED,
logs: { enabled: true, entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'], language: 'en' },
logs: {
enabled: true,
entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'],
language: 'en',
customPatterns: [],
},
})
})
@@ -46,7 +51,7 @@ describe('resolveEffectivePiiRedaction', () => {
expect(result).toEqual({
input: DISABLED,
blockOutputs: DISABLED,
logs: { enabled: true, entityTypes: ['US_SSN'], language: 'en' },
logs: { enabled: true, entityTypes: ['US_SSN'], language: 'en', customPatterns: [] },
})
})
@@ -57,7 +62,12 @@ describe('resolveEffectivePiiRedaction', () => {
]),
workspaceId: 'ws-1',
})
expect(result.logs).toEqual({ enabled: true, entityTypes: ['ES_NIF'], language: 'es' })
expect(result.logs).toEqual({
enabled: true,
entityTypes: ['ES_NIF'],
language: 'es',
customPatterns: [],
})
})
it('falls back to en when a stored language is unsupported/stale', () => {
@@ -67,7 +77,12 @@ describe('resolveEffectivePiiRedaction', () => {
]),
workspaceId: 'ws-1',
})
expect(result.logs).toEqual({ enabled: true, entityTypes: ['EMAIL_ADDRESS'], language: 'en' })
expect(result.logs).toEqual({
enabled: true,
entityTypes: ['EMAIL_ADDRESS'],
language: 'en',
customPatterns: [],
})
})
it('exempts a workspace when its specific flat rule has no entity types', () => {
@@ -102,9 +117,19 @@ describe('resolveEffectivePiiRedaction', () => {
workspaceId: 'ws-1',
})
expect(result).toEqual({
input: { enabled: true, entityTypes: ['PERSON'], language: 'es' },
blockOutputs: { enabled: true, entityTypes: ['EMAIL_ADDRESS'], language: 'en' },
logs: { enabled: true, entityTypes: ['US_SSN', 'PHONE_NUMBER'], language: 'en' },
input: { enabled: true, entityTypes: ['PERSON'], language: 'es', customPatterns: [] },
blockOutputs: {
enabled: true,
entityTypes: ['EMAIL_ADDRESS'],
language: 'en',
customPatterns: [],
},
logs: {
enabled: true,
entityTypes: ['US_SSN', 'PHONE_NUMBER'],
language: 'en',
customPatterns: [],
},
})
})
@@ -125,7 +150,12 @@ describe('resolveEffectivePiiRedaction', () => {
})
expect(result.input).toEqual(DISABLED)
expect(result.blockOutputs).toEqual(DISABLED)
expect(result.logs).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' })
expect(result.logs).toEqual({
enabled: true,
entityTypes: ['PERSON'],
language: 'en',
customPatterns: [],
})
})
it('strips spaCy-NER entities from blockOutputs at resolve time (regex-only)', () => {
@@ -183,10 +213,51 @@ describe('resolveEffectivePiiRedaction', () => {
]),
workspaceId: 'ws-1',
})
expect(result.input).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' })
expect(result.input).toEqual({
enabled: true,
entityTypes: ['PERSON'],
language: 'en',
customPatterns: [],
})
// The all rule's logs entity types are NOT unioned in.
expect(result.logs).toEqual(DISABLED)
})
it('carries custom patterns through each stage (blockOutputs strips NER but keeps them)', () => {
const result = resolveEffectivePiiRedaction({
orgSettings: settings([
{
id: 'r-1',
workspaceId: 'ws-1',
stages: {
input: {
enabled: true,
entityTypes: [],
customPatterns: [{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: '<EMP>' }],
},
blockOutputs: {
enabled: true,
entityTypes: ['PERSON'],
customPatterns: [{ name: 'Tck', regex: 'TCK-\\d+', replacement: '<TCK>' }],
},
logs: stage(false, []),
},
},
]),
workspaceId: 'ws-1',
})
// Input: enabled by custom pattern alone (no entity types).
expect(result.input.enabled).toBe(true)
expect(result.input.customPatterns).toEqual([
{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: '<EMP>' },
])
// Block outputs: NER stripped, but the custom pattern keeps the stage enabled.
expect(result.blockOutputs.entityTypes).toEqual([])
expect(result.blockOutputs.enabled).toBe(true)
expect(result.blockOutputs.customPatterns).toEqual([
{ name: 'Tck', regex: 'TCK-\\d+', replacement: '<TCK>' },
])
})
})
it('is the default when no rule matches and there is no all rule', () => {
+9 -2
View File
@@ -1,7 +1,8 @@
import type { DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema'
import type { CustomPiiPattern, DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema'
import {
coercePiiLanguage,
DEFAULT_PII_LANGUAGE,
sanitizeCustomPatterns,
stripNerEntities,
} from '@/lib/guardrails/pii-entities'
@@ -12,6 +13,8 @@ export interface EffectivePiiStage {
entityTypes: string[]
/** Language whose Presidio recognizers apply when masking. */
language: string
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns: CustomPiiPattern[]
}
/**
@@ -29,6 +32,7 @@ const DISABLED_STAGE: EffectivePiiStage = {
enabled: false,
entityTypes: [],
language: DEFAULT_PII_LANGUAGE,
customPatterns: [],
}
export const DEFAULT_PII_REDACTION: EffectivePiiRedaction = {
@@ -58,11 +62,13 @@ function toEffectiveStage(
const types = opts?.regexOnly
? stripNerEntities(sanitizeEntityTypes(policy?.entityTypes))
: sanitizeEntityTypes(policy?.entityTypes)
if (!policy?.enabled || types.length === 0) return DISABLED_STAGE
const customPatterns = sanitizeCustomPatterns(policy?.customPatterns)
if (!policy?.enabled || (types.length === 0 && customPatterns.length === 0)) return DISABLED_STAGE
return {
enabled: true,
entityTypes: types,
language: coercePiiLanguage(policy.language) ?? DEFAULT_PII_LANGUAGE,
customPatterns,
}
}
@@ -99,6 +105,7 @@ export function resolveEffectivePiiRedaction(params: {
enabled: true,
entityTypes: types,
language: coercePiiLanguage(rule.language) ?? DEFAULT_PII_LANGUAGE,
customPatterns: [],
},
}
}
+7 -4
View File
@@ -4,6 +4,7 @@ import { env } from '@/lib/core/config/env'
import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { getInternalApiBaseUrl } from '@/lib/core/utils/urls'
import { chunkIndicesByBudget } from '@/lib/guardrails/pii-batching'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
/**
* Max in-flight mask-batch requests per call. Each request is a CPU-heavy NER
@@ -34,7 +35,8 @@ const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64
export async function maskPIIBatchViaHttp(
texts: string[],
entityTypes: string[],
language?: string
language?: string,
customPatterns?: CustomPiiPattern[]
): Promise<string[]> {
if (texts.length === 0) return []
@@ -43,7 +45,7 @@ export async function maskPIIBatchViaHttp(
await mapWithConcurrency(chunkIndicesByBudget(texts), CHUNK_CONCURRENCY, async (indices) => {
const chunk = indices.map((i) => texts[i])
const out = await postChunk(url, chunk, entityTypes, language)
const out = await postChunk(url, chunk, entityTypes, language, customPatterns)
if (out.length !== chunk.length) {
throw new Error('PII mask-batch returned an unexpected result')
}
@@ -59,7 +61,8 @@ async function postChunk(
url: string,
texts: string[],
entityTypes: string[],
language: string | undefined
language: string | undefined,
customPatterns: CustomPiiPattern[] | undefined
): Promise<string[]> {
// Mint per request: a single token (5min TTL) can expire mid-batch when a
// large execution fans out into many sequential chunk requests.
@@ -72,7 +75,7 @@ async function postChunk(
'content-type': 'application/json',
authorization: `Bearer ${token}`,
},
body: JSON.stringify({ texts, entityTypes, language }),
body: JSON.stringify({ texts, entityTypes, language, customPatterns }),
})
if (!response.ok) {
@@ -3,9 +3,11 @@
*/
import { describe, expect, it } from 'vitest'
import {
emptyStagePolicy,
getEntityGroupsForLanguage,
NER_PII_ENTITIES,
normalizeRuleStages,
sanitizeCustomPatterns,
stripNerEntities,
} from '@/lib/guardrails/pii-entities'
@@ -88,4 +90,75 @@ describe('normalizeRuleStages', () => {
expect(stages.blockOutputs.entityTypes).toEqual([])
expect(stages.blockOutputs.enabled).toBe(false)
})
it('keeps blockOutputs enabled when custom patterns survive the NER strip', () => {
const stages = normalizeRuleStages({
stages: {
input: { enabled: false, entityTypes: [] },
blockOutputs: {
enabled: true,
entityTypes: ['PERSON'],
customPatterns: [{ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' }],
},
logs: { enabled: false, entityTypes: [] },
},
})
expect(stages.blockOutputs.entityTypes).toEqual([])
expect(stages.blockOutputs.customPatterns).toEqual([
{ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' },
])
expect(stages.blockOutputs.enabled).toBe(true)
})
it('sanitizes stored custom patterns on every stage', () => {
const stages = normalizeRuleStages({
stages: {
input: {
enabled: true,
entityTypes: [],
customPatterns: [
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
// Malformed rows are dropped.
{ name: 'no regex', regex: '', replacement: 'x' } as never,
],
},
blockOutputs: { enabled: false, entityTypes: [] },
logs: { enabled: false, entityTypes: [] },
},
})
expect(stages.input.customPatterns).toEqual([
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
])
expect(stages.input.enabled).toBe(true)
})
})
describe('emptyStagePolicy', () => {
it('starts disabled with no entity types and no custom patterns', () => {
expect(emptyStagePolicy()).toEqual({
enabled: false,
entityTypes: [],
language: 'en',
customPatterns: [],
})
})
})
describe('sanitizeCustomPatterns', () => {
it('drops non-arrays and malformed rows, coercing missing fields', () => {
expect(sanitizeCustomPatterns(undefined)).toEqual([])
expect(sanitizeCustomPatterns('nope')).toEqual([])
expect(
sanitizeCustomPatterns([
{ name: 'A', regex: 'a+', replacement: '<A>' },
{ regex: 'b+' },
{ name: 'no regex', regex: '', replacement: 'x' },
null,
{ name: 'bad', regex: 42 },
])
).toEqual([
{ name: 'A', regex: 'a+', replacement: '<A>' },
{ name: '', regex: 'b+', replacement: '' },
])
})
})
+38 -3
View File
@@ -273,11 +273,25 @@ export function getEntityGroupsForLanguage(language: PIILanguage, opts?: { regex
export const PII_STAGES = ['input', 'blockOutputs', 'logs'] as const
export type PiiStageKey = (typeof PII_STAGES)[number]
/**
* A user-supplied custom regex pattern. Matches are replaced with `replacement`
* wrapped in angle brackets (e.g. `EMPLOYEE_ID` → `<EMPLOYEE_ID>`), mirroring the
* built-in Presidio tokens; the internal entity id is never surfaced. `name` is a
* human label only.
*/
export interface CustomPiiPattern {
name: string
regex: string
replacement: string
}
/** Per-stage redaction policy. `enabled: false` makes the stage a no-op. */
export interface PiiStagePolicy {
enabled: boolean
entityTypes: string[]
language: PIILanguage
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
}
export type PiiStages = Record<PiiStageKey, PiiStagePolicy>
@@ -320,7 +334,24 @@ export const RISKY_PII_ENTITIES: ReadonlySet<PIIEntityType> = new Set<PIIEntityT
/** A fully-disabled stage policy for new drafts. */
export function emptyStagePolicy(): PiiStagePolicy {
return { enabled: false, entityTypes: [], language: DEFAULT_PII_LANGUAGE }
return { enabled: false, entityTypes: [], language: DEFAULT_PII_LANGUAGE, customPatterns: [] }
}
/** Coerce an untrusted value into a clean `CustomPiiPattern[]` (drops malformed rows). */
export function sanitizeCustomPatterns(value: unknown): CustomPiiPattern[] {
if (!Array.isArray(value)) return []
const out: CustomPiiPattern[] = []
for (const raw of value) {
if (!raw || typeof raw !== 'object') continue
const { name, regex, replacement } = raw as Record<string, unknown>
if (typeof regex !== 'string' || regex.length === 0) continue
out.push({
name: typeof name === 'string' ? name : '',
regex,
replacement: typeof replacement === 'string' ? replacement : '',
})
}
return out
}
/** A fully-disabled stage set for new drafts. */
@@ -348,11 +379,13 @@ export function normalizeRuleStages(rule: {
? policy.entityTypes.filter((t): t is string => typeof t === 'string')
: [],
language: coercePiiLanguage(policy?.language) ?? DEFAULT_PII_LANGUAGE,
customPatterns: sanitizeCustomPatterns(policy?.customPatterns),
})
if (rule.stages) {
// Block outputs are regex-only (no spaCy NER) — strip NER from any stored
// rule so hydrated drafts never carry it; a stage left empty becomes disabled.
// rule so hydrated drafts never carry it; a stage with neither regex entities
// nor custom patterns becomes disabled.
const blockOutputs = sanitize(rule.stages.blockOutputs)
const blockOutputsEntities = stripNerEntities(blockOutputs.entityTypes)
return {
@@ -360,7 +393,9 @@ export function normalizeRuleStages(rule: {
blockOutputs: {
...blockOutputs,
entityTypes: blockOutputsEntities,
enabled: blockOutputs.enabled && blockOutputsEntities.length > 0,
enabled:
blockOutputs.enabled &&
(blockOutputsEntities.length > 0 || (blockOutputs.customPatterns?.length ?? 0) > 0),
},
logs: sanitize(rule.stages.logs),
}
+87 -21
View File
@@ -3,9 +3,37 @@ import { getErrorMessage } from '@sim/utils/errors'
import { env } from '@/lib/core/config/env'
import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { chunkIndicesByBudget } from '@/lib/guardrails/pii-batching'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
const logger = createLogger('PIIValidator')
/**
* Entity list for the batch (data-retention) paths, where an empty selection with
* custom patterns means "redact ONLY these custom patterns" (the user unchecked
* every built-in entity). An explicit empty array is sent so the server detects
* only the custom entities, never "all". With neither, `undefined` preserves the
* legacy "detect all" default.
*
* The guardrails single-text path uses the opposite convention — empty selection
* means "detect all" — so it does NOT use this helper (see {@link analyze}).
*/
function resolveBatchEntities(
entityTypes: string[],
patterns?: CustomPiiPattern[]
): string[] | undefined {
if (entityTypes.length > 0) return entityTypes
if ((patterns?.length ?? 0) > 0) return []
return undefined
}
/** Map a detected entity type back to its user-facing custom-pattern name, if it is one. */
function displayEntityType(type: string, patterns?: CustomPiiPattern[]): string {
const match = /^CUSTOM_(\d+)$/.exec(type)
if (!match) return type
const pattern = patterns?.[Number(match[1])]
return pattern?.name || type
}
/**
* Concurrent chunk requests in flight from a single mask-batch call. Each chunk is
* itself a batched service call (spaCy `nlp.pipe` over many strings). Default 4;
@@ -22,6 +50,8 @@ export interface PIIValidationInput {
entityTypes: string[] // e.g., ["PERSON", "EMAIL_ADDRESS", "CREDIT_CARD"]
mode: 'block' | 'mask' // block = fail if PII found, mask = return masked text
language?: string // default: "en"
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
requestId: string
}
@@ -54,15 +84,25 @@ interface AnalyzerSpan {
async function analyze(
text: string,
entityTypes: string[],
language: string
language: string,
patterns?: CustomPiiPattern[]
): Promise<AnalyzerSpan[]> {
// Guardrails convention: an empty selection means "detect all". Sending no
// `entities` keeps that, and the server still runs the custom recognizers under
// detect-all — so a custom pattern augments the built-in detectors, never
// silently replaces them.
const entities = entityTypes.length > 0 ? entityTypes : undefined
// boundary-raw-fetch: internal call to the Presidio analyzer service via PII_URL
const response = await fetch(`${PII_URL}/analyze`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ text, language, ...(entities ? { entities } : {}) }),
body: JSON.stringify({
text,
language,
...(entities ? { entities } : {}),
...(patterns?.length ? { patterns } : {}),
}),
})
if (!response.ok) {
const detail = await response.text().catch(() => '')
@@ -79,15 +119,21 @@ async function analyze(
async function analyzeBatch(
texts: string[],
entityTypes: string[],
language: string
language: string,
patterns?: CustomPiiPattern[]
): Promise<AnalyzerSpan[][]> {
const entities = entityTypes.length > 0 ? entityTypes : undefined
const entities = resolveBatchEntities(entityTypes, patterns)
// boundary-raw-fetch: internal call to the Presidio analyzer service via PII_URL
const response = await fetch(`${PII_URL}/analyze_batch`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ texts, language, ...(entities ? { entities } : {}) }),
body: JSON.stringify({
texts,
language,
...(entities ? { entities } : {}),
...(patterns?.length ? { patterns } : {}),
}),
})
if (!response.ok) {
const detail = await response.text().catch(() => '')
@@ -107,14 +153,17 @@ interface AnonymizeBatchItem {
* items with no spans (those texts pass through unchanged). Returns masked text
* per item, in order. Throws on failure.
*/
async function anonymizeBatch(items: AnonymizeBatchItem[]): Promise<string[]> {
async function anonymizeBatch(
items: AnonymizeBatchItem[],
patterns?: CustomPiiPattern[]
): Promise<string[]> {
if (items.length === 0) return []
// boundary-raw-fetch: internal call to the Presidio anonymizer service via PII_URL
const response = await fetch(`${PII_URL}/anonymize_batch`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ items }),
body: JSON.stringify({ items, ...(patterns?.length ? { patterns } : {}) }),
})
if (!response.ok) {
const detail = await response.text().catch(() => '')
@@ -144,15 +193,21 @@ let combinedRedactAvailable = true
async function redactBatch(
texts: string[],
entityTypes: string[],
language: string
language: string,
patterns?: CustomPiiPattern[]
): Promise<string[] | null> {
const entities = entityTypes.length > 0 ? entityTypes : undefined
const entities = resolveBatchEntities(entityTypes, patterns)
// boundary-raw-fetch: internal call to the Presidio combined redact service via PII_URL
const response = await fetch(`${PII_URL}/redact_batch`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ texts, language, ...(entities ? { entities } : {}) }),
body: JSON.stringify({
texts,
language,
...(entities ? { entities } : {}),
...(patterns?.length ? { patterns } : {}),
}),
})
if (response.status === 404) return null
if (!response.ok) {
@@ -172,14 +227,22 @@ async function redactBatch(
* Mask spans via the Presidio anonymizer service. Omitting `anonymizers` uses the
* default `replace` operator, which yields `<ENTITY_TYPE>`. Throws on failure.
*/
async function anonymize(text: string, spans: AnalyzerSpan[]): Promise<string> {
async function anonymize(
text: string,
spans: AnalyzerSpan[],
patterns?: CustomPiiPattern[]
): Promise<string> {
if (spans.length === 0) return text
// boundary-raw-fetch: internal call to the Presidio anonymizer service via PII_URL
const response = await fetch(`${PII_URL}/anonymize`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ text, analyzer_results: spans }),
body: JSON.stringify({
text,
analyzer_results: spans,
...(patterns?.length ? { patterns } : {}),
}),
})
if (!response.ok) {
const detail = await response.text().catch(() => '')
@@ -196,20 +259,21 @@ async function anonymize(text: string, spans: AnalyzerSpan[]): Promise<string> {
* - mask: passes and returns masked text with PII replaced by `<ENTITY_TYPE>`
*/
export async function validatePII(input: PIIValidationInput): Promise<PIIValidationResult> {
const { text, entityTypes, mode, language = 'en', requestId } = input
const { text, entityTypes, mode, language = 'en', customPatterns, requestId } = input
logger.info(`[${requestId}] Starting PII validation`, {
textLength: text.length,
entityTypes,
mode,
language,
customPatternCount: customPatterns?.length ?? 0,
})
try {
const spans = await analyze(text, entityTypes, language)
const spans = await analyze(text, entityTypes, language, customPatterns)
const detectedEntities: DetectedPIIEntity[] = spans.map((s) => ({
type: s.entity_type,
type: displayEntityType(s.entity_type, customPatterns),
start: s.start,
end: s.end,
score: s.score,
@@ -234,8 +298,9 @@ export async function validatePII(input: PIIValidationInput): Promise<PIIValidat
return { passed: false, error: `PII detected: ${summary}`, detectedEntities }
}
// mask mode: the anonymizer replaces every span with `<ENTITY_TYPE>`.
const maskedText = await anonymize(text, spans)
// mask mode: the anonymizer replaces every span with `<ENTITY_TYPE>` (or the
// pattern's `replacement` for custom-pattern spans).
const maskedText = await anonymize(text, spans, customPatterns)
logger.info(`[${requestId}] PII validation completed`, {
passed: true,
detectedCount: detectedEntities.length,
@@ -266,7 +331,8 @@ export async function validatePII(input: PIIValidationInput): Promise<PIIValidat
export async function maskPIIBatch(
texts: string[],
entityTypes: string[],
language = 'en'
language = 'en',
customPatterns?: CustomPiiPattern[]
): Promise<string[]> {
if (texts.length === 0) return []
@@ -276,7 +342,7 @@ export async function maskPIIBatch(
const chunkTexts = indices.map((i) => texts[i])
if (combinedRedactAvailable) {
const masked = await redactBatch(chunkTexts, entityTypes, language)
const masked = await redactBatch(chunkTexts, entityTypes, language, customPatterns)
if (masked) {
indices.forEach((originalIndex, pos) => {
result[originalIndex] = masked[pos]
@@ -287,7 +353,7 @@ export async function maskPIIBatch(
combinedRedactAvailable = false
}
const spansPerText = await analyzeBatch(chunkTexts, entityTypes, language)
const spansPerText = await analyzeBatch(chunkTexts, entityTypes, language, customPatterns)
// A short/misaligned batch response would silently leave the unmatched
// strings unmasked (fail-open). Throw so the caller applies its fail-safe
@@ -310,7 +376,7 @@ export async function maskPIIBatch(
anonymizePositions.push(pos)
})
const masked = await anonymizeBatch(toAnonymize)
const masked = await anonymizeBatch(toAnonymize, customPatterns)
if (masked.length !== toAnonymize.length) {
throw new Error(
`Presidio anonymize_batch returned ${masked.length} result(s) for ${toAnonymize.length} input(s)`
+29
View File
@@ -8,6 +8,35 @@ export interface ValidationResult {
error?: string
}
/** Result of validating a regex pattern's syntax and safety (independent of any input). */
export interface RegexPatternValidation {
valid: boolean
error?: string
}
/**
* Validate a regex pattern's syntax and safety without matching it against input:
* it must compile (`new RegExp`) and pass `safe-regex2`'s catastrophic-backtracking
* screen. Shared by the custom-pattern editor UI and any pre-flight boundary check.
*/
export function validateRegexPattern(pattern: string): RegexPatternValidation {
if (pattern.length === 0) {
return { valid: false, error: 'Pattern cannot be empty' }
}
try {
new RegExp(pattern)
} catch (error) {
return { valid: false, error: `Invalid regex: ${(error as Error).message}` }
}
if (!safe(pattern)) {
return {
valid: false,
error: 'Pattern rejected: potentially unsafe (catastrophic backtracking)',
}
}
return { valid: true }
}
export function validateRegex(inputStr: string, pattern: string): ValidationResult {
let regex: RegExp
try {
+2
View File
@@ -669,6 +669,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
const working = await redactLargeValueRefs(payload, {
entityTypes: config.entityTypes,
language: config.language,
customPatterns: config.customPatterns,
store: {
workspaceId,
workflowId: storeContext.workflowId ?? undefined,
@@ -680,6 +681,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
return redactPIIFromExecution(working, {
entityTypes: config.entityTypes,
language: config.language,
customPatterns: config.customPatterns,
})
}
@@ -9,6 +9,7 @@ import { isLargeValueRef, type LargeValueRef } from '@/lib/execution/payloads/la
import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
import type { LargeValueStoreContext } from '@/lib/execution/payloads/store'
import { materializeLargeValueRef } from '@/lib/execution/payloads/store'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
import {
PiiRedactionError,
type PiiRedactionFailureMode,
@@ -23,6 +24,8 @@ export interface RedactLargeValueRefsOptions {
/** Presidio entity types to mask. Empty = redact all detected PII. */
entityTypes: string[]
language: string
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
/** Storage scope for materializing and re-storing the masked values. */
store: LargeValueStoreContext
/**
@@ -185,6 +188,7 @@ async function maskAndReStore(
const masked = await redactObjectStrings(nested, {
entityTypes: options.entityTypes,
language: options.language,
customPatterns: options.customPatterns,
onFailure: options.onFailure ?? 'scrub',
})
return compactExecutionPayload(masked, { ...options.store, requireDurable: true })
+9 -1
View File
@@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors'
import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
import { maskPIIBatchViaHttp } from '@/lib/guardrails/mask-client'
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
const logger = createLogger('PiiRedaction')
@@ -23,6 +24,8 @@ export interface PiiRedactionOptions {
/** Presidio entity types to mask. Empty = redact all detected PII. */
entityTypes: string[]
language?: string
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
/** Failure handling. Defaults to `'scrub'`. */
onFailure?: PiiRedactionFailureMode
}
@@ -154,7 +157,12 @@ async function maskCollected(
try {
// Presidio runs only in the app container; the persist + execution paths also
// run in the trigger.dev runtime, so masking always goes over HTTP to the app.
const masked = await maskPIIBatchViaHttp(collected, options.entityTypes, language)
const masked = await maskPIIBatchViaHttp(
collected,
options.entityTypes,
language,
options.customPatterns
)
return { masked, scrubbed: false }
} catch (error) {
logger.error('PII masking failed', {
@@ -693,6 +693,7 @@ async function executeWorkflowCoreImpl(
const inputOpts = {
entityTypes: piiRedaction.input.entityTypes,
language: piiRedaction.input.language,
customPatterns: piiRedaction.input.customPatterns,
onFailure: 'throw' as const,
}
processedInput = await redactLargeValueRefsInValue(processedInput, {
@@ -722,6 +723,7 @@ async function executeWorkflowCoreImpl(
const blockOutputOpts = {
entityTypes: piiRedaction.blockOutputs.entityTypes,
language: piiRedaction.blockOutputs.language,
customPatterns: piiRedaction.blockOutputs.customPatterns,
onFailure: 'throw' as const,
}
const largeRefOpts = {
@@ -0,0 +1,62 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { guardrailsValidateContract } from '@/lib/api/contracts/hotspots'
import { guardrailsValidateTool } from '@/tools/guardrails/validate'
// The block layer serializes an empty checkbox / table subBlock to `null`; the
// tool's body builder must produce a shape the contract accepts (undefined, not null).
const buildBody = (params: Record<string, unknown>) =>
guardrailsValidateTool.request.body?.(params as never) as Record<string, unknown>
describe('guardrailsValidateTool.request.body', () => {
it('coerces a null entity-type checkbox to omitted, and the contract accepts it', () => {
const body = buildBody({ input: 'x', validationType: 'pii', piiEntityTypes: null })
expect(body.piiEntityTypes).toBeUndefined()
expect(guardrailsValidateContract.body.safeParse(body).success).toBe(true)
})
it('passes a real entity-type array through unchanged', () => {
const body = buildBody({
input: 'x',
validationType: 'pii',
piiEntityTypes: ['EMAIL_ADDRESS'],
})
expect(body.piiEntityTypes).toEqual(['EMAIL_ADDRESS'])
})
it('maps custom-pattern table rows to the wire shape and validates against the contract', () => {
const body = buildBody({
input: 'x',
validationType: 'pii',
piiEntityTypes: null,
piiCustomPatterns: [
{ cells: { Name: 'Emp', Pattern: 'EMP-\\d{6}', Replacement: 'EMPLOYEE_ID' } },
],
})
expect(body.piiCustomPatterns).toEqual([
{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: 'EMPLOYEE_ID' },
])
expect(guardrailsValidateContract.body.safeParse(body).success).toBe(true)
})
it('omits custom patterns when the table is empty/null', () => {
const body = buildBody({
input: 'x',
validationType: 'pii',
piiEntityTypes: null,
piiCustomPatterns: null,
})
expect(body.piiCustomPatterns).toBeUndefined()
})
it('regression guard: the contract rejects the raw null the block emits (why we coerce)', () => {
const parsed = guardrailsValidateContract.body.safeParse({
input: 'x',
validationType: 'pii',
piiEntityTypes: null,
})
expect(parsed.success).toBe(false)
})
})
+37 -1
View File
@@ -1,5 +1,14 @@
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
import type { ToolConfig } from '@/tools/types'
/** A row from the `piiCustomPatterns` table subBlock (cells keyed by column header). */
interface CustomPatternRow {
cells?: { Name?: string; Pattern?: string; Replacement?: string }
Name?: string
Pattern?: string
Replacement?: string
}
export interface GuardrailsValidateInput {
input: string
validationType: 'json' | 'regex' | 'hallucination' | 'pii'
@@ -20,12 +29,29 @@ export interface GuardrailsValidateInput {
piiEntityTypes?: string[]
piiMode?: string
piiLanguage?: string
piiCustomPatterns?: CustomPatternRow[]
_context?: {
workflowId?: string
workspaceId?: string
}
}
/** Map the raw table rows into the wire shape, dropping rows with no pattern. */
function toCustomPatterns(rows: CustomPatternRow[] | undefined): CustomPiiPattern[] | undefined {
if (!Array.isArray(rows)) return undefined
const patterns: CustomPiiPattern[] = []
for (const row of rows) {
const regex = (row?.cells?.Pattern ?? row?.Pattern ?? '').trim()
if (!regex) continue
patterns.push({
name: (row?.cells?.Name ?? row?.Name ?? '').trim(),
regex,
replacement: row?.cells?.Replacement ?? row?.Replacement ?? '',
})
}
return patterns.length > 0 ? patterns : undefined
}
export interface GuardrailsValidateOutput {
success: boolean
output: {
@@ -116,6 +142,12 @@ export const guardrailsValidateTool: ToolConfig<GuardrailsValidateInput, Guardra
visibility: 'user-only',
description: 'Language for PII detection (default: en)',
},
piiCustomPatterns: {
type: 'array',
required: false,
visibility: 'user-only',
description: 'Custom regex patterns to detect and replace (name, pattern, replacement)',
},
},
outputs: {
@@ -182,9 +214,13 @@ export const guardrailsValidateTool: ToolConfig<GuardrailsValidateInput, Guardra
bedrockAccessKeyId: params.bedrockAccessKeyId,
bedrockSecretKey: params.bedrockSecretKey,
bedrockRegion: params.bedrockRegion,
piiEntityTypes: params.piiEntityTypes,
// An empty entity-type checkbox serializes to null; the contract's array
// field accepts undefined (omitted), not null — so coerce. This is the
// common shape when only custom patterns are configured.
piiEntityTypes: Array.isArray(params.piiEntityTypes) ? params.piiEntityTypes : undefined,
piiMode: params.piiMode,
piiLanguage: params.piiLanguage,
piiCustomPatterns: toCustomPatterns(params.piiCustomPatterns),
workflowId: params._context?.workflowId,
workspaceId: params._context?.workspaceId,
}),
+2
View File
@@ -25,6 +25,7 @@ import {
gmailUntrashThreadV2Tool,
gmailUpdateLabelV2Tool,
} from '@/tools/gmail'
import { guardrailsValidateTool } from '@/tools/guardrails'
import { httpRequestTool } from '@/tools/http'
import {
slackAddReactionTool,
@@ -85,6 +86,7 @@ import type { ToolConfig } from '@/tools/types'
export const tools: Record<string, ToolConfig> = {
http_request: httpRequestTool,
function_execute: functionExecuteTool,
guardrails_validate: guardrailsValidateTool,
gmail_send_v2: gmailSendV2Tool,
gmail_read_v2: gmailReadV2Tool,
gmail_search_v2: gmailSearchV2Tool,
+8
View File
@@ -46,6 +46,14 @@ USER pii
# 5001 avoids colliding with the app's 3000 in local/compose runs on one host.
EXPOSE 5001
# Per-pattern regex match timeout (Presidio's `regex`-module `finditer(timeout=...)`),
# an interactive backstop against a catastrophic user-supplied custom regex. Presidio
# logs and skips a pattern that exceeds it — the request stays up rather than hanging.
# Set well below the 60s library default so a pathological pattern can't stall a worker.
# MUST be an integer — Presidio parses it with `int()`, so a float (e.g. 1.5) crashes
# the service at import.
ENV REGEX_TIMEOUT_SECONDS=2
# start-period covers the model cold start. With PII_WORKERS>1 each worker loads
# the five spaCy models independently and in parallel, so allow generous headroom
# (memory-bandwidth contention stretches the wall-time beyond the single-worker case).
+9
View File
@@ -1153,6 +1153,13 @@ export const chat = pgTable(
}
)
/** A user-supplied custom regex pattern; matches are replaced verbatim with `replacement`. */
export interface CustomPiiPattern {
name: string
regex: string
replacement: string
}
/** Per-stage PII redaction policy stored on a {@link PiiRedactionRule}. */
export interface PiiStagePolicy {
enabled: boolean
@@ -1160,6 +1167,8 @@ export interface PiiStagePolicy {
entityTypes: string[]
/** Language whose Presidio recognizers apply (e.g. 'en', 'es'); defaults to English. */
language?: string
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
customPatterns?: CustomPiiPattern[]
}
/**