mirror of
https://github.com/simstudioai/sim.git
synced 2026-08-30 17:05:18 +08:00
feat(pii): custom user-supplied regex patterns for redaction (#5732)
* feat(pii): custom user-supplied regex patterns for redaction * fix(pii): enforce custom-regex syntax + safety at the boundary schema * improvement(pii): always wrap custom-pattern redaction token in angle brackets * chore(pii): register guardrails_validate in the dev minimal tool registry * fix(pii): coerce empty guardrails entity-type checkbox (null) so the contract accepts it * fix(pii): keep detect-all when a custom pattern is added; custom patterns win overlaps
This commit is contained in:
+126
-12
@@ -10,7 +10,8 @@ import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
import regex as regex_module
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from presidio_analyzer import (
|
||||
AnalyzerEngine,
|
||||
BatchAnalyzerEngine,
|
||||
@@ -220,9 +221,11 @@ def _analyze_one(
|
||||
entities: list[str] | None,
|
||||
score_threshold: float | None,
|
||||
return_decision_process: bool = False,
|
||||
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
|
||||
):
|
||||
# Regex-only requests reuse a blank NlpArtifacts to skip the spaCy NLP pass;
|
||||
# otherwise analyze() computes artifacts (runs spaCy) as usual.
|
||||
# otherwise analyze() computes artifacts (runs spaCy) as usual. Custom-pattern
|
||||
# recognizers are regex-based, so they run fine against the blank artifacts.
|
||||
nlp_artifacts = (
|
||||
_BLANK_ARTIFACTS.get(language) if _regex_only(entities, score_threshold) else None
|
||||
)
|
||||
@@ -233,6 +236,7 @@ def _analyze_one(
|
||||
score_threshold=score_threshold,
|
||||
return_decision_process=return_decision_process,
|
||||
nlp_artifacts=nlp_artifacts,
|
||||
ad_hoc_recognizers=ad_hoc_recognizers or None,
|
||||
)
|
||||
|
||||
|
||||
@@ -241,6 +245,7 @@ def _analyze_many(
|
||||
language: str,
|
||||
entities: list[str] | None,
|
||||
score_threshold: float | None,
|
||||
ad_hoc_recognizers: list[PatternRecognizer] | None = None,
|
||||
):
|
||||
"""Analyze many texts, skipping the spaCy pass for regex-only requests."""
|
||||
if _regex_only(entities, score_threshold):
|
||||
@@ -252,6 +257,7 @@ def _analyze_many(
|
||||
entities=entities,
|
||||
score_threshold=score_threshold,
|
||||
nlp_artifacts=blank,
|
||||
ad_hoc_recognizers=ad_hoc_recognizers or None,
|
||||
)
|
||||
for text in texts
|
||||
]
|
||||
@@ -261,12 +267,92 @@ def _analyze_many(
|
||||
language=language,
|
||||
entities=entities or None,
|
||||
score_threshold=score_threshold,
|
||||
ad_hoc_recognizers=ad_hoc_recognizers or None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
app = FastAPI(title="Sim Presidio", docs_url=None, redoc_url=None)
|
||||
|
||||
# Internal entity id assigned to the i-th user-supplied custom pattern. Never
|
||||
# surfaced: the anonymizer maps it back to the pattern's chosen `replacement`, and
|
||||
# callers relabel any leftover CUSTOM_<i> span to the pattern's display name.
|
||||
CUSTOM_ENTITY_PREFIX = "CUSTOM_"
|
||||
|
||||
|
||||
class CustomPattern(BaseModel):
|
||||
"""A user-supplied regex pattern. Matches are replaced with `replacement`,
|
||||
wrapped in angle brackets (see `_wrap_token`)."""
|
||||
|
||||
regex: str
|
||||
replacement: str = ""
|
||||
name: str = ""
|
||||
|
||||
|
||||
def _wrap_token(replacement: str) -> str:
|
||||
"""Wrap the redaction token in angle brackets so custom matches read like the
|
||||
built-in Presidio tokens (`<PERSON>`, `<EMAIL_ADDRESS>`). A value the user
|
||||
already bracketed is left as-is so it never double-wraps to `<<X>>`."""
|
||||
if len(replacement) >= 2 and replacement.startswith("<") and replacement.endswith(">"):
|
||||
return replacement
|
||||
return f"<{replacement}>"
|
||||
|
||||
|
||||
def custom_operators(patterns: list[CustomPattern] | None) -> dict[str, dict[str, Any]]:
|
||||
"""Raw replace-operator per custom pattern, keyed by its internal entity id."""
|
||||
return {
|
||||
f"{CUSTOM_ENTITY_PREFIX}{i}": {"type": "replace", "new_value": _wrap_token(p.replacement)}
|
||||
for i, p in enumerate(patterns or [])
|
||||
}
|
||||
|
||||
|
||||
def build_custom_recognizers(
|
||||
patterns: list[CustomPattern] | None, language: str
|
||||
) -> tuple[list[PatternRecognizer], list[str]]:
|
||||
"""Ad-hoc PatternRecognizers + their entity ids for the given custom patterns.
|
||||
|
||||
Each regex is precompiled so a malformed pattern fails fast as a 400 rather
|
||||
than surfacing later as an opaque analyze-time 500."""
|
||||
recognizers: list[PatternRecognizer] = []
|
||||
entity_ids: list[str] = []
|
||||
for i, p in enumerate(patterns or []):
|
||||
try:
|
||||
regex_module.compile(p.regex)
|
||||
except regex_module.error as exc:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Invalid custom pattern regex: {exc}"
|
||||
) from exc
|
||||
entity = f"{CUSTOM_ENTITY_PREFIX}{i}"
|
||||
recognizers.append(
|
||||
PatternRecognizer(
|
||||
supported_entity=entity,
|
||||
# Score 1.0 so a user's explicit pattern wins any overlap with a
|
||||
# built-in detector (e.g. spaCy tagging "EMP-123456" as ORGANIZATION
|
||||
# under detect-all). Presidio resolves overlapping spans by score, so
|
||||
# the custom replacement — not the built-in token — is applied.
|
||||
patterns=[Pattern(name=p.name or entity, regex=p.regex, score=1.0)],
|
||||
supported_language=language,
|
||||
)
|
||||
)
|
||||
entity_ids.append(entity)
|
||||
return recognizers, entity_ids
|
||||
|
||||
|
||||
def resolve_entities(
|
||||
req_entities: list[str] | None, custom_entity_ids: list[str]
|
||||
) -> list[str] | None:
|
||||
"""Effective entity filter.
|
||||
|
||||
`None` means detect-all built-ins (the guardrails "empty selection = detect
|
||||
everything" convention); the ad-hoc custom recognizers still fire under `None`,
|
||||
so adding a custom pattern augments detect-all rather than silently disabling
|
||||
the built-in detectors. An explicit list — including the empty list, which is
|
||||
the data-retention "only these custom patterns" shape — is used verbatim, with
|
||||
the custom ids appended."""
|
||||
if req_entities is None:
|
||||
return None
|
||||
return list(req_entities) + custom_entity_ids
|
||||
|
||||
|
||||
class AnalyzeRequest(BaseModel):
|
||||
text: str
|
||||
@@ -274,6 +360,7 @@ class AnalyzeRequest(BaseModel):
|
||||
entities: list[str] | None = None
|
||||
score_threshold: float | None = None
|
||||
return_decision_process: bool = False
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
class AnalyzeBatchRequest(BaseModel):
|
||||
@@ -281,6 +368,7 @@ class AnalyzeBatchRequest(BaseModel):
|
||||
language: str = "en"
|
||||
entities: list[str] | None = None
|
||||
score_threshold: float | None = None
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
class AnonymizeRequest(BaseModel):
|
||||
@@ -288,6 +376,7 @@ class AnonymizeRequest(BaseModel):
|
||||
analyzer_results: list[dict[str, Any]] = []
|
||||
anonymizers: dict[str, dict[str, Any]] | None = None
|
||||
operators: dict[str, dict[str, Any]] | None = None
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
class AnonymizeBatchItem(BaseModel):
|
||||
@@ -299,6 +388,7 @@ class AnonymizeBatchRequest(BaseModel):
|
||||
items: list[AnonymizeBatchItem] = []
|
||||
anonymizers: dict[str, dict[str, Any]] | None = None
|
||||
operators: dict[str, dict[str, Any]] | None = None
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
class RedactRequest(BaseModel):
|
||||
@@ -308,6 +398,7 @@ class RedactRequest(BaseModel):
|
||||
score_threshold: float | None = None
|
||||
anonymizers: dict[str, dict[str, Any]] | None = None
|
||||
operators: dict[str, dict[str, Any]] | None = None
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
class RedactBatchRequest(BaseModel):
|
||||
@@ -317,6 +408,7 @@ class RedactBatchRequest(BaseModel):
|
||||
score_threshold: float | None = None
|
||||
anonymizers: dict[str, dict[str, Any]] | None = None
|
||||
operators: dict[str, dict[str, Any]] | None = None
|
||||
patterns: list[CustomPattern] | None = None
|
||||
|
||||
|
||||
def build_operators(
|
||||
@@ -332,6 +424,17 @@ def build_operators(
|
||||
return operators
|
||||
|
||||
|
||||
def resolve_operators(
|
||||
anonymizers: dict[str, dict[str, Any]] | None,
|
||||
operators: dict[str, dict[str, Any]] | None,
|
||||
patterns: list[CustomPattern] | None,
|
||||
) -> dict[str, OperatorConfig] | None:
|
||||
"""Merge the caller's operators with the per-custom-pattern replace operators."""
|
||||
raw = dict(anonymizers or operators or {})
|
||||
raw.update(custom_operators(patterns))
|
||||
return build_operators(raw)
|
||||
|
||||
|
||||
def run_anonymize(
|
||||
text: str,
|
||||
raw_results: list[dict[str, Any]],
|
||||
@@ -366,12 +469,15 @@ def supported_entities(language: str = "en") -> list[str]:
|
||||
@app.post("/analyze")
|
||||
def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]:
|
||||
started = time.perf_counter()
|
||||
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
|
||||
entities = resolve_entities(req.entities, custom_ids)
|
||||
results = _analyze_one(
|
||||
req.text,
|
||||
req.language,
|
||||
req.entities,
|
||||
entities,
|
||||
req.score_threshold,
|
||||
req.return_decision_process,
|
||||
recognizers,
|
||||
)
|
||||
logger.info(
|
||||
"analyze lang=%s chars=%d entities=%d duration_ms=%.1f",
|
||||
@@ -387,14 +493,16 @@ def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]:
|
||||
def analyze_batch(req: AnalyzeBatchRequest) -> list[list[dict[str, Any]]]:
|
||||
"""Analyze many texts in one pass (spaCy nlp.pipe), returning one span list
|
||||
per input in request order — the batched counterpart to /analyze."""
|
||||
results = _analyze_many(req.texts, req.language, req.entities, req.score_threshold)
|
||||
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
|
||||
entities = resolve_entities(req.entities, custom_ids)
|
||||
results = _analyze_many(req.texts, req.language, entities, req.score_threshold, recognizers)
|
||||
return [[r.to_dict() for r in per_text] for per_text in results]
|
||||
|
||||
|
||||
@app.post("/anonymize")
|
||||
def anonymize(req: AnonymizeRequest) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
operators = build_operators(req.anonymizers or req.operators)
|
||||
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
|
||||
result = run_anonymize(req.text, req.analyzer_results, operators)
|
||||
logger.info(
|
||||
"anonymize chars=%d spans=%d duration_ms=%.1f",
|
||||
@@ -422,7 +530,7 @@ def anonymize_batch(req: AnonymizeBatchRequest) -> dict[str, list[str]]:
|
||||
"""Mask many texts in one pass, returning masked text per item in request
|
||||
order — the batched counterpart to /anonymize. Anonymization is pure string
|
||||
work (no NLP), so callers should send only items with detected spans."""
|
||||
operators = build_operators(req.anonymizers or req.operators)
|
||||
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
|
||||
return {
|
||||
"texts": [
|
||||
run_anonymize(item.text, item.analyzer_results, operators).text
|
||||
@@ -438,8 +546,12 @@ def redact(req: RedactRequest) -> dict[str, str]:
|
||||
with no detected PII passes through unchanged. The analyzer results feed the
|
||||
anonymizer directly (no dict round-trip)."""
|
||||
started = time.perf_counter()
|
||||
operators = build_operators(req.anonymizers or req.operators)
|
||||
results = _analyze_one(req.text, req.language, req.entities, req.score_threshold)
|
||||
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
|
||||
entities = resolve_entities(req.entities, custom_ids)
|
||||
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
|
||||
results = _analyze_one(
|
||||
req.text, req.language, entities, req.score_threshold, ad_hoc_recognizers=recognizers
|
||||
)
|
||||
text = (
|
||||
req.text
|
||||
if not results
|
||||
@@ -466,8 +578,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
|
||||
the anonymizer directly (no dict round-trip), and anonymization runs only on
|
||||
texts that actually matched."""
|
||||
started = time.perf_counter()
|
||||
operators = build_operators(req.anonymizers or req.operators)
|
||||
analyzed = _analyze_many(req.texts, req.language, req.entities, req.score_threshold)
|
||||
recognizers, custom_ids = build_custom_recognizers(req.patterns, req.language)
|
||||
entities = resolve_entities(req.entities, custom_ids)
|
||||
operators = resolve_operators(req.anonymizers, req.operators, req.patterns)
|
||||
analyzed = _analyze_many(req.texts, req.language, entities, req.score_threshold, recognizers)
|
||||
masked: list[str] = []
|
||||
total_spans = 0
|
||||
for text, per_text in zip(req.texts, analyzed):
|
||||
@@ -484,8 +598,8 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
|
||||
"redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
|
||||
req.language,
|
||||
len(req.texts),
|
||||
len(req.entities) if req.entities else "all",
|
||||
"skip" if _regex_only(req.entities, req.score_threshold) else "full",
|
||||
len(entities) if entities else "all",
|
||||
"skip" if _regex_only(entities, req.score_threshold) else "full",
|
||||
total_spans,
|
||||
(time.perf_counter() - started) * 1000,
|
||||
)
|
||||
|
||||
@@ -52,7 +52,12 @@ describe('POST /api/guardrails/mask-batch', () => {
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json.masked).toEqual(['M(a@b.com)', 'M(hello)'])
|
||||
expect(mockMaskPIIBatch).toHaveBeenCalledWith(['a@b.com', 'hello'], ['EMAIL_ADDRESS'], 'en')
|
||||
expect(mockMaskPIIBatch).toHaveBeenCalledWith(
|
||||
['a@b.com', 'hello'],
|
||||
['EMAIL_ADDRESS'],
|
||||
'en',
|
||||
undefined
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an invalid body with 400', async () => {
|
||||
|
||||
@@ -25,11 +25,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const parsed = await parseRequest(guardrailsMaskBatchContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { texts, entityTypes, language } = parsed.data.body
|
||||
const { texts, entityTypes, language, customPatterns } = parsed.data.body
|
||||
|
||||
try {
|
||||
const startedAt = performance.now()
|
||||
const masked = await maskPIIBatch(texts, entityTypes, language)
|
||||
const masked = await maskPIIBatch(texts, entityTypes, language, customPatterns)
|
||||
logger.info('Masked PII batch', {
|
||||
count: texts.length,
|
||||
durationMs: Math.round(performance.now() - startedAt),
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
import { validateHallucination } from '@/lib/guardrails/validate_hallucination'
|
||||
import { validateJson } from '@/lib/guardrails/validate_json'
|
||||
import { validatePII } from '@/lib/guardrails/validate_pii'
|
||||
@@ -63,6 +64,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
piiEntityTypes,
|
||||
piiMode,
|
||||
piiLanguage,
|
||||
piiCustomPatterns,
|
||||
} = body
|
||||
|
||||
if (!validationType) {
|
||||
@@ -280,6 +282,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
piiEntityTypes,
|
||||
piiMode,
|
||||
piiLanguage,
|
||||
piiCustomPatterns,
|
||||
authHeaders,
|
||||
requestId
|
||||
)
|
||||
@@ -392,6 +395,7 @@ async function executeValidation(
|
||||
piiEntityTypes: string[] | undefined,
|
||||
piiMode: string | undefined,
|
||||
piiLanguage: string | undefined,
|
||||
piiCustomPatterns: CustomPiiPattern[] | undefined,
|
||||
authHeaders: { cookie?: string; authorization?: string; billingAttribution?: string } | undefined,
|
||||
requestId: string
|
||||
): Promise<{
|
||||
@@ -450,6 +454,7 @@ async function executeValidation(
|
||||
entityTypes: piiEntityTypes || [], // Empty array = detect all PII types
|
||||
mode: (piiMode as 'block' | 'mask') || 'block', // Default to block mode
|
||||
language: piiLanguage || 'en',
|
||||
customPatterns: piiCustomPatterns,
|
||||
requestId,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -214,6 +214,17 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
|
||||
},
|
||||
dependsOn: ['validationType'],
|
||||
},
|
||||
{
|
||||
id: 'piiCustomPatterns',
|
||||
title: 'Custom Patterns',
|
||||
type: 'table',
|
||||
columns: ['Name', 'Pattern', 'Replacement'],
|
||||
condition: {
|
||||
field: 'validationType',
|
||||
value: ['pii'],
|
||||
},
|
||||
dependsOn: ['validationType'],
|
||||
},
|
||||
],
|
||||
tools: {
|
||||
access: ['guardrails_validate'],
|
||||
@@ -260,6 +271,10 @@ Return ONLY the regex pattern - no explanations, no quotes, no forward slashes,
|
||||
type: 'string',
|
||||
description: 'Language for PII detection (default: en)',
|
||||
},
|
||||
piiCustomPatterns: {
|
||||
type: 'json',
|
||||
description: 'Custom regex patterns to detect and replace (name, pattern, replacement rows)',
|
||||
},
|
||||
},
|
||||
outputs: {
|
||||
input: {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @vitest-environment jsdom
|
||||
*/
|
||||
import { act } from 'react'
|
||||
import { createRoot, type Root } from 'react-dom/client'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { CustomPatternsEditor } from '@/components/pii/custom-patterns-editor'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
|
||||
let container: HTMLDivElement
|
||||
let root: Root
|
||||
|
||||
beforeEach(() => {
|
||||
container = document.createElement('div')
|
||||
document.body.appendChild(container)
|
||||
root = createRoot(container)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
act(() => root.unmount())
|
||||
container.remove()
|
||||
})
|
||||
|
||||
function row(regex: string): CustomPiiPattern {
|
||||
return { name: 'X', regex, replacement: '<X>' }
|
||||
}
|
||||
|
||||
function renderEditor(patterns: CustomPiiPattern[], onChange: (p: CustomPiiPattern[]) => void) {
|
||||
act(() => root.render(<CustomPatternsEditor patterns={patterns} onChange={onChange} />))
|
||||
}
|
||||
|
||||
function clickText(text: string) {
|
||||
const button = [...container.querySelectorAll('button')].find((b) => b.textContent === text)
|
||||
if (!button) throw new Error(`button "${text}" not found`)
|
||||
act(() => button.dispatchEvent(new MouseEvent('click', { bubbles: true })))
|
||||
}
|
||||
|
||||
describe('CustomPatternsEditor', () => {
|
||||
it('renders one input row per pattern with no error for a valid regex', () => {
|
||||
renderEditor([row('EMP-\\d{6}')], vi.fn())
|
||||
const values = [...container.querySelectorAll('input')].map((i) => i.value)
|
||||
expect(values).toContain('EMP-\\d{6}')
|
||||
expect(container.textContent).not.toMatch(/Invalid regex/)
|
||||
expect(container.textContent).not.toMatch(/potentially unsafe/)
|
||||
})
|
||||
|
||||
it('shows an inline error for a syntactically invalid regex', () => {
|
||||
renderEditor([row('(')], vi.fn())
|
||||
expect(container.textContent).toMatch(/Invalid regex/)
|
||||
})
|
||||
|
||||
it('shows an inline error for a catastrophic-backtracking pattern', () => {
|
||||
renderEditor([row('(a+)+$')], vi.fn())
|
||||
expect(container.textContent).toMatch(/potentially unsafe/)
|
||||
})
|
||||
|
||||
it('appends an empty row when "Add pattern" is clicked', () => {
|
||||
const onChange = vi.fn()
|
||||
renderEditor([row('a+')], onChange)
|
||||
clickText('Add pattern')
|
||||
expect(onChange).toHaveBeenCalledWith([
|
||||
{ name: 'X', regex: 'a+', replacement: '<X>' },
|
||||
{ name: '', regex: '', replacement: '' },
|
||||
])
|
||||
})
|
||||
|
||||
it('removes a row when its remove button is clicked', () => {
|
||||
const onChange = vi.fn()
|
||||
renderEditor([row('a+'), row('b+')], onChange)
|
||||
const remove = container.querySelector(
|
||||
'button[aria-label="Remove pattern"]'
|
||||
) as HTMLButtonElement
|
||||
act(() => remove.dispatchEvent(new MouseEvent('click', { bubbles: true })))
|
||||
expect(onChange).toHaveBeenCalledWith([{ name: 'X', regex: 'b+', replacement: '<X>' }])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,82 @@
|
||||
'use client'
|
||||
|
||||
import { Chip, ChipInput } from '@sim/emcn'
|
||||
import { Plus, Trash2 } from 'lucide-react'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
import { validateRegexPattern } from '@/lib/guardrails/validate_regex'
|
||||
|
||||
/** Matches the `.max(20)` bound on `customPatterns` in the boundary contract. */
|
||||
const MAX_PATTERNS = 20
|
||||
|
||||
interface CustomPatternsEditorProps {
|
||||
patterns: CustomPiiPattern[]
|
||||
onChange: (patterns: CustomPiiPattern[]) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Editor for user-supplied custom regex patterns. Each row is a name label, the
|
||||
* regex (validated inline for syntax + catastrophic-backtracking safety), and the
|
||||
* verbatim replacement token that matches are redacted to. Shared by the Data
|
||||
* Retention settings and any other PII-policy surface.
|
||||
*/
|
||||
export function CustomPatternsEditor({ patterns, onChange }: CustomPatternsEditorProps) {
|
||||
function updateRow(index: number, patch: Partial<CustomPiiPattern>) {
|
||||
onChange(patterns.map((pattern, i) => (i === index ? { ...pattern, ...patch } : pattern)))
|
||||
}
|
||||
|
||||
function removeRow(index: number) {
|
||||
onChange(patterns.filter((_, i) => i !== index))
|
||||
}
|
||||
|
||||
function addRow() {
|
||||
if (patterns.length >= MAX_PATTERNS) return
|
||||
onChange([...patterns, { name: '', regex: '', replacement: '' }])
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='flex flex-col gap-2'>
|
||||
{patterns.map((pattern, index) => {
|
||||
const validation = pattern.regex.length > 0 ? validateRegexPattern(pattern.regex) : null
|
||||
const error = validation && !validation.valid ? validation.error : undefined
|
||||
return (
|
||||
<div key={index} className='flex flex-col gap-1'>
|
||||
<div className='flex items-start gap-2'>
|
||||
<ChipInput
|
||||
placeholder='Name'
|
||||
value={pattern.name}
|
||||
onChange={(e) => updateRow(index, { name: e.target.value })}
|
||||
className='w-[26%]'
|
||||
/>
|
||||
<ChipInput
|
||||
placeholder='Pattern (regex)'
|
||||
value={pattern.regex}
|
||||
onChange={(e) => updateRow(index, { regex: e.target.value })}
|
||||
inputClassName='font-mono'
|
||||
error={Boolean(error)}
|
||||
className='flex-1'
|
||||
/>
|
||||
<ChipInput
|
||||
placeholder='EMPLOYEE_ID'
|
||||
value={pattern.replacement}
|
||||
onChange={(e) => updateRow(index, { replacement: e.target.value })}
|
||||
className='w-[26%]'
|
||||
/>
|
||||
<button
|
||||
type='button'
|
||||
aria-label='Remove pattern'
|
||||
onClick={() => removeRow(index)}
|
||||
className='flex size-[30px] flex-shrink-0 items-center justify-center rounded-md text-[var(--text-icon)] transition-colors hover-hover:bg-[var(--surface-active)] hover-hover:text-[var(--text-error)]'
|
||||
>
|
||||
<Trash2 className='size-[14px]' />
|
||||
</button>
|
||||
</div>
|
||||
{error && <span className='text-[var(--text-error)] text-small'>{error}</span>}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
<Chip leftIcon={Plus} onClick={addRow} disabled={patterns.length >= MAX_PATTERNS}>
|
||||
Add pattern
|
||||
</Chip>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -19,10 +19,12 @@ import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { ArrowRight, Plus } from 'lucide-react'
|
||||
import { CustomPatternsEditor } from '@/components/pii/custom-patterns-editor'
|
||||
import type { UpdateOrganizationDataRetentionBody } from '@/lib/api/contracts/organization'
|
||||
import type { RetentionOverride } from '@/lib/api/contracts/primitives'
|
||||
import { isBillingEnabled } from '@/lib/core/config/env-flags'
|
||||
import {
|
||||
type CustomPiiPattern,
|
||||
emptyPiiStages,
|
||||
getEntityGroupsForLanguage,
|
||||
isEntitySupportedForLanguage,
|
||||
@@ -35,6 +37,7 @@ import {
|
||||
type PiiStageKey,
|
||||
type PiiStagePolicy,
|
||||
type PiiStages,
|
||||
sanitizeCustomPatterns,
|
||||
stripNerEntities,
|
||||
} from '@/lib/guardrails/pii-entities'
|
||||
import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail'
|
||||
@@ -137,15 +140,18 @@ function buildRetentionOverride(workspaceId: string, draft: PolicyDraft): Retent
|
||||
}
|
||||
|
||||
/** Stable serialization of a stage set for dirty-detection. */
|
||||
function serializeStages(stages: PiiStages): Array<[PiiStageKey, boolean, string[], PIILanguage]> {
|
||||
function serializeStages(
|
||||
stages: PiiStages
|
||||
): Array<[PiiStageKey, boolean, string[], PIILanguage, CustomPiiPattern[]]> {
|
||||
return PII_STAGES.map((key) => {
|
||||
const policy = stages[key]
|
||||
return [key, policy.enabled, [...policy.entityTypes].sort(), policy.language] as [
|
||||
PiiStageKey,
|
||||
boolean,
|
||||
string[],
|
||||
PIILanguage,
|
||||
]
|
||||
return [
|
||||
key,
|
||||
policy.enabled,
|
||||
[...policy.entityTypes].sort(),
|
||||
policy.language,
|
||||
policy.customPatterns ?? [],
|
||||
] as [PiiStageKey, boolean, string[], PIILanguage, CustomPiiPattern[]]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -161,22 +167,29 @@ function normalizePolicyDraft(draft: PolicyDraft): string {
|
||||
})
|
||||
}
|
||||
|
||||
/** A stage is "on" iff it has at least one entity type selected. */
|
||||
/** A stage is "on" iff it has at least one entity type or custom pattern. */
|
||||
function stageHasContent(policy: PiiStagePolicy): boolean {
|
||||
return policy.entityTypes.length > 0
|
||||
return policy.entityTypes.length > 0 || (policy.customPatterns?.length ?? 0) > 0
|
||||
}
|
||||
|
||||
function anyStageHasContent(stages: PiiStages): boolean {
|
||||
return PII_STAGES.some((key) => stageHasContent(stages[key]))
|
||||
}
|
||||
|
||||
/** Persist-time guarantee that `enabled` mirrors "has entity types" for every stage. */
|
||||
/** Persist-time guarantee that `enabled` mirrors "has content" for every stage. */
|
||||
function withSyncedEnabled(stages: PiiStages): PiiStages {
|
||||
return PII_STAGES.reduce((acc, key) => {
|
||||
// Block outputs are regex-only — strip any NER before persisting.
|
||||
const entityTypes =
|
||||
key === 'blockOutputs' ? stripNerEntities(stages[key].entityTypes) : stages[key].entityTypes
|
||||
acc[key] = { ...stages[key], entityTypes, enabled: entityTypes.length > 0 }
|
||||
// Drop half-typed rows (empty regex) so the boundary contract never rejects the save.
|
||||
const customPatterns = sanitizeCustomPatterns(stages[key].customPatterns)
|
||||
acc[key] = {
|
||||
...stages[key],
|
||||
entityTypes,
|
||||
customPatterns,
|
||||
enabled: entityTypes.length > 0 || customPatterns.length > 0,
|
||||
}
|
||||
return acc
|
||||
}, {} as PiiStages)
|
||||
}
|
||||
@@ -195,7 +208,8 @@ function stageSummary(stages: PiiStages): string {
|
||||
}
|
||||
return PII_STAGES.map((key) => {
|
||||
const policy = stages[key]
|
||||
return `${short[key]} ${stageHasContent(policy) ? policy.entityTypes.length : 'off'}`
|
||||
const count = policy.entityTypes.length + (policy.customPatterns?.length ?? 0)
|
||||
return `${short[key]} ${stageHasContent(policy) ? count : 'off'}`
|
||||
}).join(' · ')
|
||||
}
|
||||
|
||||
@@ -343,8 +357,10 @@ function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanel
|
||||
regexOnly: stageKey === 'blockOutputs',
|
||||
})
|
||||
|
||||
function update(entityTypes: string[], language = value.language) {
|
||||
onChange({ ...value, language, entityTypes, enabled: entityTypes.length > 0 })
|
||||
function update(patch: Partial<PiiStagePolicy>) {
|
||||
const merged = { ...value, ...patch }
|
||||
const enabled = merged.entityTypes.length > 0 || (merged.customPatterns?.length ?? 0) > 0
|
||||
onChange({ ...merged, enabled })
|
||||
}
|
||||
|
||||
return (
|
||||
@@ -363,20 +379,38 @@ function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanel
|
||||
<EntityCheckboxGrid
|
||||
groups={groups}
|
||||
selected={value.entityTypes}
|
||||
onChange={(entityTypes) => update(entityTypes)}
|
||||
onChange={(entityTypes) => update({ entityTypes })}
|
||||
belowSearch={
|
||||
<div className='flex items-center justify-between gap-3'>
|
||||
<span className='text-[var(--text-muted)] text-small'>Language</span>
|
||||
<PiiLanguageSelect
|
||||
value={value.language}
|
||||
onChange={(language) =>
|
||||
update(pruneEntitiesForLanguage(value.entityTypes, language), language)
|
||||
update({
|
||||
language,
|
||||
entityTypes: pruneEntitiesForLanguage(value.entityTypes, language),
|
||||
})
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className='flex flex-col gap-2'>
|
||||
<div className='flex items-center gap-1.5'>
|
||||
<span className='text-[var(--text-muted)] text-small'>Custom patterns</span>
|
||||
<Info side='top' align='start'>
|
||||
Redact anything a regular expression can match (employee ids, internal urls, ticket
|
||||
numbers). Each match is replaced with its replacement text, wrapped in angle brackets
|
||||
(e.g. EMPLOYEE_ID → <EMPLOYEE_ID>).
|
||||
</Info>
|
||||
</div>
|
||||
<CustomPatternsEditor
|
||||
patterns={value.customPatterns ?? []}
|
||||
onChange={(customPatterns) => update({ customPatterns })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -529,7 +563,10 @@ function PolicyDetail({
|
||||
[effectiveStage]: {
|
||||
...draft.piiStages[effectiveStage],
|
||||
entityTypes: [],
|
||||
enabled: false,
|
||||
// Clearing entity types leaves any custom patterns intact,
|
||||
// so the stage stays enabled while patterns remain.
|
||||
enabled:
|
||||
(draft.piiStages[effectiveStage].customPatterns?.length ?? 0) > 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -236,6 +236,7 @@ export class BlockExecutor {
|
||||
const redactionOptions = {
|
||||
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
|
||||
language: ctx.piiBlockOutputRedaction.language,
|
||||
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
|
||||
onFailure: 'throw' as const,
|
||||
}
|
||||
// Tools like the function executor offload large outputs to large-value
|
||||
@@ -889,6 +890,7 @@ export class BlockExecutor {
|
||||
fullContent = await redactObjectStrings(fullContent, {
|
||||
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
|
||||
language: ctx.piiBlockOutputRedaction.language,
|
||||
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
|
||||
onFailure: 'throw',
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Edge } from 'reactflow'
|
||||
import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution'
|
||||
import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
import type { NodeMetadata } from '@/executor/dag/types'
|
||||
import type {
|
||||
BlockLog,
|
||||
@@ -159,6 +160,8 @@ export interface PiiBlockOutputRedaction {
|
||||
entityTypes: string[]
|
||||
/** Language whose Presidio recognizers apply. */
|
||||
language: string
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
}
|
||||
|
||||
export interface ContextExtensions {
|
||||
|
||||
@@ -134,6 +134,7 @@ export class Memory {
|
||||
content: await redactObjectStrings(message.content, {
|
||||
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
|
||||
language: ctx.piiBlockOutputRedaction.language,
|
||||
customPatterns: ctx.piiBlockOutputRedaction.customPatterns,
|
||||
onFailure: 'throw',
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { z } from 'zod'
|
||||
import { unknownRecordSchema } from '@/lib/api/contracts/primitives'
|
||||
import { customPatternSchema, unknownRecordSchema } from '@/lib/api/contracts/primitives'
|
||||
import { defineRouteContract } from '@/lib/api/contracts/types'
|
||||
import { DEFAULT_CODE_LANGUAGE } from '@/lib/execution/languages'
|
||||
export const guardrailsValidateContract = defineRouteContract({
|
||||
@@ -26,6 +26,7 @@ export const guardrailsValidateContract = defineRouteContract({
|
||||
piiEntityTypes: z.array(z.string()).optional(),
|
||||
piiMode: z.string().optional(),
|
||||
piiLanguage: z.string().optional(),
|
||||
piiCustomPatterns: z.array(customPatternSchema).max(20).optional(),
|
||||
}),
|
||||
response: {
|
||||
mode: 'json',
|
||||
@@ -49,6 +50,7 @@ const guardrailsMaskBatchBodySchema = z.object({
|
||||
texts: z.array(z.string()).max(100_000),
|
||||
entityTypes: z.array(z.string().min(1, 'Entity type cannot be empty')).max(200),
|
||||
language: z.string().min(1).max(20).optional(),
|
||||
customPatterns: z.array(customPatternSchema).max(20).optional(),
|
||||
})
|
||||
|
||||
const guardrailsMaskBatchResponseSchema = z.object({
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
customPatternSchema,
|
||||
piiStagePolicySchema,
|
||||
piiStagesSchema,
|
||||
} from '@/lib/api/contracts/primitives'
|
||||
|
||||
describe('customPatternSchema', () => {
|
||||
it('accepts a well-formed pattern', () => {
|
||||
expect(
|
||||
customPatternSchema.parse({ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' })
|
||||
).toEqual({ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' })
|
||||
})
|
||||
|
||||
it('rejects an empty regex', () => {
|
||||
expect(customPatternSchema.safeParse({ name: 'x', regex: '', replacement: '' }).success).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an over-long regex', () => {
|
||||
expect(
|
||||
customPatternSchema.safeParse({ name: '', regex: 'a'.repeat(513), replacement: '' }).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects a syntactically invalid regex at the boundary (not just in the editor)', () => {
|
||||
const parsed = customPatternSchema.safeParse({ name: 'bad', regex: '(', replacement: '' })
|
||||
expect(parsed.success).toBe(false)
|
||||
if (!parsed.success) {
|
||||
expect(parsed.error.issues[0].path).toEqual(['regex'])
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects a catastrophic-backtracking regex at the boundary', () => {
|
||||
expect(
|
||||
customPatternSchema.safeParse({ name: 'evil', regex: '(a+)+$', replacement: '' }).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('piiStagePolicySchema', () => {
|
||||
it('allows an enabled stage with only custom patterns (no entity types)', () => {
|
||||
const parsed = piiStagePolicySchema.safeParse({
|
||||
enabled: true,
|
||||
entityTypes: [],
|
||||
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
|
||||
})
|
||||
expect(parsed.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects an enabled stage with no entity types and no custom patterns', () => {
|
||||
const parsed = piiStagePolicySchema.safeParse({ enabled: true, entityTypes: [] })
|
||||
expect(parsed.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('piiStagesSchema', () => {
|
||||
it('keeps custom patterns on blockOutputs while stripping NER, staying enabled', () => {
|
||||
const parsed = piiStagesSchema.parse({
|
||||
input: { enabled: false, entityTypes: [] },
|
||||
blockOutputs: {
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON', 'EMAIL_ADDRESS'],
|
||||
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
|
||||
},
|
||||
logs: { enabled: false, entityTypes: [] },
|
||||
})
|
||||
expect(parsed.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS'])
|
||||
expect(parsed.blockOutputs.customPatterns).toEqual([
|
||||
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
|
||||
])
|
||||
expect(parsed.blockOutputs.enabled).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps blockOutputs enabled when only custom patterns survive the NER strip', () => {
|
||||
const parsed = piiStagesSchema.parse({
|
||||
input: { enabled: false, entityTypes: [] },
|
||||
blockOutputs: {
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON'],
|
||||
customPatterns: [{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' }],
|
||||
},
|
||||
logs: { enabled: false, entityTypes: [] },
|
||||
})
|
||||
expect(parsed.blockOutputs.entityTypes).toEqual([])
|
||||
expect(parsed.blockOutputs.enabled).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import { PII_LANGUAGE_CODES, stripNerEntities } from '@/lib/guardrails/pii-entities'
|
||||
import { validateRegexPattern } from '@/lib/guardrails/validate_regex'
|
||||
|
||||
export const unknownRecordSchema = z.record(z.string(), z.unknown())
|
||||
|
||||
@@ -119,6 +120,37 @@ export const userFileSchema = z
|
||||
* expressible policy, so `enabled: true` with an empty list (which would resolve
|
||||
* to off and silently skip masking) is rejected at the boundary.
|
||||
*/
|
||||
/**
|
||||
* A user-supplied custom regex pattern. `name` is a label; `regex` is matched
|
||||
* against text; matches are replaced with `replacement` wrapped in angle brackets
|
||||
* (`EMPLOYEE_ID` → `<EMPLOYEE_ID>`). Bounds guard the Presidio boundary
|
||||
* (ReDoS/oversized payloads).
|
||||
*
|
||||
* The `regex` is validated for both syntax and catastrophic-backtracking safety
|
||||
* here at the write boundary — not just in the editor — so an invalid or unsafe
|
||||
* pattern can never be persisted or reach Presidio (where it would abort the
|
||||
* batch on a 400, or time out and silently fail open, leaving PII unredacted).
|
||||
*/
|
||||
export const customPatternSchema = z.object({
|
||||
name: z.string().max(100, 'Pattern name is too long'),
|
||||
regex: z
|
||||
.string()
|
||||
.min(1, 'Pattern cannot be empty')
|
||||
.max(512, 'Pattern is too long')
|
||||
.superRefine((regex, ctx) => {
|
||||
const result = validateRegexPattern(regex)
|
||||
if (!result.valid) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: result.error ?? 'Invalid regex pattern',
|
||||
})
|
||||
}
|
||||
}),
|
||||
replacement: z.string().max(100, 'Replacement is too long'),
|
||||
})
|
||||
|
||||
export type CustomPiiPattern = z.output<typeof customPatternSchema>
|
||||
|
||||
export const piiStagePolicySchema = z
|
||||
.object({
|
||||
enabled: z.boolean(),
|
||||
@@ -126,11 +158,17 @@ export const piiStagePolicySchema = z
|
||||
entityTypes: z.array(z.string().min(1, 'Entity type cannot be empty')).max(100),
|
||||
/** Language whose Presidio recognizers apply; defaults to English. */
|
||||
language: z.enum(PII_LANGUAGE_CODES).optional(),
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns: z.array(customPatternSchema).max(20).optional(),
|
||||
})
|
||||
.refine((stage) => !stage.enabled || stage.entityTypes.length > 0, {
|
||||
message: 'An enabled redaction stage must select at least one entity type.',
|
||||
path: ['entityTypes'],
|
||||
})
|
||||
.refine(
|
||||
(stage) =>
|
||||
!stage.enabled || stage.entityTypes.length > 0 || (stage.customPatterns?.length ?? 0) > 0,
|
||||
{
|
||||
message: 'An enabled redaction stage must select at least one entity type or custom pattern.',
|
||||
path: ['entityTypes'],
|
||||
}
|
||||
)
|
||||
|
||||
export type PiiStagePolicy = z.output<typeof piiStagePolicySchema>
|
||||
|
||||
@@ -150,12 +188,14 @@ export const piiStagesSchema = z
|
||||
})
|
||||
.transform((stages) => {
|
||||
const entityTypes = stripNerEntities(stages.blockOutputs.entityTypes)
|
||||
const customPatterns = stages.blockOutputs.customPatterns ?? []
|
||||
return {
|
||||
...stages,
|
||||
blockOutputs: {
|
||||
...stages.blockOutputs,
|
||||
entityTypes,
|
||||
enabled: stages.blockOutputs.enabled && entityTypes.length > 0,
|
||||
enabled:
|
||||
stages.blockOutputs.enabled && (entityTypes.length > 0 || customPatterns.length > 0),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -13,7 +13,7 @@ function settings(rules: PiiRedactionRule[]): DataRetentionSettings {
|
||||
return { piiRedaction: { rules } }
|
||||
}
|
||||
|
||||
const DISABLED = { enabled: false, entityTypes: [], language: 'en' }
|
||||
const DISABLED = { enabled: false, entityTypes: [], language: 'en', customPatterns: [] }
|
||||
|
||||
describe('resolveEffectivePiiRedaction', () => {
|
||||
const allRule: PiiRedactionRule = {
|
||||
@@ -31,7 +31,12 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
expect(result).toEqual({
|
||||
input: DISABLED,
|
||||
blockOutputs: DISABLED,
|
||||
logs: { enabled: true, entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'], language: 'en' },
|
||||
logs: {
|
||||
enabled: true,
|
||||
entityTypes: ['EMAIL_ADDRESS', 'PHONE_NUMBER'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -46,7 +51,7 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
expect(result).toEqual({
|
||||
input: DISABLED,
|
||||
blockOutputs: DISABLED,
|
||||
logs: { enabled: true, entityTypes: ['US_SSN'], language: 'en' },
|
||||
logs: { enabled: true, entityTypes: ['US_SSN'], language: 'en', customPatterns: [] },
|
||||
})
|
||||
})
|
||||
|
||||
@@ -57,7 +62,12 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
]),
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
expect(result.logs).toEqual({ enabled: true, entityTypes: ['ES_NIF'], language: 'es' })
|
||||
expect(result.logs).toEqual({
|
||||
enabled: true,
|
||||
entityTypes: ['ES_NIF'],
|
||||
language: 'es',
|
||||
customPatterns: [],
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to en when a stored language is unsupported/stale', () => {
|
||||
@@ -67,7 +77,12 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
]),
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
expect(result.logs).toEqual({ enabled: true, entityTypes: ['EMAIL_ADDRESS'], language: 'en' })
|
||||
expect(result.logs).toEqual({
|
||||
enabled: true,
|
||||
entityTypes: ['EMAIL_ADDRESS'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
})
|
||||
})
|
||||
|
||||
it('exempts a workspace when its specific flat rule has no entity types', () => {
|
||||
@@ -102,9 +117,19 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
expect(result).toEqual({
|
||||
input: { enabled: true, entityTypes: ['PERSON'], language: 'es' },
|
||||
blockOutputs: { enabled: true, entityTypes: ['EMAIL_ADDRESS'], language: 'en' },
|
||||
logs: { enabled: true, entityTypes: ['US_SSN', 'PHONE_NUMBER'], language: 'en' },
|
||||
input: { enabled: true, entityTypes: ['PERSON'], language: 'es', customPatterns: [] },
|
||||
blockOutputs: {
|
||||
enabled: true,
|
||||
entityTypes: ['EMAIL_ADDRESS'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
},
|
||||
logs: {
|
||||
enabled: true,
|
||||
entityTypes: ['US_SSN', 'PHONE_NUMBER'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -125,7 +150,12 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
})
|
||||
expect(result.input).toEqual(DISABLED)
|
||||
expect(result.blockOutputs).toEqual(DISABLED)
|
||||
expect(result.logs).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' })
|
||||
expect(result.logs).toEqual({
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
})
|
||||
})
|
||||
|
||||
it('strips spaCy-NER entities from blockOutputs at resolve time (regex-only)', () => {
|
||||
@@ -183,10 +213,51 @@ describe('resolveEffectivePiiRedaction', () => {
|
||||
]),
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
expect(result.input).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' })
|
||||
expect(result.input).toEqual({
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON'],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
})
|
||||
// The all rule's logs entity types are NOT unioned in.
|
||||
expect(result.logs).toEqual(DISABLED)
|
||||
})
|
||||
|
||||
it('carries custom patterns through each stage (blockOutputs strips NER but keeps them)', () => {
|
||||
const result = resolveEffectivePiiRedaction({
|
||||
orgSettings: settings([
|
||||
{
|
||||
id: 'r-1',
|
||||
workspaceId: 'ws-1',
|
||||
stages: {
|
||||
input: {
|
||||
enabled: true,
|
||||
entityTypes: [],
|
||||
customPatterns: [{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: '<EMP>' }],
|
||||
},
|
||||
blockOutputs: {
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON'],
|
||||
customPatterns: [{ name: 'Tck', regex: 'TCK-\\d+', replacement: '<TCK>' }],
|
||||
},
|
||||
logs: stage(false, []),
|
||||
},
|
||||
},
|
||||
]),
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
// Input: enabled by custom pattern alone (no entity types).
|
||||
expect(result.input.enabled).toBe(true)
|
||||
expect(result.input.customPatterns).toEqual([
|
||||
{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: '<EMP>' },
|
||||
])
|
||||
// Block outputs: NER stripped, but the custom pattern keeps the stage enabled.
|
||||
expect(result.blockOutputs.entityTypes).toEqual([])
|
||||
expect(result.blockOutputs.enabled).toBe(true)
|
||||
expect(result.blockOutputs.customPatterns).toEqual([
|
||||
{ name: 'Tck', regex: 'TCK-\\d+', replacement: '<TCK>' },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
it('is the default when no rule matches and there is no all rule', () => {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import type { DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema'
|
||||
import type { CustomPiiPattern, DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema'
|
||||
import {
|
||||
coercePiiLanguage,
|
||||
DEFAULT_PII_LANGUAGE,
|
||||
sanitizeCustomPatterns,
|
||||
stripNerEntities,
|
||||
} from '@/lib/guardrails/pii-entities'
|
||||
|
||||
@@ -12,6 +13,8 @@ export interface EffectivePiiStage {
|
||||
entityTypes: string[]
|
||||
/** Language whose Presidio recognizers apply when masking. */
|
||||
language: string
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns: CustomPiiPattern[]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -29,6 +32,7 @@ const DISABLED_STAGE: EffectivePiiStage = {
|
||||
enabled: false,
|
||||
entityTypes: [],
|
||||
language: DEFAULT_PII_LANGUAGE,
|
||||
customPatterns: [],
|
||||
}
|
||||
|
||||
export const DEFAULT_PII_REDACTION: EffectivePiiRedaction = {
|
||||
@@ -58,11 +62,13 @@ function toEffectiveStage(
|
||||
const types = opts?.regexOnly
|
||||
? stripNerEntities(sanitizeEntityTypes(policy?.entityTypes))
|
||||
: sanitizeEntityTypes(policy?.entityTypes)
|
||||
if (!policy?.enabled || types.length === 0) return DISABLED_STAGE
|
||||
const customPatterns = sanitizeCustomPatterns(policy?.customPatterns)
|
||||
if (!policy?.enabled || (types.length === 0 && customPatterns.length === 0)) return DISABLED_STAGE
|
||||
return {
|
||||
enabled: true,
|
||||
entityTypes: types,
|
||||
language: coercePiiLanguage(policy.language) ?? DEFAULT_PII_LANGUAGE,
|
||||
customPatterns,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +105,7 @@ export function resolveEffectivePiiRedaction(params: {
|
||||
enabled: true,
|
||||
entityTypes: types,
|
||||
language: coercePiiLanguage(rule.language) ?? DEFAULT_PII_LANGUAGE,
|
||||
customPatterns: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { env } from '@/lib/core/config/env'
|
||||
import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
|
||||
import { getInternalApiBaseUrl } from '@/lib/core/utils/urls'
|
||||
import { chunkIndicesByBudget } from '@/lib/guardrails/pii-batching'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
|
||||
/**
|
||||
* Max in-flight mask-batch requests per call. Each request is a CPU-heavy NER
|
||||
@@ -34,7 +35,8 @@ const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64
|
||||
export async function maskPIIBatchViaHttp(
|
||||
texts: string[],
|
||||
entityTypes: string[],
|
||||
language?: string
|
||||
language?: string,
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
): Promise<string[]> {
|
||||
if (texts.length === 0) return []
|
||||
|
||||
@@ -43,7 +45,7 @@ export async function maskPIIBatchViaHttp(
|
||||
|
||||
await mapWithConcurrency(chunkIndicesByBudget(texts), CHUNK_CONCURRENCY, async (indices) => {
|
||||
const chunk = indices.map((i) => texts[i])
|
||||
const out = await postChunk(url, chunk, entityTypes, language)
|
||||
const out = await postChunk(url, chunk, entityTypes, language, customPatterns)
|
||||
if (out.length !== chunk.length) {
|
||||
throw new Error('PII mask-batch returned an unexpected result')
|
||||
}
|
||||
@@ -59,7 +61,8 @@ async function postChunk(
|
||||
url: string,
|
||||
texts: string[],
|
||||
entityTypes: string[],
|
||||
language: string | undefined
|
||||
language: string | undefined,
|
||||
customPatterns: CustomPiiPattern[] | undefined
|
||||
): Promise<string[]> {
|
||||
// Mint per request: a single token (5min TTL) can expire mid-batch when a
|
||||
// large execution fans out into many sequential chunk requests.
|
||||
@@ -72,7 +75,7 @@ async function postChunk(
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify({ texts, entityTypes, language }),
|
||||
body: JSON.stringify({ texts, entityTypes, language, customPatterns }),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
emptyStagePolicy,
|
||||
getEntityGroupsForLanguage,
|
||||
NER_PII_ENTITIES,
|
||||
normalizeRuleStages,
|
||||
sanitizeCustomPatterns,
|
||||
stripNerEntities,
|
||||
} from '@/lib/guardrails/pii-entities'
|
||||
|
||||
@@ -88,4 +90,75 @@ describe('normalizeRuleStages', () => {
|
||||
expect(stages.blockOutputs.entityTypes).toEqual([])
|
||||
expect(stages.blockOutputs.enabled).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps blockOutputs enabled when custom patterns survive the NER strip', () => {
|
||||
const stages = normalizeRuleStages({
|
||||
stages: {
|
||||
input: { enabled: false, entityTypes: [] },
|
||||
blockOutputs: {
|
||||
enabled: true,
|
||||
entityTypes: ['PERSON'],
|
||||
customPatterns: [{ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' }],
|
||||
},
|
||||
logs: { enabled: false, entityTypes: [] },
|
||||
},
|
||||
})
|
||||
expect(stages.blockOutputs.entityTypes).toEqual([])
|
||||
expect(stages.blockOutputs.customPatterns).toEqual([
|
||||
{ name: 'Employee ID', regex: 'EMP-\\d{6}', replacement: '<EMP>' },
|
||||
])
|
||||
expect(stages.blockOutputs.enabled).toBe(true)
|
||||
})
|
||||
|
||||
it('sanitizes stored custom patterns on every stage', () => {
|
||||
const stages = normalizeRuleStages({
|
||||
stages: {
|
||||
input: {
|
||||
enabled: true,
|
||||
entityTypes: [],
|
||||
customPatterns: [
|
||||
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
|
||||
// Malformed rows are dropped.
|
||||
{ name: 'no regex', regex: '', replacement: 'x' } as never,
|
||||
],
|
||||
},
|
||||
blockOutputs: { enabled: false, entityTypes: [] },
|
||||
logs: { enabled: false, entityTypes: [] },
|
||||
},
|
||||
})
|
||||
expect(stages.input.customPatterns).toEqual([
|
||||
{ name: 'Ticket', regex: 'TCK-\\d+', replacement: '<TICKET>' },
|
||||
])
|
||||
expect(stages.input.enabled).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('emptyStagePolicy', () => {
|
||||
it('starts disabled with no entity types and no custom patterns', () => {
|
||||
expect(emptyStagePolicy()).toEqual({
|
||||
enabled: false,
|
||||
entityTypes: [],
|
||||
language: 'en',
|
||||
customPatterns: [],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeCustomPatterns', () => {
|
||||
it('drops non-arrays and malformed rows, coercing missing fields', () => {
|
||||
expect(sanitizeCustomPatterns(undefined)).toEqual([])
|
||||
expect(sanitizeCustomPatterns('nope')).toEqual([])
|
||||
expect(
|
||||
sanitizeCustomPatterns([
|
||||
{ name: 'A', regex: 'a+', replacement: '<A>' },
|
||||
{ regex: 'b+' },
|
||||
{ name: 'no regex', regex: '', replacement: 'x' },
|
||||
null,
|
||||
{ name: 'bad', regex: 42 },
|
||||
])
|
||||
).toEqual([
|
||||
{ name: 'A', regex: 'a+', replacement: '<A>' },
|
||||
{ name: '', regex: 'b+', replacement: '' },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -273,11 +273,25 @@ export function getEntityGroupsForLanguage(language: PIILanguage, opts?: { regex
|
||||
export const PII_STAGES = ['input', 'blockOutputs', 'logs'] as const
|
||||
export type PiiStageKey = (typeof PII_STAGES)[number]
|
||||
|
||||
/**
|
||||
* A user-supplied custom regex pattern. Matches are replaced with `replacement`
|
||||
* wrapped in angle brackets (e.g. `EMPLOYEE_ID` → `<EMPLOYEE_ID>`), mirroring the
|
||||
* built-in Presidio tokens; the internal entity id is never surfaced. `name` is a
|
||||
* human label only.
|
||||
*/
|
||||
export interface CustomPiiPattern {
|
||||
name: string
|
||||
regex: string
|
||||
replacement: string
|
||||
}
|
||||
|
||||
/** Per-stage redaction policy. `enabled: false` makes the stage a no-op. */
|
||||
export interface PiiStagePolicy {
|
||||
enabled: boolean
|
||||
entityTypes: string[]
|
||||
language: PIILanguage
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
}
|
||||
|
||||
export type PiiStages = Record<PiiStageKey, PiiStagePolicy>
|
||||
@@ -320,7 +334,24 @@ export const RISKY_PII_ENTITIES: ReadonlySet<PIIEntityType> = new Set<PIIEntityT
|
||||
|
||||
/** A fully-disabled stage policy for new drafts. */
|
||||
export function emptyStagePolicy(): PiiStagePolicy {
|
||||
return { enabled: false, entityTypes: [], language: DEFAULT_PII_LANGUAGE }
|
||||
return { enabled: false, entityTypes: [], language: DEFAULT_PII_LANGUAGE, customPatterns: [] }
|
||||
}
|
||||
|
||||
/** Coerce an untrusted value into a clean `CustomPiiPattern[]` (drops malformed rows). */
|
||||
export function sanitizeCustomPatterns(value: unknown): CustomPiiPattern[] {
|
||||
if (!Array.isArray(value)) return []
|
||||
const out: CustomPiiPattern[] = []
|
||||
for (const raw of value) {
|
||||
if (!raw || typeof raw !== 'object') continue
|
||||
const { name, regex, replacement } = raw as Record<string, unknown>
|
||||
if (typeof regex !== 'string' || regex.length === 0) continue
|
||||
out.push({
|
||||
name: typeof name === 'string' ? name : '',
|
||||
regex,
|
||||
replacement: typeof replacement === 'string' ? replacement : '',
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A fully-disabled stage set for new drafts. */
|
||||
@@ -348,11 +379,13 @@ export function normalizeRuleStages(rule: {
|
||||
? policy.entityTypes.filter((t): t is string => typeof t === 'string')
|
||||
: [],
|
||||
language: coercePiiLanguage(policy?.language) ?? DEFAULT_PII_LANGUAGE,
|
||||
customPatterns: sanitizeCustomPatterns(policy?.customPatterns),
|
||||
})
|
||||
|
||||
if (rule.stages) {
|
||||
// Block outputs are regex-only (no spaCy NER) — strip NER from any stored
|
||||
// rule so hydrated drafts never carry it; a stage left empty becomes disabled.
|
||||
// rule so hydrated drafts never carry it; a stage with neither regex entities
|
||||
// nor custom patterns becomes disabled.
|
||||
const blockOutputs = sanitize(rule.stages.blockOutputs)
|
||||
const blockOutputsEntities = stripNerEntities(blockOutputs.entityTypes)
|
||||
return {
|
||||
@@ -360,7 +393,9 @@ export function normalizeRuleStages(rule: {
|
||||
blockOutputs: {
|
||||
...blockOutputs,
|
||||
entityTypes: blockOutputsEntities,
|
||||
enabled: blockOutputs.enabled && blockOutputsEntities.length > 0,
|
||||
enabled:
|
||||
blockOutputs.enabled &&
|
||||
(blockOutputsEntities.length > 0 || (blockOutputs.customPatterns?.length ?? 0) > 0),
|
||||
},
|
||||
logs: sanitize(rule.stages.logs),
|
||||
}
|
||||
|
||||
@@ -3,9 +3,37 @@ import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
|
||||
import { chunkIndicesByBudget } from '@/lib/guardrails/pii-batching'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
|
||||
const logger = createLogger('PIIValidator')
|
||||
|
||||
/**
|
||||
* Entity list for the batch (data-retention) paths, where an empty selection with
|
||||
* custom patterns means "redact ONLY these custom patterns" (the user unchecked
|
||||
* every built-in entity). An explicit empty array is sent so the server detects
|
||||
* only the custom entities, never "all". With neither, `undefined` preserves the
|
||||
* legacy "detect all" default.
|
||||
*
|
||||
* The guardrails single-text path uses the opposite convention — empty selection
|
||||
* means "detect all" — so it does NOT use this helper (see {@link analyze}).
|
||||
*/
|
||||
function resolveBatchEntities(
|
||||
entityTypes: string[],
|
||||
patterns?: CustomPiiPattern[]
|
||||
): string[] | undefined {
|
||||
if (entityTypes.length > 0) return entityTypes
|
||||
if ((patterns?.length ?? 0) > 0) return []
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Map a detected entity type back to its user-facing custom-pattern name, if it is one. */
|
||||
function displayEntityType(type: string, patterns?: CustomPiiPattern[]): string {
|
||||
const match = /^CUSTOM_(\d+)$/.exec(type)
|
||||
if (!match) return type
|
||||
const pattern = patterns?.[Number(match[1])]
|
||||
return pattern?.name || type
|
||||
}
|
||||
|
||||
/**
|
||||
* Concurrent chunk requests in flight from a single mask-batch call. Each chunk is
|
||||
* itself a batched service call (spaCy `nlp.pipe` over many strings). Default 4;
|
||||
@@ -22,6 +50,8 @@ export interface PIIValidationInput {
|
||||
entityTypes: string[] // e.g., ["PERSON", "EMAIL_ADDRESS", "CREDIT_CARD"]
|
||||
mode: 'block' | 'mask' // block = fail if PII found, mask = return masked text
|
||||
language?: string // default: "en"
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
requestId: string
|
||||
}
|
||||
|
||||
@@ -54,15 +84,25 @@ interface AnalyzerSpan {
|
||||
async function analyze(
|
||||
text: string,
|
||||
entityTypes: string[],
|
||||
language: string
|
||||
language: string,
|
||||
patterns?: CustomPiiPattern[]
|
||||
): Promise<AnalyzerSpan[]> {
|
||||
// Guardrails convention: an empty selection means "detect all". Sending no
|
||||
// `entities` keeps that, and the server still runs the custom recognizers under
|
||||
// detect-all — so a custom pattern augments the built-in detectors, never
|
||||
// silently replaces them.
|
||||
const entities = entityTypes.length > 0 ? entityTypes : undefined
|
||||
|
||||
// boundary-raw-fetch: internal call to the Presidio analyzer service via PII_URL
|
||||
const response = await fetch(`${PII_URL}/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ text, language, ...(entities ? { entities } : {}) }),
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
language,
|
||||
...(entities ? { entities } : {}),
|
||||
...(patterns?.length ? { patterns } : {}),
|
||||
}),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const detail = await response.text().catch(() => '')
|
||||
@@ -79,15 +119,21 @@ async function analyze(
|
||||
async function analyzeBatch(
|
||||
texts: string[],
|
||||
entityTypes: string[],
|
||||
language: string
|
||||
language: string,
|
||||
patterns?: CustomPiiPattern[]
|
||||
): Promise<AnalyzerSpan[][]> {
|
||||
const entities = entityTypes.length > 0 ? entityTypes : undefined
|
||||
const entities = resolveBatchEntities(entityTypes, patterns)
|
||||
|
||||
// boundary-raw-fetch: internal call to the Presidio analyzer service via PII_URL
|
||||
const response = await fetch(`${PII_URL}/analyze_batch`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ texts, language, ...(entities ? { entities } : {}) }),
|
||||
body: JSON.stringify({
|
||||
texts,
|
||||
language,
|
||||
...(entities ? { entities } : {}),
|
||||
...(patterns?.length ? { patterns } : {}),
|
||||
}),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const detail = await response.text().catch(() => '')
|
||||
@@ -107,14 +153,17 @@ interface AnonymizeBatchItem {
|
||||
* items with no spans (those texts pass through unchanged). Returns masked text
|
||||
* per item, in order. Throws on failure.
|
||||
*/
|
||||
async function anonymizeBatch(items: AnonymizeBatchItem[]): Promise<string[]> {
|
||||
async function anonymizeBatch(
|
||||
items: AnonymizeBatchItem[],
|
||||
patterns?: CustomPiiPattern[]
|
||||
): Promise<string[]> {
|
||||
if (items.length === 0) return []
|
||||
|
||||
// boundary-raw-fetch: internal call to the Presidio anonymizer service via PII_URL
|
||||
const response = await fetch(`${PII_URL}/anonymize_batch`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ items }),
|
||||
body: JSON.stringify({ items, ...(patterns?.length ? { patterns } : {}) }),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const detail = await response.text().catch(() => '')
|
||||
@@ -144,15 +193,21 @@ let combinedRedactAvailable = true
|
||||
async function redactBatch(
|
||||
texts: string[],
|
||||
entityTypes: string[],
|
||||
language: string
|
||||
language: string,
|
||||
patterns?: CustomPiiPattern[]
|
||||
): Promise<string[] | null> {
|
||||
const entities = entityTypes.length > 0 ? entityTypes : undefined
|
||||
const entities = resolveBatchEntities(entityTypes, patterns)
|
||||
|
||||
// boundary-raw-fetch: internal call to the Presidio combined redact service via PII_URL
|
||||
const response = await fetch(`${PII_URL}/redact_batch`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ texts, language, ...(entities ? { entities } : {}) }),
|
||||
body: JSON.stringify({
|
||||
texts,
|
||||
language,
|
||||
...(entities ? { entities } : {}),
|
||||
...(patterns?.length ? { patterns } : {}),
|
||||
}),
|
||||
})
|
||||
if (response.status === 404) return null
|
||||
if (!response.ok) {
|
||||
@@ -172,14 +227,22 @@ async function redactBatch(
|
||||
* Mask spans via the Presidio anonymizer service. Omitting `anonymizers` uses the
|
||||
* default `replace` operator, which yields `<ENTITY_TYPE>`. Throws on failure.
|
||||
*/
|
||||
async function anonymize(text: string, spans: AnalyzerSpan[]): Promise<string> {
|
||||
async function anonymize(
|
||||
text: string,
|
||||
spans: AnalyzerSpan[],
|
||||
patterns?: CustomPiiPattern[]
|
||||
): Promise<string> {
|
||||
if (spans.length === 0) return text
|
||||
|
||||
// boundary-raw-fetch: internal call to the Presidio anonymizer service via PII_URL
|
||||
const response = await fetch(`${PII_URL}/anonymize`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ text, analyzer_results: spans }),
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
analyzer_results: spans,
|
||||
...(patterns?.length ? { patterns } : {}),
|
||||
}),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const detail = await response.text().catch(() => '')
|
||||
@@ -196,20 +259,21 @@ async function anonymize(text: string, spans: AnalyzerSpan[]): Promise<string> {
|
||||
* - mask: passes and returns masked text with PII replaced by `<ENTITY_TYPE>`
|
||||
*/
|
||||
export async function validatePII(input: PIIValidationInput): Promise<PIIValidationResult> {
|
||||
const { text, entityTypes, mode, language = 'en', requestId } = input
|
||||
const { text, entityTypes, mode, language = 'en', customPatterns, requestId } = input
|
||||
|
||||
logger.info(`[${requestId}] Starting PII validation`, {
|
||||
textLength: text.length,
|
||||
entityTypes,
|
||||
mode,
|
||||
language,
|
||||
customPatternCount: customPatterns?.length ?? 0,
|
||||
})
|
||||
|
||||
try {
|
||||
const spans = await analyze(text, entityTypes, language)
|
||||
const spans = await analyze(text, entityTypes, language, customPatterns)
|
||||
|
||||
const detectedEntities: DetectedPIIEntity[] = spans.map((s) => ({
|
||||
type: s.entity_type,
|
||||
type: displayEntityType(s.entity_type, customPatterns),
|
||||
start: s.start,
|
||||
end: s.end,
|
||||
score: s.score,
|
||||
@@ -234,8 +298,9 @@ export async function validatePII(input: PIIValidationInput): Promise<PIIValidat
|
||||
return { passed: false, error: `PII detected: ${summary}`, detectedEntities }
|
||||
}
|
||||
|
||||
// mask mode: the anonymizer replaces every span with `<ENTITY_TYPE>`.
|
||||
const maskedText = await anonymize(text, spans)
|
||||
// mask mode: the anonymizer replaces every span with `<ENTITY_TYPE>` (or the
|
||||
// pattern's `replacement` for custom-pattern spans).
|
||||
const maskedText = await anonymize(text, spans, customPatterns)
|
||||
logger.info(`[${requestId}] PII validation completed`, {
|
||||
passed: true,
|
||||
detectedCount: detectedEntities.length,
|
||||
@@ -266,7 +331,8 @@ export async function validatePII(input: PIIValidationInput): Promise<PIIValidat
|
||||
export async function maskPIIBatch(
|
||||
texts: string[],
|
||||
entityTypes: string[],
|
||||
language = 'en'
|
||||
language = 'en',
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
): Promise<string[]> {
|
||||
if (texts.length === 0) return []
|
||||
|
||||
@@ -276,7 +342,7 @@ export async function maskPIIBatch(
|
||||
const chunkTexts = indices.map((i) => texts[i])
|
||||
|
||||
if (combinedRedactAvailable) {
|
||||
const masked = await redactBatch(chunkTexts, entityTypes, language)
|
||||
const masked = await redactBatch(chunkTexts, entityTypes, language, customPatterns)
|
||||
if (masked) {
|
||||
indices.forEach((originalIndex, pos) => {
|
||||
result[originalIndex] = masked[pos]
|
||||
@@ -287,7 +353,7 @@ export async function maskPIIBatch(
|
||||
combinedRedactAvailable = false
|
||||
}
|
||||
|
||||
const spansPerText = await analyzeBatch(chunkTexts, entityTypes, language)
|
||||
const spansPerText = await analyzeBatch(chunkTexts, entityTypes, language, customPatterns)
|
||||
|
||||
// A short/misaligned batch response would silently leave the unmatched
|
||||
// strings unmasked (fail-open). Throw so the caller applies its fail-safe
|
||||
@@ -310,7 +376,7 @@ export async function maskPIIBatch(
|
||||
anonymizePositions.push(pos)
|
||||
})
|
||||
|
||||
const masked = await anonymizeBatch(toAnonymize)
|
||||
const masked = await anonymizeBatch(toAnonymize, customPatterns)
|
||||
if (masked.length !== toAnonymize.length) {
|
||||
throw new Error(
|
||||
`Presidio anonymize_batch returned ${masked.length} result(s) for ${toAnonymize.length} input(s)`
|
||||
|
||||
@@ -8,6 +8,35 @@ export interface ValidationResult {
|
||||
error?: string
|
||||
}
|
||||
|
||||
/** Result of validating a regex pattern's syntax and safety (independent of any input). */
|
||||
export interface RegexPatternValidation {
|
||||
valid: boolean
|
||||
error?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a regex pattern's syntax and safety without matching it against input:
|
||||
* it must compile (`new RegExp`) and pass `safe-regex2`'s catastrophic-backtracking
|
||||
* screen. Shared by the custom-pattern editor UI and any pre-flight boundary check.
|
||||
*/
|
||||
export function validateRegexPattern(pattern: string): RegexPatternValidation {
|
||||
if (pattern.length === 0) {
|
||||
return { valid: false, error: 'Pattern cannot be empty' }
|
||||
}
|
||||
try {
|
||||
new RegExp(pattern)
|
||||
} catch (error) {
|
||||
return { valid: false, error: `Invalid regex: ${(error as Error).message}` }
|
||||
}
|
||||
if (!safe(pattern)) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Pattern rejected: potentially unsafe (catastrophic backtracking)',
|
||||
}
|
||||
}
|
||||
return { valid: true }
|
||||
}
|
||||
|
||||
export function validateRegex(inputStr: string, pattern: string): ValidationResult {
|
||||
let regex: RegExp
|
||||
try {
|
||||
|
||||
@@ -669,6 +669,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
||||
const working = await redactLargeValueRefs(payload, {
|
||||
entityTypes: config.entityTypes,
|
||||
language: config.language,
|
||||
customPatterns: config.customPatterns,
|
||||
store: {
|
||||
workspaceId,
|
||||
workflowId: storeContext.workflowId ?? undefined,
|
||||
@@ -680,6 +681,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
|
||||
return redactPIIFromExecution(working, {
|
||||
entityTypes: config.entityTypes,
|
||||
language: config.language,
|
||||
customPatterns: config.customPatterns,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import { isLargeValueRef, type LargeValueRef } from '@/lib/execution/payloads/la
|
||||
import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
|
||||
import type { LargeValueStoreContext } from '@/lib/execution/payloads/store'
|
||||
import { materializeLargeValueRef } from '@/lib/execution/payloads/store'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
import {
|
||||
PiiRedactionError,
|
||||
type PiiRedactionFailureMode,
|
||||
@@ -23,6 +24,8 @@ export interface RedactLargeValueRefsOptions {
|
||||
/** Presidio entity types to mask. Empty = redact all detected PII. */
|
||||
entityTypes: string[]
|
||||
language: string
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
/** Storage scope for materializing and re-storing the masked values. */
|
||||
store: LargeValueStoreContext
|
||||
/**
|
||||
@@ -185,6 +188,7 @@ async function maskAndReStore(
|
||||
const masked = await redactObjectStrings(nested, {
|
||||
entityTypes: options.entityTypes,
|
||||
language: options.language,
|
||||
customPatterns: options.customPatterns,
|
||||
onFailure: options.onFailure ?? 'scrub',
|
||||
})
|
||||
return compactExecutionPayload(masked, { ...options.store, requireDurable: true })
|
||||
|
||||
@@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
|
||||
import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
|
||||
import { maskPIIBatchViaHttp } from '@/lib/guardrails/mask-client'
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
|
||||
const logger = createLogger('PiiRedaction')
|
||||
|
||||
@@ -23,6 +24,8 @@ export interface PiiRedactionOptions {
|
||||
/** Presidio entity types to mask. Empty = redact all detected PII. */
|
||||
entityTypes: string[]
|
||||
language?: string
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
/** Failure handling. Defaults to `'scrub'`. */
|
||||
onFailure?: PiiRedactionFailureMode
|
||||
}
|
||||
@@ -154,7 +157,12 @@ async function maskCollected(
|
||||
try {
|
||||
// Presidio runs only in the app container; the persist + execution paths also
|
||||
// run in the trigger.dev runtime, so masking always goes over HTTP to the app.
|
||||
const masked = await maskPIIBatchViaHttp(collected, options.entityTypes, language)
|
||||
const masked = await maskPIIBatchViaHttp(
|
||||
collected,
|
||||
options.entityTypes,
|
||||
language,
|
||||
options.customPatterns
|
||||
)
|
||||
return { masked, scrubbed: false }
|
||||
} catch (error) {
|
||||
logger.error('PII masking failed', {
|
||||
|
||||
@@ -693,6 +693,7 @@ async function executeWorkflowCoreImpl(
|
||||
const inputOpts = {
|
||||
entityTypes: piiRedaction.input.entityTypes,
|
||||
language: piiRedaction.input.language,
|
||||
customPatterns: piiRedaction.input.customPatterns,
|
||||
onFailure: 'throw' as const,
|
||||
}
|
||||
processedInput = await redactLargeValueRefsInValue(processedInput, {
|
||||
@@ -722,6 +723,7 @@ async function executeWorkflowCoreImpl(
|
||||
const blockOutputOpts = {
|
||||
entityTypes: piiRedaction.blockOutputs.entityTypes,
|
||||
language: piiRedaction.blockOutputs.language,
|
||||
customPatterns: piiRedaction.blockOutputs.customPatterns,
|
||||
onFailure: 'throw' as const,
|
||||
}
|
||||
const largeRefOpts = {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { guardrailsValidateContract } from '@/lib/api/contracts/hotspots'
|
||||
import { guardrailsValidateTool } from '@/tools/guardrails/validate'
|
||||
|
||||
// The block layer serializes an empty checkbox / table subBlock to `null`; the
|
||||
// tool's body builder must produce a shape the contract accepts (undefined, not null).
|
||||
const buildBody = (params: Record<string, unknown>) =>
|
||||
guardrailsValidateTool.request.body?.(params as never) as Record<string, unknown>
|
||||
|
||||
describe('guardrailsValidateTool.request.body', () => {
|
||||
it('coerces a null entity-type checkbox to omitted, and the contract accepts it', () => {
|
||||
const body = buildBody({ input: 'x', validationType: 'pii', piiEntityTypes: null })
|
||||
expect(body.piiEntityTypes).toBeUndefined()
|
||||
expect(guardrailsValidateContract.body.safeParse(body).success).toBe(true)
|
||||
})
|
||||
|
||||
it('passes a real entity-type array through unchanged', () => {
|
||||
const body = buildBody({
|
||||
input: 'x',
|
||||
validationType: 'pii',
|
||||
piiEntityTypes: ['EMAIL_ADDRESS'],
|
||||
})
|
||||
expect(body.piiEntityTypes).toEqual(['EMAIL_ADDRESS'])
|
||||
})
|
||||
|
||||
it('maps custom-pattern table rows to the wire shape and validates against the contract', () => {
|
||||
const body = buildBody({
|
||||
input: 'x',
|
||||
validationType: 'pii',
|
||||
piiEntityTypes: null,
|
||||
piiCustomPatterns: [
|
||||
{ cells: { Name: 'Emp', Pattern: 'EMP-\\d{6}', Replacement: 'EMPLOYEE_ID' } },
|
||||
],
|
||||
})
|
||||
expect(body.piiCustomPatterns).toEqual([
|
||||
{ name: 'Emp', regex: 'EMP-\\d{6}', replacement: 'EMPLOYEE_ID' },
|
||||
])
|
||||
expect(guardrailsValidateContract.body.safeParse(body).success).toBe(true)
|
||||
})
|
||||
|
||||
it('omits custom patterns when the table is empty/null', () => {
|
||||
const body = buildBody({
|
||||
input: 'x',
|
||||
validationType: 'pii',
|
||||
piiEntityTypes: null,
|
||||
piiCustomPatterns: null,
|
||||
})
|
||||
expect(body.piiCustomPatterns).toBeUndefined()
|
||||
})
|
||||
|
||||
it('regression guard: the contract rejects the raw null the block emits (why we coerce)', () => {
|
||||
const parsed = guardrailsValidateContract.body.safeParse({
|
||||
input: 'x',
|
||||
validationType: 'pii',
|
||||
piiEntityTypes: null,
|
||||
})
|
||||
expect(parsed.success).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,14 @@
|
||||
import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
/** A row from the `piiCustomPatterns` table subBlock (cells keyed by column header). */
|
||||
interface CustomPatternRow {
|
||||
cells?: { Name?: string; Pattern?: string; Replacement?: string }
|
||||
Name?: string
|
||||
Pattern?: string
|
||||
Replacement?: string
|
||||
}
|
||||
|
||||
export interface GuardrailsValidateInput {
|
||||
input: string
|
||||
validationType: 'json' | 'regex' | 'hallucination' | 'pii'
|
||||
@@ -20,12 +29,29 @@ export interface GuardrailsValidateInput {
|
||||
piiEntityTypes?: string[]
|
||||
piiMode?: string
|
||||
piiLanguage?: string
|
||||
piiCustomPatterns?: CustomPatternRow[]
|
||||
_context?: {
|
||||
workflowId?: string
|
||||
workspaceId?: string
|
||||
}
|
||||
}
|
||||
|
||||
/** Map the raw table rows into the wire shape, dropping rows with no pattern. */
|
||||
function toCustomPatterns(rows: CustomPatternRow[] | undefined): CustomPiiPattern[] | undefined {
|
||||
if (!Array.isArray(rows)) return undefined
|
||||
const patterns: CustomPiiPattern[] = []
|
||||
for (const row of rows) {
|
||||
const regex = (row?.cells?.Pattern ?? row?.Pattern ?? '').trim()
|
||||
if (!regex) continue
|
||||
patterns.push({
|
||||
name: (row?.cells?.Name ?? row?.Name ?? '').trim(),
|
||||
regex,
|
||||
replacement: row?.cells?.Replacement ?? row?.Replacement ?? '',
|
||||
})
|
||||
}
|
||||
return patterns.length > 0 ? patterns : undefined
|
||||
}
|
||||
|
||||
export interface GuardrailsValidateOutput {
|
||||
success: boolean
|
||||
output: {
|
||||
@@ -116,6 +142,12 @@ export const guardrailsValidateTool: ToolConfig<GuardrailsValidateInput, Guardra
|
||||
visibility: 'user-only',
|
||||
description: 'Language for PII detection (default: en)',
|
||||
},
|
||||
piiCustomPatterns: {
|
||||
type: 'array',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Custom regex patterns to detect and replace (name, pattern, replacement)',
|
||||
},
|
||||
},
|
||||
|
||||
outputs: {
|
||||
@@ -182,9 +214,13 @@ export const guardrailsValidateTool: ToolConfig<GuardrailsValidateInput, Guardra
|
||||
bedrockAccessKeyId: params.bedrockAccessKeyId,
|
||||
bedrockSecretKey: params.bedrockSecretKey,
|
||||
bedrockRegion: params.bedrockRegion,
|
||||
piiEntityTypes: params.piiEntityTypes,
|
||||
// An empty entity-type checkbox serializes to null; the contract's array
|
||||
// field accepts undefined (omitted), not null — so coerce. This is the
|
||||
// common shape when only custom patterns are configured.
|
||||
piiEntityTypes: Array.isArray(params.piiEntityTypes) ? params.piiEntityTypes : undefined,
|
||||
piiMode: params.piiMode,
|
||||
piiLanguage: params.piiLanguage,
|
||||
piiCustomPatterns: toCustomPatterns(params.piiCustomPatterns),
|
||||
workflowId: params._context?.workflowId,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
}),
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
gmailUntrashThreadV2Tool,
|
||||
gmailUpdateLabelV2Tool,
|
||||
} from '@/tools/gmail'
|
||||
import { guardrailsValidateTool } from '@/tools/guardrails'
|
||||
import { httpRequestTool } from '@/tools/http'
|
||||
import {
|
||||
slackAddReactionTool,
|
||||
@@ -85,6 +86,7 @@ import type { ToolConfig } from '@/tools/types'
|
||||
export const tools: Record<string, ToolConfig> = {
|
||||
http_request: httpRequestTool,
|
||||
function_execute: functionExecuteTool,
|
||||
guardrails_validate: guardrailsValidateTool,
|
||||
gmail_send_v2: gmailSendV2Tool,
|
||||
gmail_read_v2: gmailReadV2Tool,
|
||||
gmail_search_v2: gmailSearchV2Tool,
|
||||
|
||||
@@ -46,6 +46,14 @@ USER pii
|
||||
# 5001 avoids colliding with the app's 3000 in local/compose runs on one host.
|
||||
EXPOSE 5001
|
||||
|
||||
# Per-pattern regex match timeout (Presidio's `regex`-module `finditer(timeout=...)`),
|
||||
# an interactive backstop against a catastrophic user-supplied custom regex. Presidio
|
||||
# logs and skips a pattern that exceeds it — the request stays up rather than hanging.
|
||||
# Set well below the 60s library default so a pathological pattern can't stall a worker.
|
||||
# MUST be an integer — Presidio parses it with `int()`, so a float (e.g. 1.5) crashes
|
||||
# the service at import.
|
||||
ENV REGEX_TIMEOUT_SECONDS=2
|
||||
|
||||
# start-period covers the model cold start. With PII_WORKERS>1 each worker loads
|
||||
# the five spaCy models independently and in parallel, so allow generous headroom
|
||||
# (memory-bandwidth contention stretches the wall-time beyond the single-worker case).
|
||||
|
||||
@@ -1153,6 +1153,13 @@ export const chat = pgTable(
|
||||
}
|
||||
)
|
||||
|
||||
/** A user-supplied custom regex pattern; matches are replaced verbatim with `replacement`. */
|
||||
export interface CustomPiiPattern {
|
||||
name: string
|
||||
regex: string
|
||||
replacement: string
|
||||
}
|
||||
|
||||
/** Per-stage PII redaction policy stored on a {@link PiiRedactionRule}. */
|
||||
export interface PiiStagePolicy {
|
||||
enabled: boolean
|
||||
@@ -1160,6 +1167,8 @@ export interface PiiStagePolicy {
|
||||
entityTypes: string[]
|
||||
/** Language whose Presidio recognizers apply (e.g. 'en', 'es'); defaults to English. */
|
||||
language?: string
|
||||
/** User-supplied custom regex patterns applied alongside `entityTypes`. */
|
||||
customPatterns?: CustomPiiPattern[]
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user