fix(knowledge): document tag filter matches case-insensitively and by calendar day (#5221)

The Knowledge Base document list applied tag filters with case-sensitive text
equality and compared date tags against a midnight-UTC timestamp, so text
filters missed on any casing difference and date eq never matched a stored
timestamp — both silently returned empty results.

Align the document-list filter with the knowledge search filter semantics:
text eq/neq are now case-insensitive (LOWER) and date comparisons run on the
calendar day (::date). Extract the predicate builder into its own
single-responsibility module with unit coverage, and tidy the filter popover's
secondary labels to the caption text size for chip-design consistency.
This commit is contained in:
Waleed
2026-06-26 10:18:23 -07:00
committed by GitHub
parent a1d5870681
commit 365d8be02b
5 changed files with 341 additions and 161 deletions
@@ -23,8 +23,8 @@ import {
getProcessingConfig,
KnowledgeBaseFileOwnershipError,
processDocumentsWithQueue,
type TagFilterCondition,
} from '@/lib/knowledge/documents/service'
import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter'
import { captureServerEvent } from '@/lib/posthog/server'
import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
@@ -926,7 +926,7 @@ export function KnowledgeBase({
setSelectedDocuments(new Set())
setIsSelectAllMode(false)
}}
className='-mr-1 h-auto px-1 py-0.5 text-[var(--text-muted)] text-xs hover-hover:text-[var(--text-secondary)]'
className='-mr-1 h-auto px-1 py-0.5 text-[var(--text-muted)] text-caption hover-hover:text-[var(--text-secondary)]'
>
Clear
</Button>
@@ -1499,7 +1499,7 @@ function TagFilterValueControl({ entry, onChange }: TagFilterValueControlProps)
fullWidth
flush
/>
<span className='flex-shrink-0 text-[var(--text-muted)] text-xs'>to</span>
<span className='flex-shrink-0 text-[var(--text-muted)] text-caption'>to</span>
<ChipDatePicker
value={entry.valueTo || undefined}
onChange={(value) => onChange({ valueTo: value })}
@@ -1530,7 +1530,7 @@ function TagFilterValueControl({ entry, onChange }: TagFilterValueControlProps)
onChange={(event) => onChange({ value: event.target.value })}
placeholder='From'
/>
<span className='flex-shrink-0 text-[var(--text-muted)] text-xs'>to</span>
<span className='flex-shrink-0 text-[var(--text-muted)] text-caption'>to</span>
<ChipInput
value={entry.valueTo}
onChange={(event) => onChange({ valueTo: event.target.value })}
@@ -1625,7 +1625,7 @@ function TagFilterSection({ tagDefinitions, entries, onChange }: TagFilterSectio
{activeCount > 0 && (
<Button
variant='ghost'
className='-mr-1 h-auto px-1 py-0.5 text-[var(--text-muted)] text-xs hover-hover:text-[var(--text-secondary)]'
className='-mr-1 h-auto px-1 py-0.5 text-[var(--text-muted)] text-caption hover-hover:text-[var(--text-secondary)]'
onClick={() => onChange([])}
>
Clear all
@@ -1648,7 +1648,7 @@ function TagFilterSection({ tagDefinitions, entries, onChange }: TagFilterSectio
<div key={entry.id} className='flex flex-col gap-2'>
{index > 0 && (
<div className='flex items-center gap-2'>
<span className='shrink-0 text-[var(--text-muted)] text-xs leading-none'>
<span className='shrink-0 text-[var(--text-muted)] text-caption leading-none'>
and
</span>
<div className='h-px flex-1 bg-[var(--border-1)]' />
+5 -155
View File
@@ -12,22 +12,7 @@ import { sha256Hex } from '@sim/security/hash'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { tasks } from '@trigger.dev/sdk'
import {
and,
asc,
desc,
eq,
gt,
gte,
inArray,
isNotNull,
isNull,
lt,
lte,
ne,
type SQL,
sql,
} from 'drizzle-orm'
import { and, asc, desc, eq, inArray, isNotNull, isNull, type SQL, sql } from 'drizzle-orm'
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
import { recordUsage } from '@/lib/billing/core/usage-log'
import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
@@ -36,6 +21,10 @@ import { resolveTriggerRegion } from '@/lib/core/async-jobs/region'
import { env, envNumber } from '@/lib/core/config/env'
import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags'
import { processDocument } from '@/lib/knowledge/documents/document-processor'
import {
buildTagFilterCondition,
type TagFilterCondition,
} from '@/lib/knowledge/documents/tag-filter'
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models'
import { generateEmbeddings } from '@/lib/knowledge/embeddings'
@@ -997,145 +986,6 @@ export async function createDocumentRecords(
})
}
export interface TagFilterCondition {
tagSlot: string
fieldType: 'text' | 'number' | 'date' | 'boolean'
operator: string
value: unknown
valueTo?: unknown
}
const ALLOWED_TAG_SLOTS = new Set([
'tag1',
'tag2',
'tag3',
'tag4',
'tag5',
'tag6',
'tag7',
'number1',
'number2',
'number3',
'number4',
'number5',
'date1',
'date2',
'boolean1',
'boolean2',
'boolean3',
])
function escapeLikePattern(s: string): string {
return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')
}
function buildTagFilterCondition(filter: TagFilterCondition): SQL | undefined {
if (!ALLOWED_TAG_SLOTS.has(filter.tagSlot)) return undefined
const col = document[filter.tagSlot as keyof typeof document]
if (filter.fieldType === 'text') {
const v = String(filter.value ?? '')
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.tag1, v)
case 'neq':
return ne(col as typeof document.tag1, v)
case 'contains': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
}
case 'not_contains': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) NOT LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
}
case 'starts_with': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`${escaped}%`}) ESCAPE '\\'`
}
case 'ends_with': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}`}) ESCAPE '\\'`
}
default:
return undefined
}
}
if (filter.fieldType === 'number') {
const num = Number(filter.value)
if (Number.isNaN(num)) return undefined
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.number1, num)
case 'neq':
return ne(col as typeof document.number1, num)
case 'gt':
return gt(col as typeof document.number1, num)
case 'gte':
return gte(col as typeof document.number1, num)
case 'lt':
return lt(col as typeof document.number1, num)
case 'lte':
return lte(col as typeof document.number1, num)
case 'between': {
const numTo = Number(filter.valueTo)
if (Number.isNaN(numTo)) return undefined
return and(
gte(col as typeof document.number1, num),
lte(col as typeof document.number1, numTo)
)
}
default:
return undefined
}
}
if (filter.fieldType === 'date') {
const v = String(filter.value ?? '')
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.date1, new Date(v))
case 'neq':
return ne(col as typeof document.date1, new Date(v))
case 'gt':
return gt(col as typeof document.date1, new Date(v))
case 'gte':
return gte(col as typeof document.date1, new Date(v))
case 'lt':
return lt(col as typeof document.date1, new Date(v))
case 'lte':
return lte(col as typeof document.date1, new Date(v))
case 'between': {
if (!filter.valueTo) return undefined
const valueTo = String(filter.valueTo)
return and(
gte(col as typeof document.date1, new Date(v)),
lte(col as typeof document.date1, new Date(valueTo))
)
}
default:
return undefined
}
}
if (filter.fieldType === 'boolean') {
const boolVal =
typeof filter.value === 'boolean' ? filter.value : parseBooleanValue(String(filter.value))
if (boolVal === null) return undefined
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.boolean1, boolVal)
case 'neq':
return ne(col as typeof document.boolean1, boolVal)
default:
return undefined
}
}
return undefined
}
export async function getDocuments(
knowledgeBaseId: string,
options: {
@@ -0,0 +1,177 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { buildTagFilterCondition } from '@/lib/knowledge/documents/tag-filter'
/**
* The global `drizzle-orm` mock renders `sql` fragments to a `?`-placeholder
* string via `toSQL()` and returns plain `{ type, left, right }` objects for the
* comparison operators, so we can assert the exact predicate each filter builds.
*/
function rendered(condition: ReturnType<typeof buildTagFilterCondition>) {
return (condition as unknown as { toSQL: () => { sql: string; params: unknown[] } }).toSQL()
}
describe('buildTagFilterCondition', () => {
it('ignores unknown tag slots', () => {
expect(
buildTagFilterCondition({
tagSlot: 'not_a_real_slot',
fieldType: 'text',
operator: 'eq',
value: 'x',
})
).toBeUndefined()
})
describe('text', () => {
it('matches eq case-insensitively', () => {
const { sql, params } = rendered(
buildTagFilterCondition({
tagSlot: 'tag1',
fieldType: 'text',
operator: 'eq',
value: 'Ada Lovelace',
})
)
expect(sql).toBe('LOWER(?) = LOWER(?)')
expect(params).toEqual(['tag1', 'Ada Lovelace'])
})
it('matches neq case-insensitively', () => {
const { sql, params } = rendered(
buildTagFilterCondition({
tagSlot: 'tag2',
fieldType: 'text',
operator: 'neq',
value: 'Spreadsheet',
})
)
expect(sql).toBe('LOWER(?) != LOWER(?)')
expect(params).toEqual(['tag2', 'Spreadsheet'])
})
it('escapes LIKE wildcards in contains', () => {
const { params } = rendered(
buildTagFilterCondition({
tagSlot: 'tag1',
fieldType: 'text',
operator: 'contains',
value: '50%_off',
})
)
expect(params).toContain('%50\\%\\_off%')
})
it('returns undefined for an unsupported operator', () => {
expect(
buildTagFilterCondition({
tagSlot: 'tag1',
fieldType: 'text',
operator: 'gt',
value: 'x',
})
).toBeUndefined()
})
})
describe('date', () => {
it('compares eq on the calendar day', () => {
const { sql, params } = rendered(
buildTagFilterCondition({
tagSlot: 'date1',
fieldType: 'date',
operator: 'eq',
value: '2026-04-21',
})
)
expect(sql).toBe('?::date = ?::date')
expect(params).toEqual(['date1', '2026-04-21'])
})
it('compares range bounds on the calendar day', () => {
const condition = buildTagFilterCondition({
tagSlot: 'date1',
fieldType: 'date',
operator: 'between',
value: '2026-04-01',
valueTo: '2026-04-30',
}) as unknown as { type: string; conditions: unknown[] }
expect(condition.type).toBe('and')
expect(condition.conditions).toHaveLength(2)
expect(rendered(condition.conditions[0] as never).sql).toBe('?::date >= ?::date')
expect(rendered(condition.conditions[1] as never).sql).toBe('?::date <= ?::date')
})
it('ignores values that are not YYYY-MM-DD', () => {
expect(
buildTagFilterCondition({
tagSlot: 'date1',
fieldType: 'date',
operator: 'eq',
value: 'not-a-date',
})
).toBeUndefined()
})
it('ignores a between filter missing its upper bound', () => {
expect(
buildTagFilterCondition({
tagSlot: 'date1',
fieldType: 'date',
operator: 'between',
value: '2026-04-01',
})
).toBeUndefined()
})
})
describe('number', () => {
it('builds an equality comparison', () => {
expect(
buildTagFilterCondition({
tagSlot: 'number1',
fieldType: 'number',
operator: 'eq',
value: '42',
})
).toEqual({ type: 'eq', left: 'number1', right: 42 })
})
it('ignores non-numeric values', () => {
expect(
buildTagFilterCondition({
tagSlot: 'number1',
fieldType: 'number',
operator: 'eq',
value: 'abc',
})
).toBeUndefined()
})
})
describe('boolean', () => {
it('parses string values', () => {
expect(
buildTagFilterCondition({
tagSlot: 'boolean1',
fieldType: 'boolean',
operator: 'eq',
value: 'true',
})
).toEqual({ type: 'eq', left: 'boolean1', right: true })
})
it('ignores values that are not boolean-like', () => {
expect(
buildTagFilterCondition({
tagSlot: 'boolean1',
fieldType: 'boolean',
operator: 'eq',
value: 'maybe',
})
).toBeUndefined()
})
})
})
@@ -0,0 +1,153 @@
import { document } from '@sim/db/schema'
import { and, eq, gt, gte, lt, lte, ne, type SQL, sql } from 'drizzle-orm'
import { parseBooleanValue } from '@/lib/knowledge/tags/utils'
/**
* A single tag filter applied to a document list query.
*/
export interface TagFilterCondition {
tagSlot: string
fieldType: 'text' | 'number' | 'date' | 'boolean'
operator: string
value: unknown
valueTo?: unknown
}
const ALLOWED_TAG_SLOTS = new Set([
'tag1',
'tag2',
'tag3',
'tag4',
'tag5',
'tag6',
'tag7',
'number1',
'number2',
'number3',
'number4',
'number5',
'date1',
'date2',
'boolean1',
'boolean2',
'boolean3',
])
const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/
function escapeLikePattern(s: string): string {
return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')
}
/**
* Builds a SQL predicate for a single tag filter against the document table.
*
* Text comparisons are case-insensitive and date comparisons are evaluated on
* the calendar day, matching the semantics of the knowledge base search filter
* (`app/api/knowledge/search/utils.ts`). Returns `undefined` when the slot,
* operator, or value is not usable so the caller can skip the condition.
*/
export function buildTagFilterCondition(filter: TagFilterCondition): SQL | undefined {
if (!ALLOWED_TAG_SLOTS.has(filter.tagSlot)) return undefined
const col = document[filter.tagSlot as keyof typeof document]
if (filter.fieldType === 'text') {
const v = String(filter.value ?? '')
switch (filter.operator) {
case 'eq':
return sql`LOWER(${col}) = LOWER(${v})`
case 'neq':
return sql`LOWER(${col}) != LOWER(${v})`
case 'contains': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
}
case 'not_contains': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) NOT LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
}
case 'starts_with': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`${escaped}%`}) ESCAPE '\\'`
}
case 'ends_with': {
const escaped = escapeLikePattern(v)
return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}`}) ESCAPE '\\'`
}
default:
return undefined
}
}
if (filter.fieldType === 'number') {
const num = Number(filter.value)
if (Number.isNaN(num)) return undefined
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.number1, num)
case 'neq':
return ne(col as typeof document.number1, num)
case 'gt':
return gt(col as typeof document.number1, num)
case 'gte':
return gte(col as typeof document.number1, num)
case 'lt':
return lt(col as typeof document.number1, num)
case 'lte':
return lte(col as typeof document.number1, num)
case 'between': {
const numTo = Number(filter.valueTo)
if (Number.isNaN(numTo)) return undefined
return and(
gte(col as typeof document.number1, num),
lte(col as typeof document.number1, numTo)
)
}
default:
return undefined
}
}
if (filter.fieldType === 'date') {
const v = String(filter.value ?? '')
if (!DATE_ONLY_PATTERN.test(v)) return undefined
switch (filter.operator) {
case 'eq':
return sql`${col}::date = ${v}::date`
case 'neq':
return sql`${col}::date != ${v}::date`
case 'gt':
return sql`${col}::date > ${v}::date`
case 'gte':
return sql`${col}::date >= ${v}::date`
case 'lt':
return sql`${col}::date < ${v}::date`
case 'lte':
return sql`${col}::date <= ${v}::date`
case 'between': {
const valueTo = String(filter.valueTo ?? '')
if (!DATE_ONLY_PATTERN.test(valueTo)) return undefined
return and(sql`${col}::date >= ${v}::date`, sql`${col}::date <= ${valueTo}::date`)
}
default:
return undefined
}
}
if (filter.fieldType === 'boolean') {
const boolVal =
typeof filter.value === 'boolean' ? filter.value : parseBooleanValue(String(filter.value))
if (boolVal === null) return undefined
switch (filter.operator) {
case 'eq':
return eq(col as typeof document.boolean1, boolVal)
case 'neq':
return ne(col as typeof document.boolean1, boolVal)
default:
return undefined
}
}
return undefined
}