feat(risk-control): record matched keyword in keyword-block logs

The risk control center's moderation log records had no field for the
keyword that triggered a keyword block, so the admin UI couldn't show
which keyword was hit (only the application slog logged it).

- migration 156: add matched_keyword column to content_moderation_logs
- ContentModerationLog gains MatchedKeyword; set it on keyword block
- repo CreateLog/ListLogs persist and read the column
- frontend: show "命中关键词" inline in the log table and detail modal
- i18n: add matchedKeyword (zh/en)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
bwlc
2026-06-26 10:43:05 +08:00
co-authored by Claude Opus 4.8
parent e5f38a6f60
commit 815bc6c9b5
8 changed files with 21 additions and 4 deletions
@@ -53,17 +53,17 @@ INSERT INTO content_moderation_logs (
request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name,
endpoint, provider, model, mode, action, flagged, highest_category, highest_score,
category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error,
violation_count, auto_banned, email_sent, queue_delay_ms
violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword
) VALUES (
$1, $2, $3, $4, $5, $6, $7,
$8, $9, $10, $11, $12, $13, $14, $15,
$16::jsonb, $17::jsonb, $18, $19, $20,
$21, $22, $23, $24
$21, $22, $23, $24, $25
) RETURNING id, created_at`,
log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName,
log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore,
string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error,
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS),
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword,
).Scan(&log.ID, &log.CreatedAt)
if err != nil {
return fmt.Errorf("insert content moderation log: %w", err)
@@ -97,7 +97,7 @@ SELECT
l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name,
l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score,
l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error,
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.created_at
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at
FROM content_moderation_logs l
LEFT JOIN users u ON u.id = l.user_id `+whereSQL+`
ORDER BY l.created_at DESC, l.id DESC
@@ -141,6 +141,7 @@ LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)),
&item.EmailSent,
&item.UserStatus,
&queueDelay,
&item.MatchedKeyword,
&item.CreatedAt,
); err != nil {
return nil, nil, fmt.Errorf("scan content moderation log: %w", err)
@@ -390,6 +390,7 @@ type ContentModerationLog struct {
Flagged bool `json:"flagged"`
HighestCategory string `json:"highest_category"`
HighestScore float64 `json:"highest_score"`
MatchedKeyword string `json:"matched_keyword"`
CategoryScores map[string]float64 `json:"category_scores"`
ThresholdSnapshot map[string]float64 `json:"threshold_snapshot"`
InputExcerpt string `json:"input_excerpt"`
@@ -895,6 +896,7 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
"keyword", keyword)
scores := map[string]float64{contentModerationKeywordCategory: 1.0}
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
log.MatchedKeyword = keyword
s.enqueueRecord(input, cfg, log, hashText, false, true)
return &ContentModerationDecision{
Allowed: false,
@@ -485,6 +485,7 @@ func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T
require.True(t, logs[0].Flagged)
require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
}
func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
@@ -0,0 +1,3 @@
-- 风控中心:记录关键词拦截命中的具体关键词
ALTER TABLE content_moderation_logs ADD COLUMN IF NOT EXISTS matched_keyword VARCHAR(255) NOT NULL DEFAULT '';
+1
View File
@@ -182,6 +182,7 @@ export interface ContentModerationLog {
flagged: boolean
highest_category: string
highest_score: number
matched_keyword: string
category_scores: Record<string, number>
threshold_snapshot: Record<string, number>
input_excerpt: string
+1
View File
@@ -2648,6 +2648,7 @@ export default {
unbanFailed: 'Failed to unban user',
inputDetailTitle: 'Input Summary Detail',
inputDetailContent: 'Full Content',
matchedKeyword: 'Matched Keyword',
queueDelay: 'Queued {ms} ms',
allGroups: 'All Groups',
allGroupsHint: 'Auditing all groups',
+1
View File
@@ -2725,6 +2725,7 @@ export default {
unbanFailed: '解封用户失败',
inputDetailTitle: '输入摘要详情',
inputDetailContent: '完整内容',
matchedKeyword: '命中关键词',
queueDelay: '排队 {ms} ms',
allGroups: '全部分组',
allGroupsHint: '当前审计全部分组',
@@ -317,6 +317,9 @@
<td class="whitespace-nowrap px-5 py-4 text-sm text-gray-700 dark:text-gray-300">
<div>{{ row.highest_category || '-' }}</div>
<div class="text-xs text-gray-400">{{ percent(row.highest_score) }}</div>
<div v-if="row.matched_keyword" class="mt-0.5 text-xs font-medium text-red-600 dark:text-red-300" :title="t('admin.riskControl.matchedKeyword') + ': ' + row.matched_keyword">
{{ t('admin.riskControl.matchedKeyword') }}: {{ row.matched_keyword }}
</div>
</td>
<td class="whitespace-nowrap px-5 py-4 text-sm text-gray-700 dark:text-gray-300">
<div>{{ violationCountText(row) }}</div>
@@ -1078,6 +1081,10 @@
{{ inputDetailRow.highest_category || '-' }} / {{ percent(inputDetailRow.highest_score) }}
</p>
</div>
<div v-if="inputDetailRow.matched_keyword" class="rounded-lg border border-red-100 bg-red-50 p-4 dark:border-red-900/60 dark:bg-red-900/20">
<p class="text-xs font-medium text-red-500 dark:text-red-300">{{ t('admin.riskControl.matchedKeyword') }}</p>
<p class="mt-1 truncate text-sm font-semibold text-red-700 dark:text-red-200" :title="inputDetailRow.matched_keyword">{{ inputDetailRow.matched_keyword }}</p>
</div>
</div>
<div class="rounded-xl border border-gray-100 bg-white p-4 shadow-sm dark:border-dark-700 dark:bg-dark-800">