diff --git a/backend/internal/repository/content_moderation_repo.go b/backend/internal/repository/content_moderation_repo.go index 9a40b3c2fd..369951eb4b 100644 --- a/backend/internal/repository/content_moderation_repo.go +++ b/backend/internal/repository/content_moderation_repo.go @@ -53,17 +53,17 @@ INSERT INTO content_moderation_logs ( request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name, endpoint, provider, model, mode, action, flagged, highest_category, highest_score, category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error, - violation_count, auto_banned, email_sent, queue_delay_ms + violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16::jsonb, $17::jsonb, $18, $19, $20, - $21, $22, $23, $24 + $21, $22, $23, $24, $25 ) RETURNING id, created_at`, log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName, log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore, string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error, - log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), + log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword, ).Scan(&log.ID, &log.CreatedAt) if err != nil { return fmt.Errorf("insert content moderation log: %w", err) @@ -97,7 +97,7 @@ SELECT l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name, l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score, l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error, - l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.created_at + l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at FROM content_moderation_logs l LEFT JOIN users u ON u.id = l.user_id `+whereSQL+` ORDER BY l.created_at DESC, l.id DESC @@ -141,6 +141,7 @@ LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)), &item.EmailSent, &item.UserStatus, &queueDelay, + &item.MatchedKeyword, &item.CreatedAt, ); err != nil { return nil, nil, fmt.Errorf("scan content moderation log: %w", err) diff --git a/backend/internal/service/content_moderation.go b/backend/internal/service/content_moderation.go index 0ce4355fe7..6d3b91d205 100644 --- a/backend/internal/service/content_moderation.go +++ b/backend/internal/service/content_moderation.go @@ -390,6 +390,7 @@ type ContentModerationLog struct { Flagged bool `json:"flagged"` HighestCategory string `json:"highest_category"` HighestScore float64 `json:"highest_score"` + MatchedKeyword string `json:"matched_keyword"` CategoryScores map[string]float64 `json:"category_scores"` ThresholdSnapshot map[string]float64 `json:"threshold_snapshot"` InputExcerpt string `json:"input_excerpt"` @@ -895,6 +896,7 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer "keyword", keyword) scores := map[string]float64{contentModerationKeywordCategory: 1.0} log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "") + log.MatchedKeyword = keyword s.enqueueRecord(input, cfg, log, hashText, false, true) return &ContentModerationDecision{ Allowed: false, diff --git a/backend/internal/service/content_moderation_test.go b/backend/internal/service/content_moderation_test.go index 3336745647..d33101d5d1 100644 --- a/backend/internal/service/content_moderation_test.go +++ b/backend/internal/service/content_moderation_test.go @@ -485,6 +485,7 @@ func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T require.True(t, logs[0].Flagged) require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action) require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory) + require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit") } func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) { diff --git a/backend/migrations/156_content_moderation_matched_keyword.sql b/backend/migrations/156_content_moderation_matched_keyword.sql new file mode 100644 index 0000000000..ce2652da46 --- /dev/null +++ b/backend/migrations/156_content_moderation_matched_keyword.sql @@ -0,0 +1,3 @@ +-- 风控中心：记录关键词拦截命中的具体关键词 + +ALTER TABLE content_moderation_logs ADD COLUMN IF NOT EXISTS matched_keyword VARCHAR(255) NOT NULL DEFAULT ''; diff --git a/frontend/src/api/admin/riskControl.ts b/frontend/src/api/admin/riskControl.ts index 5cce4c1827..d96845d1a0 100644 --- a/frontend/src/api/admin/riskControl.ts +++ b/frontend/src/api/admin/riskControl.ts @@ -182,6 +182,7 @@ export interface ContentModerationLog { flagged: boolean highest_category: string highest_score: number + matched_keyword: string category_scores: Record threshold_snapshot: Record input_excerpt: string diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 0ac93a9811..f6fe7acc56 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -2648,6 +2648,7 @@ export default { unbanFailed: 'Failed to unban user', inputDetailTitle: 'Input Summary Detail', inputDetailContent: 'Full Content', + matchedKeyword: 'Matched Keyword', queueDelay: 'Queued {ms} ms', allGroups: 'All Groups', allGroupsHint: 'Auditing all groups', diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts index ec9c51a171..27bb8427e5 100644 --- a/frontend/src/i18n/locales/zh.ts +++ b/frontend/src/i18n/locales/zh.ts @@ -2725,6 +2725,7 @@ export default { unbanFailed: '解封用户失败', inputDetailTitle: '输入摘要详情', inputDetailContent: '完整内容', + matchedKeyword: '命中关键词', queueDelay: '排队 {ms} ms', allGroups: '全部分组', allGroupsHint: '当前审计全部分组', diff --git a/frontend/src/views/admin/RiskControlView.vue b/frontend/src/views/admin/RiskControlView.vue index fbb33a2673..78bd9b93fb 100644 --- a/frontend/src/views/admin/RiskControlView.vue +++ b/frontend/src/views/admin/RiskControlView.vue @@ -317,6 +317,9 @@
{{ row.highest_category || '-' }}
{{ percent(row.highest_score) }}
+
+ {{ t('admin.riskControl.matchedKeyword') }}: {{ row.matched_keyword }} +
{{ violationCountText(row) }}
@@ -1078,6 +1081,10 @@ {{ inputDetailRow.highest_category || '-' }} / {{ percent(inputDetailRow.highest_score) }}

+
+

{{ t('admin.riskControl.matchedKeyword') }}

+

{{ inputDetailRow.matched_keyword }}

+