mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-21 14:19:18 +08:00
feat(risk-control): record matched keyword in keyword-block logs
The risk control center's moderation log records had no field for the keyword that triggered a keyword block, so the admin UI couldn't show which keyword was hit (only the application slog logged it). - migration 156: add matched_keyword column to content_moderation_logs - ContentModerationLog gains MatchedKeyword; set it on keyword block - repo CreateLog/ListLogs persist and read the column - frontend: show "命中关键词" inline in the log table and detail modal - i18n: add matchedKeyword (zh/en) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -53,17 +53,17 @@ INSERT INTO content_moderation_logs (
|
||||
request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name,
|
||||
endpoint, provider, model, mode, action, flagged, highest_category, highest_score,
|
||||
category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error,
|
||||
violation_count, auto_banned, email_sent, queue_delay_ms
|
||||
violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7,
|
||||
$8, $9, $10, $11, $12, $13, $14, $15,
|
||||
$16::jsonb, $17::jsonb, $18, $19, $20,
|
||||
$21, $22, $23, $24
|
||||
$21, $22, $23, $24, $25
|
||||
) RETURNING id, created_at`,
|
||||
log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName,
|
||||
log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore,
|
||||
string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error,
|
||||
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS),
|
||||
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword,
|
||||
).Scan(&log.ID, &log.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert content moderation log: %w", err)
|
||||
@@ -97,7 +97,7 @@ SELECT
|
||||
l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name,
|
||||
l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score,
|
||||
l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error,
|
||||
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.created_at
|
||||
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at
|
||||
FROM content_moderation_logs l
|
||||
LEFT JOIN users u ON u.id = l.user_id `+whereSQL+`
|
||||
ORDER BY l.created_at DESC, l.id DESC
|
||||
@@ -141,6 +141,7 @@ LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)),
|
||||
&item.EmailSent,
|
||||
&item.UserStatus,
|
||||
&queueDelay,
|
||||
&item.MatchedKeyword,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan content moderation log: %w", err)
|
||||
|
||||
@@ -390,6 +390,7 @@ type ContentModerationLog struct {
|
||||
Flagged bool `json:"flagged"`
|
||||
HighestCategory string `json:"highest_category"`
|
||||
HighestScore float64 `json:"highest_score"`
|
||||
MatchedKeyword string `json:"matched_keyword"`
|
||||
CategoryScores map[string]float64 `json:"category_scores"`
|
||||
ThresholdSnapshot map[string]float64 `json:"threshold_snapshot"`
|
||||
InputExcerpt string `json:"input_excerpt"`
|
||||
@@ -895,6 +896,7 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
"keyword", keyword)
|
||||
scores := map[string]float64{contentModerationKeywordCategory: 1.0}
|
||||
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
|
||||
log.MatchedKeyword = keyword
|
||||
s.enqueueRecord(input, cfg, log, hashText, false, true)
|
||||
return &ContentModerationDecision{
|
||||
Allowed: false,
|
||||
|
||||
@@ -485,6 +485,7 @@ func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T
|
||||
require.True(t, logs[0].Flagged)
|
||||
require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
|
||||
require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
|
||||
require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
|
||||
}
|
||||
|
||||
func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
-- 风控中心:记录关键词拦截命中的具体关键词
|
||||
|
||||
ALTER TABLE content_moderation_logs ADD COLUMN IF NOT EXISTS matched_keyword VARCHAR(255) NOT NULL DEFAULT '';
|
||||
@@ -182,6 +182,7 @@ export interface ContentModerationLog {
|
||||
flagged: boolean
|
||||
highest_category: string
|
||||
highest_score: number
|
||||
matched_keyword: string
|
||||
category_scores: Record<string, number>
|
||||
threshold_snapshot: Record<string, number>
|
||||
input_excerpt: string
|
||||
|
||||
@@ -2648,6 +2648,7 @@ export default {
|
||||
unbanFailed: 'Failed to unban user',
|
||||
inputDetailTitle: 'Input Summary Detail',
|
||||
inputDetailContent: 'Full Content',
|
||||
matchedKeyword: 'Matched Keyword',
|
||||
queueDelay: 'Queued {ms} ms',
|
||||
allGroups: 'All Groups',
|
||||
allGroupsHint: 'Auditing all groups',
|
||||
|
||||
@@ -2725,6 +2725,7 @@ export default {
|
||||
unbanFailed: '解封用户失败',
|
||||
inputDetailTitle: '输入摘要详情',
|
||||
inputDetailContent: '完整内容',
|
||||
matchedKeyword: '命中关键词',
|
||||
queueDelay: '排队 {ms} ms',
|
||||
allGroups: '全部分组',
|
||||
allGroupsHint: '当前审计全部分组',
|
||||
|
||||
@@ -317,6 +317,9 @@
|
||||
<td class="whitespace-nowrap px-5 py-4 text-sm text-gray-700 dark:text-gray-300">
|
||||
<div>{{ row.highest_category || '-' }}</div>
|
||||
<div class="text-xs text-gray-400">{{ percent(row.highest_score) }}</div>
|
||||
<div v-if="row.matched_keyword" class="mt-0.5 text-xs font-medium text-red-600 dark:text-red-300" :title="t('admin.riskControl.matchedKeyword') + ': ' + row.matched_keyword">
|
||||
{{ t('admin.riskControl.matchedKeyword') }}: {{ row.matched_keyword }}
|
||||
</div>
|
||||
</td>
|
||||
<td class="whitespace-nowrap px-5 py-4 text-sm text-gray-700 dark:text-gray-300">
|
||||
<div>{{ violationCountText(row) }}</div>
|
||||
@@ -1078,6 +1081,10 @@
|
||||
{{ inputDetailRow.highest_category || '-' }} / {{ percent(inputDetailRow.highest_score) }}
|
||||
</p>
|
||||
</div>
|
||||
<div v-if="inputDetailRow.matched_keyword" class="rounded-lg border border-red-100 bg-red-50 p-4 dark:border-red-900/60 dark:bg-red-900/20">
|
||||
<p class="text-xs font-medium text-red-500 dark:text-red-300">{{ t('admin.riskControl.matchedKeyword') }}</p>
|
||||
<p class="mt-1 truncate text-sm font-semibold text-red-700 dark:text-red-200" :title="inputDetailRow.matched_keyword">{{ inputDetailRow.matched_keyword }}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="rounded-xl border border-gray-100 bg-white p-4 shadow-sm dark:border-dark-700 dark:bg-dark-800">
|
||||
|
||||
Reference in New Issue
Block a user