mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-24 16:05:44 +08:00
feat(risk-control): record matched keyword in keyword-block logs
The risk control center's moderation log records had no field for the keyword that triggered a keyword block, so the admin UI couldn't show which keyword was hit (only the application slog logged it). - migration 156: add matched_keyword column to content_moderation_logs - ContentModerationLog gains MatchedKeyword; set it on keyword block - repo CreateLog/ListLogs persist and read the column - frontend: show "命中关键词" inline in the log table and detail modal - i18n: add matchedKeyword (zh/en) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -53,17 +53,17 @@ INSERT INTO content_moderation_logs (
|
||||
request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name,
|
||||
endpoint, provider, model, mode, action, flagged, highest_category, highest_score,
|
||||
category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error,
|
||||
violation_count, auto_banned, email_sent, queue_delay_ms
|
||||
violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7,
|
||||
$8, $9, $10, $11, $12, $13, $14, $15,
|
||||
$16::jsonb, $17::jsonb, $18, $19, $20,
|
||||
$21, $22, $23, $24
|
||||
$21, $22, $23, $24, $25
|
||||
) RETURNING id, created_at`,
|
||||
log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName,
|
||||
log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore,
|
||||
string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error,
|
||||
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS),
|
||||
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword,
|
||||
).Scan(&log.ID, &log.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert content moderation log: %w", err)
|
||||
@@ -97,7 +97,7 @@ SELECT
|
||||
l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name,
|
||||
l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score,
|
||||
l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error,
|
||||
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.created_at
|
||||
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at
|
||||
FROM content_moderation_logs l
|
||||
LEFT JOIN users u ON u.id = l.user_id `+whereSQL+`
|
||||
ORDER BY l.created_at DESC, l.id DESC
|
||||
@@ -141,6 +141,7 @@ LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)),
|
||||
&item.EmailSent,
|
||||
&item.UserStatus,
|
||||
&queueDelay,
|
||||
&item.MatchedKeyword,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan content moderation log: %w", err)
|
||||
|
||||
@@ -390,6 +390,7 @@ type ContentModerationLog struct {
|
||||
Flagged bool `json:"flagged"`
|
||||
HighestCategory string `json:"highest_category"`
|
||||
HighestScore float64 `json:"highest_score"`
|
||||
MatchedKeyword string `json:"matched_keyword"`
|
||||
CategoryScores map[string]float64 `json:"category_scores"`
|
||||
ThresholdSnapshot map[string]float64 `json:"threshold_snapshot"`
|
||||
InputExcerpt string `json:"input_excerpt"`
|
||||
@@ -895,6 +896,7 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
"keyword", keyword)
|
||||
scores := map[string]float64{contentModerationKeywordCategory: 1.0}
|
||||
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
|
||||
log.MatchedKeyword = keyword
|
||||
s.enqueueRecord(input, cfg, log, hashText, false, true)
|
||||
return &ContentModerationDecision{
|
||||
Allowed: false,
|
||||
|
||||
@@ -485,6 +485,7 @@ func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T
|
||||
require.True(t, logs[0].Flagged)
|
||||
require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
|
||||
require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
|
||||
require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
|
||||
}
|
||||
|
||||
func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user