mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-01 14:53:07 +08:00
fix(i18n): use WEKNORA_LANGUAGE env for prompt language fallback instead of hardcoded zh-CN
- Add DefaultLanguage() that reads WEKNORA_LANGUAGE env with zh-CN fallback - Replace hardcoded zh-CN in LanguageNameFromContext with DefaultLanguage() - Unify middleware/language.go to use types.DefaultLanguage() - Add Language field to DocumentProcessPayload (8 enqueue sites + worker restore) - Add Language field to ImageMultimodalPayload (enqueue + worker restore) - Propagate language from ImageMultimodalPayload to QuestionGenerationPayload
This commit is contained in:
@@ -79,6 +79,9 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
payload.ChunkID, payload.ImageURL, payload.EnableOCR, payload.EnableCaption)
|
||||
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
||||
}
|
||||
|
||||
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
@@ -437,6 +440,7 @@ func (s *ImageMultimodalService) enqueueQuestionGenerationIfEnabled(ctx context.
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
QuestionCount: questionCount,
|
||||
Language: payload.Language,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
|
||||
@@ -383,6 +383,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -393,6 +394,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -557,6 +559,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -565,6 +568,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -770,6 +774,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -780,6 +785,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -988,6 +994,7 @@ func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Contex
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -996,6 +1003,7 @@ func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Contex
|
||||
EnableMultimodel: false, // 文本段落不支持多模态
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -2704,6 +2712,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: existing.ID,
|
||||
@@ -2714,6 +2723,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -2754,6 +2764,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: existing.ID,
|
||||
@@ -2764,6 +2775,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -2799,6 +2811,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: existing.ID,
|
||||
@@ -2807,6 +2820,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -7380,6 +7394,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
ctx = logger.WithRequestID(ctx, payload.RequestId)
|
||||
ctx = logger.WithField(ctx, "document_process", payload.KnowledgeID)
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
||||
}
|
||||
|
||||
// 获取任务重试信息,用于判断是否是最后一次重试
|
||||
retryCount, _ := asynq.GetRetryCount(ctx)
|
||||
@@ -7826,6 +7843,7 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
|
||||
chunkID = chunks[0].ChunkID
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
payload := types.ImageMultimodalPayload{
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -7834,6 +7852,7 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
|
||||
ImageURL: img.ServingURL,
|
||||
EnableOCR: true,
|
||||
EnableCaption: true,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
@@ -9026,6 +9045,7 @@ func (s *knowledgeService) moveKnowledgeReparse(
|
||||
}
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
@@ -9036,6 +9056,7 @@ func (s *knowledgeService) moveKnowledgeReparse(
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
Language: lang,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
|
||||
@@ -2,43 +2,46 @@ package middleware
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// DefaultLanguage is the fallback language when no preference is specified.
|
||||
const DefaultLanguage = "zh-CN"
|
||||
|
||||
// Language extracts the user's language preference and injects it into the request context.
|
||||
//
|
||||
// Priority (highest to lowest):
|
||||
// 1. Accept-Language HTTP header (first tag, e.g. "zh-CN,zh;q=0.9" → "zh-CN")
|
||||
// 2. WEKNORA_LANGUAGE environment variable
|
||||
// 3. DefaultLanguage ("zh-CN")
|
||||
// 1. WEKNORA_LANGUAGE environment variable (deployment-level override for document processing language)
|
||||
// 2. Accept-Language HTTP header (first tag, e.g. "zh-CN,zh;q=0.9" → "zh-CN")
|
||||
// 3. "zh-CN" hardcoded fallback
|
||||
//
|
||||
// WEKNORA_LANGUAGE takes precedence over Accept-Language because the UI locale (menu language)
|
||||
// and the document processing language (question/summary generation) are separate concerns.
|
||||
// A user may prefer English UI while processing Korean documents.
|
||||
func Language() gin.HandlerFunc {
|
||||
// Read env var once at startup
|
||||
envLang := strings.TrimSpace(os.Getenv("WEKNORA_LANGUAGE"))
|
||||
// Read env var once at startup; empty string means "not configured"
|
||||
envLang := types.EnvLanguage()
|
||||
|
||||
return func(c *gin.Context) {
|
||||
// 1. WEKNORA_LANGUAGE env takes precedence (deployment-level document language)
|
||||
if envLang != "" {
|
||||
c.Set(types.LanguageContextKey.String(), envLang)
|
||||
ctx := context.WithValue(c.Request.Context(), types.LanguageContextKey, envLang)
|
||||
c.Request = c.Request.WithContext(ctx)
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
lang := ""
|
||||
|
||||
// 1. Try Accept-Language header
|
||||
// 2. Try Accept-Language header
|
||||
if acceptLang := c.GetHeader("Accept-Language"); acceptLang != "" {
|
||||
// Parse the first language tag (e.g. "zh-CN,zh;q=0.9,en;q=0.8" → "zh-CN")
|
||||
lang = parseFirstLanguageTag(acceptLang)
|
||||
}
|
||||
|
||||
// 2. Fallback to environment variable
|
||||
if lang == "" && envLang != "" {
|
||||
lang = envLang
|
||||
}
|
||||
|
||||
// 3. Fallback to default
|
||||
// 3. Fallback to hardcoded default
|
||||
if lang == "" {
|
||||
lang = DefaultLanguage
|
||||
lang = "zh-CN"
|
||||
}
|
||||
|
||||
// Inject into context
|
||||
|
||||
@@ -1,6 +1,24 @@
|
||||
package types
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// EnvLanguage returns the WEKNORA_LANGUAGE environment variable value, or empty string if unset.
|
||||
func EnvLanguage() string {
|
||||
return strings.TrimSpace(os.Getenv("WEKNORA_LANGUAGE"))
|
||||
}
|
||||
|
||||
// DefaultLanguage returns the configured default language locale.
|
||||
// It reads the WEKNORA_LANGUAGE environment variable; if unset, falls back to "zh-CN".
|
||||
func DefaultLanguage() string {
|
||||
if lang := EnvLanguage(); lang != "" {
|
||||
return lang
|
||||
}
|
||||
return "zh-CN"
|
||||
}
|
||||
|
||||
// TenantIDFromContext extracts the tenant ID from ctx.
|
||||
// Returns (0, false) when the key is absent or the value is not uint64.
|
||||
@@ -55,10 +73,11 @@ func LanguageFromContext(ctx context.Context) (string, bool) {
|
||||
|
||||
// LanguageNameFromContext returns the human-readable language name for use in prompts.
|
||||
// e.g. "zh-CN" -> "Chinese (Simplified)", "en-US" -> "English", "ko-KR" -> "Korean"
|
||||
// Falls back to DefaultLanguage() (WEKNORA_LANGUAGE env, then "zh-CN").
|
||||
func LanguageNameFromContext(ctx context.Context) string {
|
||||
lang, ok := LanguageFromContext(ctx)
|
||||
if !ok {
|
||||
lang = "zh-CN"
|
||||
lang = DefaultLanguage()
|
||||
}
|
||||
return LanguageLocaleName(lang)
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ type DocumentProcessPayload struct {
|
||||
EnableMultimodel bool `json:"enable_multimodel"`
|
||||
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
|
||||
QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
}
|
||||
|
||||
// FAQImportPayload represents the FAQ import task payload (including dry run mode)
|
||||
@@ -151,6 +152,7 @@ type ImageMultimodalPayload struct {
|
||||
ImageLocalPath string `json:"image_local_path"` // deprecated: kept for backward compat with in-flight tasks
|
||||
EnableOCR bool `json:"enable_ocr"`
|
||||
EnableCaption bool `json:"enable_caption"`
|
||||
Language string `json:"language,omitempty"` // Request locale for {{language}} in prompt templates
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
|
||||
Reference in New Issue
Block a user