feat(client): enhance knowledge file upload with process configuration support

- Updated the `CreateKnowledgeFromFile` function to accept an optional `processConfig` parameter for per-upload parse configuration overrides.
- Introduced the `KnowledgeProcessOverrides` type to define various parsing settings, including parser engine rules and chunking configurations.
- Modified the API client and related components to handle the new process configuration, improving flexibility in knowledge file uploads.
- Updated documentation and API specifications to reflect the new `process_config` parameter in the upload process.
- Added a new `UploadConfirmHost` component to facilitate user confirmation for uploads, integrating with the updated upload logic.
- Enhanced user experience by providing detailed configuration options during the upload process.
This commit is contained in:
wizardchen
2026-06-09 20:38:24 +08:00
committed by lyingbug
parent 589d2a6e52
commit ed9e780c8c
41 changed files with 3546 additions and 596 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ func ExampleUsage() {
"source": "local",
"type": "document",
}
knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), createdKB.ID, filePath, metadata, nil, "", "")
knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), createdKB.ID, filePath, metadata, nil, "", "", nil)
if err != nil {
fmt.Printf("Failed to upload knowledge file: %v\n", err)
} else {
+27
View File
@@ -83,6 +83,19 @@ var ErrDuplicateFile = errors.New("file already exists")
// ErrDuplicateURL is returned when attempting to create a knowledge entry with a URL that already exists
var ErrDuplicateURL = errors.New("URL already exists")
// KnowledgeProcessOverrides stores per-upload parse config overrides sent as process_config.
// When nil, the server uses the knowledge base defaults only.
type KnowledgeProcessOverrides struct {
ParserEngineRules []ParserEngineRule `json:"parser_engine_rules,omitempty"`
ChunkingConfig *ChunkingConfig `json:"chunking_config,omitempty"`
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
VLMConfig *VLMConfig `json:"vlm_config,omitempty"`
ASRConfig *ASRConfig `json:"asr_config,omitempty"`
QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"`
GraphEnabled *bool `json:"graph_enabled,omitempty"`
ExtractConfig *ExtractConfig `json:"extract_config,omitempty"`
}
// CreateKnowledgeFromFile creates a knowledge entry from a local file path
// Parameters:
// - knowledgeBaseID: The ID of the knowledge base
@@ -91,8 +104,10 @@ var ErrDuplicateURL = errors.New("URL already exists")
// - enableMultimodel: Optional flag to enable multimodal processing
// - customFileName: Optional custom file name (useful for folder uploads with path)
// - channel: Optional ingestion channel (e.g. "web", "api", "wechat"); empty defaults to "web"
// - processConfig: Optional parse config overrides (serialized as process_config form field)
func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
knowledgeBaseID string, filePath string, metadata map[string]string, enableMultimodel *bool, customFileName string, channel string,
processConfig *KnowledgeProcessOverrides,
) (*Knowledge, error) {
// Open the local file
file, err := os.Open(filePath)
@@ -159,6 +174,16 @@ func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
}
}
if processConfig != nil {
processConfigBytes, err := json.Marshal(processConfig)
if err != nil {
return nil, fmt.Errorf("failed to serialize process_config: %w", err)
}
if err := writer.WriteField("process_config", string(processConfigBytes)); err != nil {
return nil, fmt.Errorf("failed to write process_config field: %w", err)
}
}
// Close the multipart writer
err = writer.Close()
if err != nil {
@@ -209,6 +234,8 @@ type CreateKnowledgeFromURLRequest struct {
TagID string `json:"tag_id,omitempty"`
// Channel identifies the ingestion channel (e.g. "web", "browser_extension", "api")
Channel string `json:"channel,omitempty"`
// ProcessConfig is optional per-upload parse config overrides (KnowledgeProcessOverrides).
ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
}
// CreateKnowledgeFromURL creates a knowledge entry from a URL.
+19
View File
@@ -108,6 +108,25 @@ type GraphRelation struct {
Type string `json:"type"`
}
// ParserEngineRule maps a set of file types to a specific parser engine.
type ParserEngineRule struct {
FileTypes []string `json:"file_types"`
Engine string `json:"engine"`
}
// QuestionGenerationConfig controls LLM-generated questions per chunk during parsing.
type QuestionGenerationConfig struct {
Enabled bool `json:"enabled"`
QuestionCount int `json:"question_count"`
}
// ASRConfig represents automatic speech recognition settings for audio files.
type ASRConfig struct {
Enabled bool `json:"enabled"`
ModelID string `json:"model_id"`
Language string `json:"language,omitempty"`
}
// UnmarshalJSON keeps backward compatibility for legacy responses that still
// use `cos_config` instead of `storage_config`.
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
+3
View File
@@ -52,9 +52,12 @@
| `fileName` | string | 否 | 自定义文件名,用于"文件夹上传"时保留相对路径(如 `docs/intro.md` |
| `metadata` | string | 否 | JSON 字符串,会被反序列化为 `map[string]string` |
| `enable_multimodel` | string | 否 | `"true"` / `"false"`,是否启用图文多模态解析 |
| `process_config` | string | 否 | JSON 字符串,批次解析配置覆盖(`KnowledgeProcessOverrides`);写入 `knowledge.metadata.process_overrides`。未传时行为与现网一致 |
| `tag_id` | string | 否 | 标签 ID;传 `__untagged__` 或空字符串表示未分类 |
| `channel` | string | 否 | 来源渠道标识(写入 `channel` 字段,默认 `web`) |
`process_config` 可选字段包括:`parser_engine_rules``chunking_config``enable_multimodel``vlm_config``asr_config``question_generation_config``graph_enabled``extract_config`。若同时传 `enable_multimodel``process_config.enable_multimodel`,以 `process_config` 为准。
**请求**:
```curl
+6
View File
@@ -4775,6 +4775,12 @@ const docTemplate = `{
"description": "启用多模态处理",
"name": "enable_multimodel",
"in": "formData"
},
{
"type": "string",
"description": "处理配置JSONKnowledgeProcessOverrides",
"name": "process_config",
"in": "formData"
}
],
"responses": {
+6
View File
@@ -4768,6 +4768,12 @@
"description": "启用多模态处理",
"name": "enable_multimodel",
"in": "formData"
},
{
"type": "string",
"description": "处理配置JSONKnowledgeProcessOverrides",
"name": "process_config",
"in": "formData"
}
],
"responses": {
+4
View File
@@ -7746,6 +7746,10 @@ paths:
in: formData
name: enable_multimodel
type: boolean
- description: 处理配置JSONKnowledgeProcessOverrides
in: formData
name: process_config
type: string
produces:
- application/json
responses:
+2
View File
@@ -4,6 +4,7 @@ import { useRouter } from 'vue-router'
import { useI18n } from 'vue-i18n'
import { MessagePlugin, NotifyPlugin } from 'tdesign-vue-next'
import ManualKnowledgeEditor from '@/components/manual-knowledge-editor.vue'
import UploadConfirmHost from '@/components/UploadConfirmHost.vue'
import { useAuthStore } from '@/stores/auth'
import { useSettingsStore } from '@/stores/settings'
import { getCurrentUser, userInfoFromApi } from '@/api/auth'
@@ -257,6 +258,7 @@ onUnmounted(() => {
<div id="app">
<RouterView />
<ManualKnowledgeEditor />
<UploadConfirmHost />
</div>
</t-config-provider>
</template>
+37 -5
View File
@@ -1,4 +1,5 @@
import { get, post, put, del, postUpload, getDown } from "../../utils/request";
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess';
// 知识库管理 API(列表、创建、获取、更新、删除、复制)
export function listKnowledgeBases(params?: {
@@ -160,23 +161,51 @@ export function togglePinKnowledgeBase(id: string) {
// 知识文件 API(基于具体知识库)
// data.tag_id: 可选,指定知识所属的分类ID
export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string; [key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) {
export function uploadKnowledgeFile(
kbId: string,
data: {
file: File
tag_id?: string
fileName?: string
process_config?: KnowledgeProcessOverrides | string
[key: string]: any
} = { file: new File([], '') },
onProgress?: (progressEvent: any) => void,
) {
const formData = new FormData();
Object.keys(data).forEach(key => {
if (data[key] !== undefined) formData.append(key, data[key]);
const value = data[key];
if (value === undefined) return;
if (key === 'process_config' && value && typeof value !== 'string') {
formData.append(key, JSON.stringify(value));
} else {
formData.append(key, value);
}
});
return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress);
}
// 从URL创建知识
// data.tag_id: 可选,指定知识所属的分类ID
export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) {
export function createKnowledgeFromURL(
kbId: string,
data: { url: string; enable_multimodel?: boolean; tag_id?: string; process_config?: KnowledgeProcessOverrides },
) {
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data);
}
// 手工创建知识
// data.tag_id: 可选,指定知识所属的分类ID
export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) {
export function createManualKnowledge(
kbId: string,
data: {
title: string
content: string
status: string
tag_id?: string
process_config?: KnowledgeProcessOverrides
},
) {
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data);
}
@@ -215,7 +244,10 @@ export function getKnowledgeDetails(id: string, options?: { agent_id?: string })
return get(qs ? `/api/v1/knowledge/${id}?${qs}` : `/api/v1/knowledge/${id}`);
}
export function updateManualKnowledge(id: string, data: { title: string; content: string; status: string }) {
export function updateManualKnowledge(
id: string,
data: { title: string; content: string; status: string; process_config?: KnowledgeProcessOverrides },
) {
return put(`/api/v1/knowledge/manual/${id}`, data);
}
+4 -1
View File
@@ -6,6 +6,7 @@
:placeholder="placeholderText"
:disabled="disabled"
:loading="loading"
:status="status"
filterable
style="width: 100%;"
>
@@ -51,13 +52,15 @@ interface Props {
selectedModelId?: string
disabled?: boolean
placeholder?: string
status?: 'default' | 'success' | 'warning' | 'error'
// 可选:外部传入的所有模型列表,如果提供则不调用API
allModels?: ModelConfig[]
}
const props = withDefaults(defineProps<Props>(), {
disabled: false,
placeholder: ''
placeholder: '',
status: 'default',
})
const emit = defineEmits<{
@@ -0,0 +1,29 @@
<script setup lang="ts">
import UploadConfirmDialog from '@/views/knowledge/components/UploadConfirmDialog.vue'
import { useUploadConfirmStore } from '@/stores/uploadConfirm'
const uploadConfirmStore = useUploadConfirmStore()
const handleConfirm = (payload: Parameters<typeof uploadConfirmStore.resolveConfirm>[0]) => {
uploadConfirmStore.resolveConfirm(payload)
}
const handleCancel = () => {
uploadConfirmStore.rejectConfirm()
}
</script>
<template>
<UploadConfirmDialog
v-model:visible="uploadConfirmStore.visible"
:mode="uploadConfirmStore.mode"
:kb-info="uploadConfirmStore.kbInfo"
:files="uploadConfirmStore.files"
:urls="uploadConfirmStore.urls"
:manual-preview="uploadConfirmStore.manual"
:accept-file-types="uploadConfirmStore.acceptFileTypes"
:supported-file-types="uploadConfirmStore.supportedFileTypes"
@confirm="handleConfirm"
@cancel="handleCancel"
/>
</template>
@@ -3,7 +3,15 @@ import { ref, reactive, computed, watch, nextTick, onBeforeUnmount } from 'vue'
import { marked } from 'marked'
import { MessagePlugin } from 'tdesign-vue-next'
import { useUIStore } from '@/stores/ui'
import { listKnowledgeBases, getKnowledgeDetails, createManualKnowledge, updateManualKnowledge } from '@/api/knowledge-base'
import {
listKnowledgeBases,
getKnowledgeDetails,
getKnowledgeBaseById,
createManualKnowledge,
updateManualKnowledge,
} from '@/api/knowledge-base'
import { useUploadConfirmStore } from '@/stores/uploadConfirm'
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
import { sanitizeHTML, safeMarkdownToHTML } from '@/utils/security'
import { useI18n } from 'vue-i18n'
@@ -24,6 +32,7 @@ interface KnowledgeDetailResponse {
type ManualStatus = 'draft' | 'publish'
const uiStore = useUIStore()
const uploadConfirmStore = useUploadConfirmStore()
const { t } = useI18n()
const visible = computed({
@@ -576,11 +585,53 @@ const handleSave = async (targetStatus: ManualStatus) => {
saving.value = true
savingAction.value = targetStatus
try {
const payload: { title: string; content: string; status: string; tag_id?: string } = {
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
const payload: {
title: string
content: string
status: string
tag_id?: string
process_config?: KnowledgeProcessOverrides
} = {
title: form.title.trim(),
content: form.content,
status: targetStatus,
}
if (tagIdToUpload) {
payload.tag_id = tagIdToUpload
}
if (targetStatus === 'publish') {
let kbInfo: any
try {
const kbRes: any = await getKnowledgeBaseById(form.kbId)
kbInfo = kbRes?.data
} catch {
MessagePlugin.error(t('manualEditor.error.fetchDetailFailed'))
return
}
if (!kbInfo) {
MessagePlugin.error(t('manualEditor.error.fetchDetailFailed'))
return
}
try {
const confirmResult = await uploadConfirmStore.open({
mode: 'manual',
kbInfo,
manual: {
kbId: form.kbId,
knowledgeId: knowledgeId.value || undefined,
title: payload.title,
content: payload.content,
tagId: tagIdToUpload,
},
})
payload.process_config = confirmResult.processConfig
} catch {
return
}
}
let response: any
let knowledgeID = knowledgeId.value
let kbId = form.kbId
@@ -588,11 +639,6 @@ const handleSave = async (targetStatus: ManualStatus) => {
if (mode.value === 'edit' && knowledgeId.value) {
response = await updateManualKnowledge(knowledgeId.value, payload)
} else {
// 创建新知识时,从 store 获取当前选中的分类ID
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
if (tagIdToUpload) {
payload.tag_id = tagIdToUpload
}
response = await createManualKnowledge(form.kbId, payload)
knowledgeID = response?.data?.id || knowledgeID
kbId = form.kbId
+68
View File
@@ -636,6 +636,74 @@ export default {
chunkLoadFailed: 'Failed to load chunks',
},
uploadConfirm: {
title: 'Confirm Upload',
fileList: 'Files to upload',
parseConfig: 'Parse settings',
configNav: 'Parse settings navigation',
tabOverview: 'Overview',
overviewTitle: 'Parse settings for this batch',
overviewDesc: 'Click any row to edit',
backToOverview: 'Back to overview',
summaryChunkOverlapShort: 'Overlap {overlap}',
summaryParentChildShort: 'Parent-child',
summaryParserMode: 'Mode',
summaryParserBuiltin: 'Built-in engine (default)',
summaryChunkSize: 'Chunk size',
summaryChunkOverlap: 'Overlap',
summaryStrategy: 'Strategy',
summaryStrategyDefault: 'Default',
summaryParentChild: 'Parent-child',
summaryParentChildOn: 'On (parent {parent} / child {child})',
summaryParentChildOff: 'Off',
summaryStatus: 'Status',
summaryModel: 'Model',
summaryQuestionCount: 'Questions per chunk',
summaryQuestionCountValue: '{count}',
summaryGraphTags: 'Relation types',
summaryGraphTagsValue: '{count}',
navChunkingSummary: 'Chunk {size}',
statusOn: 'On',
statusOff: 'Off',
notSet: 'Not set',
confirm: 'Upload and parse',
cancel: 'Cancel',
uploading: 'Uploading {current}/{total}',
tabParser: 'Parser',
tabChunking: 'Chunking',
tabMultimodal: 'Multimodal',
tabAsr: 'Audio',
tabQuestion: 'Question generation',
tabGraph: 'Knowledge graph',
noFiles: 'Keep at least one file to upload',
noItems: 'Add at least one file or URL',
urlItemLabel: 'URL',
urlAdded: 'URL added',
urlDuplicate: 'This URL is already in the list',
addUrl: 'Add',
vlmModelRequired: 'This batch includes images. Enable multimodal and select a VLM model.',
multimodalRequiredForImages: 'Off (batch includes images)',
asrModelRequired: 'This batch includes audio. Enable ASR and select a speech recognition model.',
asrRequiredForAudio: 'Off (batch includes audio)',
vlmModelSelectRequired: 'Multimodal is enabled. Please select a VLM model.',
asrModelSelectRequired: 'Speech recognition is enabled. Please select an ASR model.',
continueAdd: 'Add more',
addMoreFiles: 'Add more files',
addMoreFolder: 'Add more from folder',
filesAdded: 'Added {count} file(s)',
filesAllDuplicate: 'Selected files are already in the list',
titleUrl: 'Confirm URL import',
titleManual: 'Confirm online publish',
overviewDescUrl: 'The URL will be fetched and parsed after you confirm the settings',
overviewDescManual: 'The document will be published and indexed after you confirm the settings',
confirmUrl: 'Import and parse',
confirmManual: 'Publish and parse',
urlSource: 'Import source',
manualSource: 'Document to publish',
editUrl: 'Edit URL',
manualCharCount: '{count} characters',
},
knowledgeStages: {
title: 'Processing pipeline',
root: 'Knowledge processing',
+67
View File
@@ -642,6 +642,73 @@ export default {
deleteSuccess: "지식이 성공적으로 삭제되었습니다!",
chunkLoadFailed: "청크 로드 실패",
},
uploadConfirm: {
title: "문서 업로드 확인",
fileList: "업로드할 파일",
parseConfig: "파싱 설정",
configNav: "파싱 설정 탐색",
tabOverview: "구성 개요",
overviewTitle: "이번 배치 파싱 설정",
overviewDesc: "항목을 클릭하여 설정을 수정하세요",
backToOverview: "구성 개요로 돌아가기",
summaryChunkOverlapShort: "겹침 {overlap}",
summaryParentChildShort: "부모-자식 청킹",
summaryParserMode: "모드",
summaryParserBuiltin: "내장 엔진(기본)",
summaryChunkSize: "청크 크기",
summaryChunkOverlap: "겹침",
summaryStrategy: "전략",
summaryStrategyDefault: "기본",
summaryParentChild: "부모-자식 청킹",
summaryParentChildOn: "켜짐(부모 {parent} / 자식 {child})",
summaryParentChildOff: "꺼짐",
summaryStatus: "상태",
summaryModel: "모델",
summaryQuestionCount: "청크당 질문 수",
summaryQuestionCountValue: "{count}개",
summaryGraphTags: "관계 유형",
summaryGraphTagsValue: "{count}개",
navChunkingSummary: "청크 {size}",
statusOn: "켜짐",
statusOff: "꺼짐",
notSet: "미설정",
confirm: "업로드 및 파싱 확인",
cancel: "취소",
uploading: "업로드 중 {current}/{total}",
tabParser: "파서",
tabChunking: "청킹",
tabMultimodal: "멀티모달",
tabAsr: "오디오",
tabQuestion: "질문 생성",
tabGraph: "지식 그래프",
noFiles: "업로드할 파일을 하나 이상 남겨 두세요",
noItems: "파일 또는 URL을 하나 이상 추가하세요",
urlItemLabel: "URL",
urlAdded: "URL이 추가되었습니다",
urlDuplicate: "이 URL은 이미 목록에 있습니다",
addUrl: "추가",
vlmModelRequired: "이 배치에 이미지가 포함되어 있습니다. 멀티모달을 활성화하고 VLM 모델을 선택하세요",
multimodalRequiredForImages: "꺼짐(이 배치에 이미지 포함)",
asrModelRequired: "이 배치에 오디오가 포함되어 있습니다. ASR을 활성화하고 음성 인식 모델을 선택하세요",
asrRequiredForAudio: "꺼짐(이 배치에 오디오 포함)",
vlmModelSelectRequired: "멀티모달이 활성화되었습니다. VLM 모델을 선택하세요",
asrModelSelectRequired: "음성 인식이 활성화되었습니다. ASR 모델을 선택하세요",
continueAdd: "계속 추가",
addMoreFiles: "파일 더 추가",
addMoreFolder: "폴더에서 더 추가",
filesAdded: "{count}개 파일이 추가되었습니다",
filesAllDuplicate: "선택한 파일이 이미 목록에 있습니다",
titleUrl: "URL 가져오기 확인",
titleManual: "온라인 편집 게시 확인",
overviewDescUrl: "설정 확인 후 URL을 가져와 파싱합니다",
overviewDescManual: "설정 확인 후 문서를 게시하고 인덱싱합니다",
confirmUrl: "가져오기 및 파싱",
confirmManual: "게시 및 파싱",
urlSource: "가져오기 소스",
manualSource: "게시할 문서",
editUrl: "URL 수정",
manualCharCount: "{count}자",
},
knowledgeStages: {
title: "처리 파이프라인",
root: "지식 처리",
+67
View File
@@ -606,6 +606,73 @@ export default {
cancelParseSubmitted: 'Разбор остановлен',
cancelParseFailed: 'Не удалось остановить, попробуйте позже'
},
uploadConfirm: {
title: 'Подтверждение загрузки',
fileList: 'Файлы для загрузки',
parseConfig: 'Настройки разбора',
configNav: 'Навигация по настройкам разбора',
tabOverview: 'Обзор',
overviewTitle: 'Настройки разбора для этой партии',
overviewDesc: 'Нажмите на строку, чтобы изменить',
backToOverview: 'Вернуться к обзору',
summaryChunkOverlapShort: 'Перекрытие {overlap}',
summaryParentChildShort: 'Родительско-дочернее',
summaryParserMode: 'Режим',
summaryParserBuiltin: 'Встроенный движок (по умолчанию)',
summaryChunkSize: 'Размер чанка',
summaryChunkOverlap: 'Перекрытие',
summaryStrategy: 'Стратегия',
summaryStrategyDefault: 'По умолчанию',
summaryParentChild: 'Родительско-дочернее',
summaryParentChildOn: 'Вкл. (родитель {parent} / дочерний {child})',
summaryParentChildOff: 'Выкл.',
summaryStatus: 'Статус',
summaryModel: 'Модель',
summaryQuestionCount: 'Вопросов на чанк',
summaryQuestionCountValue: '{count}',
summaryGraphTags: 'Типы связей',
summaryGraphTagsValue: '{count}',
navChunkingSummary: 'Чанк {size}',
statusOn: 'Вкл.',
statusOff: 'Выкл.',
notSet: 'Не задано',
confirm: 'Загрузить и обработать',
cancel: 'Отмена',
uploading: 'Загрузка {current}/{total}',
tabParser: 'Парсер',
tabChunking: 'Разбиение',
tabMultimodal: 'Мультимодальность',
tabAsr: 'Аудио',
tabQuestion: 'Генерация вопросов',
tabGraph: 'Граф знаний',
noFiles: 'Оставьте хотя бы один файл для загрузки',
noItems: 'Добавьте хотя бы один файл или URL',
urlItemLabel: 'URL',
urlAdded: 'URL добавлен',
urlDuplicate: 'Этот URL уже в списке',
addUrl: 'Добавить',
vlmModelRequired: 'В партии есть изображения. Включите мультимодальность и выберите модель VLM.',
multimodalRequiredForImages: 'Выкл. (в партии есть изображения)',
asrModelRequired: 'В партии есть аудио. Включите ASR и выберите модель распознавания речи.',
asrRequiredForAudio: 'Выкл. (в партии есть аудио)',
vlmModelSelectRequired: 'Мультимодальность включена. Выберите модель VLM.',
asrModelSelectRequired: 'Распознавание речи включено. Выберите модель ASR.',
continueAdd: 'Добавить ещё',
addMoreFiles: 'Добавить файлы',
addMoreFolder: 'Добавить из папки',
filesAdded: 'Добавлено файлов: {count}',
filesAllDuplicate: 'Выбранные файлы уже в списке',
titleUrl: 'Подтверждение импорта URL',
titleManual: 'Подтверждение публикации',
overviewDescUrl: 'После подтверждения настроек URL будет загружен и обработан',
overviewDescManual: 'После подтверждения настроек документ будет опубликован и проиндексирован',
confirmUrl: 'Импортировать и обработать',
confirmManual: 'Опубликовать и обработать',
urlSource: 'Источник импорта',
manualSource: 'Документ для публикации',
editUrl: 'Изменить URL',
manualCharCount: '{count} символов',
},
knowledgeStages: {
title: 'Конвейер обработки',
root: 'Обработка знаний',
+67
View File
@@ -639,6 +639,73 @@ export default {
deleteSuccess: "知识删除成功!",
chunkLoadFailed: "分块加载失败",
},
uploadConfirm: {
title: "上传文档确认",
fileList: "待上传文件",
parseConfig: "解析配置",
configNav: "解析配置导航",
tabOverview: "配置总览",
overviewTitle: "本批次解析配置",
overviewDesc: "点击任一项可修改配置",
backToOverview: "返回配置总览",
summaryChunkOverlapShort: "重叠 {overlap}",
summaryParentChildShort: "父子分块",
summaryParserMode: "模式",
summaryParserBuiltin: "内置引擎(默认)",
summaryChunkSize: "分块大小",
summaryChunkOverlap: "重叠",
summaryStrategy: "策略",
summaryStrategyDefault: "默认",
summaryParentChild: "父子分块",
summaryParentChildOn: "已开启(父 {parent} / 子 {child}",
summaryParentChildOff: "未开启",
summaryStatus: "状态",
summaryModel: "模型",
summaryQuestionCount: "每块问题数",
summaryQuestionCountValue: "{count} 个",
summaryGraphTags: "关系类型",
summaryGraphTagsValue: "{count} 个",
navChunkingSummary: "分块 {size}",
statusOn: "已开启",
statusOff: "未开启",
notSet: "未设置",
confirm: "确认上传并解析",
cancel: "取消",
uploading: "正在上传 {current}/{total}",
tabParser: "解析引擎",
tabChunking: "分块",
tabMultimodal: "多模态",
tabAsr: "音频",
tabQuestion: "问题生成",
tabGraph: "图谱",
noFiles: "请至少保留一个待上传文件",
noItems: "请至少添加一个文件或 URL",
urlItemLabel: "URL",
urlAdded: "已添加 URL",
urlDuplicate: "该 URL 已在列表中",
addUrl: "添加",
vlmModelRequired: "本批次包含图片,请启用多模态并选择 VLM 模型",
multimodalRequiredForImages: "未开启(本批次含图片)",
asrModelRequired: "本批次包含音频,请启用 ASR 并选择语音识别模型",
asrRequiredForAudio: "未开启(本批次含音频)",
vlmModelSelectRequired: "已启用多模态,请选择 VLM 模型",
asrModelSelectRequired: "已启用语音识别,请选择 ASR 模型",
continueAdd: "继续添加",
addMoreFiles: "继续添加文件",
addMoreFolder: "继续添加文件夹",
filesAdded: "已添加 {count} 个文件",
filesAllDuplicate: "所选文件已在列表中",
titleUrl: "URL 导入确认",
titleManual: "在线编辑发布确认",
overviewDescUrl: "确认解析配置后将抓取并解析该 URL",
overviewDescManual: "确认解析配置后将发布并索引该文档",
confirmUrl: "确认导入并解析",
confirmManual: "确认发布并解析",
urlSource: "导入来源",
manualSource: "待发布文档",
editUrl: "修改 URL",
manualCharCount: "{count} 个字符",
},
knowledgeStages: {
title: "处理流水线",
root: "知识处理",
+85
View File
@@ -0,0 +1,85 @@
import { defineStore } from 'pinia'
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
export type UploadConfirmMode = 'file' | 'url' | 'manual'
export interface UploadConfirmManualSource {
kbId: string
knowledgeId?: string
title: string
content: string
tagId?: string
}
export interface UploadConfirmResult {
processConfig: KnowledgeProcessOverrides
mode: UploadConfirmMode
files?: File[]
urls?: string[]
manual?: UploadConfirmManualSource
}
export interface OpenUploadConfirmOptions {
mode: UploadConfirmMode
kbInfo: any
files?: File[]
urls?: string[]
manual?: UploadConfirmManualSource
acceptFileTypes?: string
supportedFileTypes?: string[]
}
export const useUploadConfirmStore = defineStore('uploadConfirm', {
state: () => ({
visible: false,
mode: 'file' as UploadConfirmMode,
kbInfo: null as any,
files: [] as File[],
urls: [] as string[],
manual: null as UploadConfirmManualSource | null,
acceptFileTypes: '',
supportedFileTypes: [] as string[],
pendingResolve: null as ((value: UploadConfirmResult) => void) | null,
pendingReject: null as (() => void) | null,
}),
actions: {
open(options: OpenUploadConfirmOptions): Promise<UploadConfirmResult> {
return new Promise((resolve, reject) => {
this.visible = true
this.mode = options.mode
this.kbInfo = options.kbInfo
this.files = options.files ? [...options.files] : []
this.urls = options.urls ? [...options.urls] : []
this.manual = options.manual || null
this.acceptFileTypes = options.acceptFileTypes || ''
this.supportedFileTypes = options.supportedFileTypes ? [...options.supportedFileTypes] : []
this.pendingResolve = resolve
this.pendingReject = reject
})
},
resolveConfirm(payload: UploadConfirmResult) {
this.pendingResolve?.(payload)
this.reset()
},
rejectConfirm() {
this.pendingReject?.()
this.reset()
},
reset() {
this.visible = false
this.mode = 'file'
this.kbInfo = null
this.files = []
this.urls = []
this.manual = null
this.acceptFileTypes = ''
this.supportedFileTypes = []
this.pendingResolve = null
this.pendingReject = null
},
},
})
+66
View File
@@ -0,0 +1,66 @@
/** Matches backend types.KnowledgeProcessOverrides (snake_case JSON). */
export interface ParserEngineRule {
file_types: string[]
engine: string
}
export interface ChunkingConfigOverride {
chunk_size?: number
chunk_overlap?: number
separators?: string[]
enable_multimodal?: boolean
parser_engine_rules?: ParserEngineRule[]
enable_parent_child?: boolean
parent_chunk_size?: number
child_chunk_size?: number
strategy?: string
token_limit?: number
languages?: string[]
}
export interface VLMConfigOverride {
enabled?: boolean
model_id?: string
}
export interface ASRConfigOverride {
enabled?: boolean
model_id?: string
language?: string
}
export interface QuestionGenerationConfigOverride {
enabled?: boolean
question_count?: number
}
export interface GraphNodeOverride {
name: string
attributes?: string[]
}
export interface GraphRelationOverride {
node1: string
node2: string
type: string
}
export interface ExtractConfigOverride {
enabled?: boolean
text?: string
tags?: string[]
nodes?: GraphNodeOverride[]
relations?: GraphRelationOverride[]
}
export interface KnowledgeProcessOverrides {
parser_engine_rules?: ParserEngineRule[]
chunking_config?: ChunkingConfigOverride
enable_multimodel?: boolean
vlm_config?: VLMConfigOverride
asr_config?: ASRConfigOverride
question_generation_config?: QuestionGenerationConfigOverride
graph_enabled?: boolean
extract_config?: ExtractConfigOverride
}
+183 -349
View File
@@ -1,6 +1,6 @@
<script setup lang="ts">
import { ref, onMounted, onUnmounted, watch, reactive, computed, nextTick, h, type ComponentPublicInstance } from "vue";
import { MessagePlugin, Icon as TIcon } from "tdesign-vue-next";
import { ref, onMounted, onUnmounted, watch, reactive, computed, nextTick, type ComponentPublicInstance } from "vue";
import { MessagePlugin } from "tdesign-vue-next";
import DocContent from "@/components/doc-content.vue";
import KnowledgeProcessingTimeline from "@/components/knowledge-processing-timeline.vue";
import useKnowledgeBase from '@/hooks/useKnowledgeBase';
@@ -42,6 +42,9 @@ import { knowledgeSpansPayloadHasTrace } from '@/utils/knowledgeTrace';
import FAQEntryManager from './components/FAQEntryManager.vue';
import DocumentListView from './components/DocumentListView.vue';
import DocumentBatchBar from './components/DocumentBatchBar.vue';
import KbUploadSourceDropdown from './components/KbUploadSourceDropdown.vue';
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess';
import { useUploadConfirmStore, type UploadConfirmResult } from '@/stores/uploadConfirm';
import WikiBrowser from './wiki/WikiBrowser.vue';
import { getWikiStats } from '@/api/wiki';
import {
@@ -51,15 +54,14 @@ import {
} from './wikiStatusRefresh';
import { listMoveTargets, moveKnowledge, getKnowledgeMoveProgress } from '@/api/knowledge-base';
import { useI18n } from 'vue-i18n';
import { formatStringDate, kbFileTypeVerification } from '@/utils';
import { formatStringDate } from '@/utils';
import { formatFileSize } from '@/utils/files';
import type { ParserEngineInfo } from '@/api/system';
const route = useRoute();
const { t } = useI18n();
const kbId = computed(() => (route.params as any).kbId as string || '');
const kbInfo = ref<any>(null);
const uploadInputRef = ref<HTMLInputElement | null>(null);
const folderUploadInputRef = ref<HTMLInputElement | null>(null);
const uploadSourceRef = ref<InstanceType<typeof KbUploadSourceDropdown> | null>(null);
const uploading = ref(false);
const kbLoading = ref(false);
const docListLoading = ref(true);
@@ -984,7 +986,9 @@ const handleOpenURLImportDialog = (event: CustomEvent) => {
const eventKbId = event.detail.kbId;
console.log('接收到URL导入对话框打开事件,知识库ID:', eventKbId, '当前知识库ID:', kbId.value);
if (eventKbId && eventKbId === kbId.value && !isFAQ.value) {
urlDialogVisible.value = true;
if (ensureDocumentKbReady()) {
uploadSourceRef.value?.openUrlDialog();
}
}
};
@@ -1352,32 +1356,6 @@ const documentTitle = computed(() => {
return t('knowledgeEditor.document.title');
});
// 文档操作下拉菜单选项
const documentActionOptions = computed(() => [
{ content: t('upload.uploadDocument'), value: 'upload', prefixIcon: () => h(TIcon, { name: 'upload', size: '16px' }) },
{ content: t('upload.uploadFolder'), value: 'uploadFolder', prefixIcon: () => h(TIcon, { name: 'folder-add', size: '16px' }) },
{ content: t('knowledgeBase.importURL'), value: 'importURL', prefixIcon: () => h(TIcon, { name: 'link', size: '16px' }) },
{ content: t('upload.onlineEdit'), value: 'manualCreate', prefixIcon: () => h(TIcon, { name: 'edit', size: '16px' }) },
]);
// 处理文档操作下拉菜单选择
const handleDocumentActionSelect = (data: { value: string }) => {
switch (data.value) {
case 'upload':
handleDocumentUploadClick();
break;
case 'uploadFolder':
handleFolderUploadClick();
break;
case 'importURL':
handleURLImportClick();
break;
case 'manualCreate':
handleManualCreate();
break;
}
};
const ensureDocumentKbReady = () => {
if (isFAQ.value) {
MessagePlugin.warning(t('knowledgeBase.operationNotSupportedForType'));
@@ -1406,338 +1384,147 @@ const ensureDocumentKbReady = () => {
};
const handleDocumentUploadClick = () => {
if (!ensureDocumentKbReady()) return;
uploadInputRef.value?.click();
const IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
const AUDIO_EXTENSIONS = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
const uploadConfirmStore = useUploadConfirmStore();
const getFolderUploadFileName = (file: File) => {
const relativePath = (file as any).webkitRelativePath;
if (!relativePath) return undefined;
const pathParts = relativePath.split('/');
if (pathParts.length <= 2) return undefined;
const subPath = pathParts.slice(1, -1).join('/');
return `${subPath}/${file.name}`;
};
const handleFolderUploadClick = () => {
if (!ensureDocumentKbReady()) return;
folderUploadInputRef.value?.click();
};
const resetUploadInput = () => {
if (uploadInputRef.value) {
uploadInputRef.value.value = '';
}
};
const handleDocumentUpload = async (event: Event) => {
const input = event.target as HTMLInputElement;
const files = input?.files;
if (!files || files.length === 0) return;
const targetKbId = kbId.value;
if (!targetKbId) {
MessagePlugin.error(t('error.missingKbId'));
resetUploadInput();
return;
}
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
const asrEnabled = kbInfo.value?.asr_config?.enabled || false;
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
const validFiles: File[] = [];
let skippedCount = 0;
let imageFilteredCount = 0;
let videoFilteredCount = 0;
let audioFilteredCount = 0;
for (let i = 0; i < files.length; i++) {
const file = files[i];
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'];
const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
if (videoTypes.includes(fileExt)) {
videoFilteredCount++;
continue;
}
if (!vlmEnabled) {
if (imageTypes.includes(fileExt)) {
imageFilteredCount++;
continue;
}
}
if (!asrEnabled && audioTypes.includes(fileExt)) {
audioFilteredCount++;
continue;
}
if (!kbFileTypeVerification(file, files.length > 1, dynamicTypes)) {
validFiles.push(file);
const showUploadResultMessages = (
successCount: number,
failCount: number,
totalCount: number,
mode: 'document' | 'folder',
) => {
if (mode === 'folder') {
if (failCount === 0) {
MessagePlugin.success(t('knowledgeBase.uploadAllSuccess', { count: successCount }));
} else if (successCount > 0) {
MessagePlugin.warning(t('knowledgeBase.uploadPartialSuccess', { success: successCount, fail: failCount }));
} else {
skippedCount++;
MessagePlugin.error(t('knowledgeBase.uploadAllFailed'));
}
}
if (imageFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
}
if (videoFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
}
if (audioFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
}
if (validFiles.length === 0) {
if (skippedCount > 0) {
MessagePlugin.warning(t('knowledgeBase.allFilesSkippedNoEngine'));
}
resetUploadInput();
return;
}
if (skippedCount > 0) {
MessagePlugin.warning(t('knowledgeBase.filesSkippedNoEngine', { count: skippedCount }));
}
let successCount = 0;
let failCount = 0;
const totalCount = validFiles.length;
// 获取当前选中的分类ID(如果不是"未分类"则传递)
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
for (const file of validFiles) {
try {
const responseData: any = await uploadKnowledgeFile(targetKbId, { file, tag_id: tagIdToUpload });
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
if (isSuccess) {
successCount++;
} else {
failCount++;
let errorMessage = t('knowledgeBase.uploadFailed');
if (responseData?.error?.message) {
errorMessage = responseData.error.message;
} else if (responseData?.message) {
errorMessage = responseData.message;
}
if (responseData?.code === 'duplicate_file' || responseData?.error?.code === 'duplicate_file') {
errorMessage = t('knowledgeBase.fileExists');
}
if (totalCount === 1) {
MessagePlugin.error(errorMessage);
}
}
} catch (error: any) {
failCount++;
let errorMessage = error?.error?.message || error?.message || t('knowledgeBase.uploadFailed');
if (error?.code === 'duplicate_file') {
errorMessage = t('knowledgeBase.fileExists');
}
if (totalCount === 1) {
MessagePlugin.error(errorMessage);
}
}
}
// 显示上传结果
if (successCount > 0) {
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
detail: { kbId: targetKbId }
}));
}
if (totalCount === 1) {
if (successCount === 1) {
MessagePlugin.success(t('knowledgeBase.uploadSuccess'));
}
} else {
if (failCount === 0) {
MessagePlugin.success(t('knowledgeBase.allUploadSuccess', { count: successCount }));
} else if (successCount > 0) {
MessagePlugin.warning(t('knowledgeBase.partialUploadSuccess', { success: successCount, fail: failCount }));
} else {
MessagePlugin.error(t('knowledgeBase.allUploadFailed', { count: failCount }));
}
return;
}
resetUploadInput();
if (failCount === 0) {
MessagePlugin.success(t('knowledgeBase.allUploadSuccess', { count: successCount }));
} else if (successCount > 0) {
MessagePlugin.warning(t('knowledgeBase.partialUploadSuccess', { success: successCount, fail: failCount }));
} else {
MessagePlugin.error(t('knowledgeBase.allUploadFailed', { count: failCount }));
}
};
// 处理文件夹上传
const handleFolderUpload = async (event: Event) => {
const input = event.target as HTMLInputElement;
const files = input?.files;
if (!files || files.length === 0) return;
const executeUploadBatch = async (
files: File[],
options: { processConfig?: KnowledgeProcessOverrides } = {},
) => {
const targetKbId = kbId.value;
if (!targetKbId) {
MessagePlugin.error(t('error.missingKbId'));
if (input) input.value = '';
return;
if (!targetKbId || files.length === 0) {
return { successCount: 0, failCount: files.length };
}
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
const asrEnabled = kbInfo.value?.asr_config?.enabled || false;
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
const validFiles: File[] = [];
let hiddenFileCount = 0;
let imageFilteredCount = 0;
let videoFilteredCount = 0;
let audioFilteredCount = 0;
for (let i = 0; i < files.length; i++) {
const file = files[i];
const relativePath = (file as any).webkitRelativePath || file.name;
const pathParts = relativePath.split('/');
const hasHiddenComponent = pathParts.some((part: string) => part.startsWith('.'));
if (hasHiddenComponent) {
hiddenFileCount++;
continue;
}
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'];
const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
if (videoTypes.includes(fileExt)) {
videoFilteredCount++;
continue;
}
if (!vlmEnabled) {
if (imageTypes.includes(fileExt)) {
imageFilteredCount++;
continue;
}
}
if (!asrEnabled && audioTypes.includes(fileExt)) {
audioFilteredCount++;
continue;
}
if (!kbFileTypeVerification(file, true, dynamicTypes)) {
validFiles.push(file);
}
}
if (imageFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
}
if (videoFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
}
if (audioFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
}
if (validFiles.length === 0) {
MessagePlugin.warning(t('knowledgeBase.noValidFilesInFolder', { total: files.length }));
if (input) input.value = '';
return;
}
MessagePlugin.info(t('knowledgeBase.uploadingFolder', { total: validFiles.length }));
// 批量上传
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
let successCount = 0;
let failCount = 0;
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
for (const file of validFiles) {
const relativePath = (file as any).webkitRelativePath;
let fileName = file.name;
if (relativePath) {
const pathParts = relativePath.split('/');
if (pathParts.length > 2) {
const subPath = pathParts.slice(1, -1).join('/');
fileName = `${subPath}/${file.name}`;
}
}
const totalCount = files.length;
const hasFolderPaths = files.some((file) => {
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
return !!relativePath && relativePath.split('/').length > 2;
});
for (const file of files) {
try {
await uploadKnowledgeFile(targetKbId, { file, fileName, tag_id: tagIdToUpload });
successCount++;
const uploadData: {
file: File
tag_id?: string
fileName?: string
process_config?: KnowledgeProcessOverrides
} = { file, tag_id: tagIdToUpload };
const fileName = getFolderUploadFileName(file);
if (fileName) uploadData.fileName = fileName;
if (options.processConfig) {
uploadData.process_config = options.processConfig;
}
const responseData: any = await uploadKnowledgeFile(targetKbId, uploadData);
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
if (isSuccess) {
successCount++;
} else {
failCount++;
if (totalCount === 1) {
let errorMessage = t('knowledgeBase.uploadFailed');
if (responseData?.error?.message) {
errorMessage = responseData.error.message;
} else if (responseData?.message) {
errorMessage = responseData.message;
}
if (responseData?.code === 'duplicate_file' || responseData?.error?.code === 'duplicate_file') {
errorMessage = t('knowledgeBase.fileExists');
}
MessagePlugin.error(errorMessage);
}
}
} catch (error: any) {
failCount++;
if (totalCount === 1) {
let errorMessage = error?.error?.message || error?.message || t('knowledgeBase.uploadFailed');
if (error?.code === 'duplicate_file') {
errorMessage = t('knowledgeBase.fileExists');
}
MessagePlugin.error(errorMessage);
}
}
}
if (successCount > 0) {
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
detail: { kbId: targetKbId }
detail: { kbId: targetKbId },
}));
}
if (failCount === 0) {
MessagePlugin.success(t('knowledgeBase.uploadAllSuccess', { count: successCount }));
} else if (successCount > 0) {
MessagePlugin.warning(t('knowledgeBase.uploadPartialSuccess', { success: successCount, fail: failCount }));
} else {
MessagePlugin.error(t('knowledgeBase.uploadAllFailed'));
}
if (input) input.value = '';
showUploadResultMessages(successCount, failCount, totalCount, hasFolderPaths ? 'folder' : 'document');
return { successCount, failCount };
};
const handleManualCreate = () => {
if (!ensureDocumentKbReady()) return;
uiStore.openManualEditor({
mode: 'create',
kbId: kbId.value,
status: 'draft',
onSuccess: manualEditorSuccess,
});
};
// URL 导入相关
const urlDialogVisible = ref(false);
const urlInputValue = ref('');
const urlImporting = ref(false);
const handleURLImportClick = () => {
if (!ensureDocumentKbReady()) return;
urlInputValue.value = '';
urlDialogVisible.value = true;
};
const handleURLImportCancel = () => {
urlDialogVisible.value = false;
urlInputValue.value = '';
};
const handleURLImportConfirm = async () => {
const url = urlInputValue.value.trim();
if (!url) {
MessagePlugin.warning(t('knowledgeBase.urlRequired'));
return;
}
// 简单的URL格式验证
try {
new URL(url);
} catch (error) {
MessagePlugin.warning(t('knowledgeBase.invalidURL'));
return;
}
const executeUrlImport = async (url: string, processConfig?: KnowledgeProcessOverrides) => {
const targetKbId = kbId.value;
if (!targetKbId) {
MessagePlugin.error(t('error.missingKbId'));
return;
}
urlImporting.value = true;
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
try {
// 获取当前选中的分类ID
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
const responseData: any = await createKnowledgeFromURL(targetKbId, { url, tag_id: tagIdToUpload });
const responseData: any = await createKnowledgeFromURL(targetKbId, {
url,
tag_id: tagIdToUpload,
process_config: processConfig,
});
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
detail: { kbId: targetKbId }
detail: { kbId: targetKbId },
}));
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
if (isSuccess) {
MessagePlugin.success(t('knowledgeBase.urlImportSuccess'));
urlDialogVisible.value = false;
urlInputValue.value = '';
} else {
let errorMessage = t('knowledgeBase.urlImportFailed');
if (responseData?.error?.message) {
@@ -1756,11 +1543,73 @@ const handleURLImportConfirm = async () => {
errorMessage = t('knowledgeBase.urlExists');
}
MessagePlugin.error(errorMessage);
} finally {
urlImporting.value = false;
}
};
const handleUploadConfirmResult = async (result: UploadConfirmResult) => {
if (result.mode === 'manual') {
return;
}
const files = result.files || [];
const urls = result.urls || [];
const processConfig = result.processConfig;
if (files.length > 0) {
const hasFolderPaths = files.some((file) => {
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
return !!relativePath && relativePath.split('/').length > 2;
});
if (hasFolderPaths) {
MessagePlugin.info(t('knowledgeBase.uploadingFolder', { total: files.length }));
}
await executeUploadBatch(files, { processConfig });
}
for (const url of urls) {
await executeUrlImport(url, processConfig);
}
};
const openUploadConfirmDialog = async (files: File[], urls: string[] = []) => {
if (!kbInfo.value) return;
if (files.length === 0 && urls.length === 0) return;
try {
const result = await uploadConfirmStore.open({
mode: 'file',
kbInfo: kbInfo.value,
files,
urls,
acceptFileTypes: acceptFileTypes.value,
supportedFileTypes: [...supportedFileTypes.value],
});
await handleUploadConfirmResult(result);
} catch {
// cancelled
}
};
const handleUploadSourceFiles = (files: File[]) => {
if (!ensureDocumentKbReady()) return;
if (files.length === 0) return;
openUploadConfirmDialog(files);
};
const handleUploadSourceUrl = (url: string) => {
if (!ensureDocumentKbReady()) return;
openUploadConfirmDialog([], [url]);
};
const handleManualCreate = () => {
if (!ensureDocumentKbReady()) return;
uiStore.openManualEditor({
mode: 'create',
kbId: kbId.value,
status: 'draft',
onSuccess: manualEditorSuccess,
});
};
const handleOpenKBSettings = () => {
if (!kbId.value) {
MessagePlugin.warning(t('knowledgeEditor.messages.missingId'));
@@ -2158,11 +2007,6 @@ async function createNewSession(value: string): Promise<void> {
</div>
<template v-if="activeKbTab === 'documents' || !isWiki">
<input ref="uploadInputRef" type="file" class="document-upload-input"
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'"
multiple @change="handleDocumentUpload" />
<input ref="folderUploadInputRef" type="file" class="document-upload-input" webkitdirectory
@change="handleFolderUpload" />
<div class="knowledge-main">
<aside class="tag-sidebar">
<div class="sidebar-header">
@@ -2319,15 +2163,20 @@ async function createNewSession(value: string): Promise<void> {
</t-tooltip>
</div>
<div v-if="canEdit" class="doc-filter-actions">
<t-tooltip :content="$t('knowledgeBase.addDocument')" placement="top">
<t-dropdown :options="documentActionOptions" trigger="click" placement="bottom-right"
@click="handleDocumentActionSelect">
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
data-guide="kb-detail-add-doc">
<template #icon><t-icon name="file-add" size="16px" /></template>
</t-button>
</t-dropdown>
</t-tooltip>
<KbUploadSourceDropdown
ref="uploadSourceRef"
:accept-file-types="acceptFileTypes"
:supported-file-types="[...supportedFileTypes]"
include-manual
trigger-icon="file-add"
trigger-class="content-bar-icon-btn"
data-guide="kb-detail-add-doc"
:tooltip="t('knowledgeBase.addDocument')"
placement="bottom-right"
@files="handleUploadSourceFiles"
@url="handleUploadSourceUrl"
@manual="handleManualCreate"
/>
</div>
</div>
<div class="doc-scroll-container" :class="{ 'is-empty': !cardList.length && !docListLoading }" ref="knowledgeScroll"
@@ -2687,21 +2536,6 @@ async function createNewSession(value: string): Promise<void> {
@delete="confirmBatchDelete" />
</div>
</div>
<!-- URL 导入对话框 -->
<t-dialog v-model:visible="urlDialogVisible" :header="$t('knowledgeBase.importURLTitle')" :confirm-btn="{
content: $t('common.confirm'),
theme: 'primary',
loading: urlImporting,
}" :cancel-btn="{ content: $t('common.cancel') }" @confirm="handleURLImportConfirm"
@cancel="handleURLImportCancel" width="500px">
<div class="url-import-form">
<div class="url-input-label">{{ $t('knowledgeBase.urlLabel') }}</div>
<t-input v-model="urlInputValue" :placeholder="$t('knowledgeBase.urlPlaceholder')" clearable autofocus
@enter="handleURLImportConfirm" />
<div class="url-input-tip">{{ $t('knowledgeBase.urlTip') }}</div>
</div>
</t-dialog>
</div>
</div>
</template>
@@ -0,0 +1,251 @@
<template>
<div class="kb-upload-source-dropdown">
<input
ref="fileInputRef"
type="file"
class="hidden-file-input"
multiple
:accept="acceptFileTypes || undefined"
@change="(e) => handleFilesChange(e, false)"
/>
<input
ref="folderInputRef"
type="file"
class="hidden-file-input"
webkitdirectory
multiple
@change="(e) => handleFilesChange(e, true)"
/>
<t-tooltip :content="tooltipText" placement="top">
<t-dropdown
:options="dropdownOptions"
trigger="click"
:placement="placement"
@click="handleActionSelect"
>
<t-button
variant="text"
theme="default"
:class="['kb-upload-source-trigger', triggerClass]"
:data-guide="dataGuide || undefined"
size="small"
>
<template #icon><t-icon :name="triggerIcon" size="16px" /></template>
</t-button>
</t-dropdown>
</t-tooltip>
<t-dialog
v-model:visible="urlDialogVisible"
:header="t('knowledgeBase.importURLTitle')"
:confirm-btn="{ content: t('common.confirm'), theme: 'primary' }"
:cancel-btn="{ content: t('common.cancel') }"
width="500px"
@confirm="handleUrlDialogConfirm"
@cancel="handleUrlDialogCancel"
>
<div class="url-import-form">
<div class="url-input-label">{{ t('knowledgeBase.urlLabel') }}</div>
<t-input
v-model="urlInputValue"
:placeholder="t('knowledgeBase.urlPlaceholder')"
clearable
autofocus
@enter="handleUrlDialogConfirm"
/>
<div class="url-input-tip">{{ t('knowledgeBase.urlTip') }}</div>
</div>
</t-dialog>
</div>
</template>
<script setup lang="ts">
import { ref, computed, h, withDefaults } from 'vue'
import { useI18n } from 'vue-i18n'
import { MessagePlugin, Icon as TIcon } from 'tdesign-vue-next'
import { filterUploadFiles } from '../utils/uploadSources'
const props = withDefaults(defineProps<{
acceptFileTypes?: string
supportedFileTypes?: string[]
includeManual?: boolean
triggerIcon?: string
triggerClass?: string
dataGuide?: string
tooltip?: string
placement?: 'top' | 'bottom' | 'bottom-right' | 'bottom-left'
}>(), {
acceptFileTypes: '',
supportedFileTypes: () => [],
includeManual: false,
triggerIcon: 'file-add',
triggerClass: '',
dataGuide: '',
tooltip: '',
placement: 'bottom-right',
})
const emit = defineEmits<{
files: [files: File[]]
url: [url: string]
manual: []
}>()
const { t } = useI18n()
const fileInputRef = ref<HTMLInputElement | null>(null)
const folderInputRef = ref<HTMLInputElement | null>(null)
const urlDialogVisible = ref(false)
const urlInputValue = ref('')
const tooltipText = computed(() => props.tooltip || t('knowledgeBase.addDocument'))
const dropdownOptions = computed(() => {
const options = [
{
content: t('upload.uploadDocument'),
value: 'upload',
prefixIcon: () => h(TIcon, { name: 'upload', size: '16px' }),
},
{
content: t('upload.uploadFolder'),
value: 'uploadFolder',
prefixIcon: () => h(TIcon, { name: 'folder-add', size: '16px' }),
},
{
content: t('knowledgeBase.importURL'),
value: 'importURL',
prefixIcon: () => h(TIcon, { name: 'link', size: '16px' }),
},
]
if (props.includeManual) {
options.push({
content: t('upload.onlineEdit'),
value: 'manualCreate',
prefixIcon: () => h(TIcon, { name: 'edit', size: '16px' }),
})
}
return options
})
const handleActionSelect = (data: { value: string }) => {
switch (data.value) {
case 'upload':
fileInputRef.value?.click()
break
case 'uploadFolder':
folderInputRef.value?.click()
break
case 'importURL':
urlInputValue.value = ''
urlDialogVisible.value = true
break
case 'manualCreate':
emit('manual')
break
default:
break
}
}
const notifyFilterResult = (result: ReturnType<typeof filterUploadFiles>, emptyAllSkippedKey: string) => {
const { validFiles, skippedCount, videoFilteredCount } = result
if (validFiles.length === 0) {
if (skippedCount > 0) {
MessagePlugin.warning(t(emptyAllSkippedKey))
}
return false
}
if (videoFilteredCount > 0) {
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }))
}
if (skippedCount > 0) {
MessagePlugin.warning(t('knowledgeBase.filesSkippedNoEngine', { count: skippedCount }))
}
return true
}
const handleFilesChange = (event: Event, fromFolder: boolean) => {
const input = event.target as HTMLInputElement
const files = input.files
if (!files || files.length === 0) return
const result = filterUploadFiles(files, {
supportedFileTypes: props.supportedFileTypes,
fromFolder,
multiFile: files.length > 1,
})
if (!notifyFilterResult(result, 'knowledgeBase.allFilesSkippedNoEngine')) {
input.value = ''
return
}
emit('files', result.validFiles)
input.value = ''
}
const handleUrlDialogConfirm = () => {
const url = urlInputValue.value.trim()
if (!url) {
MessagePlugin.warning(t('knowledgeBase.urlRequired'))
return
}
try {
new URL(url)
} catch {
MessagePlugin.warning(t('knowledgeBase.invalidURL'))
return
}
urlDialogVisible.value = false
urlInputValue.value = ''
emit('url', url)
}
const handleUrlDialogCancel = () => {
urlDialogVisible.value = false
urlInputValue.value = ''
}
const openUrlDialog = () => {
urlInputValue.value = ''
urlDialogVisible.value = true
}
defineExpose({ openUrlDialog })
</script>
<style lang="less" scoped>
.hidden-file-input {
position: absolute;
width: 0;
height: 0;
opacity: 0;
pointer-events: none;
}
.kb-upload-source-trigger {
color: var(--td-text-color-secondary);
&:hover {
color: var(--td-brand-color);
}
}
.url-import-form {
.url-input-label {
margin-bottom: 8px;
font-size: 14px;
font-weight: 500;
color: var(--td-text-color-primary);
}
.url-input-tip {
margin-top: 8px;
font-size: 12px;
line-height: 1.5;
color: var(--td-text-color-placeholder);
}
}
</style>
File diff suppressed because it is too large Load Diff
@@ -1,10 +1,9 @@
<template>
<div class="graph-settings">
<div class="section-header">
<div class="graph-settings" :class="{ 'graph-settings--embedded': embedded }">
<div v-if="!embedded" class="section-header">
<h2>{{ t('graphSettings.title') }}</h2>
<p class="section-description">{{ t('graphSettings.description') }}</p>
<!-- Warning message when graph database is not enabled -->
<t-alert
v-if="!isGraphDatabaseEnabled"
theme="warning"
@@ -18,6 +17,15 @@
</template>
</t-alert>
</div>
<t-alert
v-else-if="!isGraphDatabaseEnabled"
theme="warning"
class="embedded-graph-alert"
>
<template #message>
<div>{{ t('graphSettings.disabledWarning') }}</div>
</template>
</t-alert>
<div v-if="isGraphDatabaseEnabled" class="settings-group">
<!-- 启用实体关系提取 -->
@@ -296,7 +304,7 @@
</template>
<script setup lang="ts">
import { ref, watch, onMounted, computed } from 'vue'
import { ref, watch, onMounted, computed, withDefaults } from 'vue'
import { MessagePlugin } from 'tdesign-vue-next'
import { useI18n } from 'vue-i18n'
import { extractTextRelations, fabriText, fabriTag, type Node, type Relation } from '@/api/initialization'
@@ -323,9 +331,12 @@ interface Props {
graphExtract: GraphExtractConfig
modelId: string
allModels?: any[]
embedded?: boolean
}
const props = defineProps<Props>()
const props = withDefaults(defineProps<Props>(), {
embedded: false,
})
const emit = defineEmits<{
'update:graphExtract': [value: GraphExtractConfig]
@@ -766,4 +777,27 @@ onMounted(async () => {
gap: 12px;
flex-wrap: wrap;
}
.graph-settings--embedded {
.embedded-graph-alert {
margin-bottom: 12px;
}
.setting-row:not(.vertical) {
flex-direction: column;
align-items: stretch;
gap: 8px;
padding: 12px 0;
}
.setting-row:not(.vertical) .setting-info {
flex: none;
max-width: none;
padding-right: 0;
}
.setting-row:not(.vertical) .setting-control {
align-self: flex-start;
}
}
</style>
@@ -1,6 +1,6 @@
<template>
<div class="kb-advanced-settings">
<div class="section-header">
<div class="kb-advanced-settings" :class="{ 'kb-advanced-settings--embedded': embedded }">
<div v-if="!embedded" class="section-header">
<h2>{{ $t('knowledgeEditor.advanced.title') }}</h2>
<p class="section-description">{{ $t('knowledgeEditor.advanced.description') }}</p>
</div>
@@ -49,7 +49,7 @@
</template>
<script setup lang="ts">
import { ref, watch } from 'vue'
import { ref, watch, withDefaults } from 'vue'
interface QuestionGenerationConfig {
enabled: boolean
@@ -60,9 +60,12 @@ interface Props {
questionGeneration?: QuestionGenerationConfig
ragEnabled?: boolean
allModels?: any[]
embedded?: boolean
}
const props = defineProps<Props>()
const props = withDefaults(defineProps<Props>(), {
embedded: false,
})
const emit = defineEmits<{
'update:questionGeneration': [value: QuestionGenerationConfig]
@@ -182,4 +185,51 @@ const handleQuestionGenerationChange = () => {
font-weight: 500;
}
.kb-advanced-settings--embedded {
.setting-row {
padding: 12px 0;
}
.setting-row:has(.t-switch) {
flex-direction: row;
align-items: center;
gap: 16px;
.setting-info {
flex: 1;
min-width: 0;
max-width: none;
padding-right: 0;
}
.setting-control {
flex: none;
align-self: center;
}
}
.subsection .setting-row {
flex-direction: column;
align-items: stretch;
gap: 8px;
.setting-info {
flex: none;
max-width: none;
padding-right: 0;
}
.setting-control {
align-self: flex-start;
}
}
.subsection {
margin-top: 0;
padding: 0;
border: none;
background: none;
}
}
</style>
@@ -1,6 +1,6 @@
<template>
<div class="kb-chunking-settings">
<div class="section-header">
<div class="kb-chunking-settings" :class="{ 'kb-chunking-settings--embedded': embedded }">
<div v-if="!embedded" class="section-header">
<div class="section-header-text">
<h2>{{ $t('knowledgeEditor.chunking.title') }}</h2>
<p class="section-description">{{ $t('knowledgeEditor.chunking.description') }}</p>
@@ -21,12 +21,12 @@
:placeholder="$t('knowledgeEditor.chunking.strategyPlaceholder')"
:clearable="true"
@change="handleStrategyChange"
style="width: 280px;"
:style="selectStyle"
/>
<!-- Test trigger sits right next to the strategy picker so users
discover it exactly when they're deciding which strategy to
use on their content. -->
<KBChunkingDebug :config="debugConfig" />
<KBChunkingDebug v-if="!embedded" :config="debugConfig" />
</div>
</div>
@@ -51,9 +51,9 @@
:min="100"
:max="4000"
:step="50"
:marks="{ 100: '100', 1000: '1000', 2000: '2000', 4000: '4000' }"
:marks="embedded ? undefined : chunkSizeMarks"
@change="handleChunkSizeChange"
style="width: 200px;"
:style="sliderStyle"
/>
<span class="value-display">{{ localChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
@@ -74,9 +74,9 @@
:min="0"
:max="500"
:step="20"
:marks="{ 0: '0', 250: '250', 500: '500' }"
:marks="embedded ? undefined : chunkOverlapMarks"
@change="handleChunkOverlapChange"
style="width: 200px;"
:style="sliderStyle"
/>
<span class="value-display">{{ localChunkOverlap }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
@@ -84,7 +84,7 @@
</div>
<!-- Separators -->
<div class="setting-row">
<div class="setting-row setting-row--separators">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.separatorsLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.separatorsDescription') }}</p>
@@ -98,13 +98,13 @@
filterable
:placeholder="$t('knowledgeEditor.chunking.separatorsPlaceholder')"
@change="handleSeparatorsChange"
style="width: 280px;"
:style="selectStyle"
/>
</div>
</div>
<!-- Parent-Child Chunking -->
<div class="setting-row">
<div class="setting-row setting-row--toggle">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.chunking.parentChildLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.chunking.parentChildDescription') }}</p>
@@ -130,9 +130,9 @@
:min="512"
:max="8192"
:step="64"
:marks="{ 512: '512', 2048: '2048', 4096: '4096', 8192: '8192' }"
:marks="embedded ? undefined : parentChunkSizeMarks"
@change="handleParentChunkSizeChange"
style="width: 200px;"
:style="sliderStyle"
/>
<span class="value-display">{{ localParentChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
@@ -152,9 +152,9 @@
:min="64"
:max="2048"
:step="32"
:marks="{ 64: '64', 384: '384', 1024: '1024', 2048: '2048' }"
:marks="embedded ? undefined : childChunkSizeMarks"
@change="handleChildChunkSizeChange"
style="width: 200px;"
:style="sliderStyle"
/>
<span class="value-display">{{ localChildChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
</div>
@@ -201,7 +201,7 @@
:disabled="advancedDisabled"
:placeholder="$t('knowledgeEditor.chunking.languagesPlaceholder')"
@change="handleLanguagesChange"
style="width: 280px;"
:style="selectStyle"
/>
</div>
</div>
@@ -212,7 +212,7 @@
</template>
<script setup lang="ts">
import { ref, watch, computed } from 'vue'
import { ref, watch, computed, withDefaults } from 'vue'
import { useI18n } from 'vue-i18n'
import { ChevronRightIcon } from 'tdesign-icons-vue-next'
import KBChunkingDebug from './KBChunkingDebug.vue'
@@ -253,9 +253,20 @@ interface ChunkingConfig {
interface Props {
config: ChunkingConfig
embedded?: boolean
}
const props = defineProps<Props>()
const props = withDefaults(defineProps<Props>(), {
embedded: false,
})
const selectStyle = computed(() => (props.embedded ? { width: '100%' } : { width: '280px' }))
const sliderStyle = computed(() => (props.embedded ? { width: '100%' } : { width: '200px' }))
const chunkSizeMarks = { 100: '100', 1000: '1000', 2000: '2000', 4000: '4000' }
const chunkOverlapMarks = { 0: '0', 250: '250', 500: '500' }
const parentChunkSizeMarks = { 512: '512', 2048: '2048', 4096: '4096', 8192: '8192' }
const childChunkSizeMarks = { 64: '64', 384: '384', 1024: '1024', 2048: '2048' }
const emit = defineEmits<{
'update:config': [value: ChunkingConfig]
@@ -563,4 +574,102 @@ const emitUpdate = () => {
// panel edge and looks detached from the rest of the form.
margin-top: 4px;
}
.kb-chunking-settings--embedded {
.setting-row {
flex-direction: column;
align-items: stretch;
gap: 10px;
padding: 14px 0;
}
.setting-row--toggle {
flex-direction: row;
align-items: center;
gap: 16px;
.setting-info {
flex: 1;
min-width: 0;
}
.setting-control {
flex: none;
width: auto;
justify-content: flex-end;
}
}
.setting-row--separators {
padding-bottom: 18px;
}
.setting-info {
flex: none;
max-width: none;
padding-right: 0;
label {
font-size: 14px;
}
.desc {
font-size: 12px;
}
}
.setting-control {
flex: none;
max-width: none;
justify-content: flex-start;
align-items: stretch;
width: 100%;
}
.strategy-control {
flex-direction: column;
align-items: stretch;
width: 100%;
}
.strategy-info-panel {
margin: -4px 0 10px;
padding: 8px 12px;
font-size: 12px;
}
.slider-container {
flex-direction: column;
align-items: stretch;
gap: 8px;
width: 100%;
justify-content: flex-start;
}
.value-display {
order: -1;
text-align: left;
min-width: 0;
}
:deep(.t-slider) {
padding-bottom: 0;
}
:deep(.t-select__wrap) {
max-width: 100%;
}
:deep(.t-tag) {
max-width: 100%;
}
.advanced-toggle {
padding-top: 10px;
}
.advanced-section .setting-row {
padding: 14px 0;
}
}
</style>
@@ -1,6 +1,6 @@
<template>
<div class="kb-parser-settings">
<div class="section-header">
<div class="kb-parser-settings" :class="{ 'kb-parser-settings--embedded': embedded }">
<div v-if="!embedded" class="section-header">
<h2>{{ $t('kbSettings.parser.title') }}</h2>
<p class="section-description">{{ $t('kbSettings.parser.description') }}</p>
</div>
@@ -33,7 +33,7 @@
<t-select
:value="getEngineForGroup(group.extensions) || undefined"
@change="(val: string) => handleEngineChange(group.extensions, val)"
style="width: 280px;"
:style="embedded ? { width: '100%' } : { width: '280px' }"
:status="hasAvailableEngine(group.extensions) ? 'default' : 'warning'"
:placeholder="$t('kbSettings.parser.noEngine')"
>
@@ -123,10 +123,16 @@ interface EngineOption {
interface Props {
parserEngineRules?: ParserEngineRule[]
/** Compact layout for upload-confirm dialog */
embedded?: boolean
/** When set, only show file-type groups matching these extensions */
relevantExtensions?: string[]
}
const props = withDefaults(defineProps<Props>(), {
parserEngineRules: () => []
parserEngineRules: () => [],
embedded: false,
relevantExtensions: () => [],
})
const emit = defineEmits<{
@@ -182,7 +188,11 @@ const fileTypeGroups = computed(() => {
})
}
return groups
const rel = props.relevantExtensions
if (!rel?.length) return groups
const relSet = new Set(rel)
const filtered = groups.filter(g => g.extensions.some(e => relSet.has(e)))
return filtered.length > 0 ? filtered : groups
})
function getEngineOptions(extensions: string[]): EngineOption[] {
@@ -468,6 +478,46 @@ watch(() => props.parserEngineRules, (v) => {
}
}
}
.kb-parser-settings--embedded {
.setting-row {
flex-direction: column;
align-items: stretch;
gap: 10px;
padding: 12px 0;
}
.setting-info {
flex: none;
max-width: none;
padding-right: 0;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 6px 10px;
}
.setting-control {
flex: none;
max-width: none;
align-items: stretch;
}
.group-label {
font-size: 14px;
margin-bottom: 0;
}
.ext-tags {
margin-top: 0;
gap: 4px;
}
.ext-tag {
font-size: 11px;
padding: 2px 6px;
}
}
</style>
<style lang="less">
@@ -0,0 +1,70 @@
import { kbFileTypeVerification } from '@/utils'
export const UPLOAD_VIDEO_EXTENSIONS = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv']
export function getUploadFileExt(file: File): string {
const dot = file.name.lastIndexOf('.')
if (dot < 0) return ''
return file.name.substring(dot + 1).toLowerCase()
}
export function getUploadFileKey(file: File): string {
const path = (file as File & { webkitRelativePath?: string }).webkitRelativePath || ''
return `${path || file.name}\0${file.size}`
}
export interface FilterUploadFilesOptions {
supportedFileTypes?: Set<string> | string[]
fromFolder?: boolean
multiFile?: boolean
}
export interface FilterUploadFilesResult {
validFiles: File[]
skippedCount: number
videoFilteredCount: number
hiddenFileCount: number
}
export function filterUploadFiles(
files: FileList | File[],
options: FilterUploadFilesOptions = {},
): FilterUploadFilesResult {
const list = Array.from(files)
const dynamicTypes = options.supportedFileTypes
? options.supportedFileTypes instanceof Set
? options.supportedFileTypes
: new Set(options.supportedFileTypes)
: undefined
const validFiles: File[] = []
let skippedCount = 0
let videoFilteredCount = 0
let hiddenFileCount = 0
const multiFile = options.multiFile ?? list.length > 1
for (const file of list) {
if (options.fromFolder) {
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath || file.name
if (relativePath.split('/').some(part => part.startsWith('.'))) {
hiddenFileCount++
continue
}
}
const fileExt = getUploadFileExt(file)
if (UPLOAD_VIDEO_EXTENSIONS.includes(fileExt)) {
videoFilteredCount++
continue
}
if (kbFileTypeVerification(file, multiFile, dynamicTypes)) {
skippedCount++
continue
}
validFiles.push(file)
}
return { validFiles, skippedCount, videoFilteredCount, hiddenFileCount }
}
@@ -839,6 +839,7 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
item.FileName, // customFileName — must include extension for file-type validation
tagID, // auto-tag from data source
channel,
nil,
)
return isUpdate, err
}
@@ -855,6 +856,7 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
item.Title,
tagID, // auto-tag from data source
channel,
nil,
)
return isUpdate, err
}
+20 -8
View File
@@ -290,10 +290,22 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
handleErr = err
return err
}
if kb.ExtractConfig == nil {
logger.Warnf(ctx, "failed to get extract config")
graphOut["skipped"] = "no_extract_config"
return err
var processOverrides *types.KnowledgeProcessOverrides
knowledgeID := p.KnowledgeID
if knowledgeID == "" {
knowledgeID = chunk.KnowledgeID
}
if knowledgeID != "" && s.knowledgeRepo != nil {
if k, kerr := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, knowledgeID); kerr == nil && k != nil {
processOverrides, _ = k.ProcessOverrides()
}
}
extractCfg := ResolveProcessConfig(kb, processOverrides).ExtractConfig
if !extractCfg.Enabled {
logger.Warnf(ctx, "extract config not enabled")
graphOut["skipped"] = "extract_disabled"
return nil
}
chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID)
@@ -305,12 +317,12 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
template := &types.PromptTemplateStructured{
Description: s.template.Description,
Tags: kb.ExtractConfig.Tags,
Tags: extractCfg.Tags,
Examples: []types.GraphData{
{
Text: kb.ExtractConfig.Text,
Node: kb.ExtractConfig.Nodes,
Relation: kb.ExtractConfig.Relations,
Text: extractCfg.Text,
Node: extractCfg.Nodes,
Relation: extractCfg.Relations,
},
},
}
@@ -201,7 +201,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
}()
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
vlmModel, vlmCfg, err := s.resolveVLM(ctx, payload.KnowledgeBaseID, payload.KnowledgeID)
if err != nil {
handleErr = fmt.Errorf("resolve VLM: %w", err)
return handleErr
@@ -210,12 +210,10 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
// legacy inline-config path) so the trace shows WHICH model handled
// this image. Without this, debugging "VLM is slow" requires a
// separate hop to the KB config.
if kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil {
if id := strings.TrimSpace(kb.VLMConfig.ModelID); id != "" {
imgOut["vlm_model_id"] = id
} else {
imgOut["vlm_model_id"] = "legacy_inline"
}
if id := strings.TrimSpace(vlmCfg.ModelID); id != "" {
imgOut["vlm_model_id"] = id
} else {
imgOut["vlm_model_id"] = "legacy_inline"
}
// Read image bytes. A provider:// URL must be resolved via FileService —
@@ -456,27 +454,36 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
// Per-upload process_overrides on the knowledge entry take precedence over KB defaults.
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID, knowledgeID string) (vlm.VLM, types.VLMConfig, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
return nil, types.VLMConfig{}, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
return nil, types.VLMConfig{}, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
var processOverrides *types.KnowledgeProcessOverrides
if knowledgeID != "" && s.knowledgeRepo != nil {
if k, kerr := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, knowledgeID); kerr == nil && k != nil {
processOverrides, _ = k.ProcessOverrides()
}
}
vlmCfg := ResolveProcessConfig(kb, processOverrides).VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
return nil, types.VLMConfig{}, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
model, err := s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
return model, vlmCfg, err
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
model, err := vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
return model, vlmCfg, err
}
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
+125 -108
View File
@@ -26,6 +26,7 @@ import (
// CreateKnowledgeFromFile creates a knowledge entry from an uploaded file
func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagID string, channel string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file")
@@ -60,62 +61,6 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, err
}
// 检查多模态配置完整性 - 只在图片文件时校验
if !IsImageType(getFileType(fileName)) {
logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation")
} else {
// 解析有效 provider:优先 KB 级别(新字段 > 旧字段),其次租户默认
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
// 根据 provider 校验租户级存储引擎配置
switch provider {
case "cos":
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
case "minio":
ok := false
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
if m.Mode == "remote" {
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
} else {
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
}
}
if !ok {
logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
}
// 检查VLM配置
if !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" {
logger.Error(ctx, "VLM model is not configured")
return nil, werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
}
logger.Info(ctx, "Image multimodal configuration validation passed")
}
// 检查音频ASR配置完整性 - 只在音频文件时校验
if IsAudioType(getFileType(fileName)) {
if !kb.ASRConfig.IsASREnabled() {
logger.Error(ctx, "ASR model is not configured")
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
}
logger.Info(ctx, "Audio ASR configuration validation passed")
}
// Validate file type
logger.Infof(ctx, "Checking file type: %s", fileName)
if !isValidFileType(fileName) {
@@ -179,6 +124,64 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, werrors.NewValidationError("文件名包含非法字符")
}
eff := ResolveProcessConfig(kb, processOverrides)
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
eff.EnableMultimodel = *enableMultimodel
}
if processOverrides != nil {
if err := ValidateProcessOverrides(ctx, kb, processOverrides, []string{getFileType(safeFilename)}); err != nil {
return nil, err
}
} else {
// 检查多模态配置完整性 - 只在图片文件时校验
if IsImageType(getFileType(safeFilename)) {
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
switch provider {
case "cos":
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
case "minio":
ok := false
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
if m.Mode == "remote" {
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
} else {
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
}
}
if !ok {
logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing")
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
}
if !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" {
logger.Error(ctx, "VLM model is not configured")
return nil, werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
}
}
if IsAudioType(getFileType(safeFilename)) {
if !kb.ASRConfig.IsASREnabled() {
logger.Error(ctx, "ASR model is not configured")
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
}
}
}
// Prepare knowledge record
logger.Info(ctx, "Preparing knowledge record")
knowledge := &types.Knowledge{
@@ -201,6 +204,13 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
Metadata: metadataJSON,
}
if processOverrides != nil {
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
logger.Errorf(ctx, "Failed to set process overrides: %v", err)
return nil, err
}
}
// Save the file to storage (use KB-level storage engine if configured)
logger.Infof(ctx, "Saving file, knowledge ID: %s", knowledge.ID)
fileSvc := s.resolveFileService(ctx, kb)
@@ -223,21 +233,12 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
// Enqueue document processing task to Asynq
logger.Info(ctx, "Enqueuing document processing task to Asynq")
enableMultimodelValue := false
if enableMultimodel != nil {
enableMultimodelValue = *enableMultimodel
} else {
enableMultimodelValue = kb.IsMultimodalEnabled()
}
enableMultimodelValue := eff.EnableMultimodel
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
questionCount := eff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -307,13 +308,16 @@ func isFileURL(rawURL, fileName, fileType string) bool {
func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagID string, channel string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from URL")
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL)
// Route to file_url logic when the URL points to a downloadable file
if isFileURL(rawURL, fileName, fileType) {
return s.createKnowledgeFromFileURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID, channel)
return s.createKnowledgeFromFileURL(
ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID, channel, processOverrides,
)
}
url := rawURL
@@ -397,6 +401,11 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
// Save knowledge record
logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID)
eff, err := ApplyKnowledgeProcessOverrides(ctx, kb, knowledge, processOverrides, []string{"html"}, enableMultimodel)
if err != nil {
return nil, err
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
return nil, err
@@ -404,21 +413,11 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
// Enqueue URL processing task to Asynq
logger.Info(ctx, "Enqueuing URL processing task to Asynq")
enableMultimodelValue := false
if enableMultimodel != nil {
enableMultimodelValue = *enableMultimodel
} else {
enableMultimodelValue = kb.IsMultimodalEnabled()
}
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableMultimodelValue := eff.EnableMultimodel
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
questionCount := eff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -515,6 +514,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
title string,
tagID string,
channel string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file URL")
logger.Infof(ctx, "Knowledge base ID: %s, file URL: %s", kbID, fileURL)
@@ -624,26 +624,32 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
knowledge.Title = displayName
}
resolvedFileType := fileType
if resolvedFileType == "" && fileName != "" {
resolvedFileType = getFileType(fileName)
}
if resolvedFileType == "" {
resolvedFileType = getFileType(extractFileNameFromURL(fileURL))
}
eff, err := ApplyKnowledgeProcessOverrides(
ctx, kb, knowledge, processOverrides, []string{resolvedFileType}, enableMultimodel,
)
if err != nil {
return nil, err
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
return nil, err
}
// Build async task payload
enableMultimodelValue := false
if enableMultimodel != nil {
enableMultimodelValue = *enableMultimodel
} else {
enableMultimodelValue = kb.IsMultimodalEnabled()
}
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableMultimodelValue := eff.EnableMultimodel
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
questionCount := eff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -775,6 +781,12 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context,
knowledge.ParseStatus = "pending"
}
if status == types.ManualKnowledgeStatusPublish {
if _, err := ApplyKnowledgeProcessOverrides(ctx, kb, knowledge, payload.ProcessConfig, nil, nil); err != nil {
return nil, err
}
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to create manual knowledge record: %v", err)
return nil, err
@@ -998,6 +1010,10 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
existing.Description = ""
existing.ProcessedAt = nil
if _, err := ApplyKnowledgeProcessOverrides(ctx, kb, existing, payload.ProcessConfig, nil, nil); err != nil {
return nil, err
}
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to persist manual knowledge before indexing: %v", err)
return nil, err
@@ -1101,26 +1117,27 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
}
}
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
// Manual content is markdown - chunk directly with Go chunker
chunkCfg := buildSplitterConfig(kb)
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
var parsed []types.ParsedChunk
opts := ProcessChunksOptions{
// When the KB has VLM enabled and we resolved remote images, pass them
// through so processChunks will enqueue image:multimodal tasks (OCR + caption).
EnableMultimodel: kb.IsMultimodalEnabled() && len(resolvedImages) > 0,
EnableMultimodel: eff.EnableMultimodel && len(resolvedImages) > 0,
StoredImages: resolvedImages,
}
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
if eff.QuestionGenerationConfig.Enabled {
opts.EnableQuestionGeneration = true
opts.QuestionCount = kb.QuestionGenerationConfig.QuestionCount
opts.QuestionCount = eff.QuestionGenerationConfig.QuestionCount
if opts.QuestionCount <= 0 {
opts.QuestionCount = 3
}
}
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
if eff.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(eff.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg)
parsed = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
@@ -138,6 +138,7 @@ func TestCreateKnowledgeFromFileDoesNotPersistWhenStorageSaveFails(t *testing.T)
"",
"",
"",
nil,
)
require.Error(t, err)
@@ -168,6 +169,7 @@ func TestCreateKnowledgeFromFilePersistsStoredFilePathOnCreate(t *testing.T) {
"",
"",
"",
nil,
)
require.NoError(t, err)
@@ -201,6 +203,7 @@ func TestCreateKnowledgeFromFileDeletesStoredFileWhenCreateFails(t *testing.T) {
"",
"",
"",
nil,
)
require.EqualError(t, err, "database unavailable")
@@ -211,6 +214,52 @@ func TestCreateKnowledgeFromFileDeletesStoredFileWhenCreateFails(t *testing.T) {
require.Equal(t, "stored/"+fileSvc.savedWithKnowledgeID, fileSvc.deletedPath)
}
func TestCreateKnowledgeFromFile_PersistsProcessOverrides(t *testing.T) {
t.Parallel()
repo := &createKnowledgeFileRepoStub{}
fileSvc := &createKnowledgeFileServiceStub{}
task := &createKnowledgeTaskEnqueuerStub{}
svc := &knowledgeService{
repo: repo,
kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}},
fileSvc: fileSvc,
task: task,
}
chunkSize := 512
overrides := &types.KnowledgeProcessOverrides{
ChunkingConfig: &types.ChunkingConfig{ChunkSize: chunkSize},
}
knowledge, err := svc.CreateKnowledgeFromFile(
newCreateKnowledgeFileContext(),
"kb-1",
newMultipartFileHeader(t, "doc.txt", "hello"),
map[string]string{"source": "test"},
nil,
"",
"",
"",
overrides,
)
require.NoError(t, err)
require.NotNil(t, knowledge)
require.Equal(t, 1, repo.createCalls)
require.NotNil(t, repo.createdKnowledge)
parsed, err := repo.createdKnowledge.ProcessOverrides()
require.NoError(t, err)
require.NotNil(t, parsed)
require.NotNil(t, parsed.ChunkingConfig)
require.Equal(t, chunkSize, parsed.ChunkingConfig.ChunkSize)
metadataMap, err := repo.createdKnowledge.Metadata.Map()
require.NoError(t, err)
require.Equal(t, "test", metadataMap["source"])
}
func newCreateKnowledgeFileContext() context.Context {
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
ctx = context.WithValue(ctx, types.TenantInfoContextKey, &types.Tenant{})
@@ -124,6 +124,9 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err)
}
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
// 2. Fetch all chunks
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
@@ -155,7 +158,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// drains is bounded by the housekeeping finalizing sweep.
willSpawnSummary := len(textChunks) > 0
willSpawnQuestion := willSpawnSummary && kb.NeedsEmbeddingModel() &&
kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled
eff.QuestionGenerationConfig.Enabled
willSpawnWiki := kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0
// Question generation now fans out one subtask per plain text chunk
@@ -184,7 +187,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
questionBatchCount := (len(questionChunks) + questionGenChunkBatchSize - 1) / questionGenChunkBatchSize
graphChunkCount := 0
if kb.IsGraphEnabled() {
if eff.GraphEnabled {
graphChunkCount = len(textChunks)
}
expectedSubtasks := 0
@@ -302,7 +305,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
"chunk_count": len(questionChunks),
})
}
enqueuedQuestionCount = s.enqueueQuestionGenerationTasks(ctx, payload, kb, attempt, questionChunks)
enqueuedQuestionCount = s.enqueueQuestionGenerationTasks(ctx, payload, eff.QuestionGenerationConfig, attempt, questionChunks)
}
}
@@ -466,18 +469,18 @@ const postprocessQuestionGroupSpanName = "postprocess.question"
func (s *KnowledgePostProcessService) enqueueQuestionGenerationTasks(
ctx context.Context,
payload types.KnowledgePostProcessPayload,
kb *types.KnowledgeBase,
qg types.QuestionGenerationConfig,
attempt int,
questionChunks []*types.Chunk,
) int {
if s.taskEnqueuer == nil || len(questionChunks) == 0 {
return 0
}
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
if !qg.Enabled {
return 0
}
questionCount := kb.QuestionGenerationConfig.QuestionCount
questionCount := qg.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
@@ -201,13 +201,17 @@ func finalizeIndexedKnowledgeState(
// identical whether callers come through this path or invoke the chunker
// directly with a zero-value config.
func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig {
return buildSplitterConfigFromChunking(kb.ChunkingConfig)
}
func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig {
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
Separators: kb.ChunkingConfig.Separators,
Strategy: kb.ChunkingConfig.Strategy,
TokenLimit: kb.ChunkingConfig.TokenLimit,
Languages: kb.ChunkingConfig.Languages,
ChunkSize: cc.ChunkSize,
ChunkOverlap: cc.ChunkOverlap,
Separators: cc.Separators,
Strategy: cc.Strategy,
TokenLimit: cc.TokenLimit,
Languages: cc.Languages,
}
if chunkCfg.ChunkSize <= 0 {
chunkCfg.ChunkSize = chunker.DefaultChunkSize
@@ -1973,6 +1977,9 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return nil, err
}
processOverrides, _ := existing.ProcessOverrides()
reparseEff := ResolveProcessConfig(kb, processOverrides)
// Keep wiki's pending queue consistent across both manual and non-manual
// paths. The destructive work (swapping old wiki contributions for new)
// happens asynchronously inside mapOneDocument — see its oldPageSlugs
@@ -2060,17 +2067,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
if existing.FilePath != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// Determine multimodal setting
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -2119,16 +2120,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
if existing.Type == "file_url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -2172,16 +2168,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
if existing.Type == "url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang, _ := types.LanguageFromContext(ctx)
@@ -2734,6 +2725,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
// Re-check abort status right before flipping to "processing" — closes
// the race where the user cancels between the entry guard above and
// this write (otherwise the worker would overwrite cancelled→processing
@@ -2775,7 +2769,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
// 检查音频ASR配置(仅对文件导入)
if payload.FilePath != "" && IsAudioType(payload.FileType) && !kb.ASRConfig.IsASREnabled() {
if payload.FilePath != "" && IsAudioType(payload.FileType) && !eff.ASRConfig.IsASREnabled() {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument audio without ASR model configured")
knowledge.ParseStatus = "failed"
@@ -2857,7 +2851,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
payload.FilePath = filePath
payload.FileName = resolvedFileName
payload.FileType = resolvedFileType
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
@@ -2866,7 +2860,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
} else if payload.URL != "" {
// URL import
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
@@ -2910,7 +2904,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
} else {
// File import
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
@@ -2921,7 +2915,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// Step 1.5: ASR transcription for audio files
if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 {
if !kb.ASRConfig.IsASREnabled() {
if !eff.ASRConfig.IsASREnabled() {
logger.Error(ctx, "Audio file detected but ASR is not configured")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "ASR model is not configured for audio transcription"
@@ -2933,7 +2927,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes",
knowledge.ID, len(convertResult.AudioData))
asrModel, err := s.modelService.GetASRModel(ctx, kb.ASRConfig.ModelID)
asrModel, err := s.modelService.GetASRModel(ctx, eff.ASRConfig.ModelID)
if err != nil {
logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err)
knowledge.ParseStatus = "failed"
@@ -3003,7 +2997,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
// Step 3: Split into chunks using Go chunker
chunkCfg := buildSplitterConfig(kb)
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
processOpts := ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration,
@@ -3016,8 +3010,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
processOpts.Metadata = convertResult.Metadata
}
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
if eff.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(eff.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
@@ -3064,6 +3058,7 @@ func (s *knowledgeService) convert(
payload types.DocumentProcessPayload,
kb *types.KnowledgeBase,
knowledge *types.Knowledge,
eff types.EffectiveProcessConfig,
isLastRetry bool,
) (*types.ReadResult, error) {
// Stage tracking: docreader. Mark the stage as running here so the
@@ -3097,13 +3092,13 @@ func (s *knowledgeService) convert(
}
}
parserEngine := kb.ChunkingConfig.ResolveParserEngine(fileType)
parserEngine := eff.ChunkingConfig.ResolveParserEngine(fileType)
if isURL {
parserEngine = kb.ChunkingConfig.ResolveParserEngine("url")
parserEngine = eff.ChunkingConfig.ResolveParserEngine("url")
}
logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v",
kb.ID, fileType, isURL, parserEngine, kb.ChunkingConfig.ParserEngineRules)
kb.ID, fileType, isURL, parserEngine, eff.ChunkingConfig.ParserEngineRules)
var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides)
if reader == nil {
@@ -0,0 +1,231 @@
package service
import (
"context"
"os"
"strings"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
)
// ResolveProcessConfig merges KB defaults with per-upload overrides for the parse pipeline.
func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgeProcessOverrides) types.EffectiveProcessConfig {
eff := types.EffectiveProcessConfig{
ChunkingConfig: kb.ChunkingConfig,
EnableMultimodel: kb.IsMultimodalEnabled(),
VLMConfig: kb.VLMConfig,
ASRConfig: kb.ASRConfig,
QuestionGenerationConfig: defaultQuestionGenerationConfig(kb),
GraphEnabled: kb.IsGraphEnabled(),
ExtractConfig: derefExtractConfig(kb.ExtractConfig),
}
if overrides == nil {
return eff
}
if overrides.ChunkingConfig != nil {
eff.ChunkingConfig = mergeChunkingConfig(eff.ChunkingConfig, overrides.ChunkingConfig)
}
if len(overrides.ParserEngineRules) > 0 {
eff.ChunkingConfig.ParserEngineRules = overrides.ParserEngineRules
}
if overrides.EnableMultimodel != nil {
eff.EnableMultimodel = *overrides.EnableMultimodel
}
if overrides.VLMConfig != nil {
eff.VLMConfig = *overrides.VLMConfig
}
if overrides.ASRConfig != nil {
eff.ASRConfig = *overrides.ASRConfig
}
if overrides.QuestionGenerationConfig != nil {
eff.QuestionGenerationConfig = *overrides.QuestionGenerationConfig
}
if overrides.GraphEnabled != nil {
eff.GraphEnabled = *overrides.GraphEnabled
}
if overrides.ExtractConfig != nil {
eff.ExtractConfig = mergeExtractConfig(eff.ExtractConfig, overrides.ExtractConfig)
}
return eff
}
// ValidateProcessOverrides validates batch overrides against file types in the upload.
func ValidateProcessOverrides(
ctx context.Context,
kb *types.KnowledgeBase,
overrides *types.KnowledgeProcessOverrides,
fileTypes []string,
) error {
if overrides == nil {
return nil
}
hasImage := false
hasAudio := false
for _, ft := range fileTypes {
if IsImageType(ft) {
hasImage = true
}
if IsAudioType(ft) {
hasAudio = true
}
}
eff := ResolveProcessConfig(kb, overrides)
if hasImage {
if err := validateImageMultimodalConfig(ctx, kb); err != nil {
return err
}
if !eff.VLMConfig.IsEnabled() {
return werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
}
}
if hasAudio && !eff.ASRConfig.IsASREnabled() {
return werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
}
return nil
}
// ApplyKnowledgeProcessOverrides validates optional overrides, persists them on the
// knowledge record, and returns the effective config for task enqueue.
func ApplyKnowledgeProcessOverrides(
ctx context.Context,
kb *types.KnowledgeBase,
knowledge *types.Knowledge,
processOverrides *types.KnowledgeProcessOverrides,
fileTypes []string,
enableMultimodel *bool,
) (types.EffectiveProcessConfig, error) {
eff := ResolveProcessConfig(kb, processOverrides)
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
eff.EnableMultimodel = *enableMultimodel
}
if processOverrides == nil {
return eff, nil
}
if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil {
return eff, err
}
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
return eff, err
}
return eff, nil
}
func defaultQuestionGenerationConfig(kb *types.KnowledgeBase) types.QuestionGenerationConfig {
if kb == nil || kb.QuestionGenerationConfig == nil {
return types.QuestionGenerationConfig{}
}
return *kb.QuestionGenerationConfig
}
func derefExtractConfig(cfg *types.ExtractConfig) types.ExtractConfig {
if cfg == nil {
return types.ExtractConfig{}
}
return *cfg
}
func mergeChunkingConfig(base types.ChunkingConfig, override *types.ChunkingConfig) types.ChunkingConfig {
if override == nil {
return base
}
result := base
if override.ChunkSize != 0 {
result.ChunkSize = override.ChunkSize
}
if override.ChunkOverlap != 0 {
result.ChunkOverlap = override.ChunkOverlap
}
if len(override.Separators) > 0 {
result.Separators = override.Separators
}
if override.EnableMultimodal {
result.EnableMultimodal = override.EnableMultimodal
}
if len(override.ParserEngineRules) > 0 {
result.ParserEngineRules = override.ParserEngineRules
}
if override.EnableParentChild {
result.EnableParentChild = override.EnableParentChild
}
if override.ParentChunkSize != 0 {
result.ParentChunkSize = override.ParentChunkSize
}
if override.ChildChunkSize != 0 {
result.ChildChunkSize = override.ChildChunkSize
}
if override.Strategy != "" {
result.Strategy = override.Strategy
}
if override.TokenLimit != 0 {
result.TokenLimit = override.TokenLimit
}
if len(override.Languages) > 0 {
result.Languages = override.Languages
}
return result
}
func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig) types.ExtractConfig {
if override == nil {
return base
}
result := base
if override.Enabled {
result.Enabled = true
}
if override.Text != "" {
result.Text = override.Text
}
if len(override.Tags) > 0 {
result.Tags = override.Tags
}
if len(override.Nodes) > 0 {
result.Nodes = override.Nodes
}
if len(override.Relations) > 0 {
result.Relations = override.Relations
}
return result
}
func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) error {
provider := kb.GetStorageProvider()
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
}
switch provider {
case "cos":
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
return werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
case "minio":
ok := false
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
if m.Mode == "remote" {
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
} else {
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
}
}
if !ok {
return werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
}
}
return nil
}
@@ -0,0 +1,245 @@
package service
import (
"context"
"testing"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
func processConfigBoolPtr(v bool) *bool {
return &v
}
func testKBWithGraphEnabled(enabled bool) *types.KnowledgeBase {
return &types.KnowledgeBase{
IndexingStrategy: types.IndexingStrategy{GraphEnabled: enabled},
ExtractConfig: &types.ExtractConfig{Enabled: enabled},
}
}
func TestResolveProcessConfig_OverridesChunkSize(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50},
}
overrides := &types.KnowledgeProcessOverrides{
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 2048},
}
eff := ResolveProcessConfig(kb, overrides)
require.Equal(t, 2048, eff.ChunkingConfig.ChunkSize)
require.Equal(t, 50, eff.ChunkingConfig.ChunkOverlap)
}
func TestResolveProcessConfig_GraphDisabled(t *testing.T) {
t.Parallel()
kb := testKBWithGraphEnabled(true)
overrides := &types.KnowledgeProcessOverrides{GraphEnabled: processConfigBoolPtr(false)}
eff := ResolveProcessConfig(kb, overrides)
require.False(t, eff.GraphEnabled)
}
func TestResolveProcessConfig_NilOverridesUsesKBDefaults(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50},
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
ASRConfig: types.ASRConfig{Enabled: true, ModelID: "asr-1"},
QuestionGenerationConfig: &types.QuestionGenerationConfig{
Enabled: true,
QuestionCount: 3,
},
IndexingStrategy: types.IndexingStrategy{GraphEnabled: true},
ExtractConfig: &types.ExtractConfig{Enabled: true, Tags: []string{"tag-a"}},
}
kb.ChunkingConfig.EnableMultimodal = true
eff := ResolveProcessConfig(kb, nil)
require.Equal(t, 512, eff.ChunkingConfig.ChunkSize)
require.Equal(t, 50, eff.ChunkingConfig.ChunkOverlap)
require.True(t, eff.EnableMultimodel)
require.Equal(t, "vlm-1", eff.VLMConfig.ModelID)
require.Equal(t, "asr-1", eff.ASRConfig.ModelID)
require.True(t, eff.QuestionGenerationConfig.Enabled)
require.Equal(t, 3, eff.QuestionGenerationConfig.QuestionCount)
require.True(t, eff.GraphEnabled)
require.True(t, eff.ExtractConfig.Enabled)
require.Equal(t, []string{"tag-a"}, eff.ExtractConfig.Tags)
}
func TestBuildSplitterConfigFromChunking_UsesEffectiveChunkingConfig(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50, Strategy: "token"},
}
overrides := &types.KnowledgeProcessOverrides{
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 1500, ChunkOverlap: 120, Strategy: "character"},
}
eff := ResolveProcessConfig(kb, overrides)
cfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
require.Equal(t, 1500, cfg.ChunkSize)
require.Equal(t, 120, cfg.ChunkOverlap)
require.Equal(t, "character", cfg.Strategy)
}
func TestEffectiveChunkingConfig_ResolveParserEngineFromOverrides(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{
ParserEngineRules: []types.ParserEngineRule{
{FileTypes: []string{"pdf"}, Engine: "builtin"},
},
},
}
overrides := &types.KnowledgeProcessOverrides{
ParserEngineRules: []types.ParserEngineRule{
{FileTypes: []string{"pdf"}, Engine: "mineru"},
},
}
eff := ResolveProcessConfig(kb, overrides)
require.Equal(t, "mineru", eff.ChunkingConfig.ResolveParserEngine("pdf"))
}
func TestResolveProcessConfig_ParserEngineRulesReplaced(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{
ParserEngineRules: []types.ParserEngineRule{
{FileTypes: []string{"pdf"}, Engine: "builtin"},
},
},
}
overrides := &types.KnowledgeProcessOverrides{
ParserEngineRules: []types.ParserEngineRule{
{FileTypes: []string{"docx"}, Engine: "custom"},
},
}
eff := ResolveProcessConfig(kb, overrides)
require.Len(t, eff.ChunkingConfig.ParserEngineRules, 1)
require.Equal(t, []string{"docx"}, eff.ChunkingConfig.ParserEngineRules[0].FileTypes)
require.Equal(t, "custom", eff.ChunkingConfig.ParserEngineRules[0].Engine)
}
func TestResolveProcessConfig_EnableMultimodelOverride(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
}
overrides := &types.KnowledgeProcessOverrides{
EnableMultimodel: processConfigBoolPtr(false),
}
eff := ResolveProcessConfig(kb, overrides)
require.False(t, eff.EnableMultimodel)
}
func TestResolveProcessConfig_ExtractConfigFieldMerge(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ExtractConfig: &types.ExtractConfig{
Enabled: true,
Text: "base text",
Tags: []string{"base-tag"},
},
}
overrides := &types.KnowledgeProcessOverrides{
ExtractConfig: &types.ExtractConfig{
Tags: []string{"override-tag"},
},
}
eff := ResolveProcessConfig(kb, overrides)
require.True(t, eff.ExtractConfig.Enabled)
require.Equal(t, "base text", eff.ExtractConfig.Text)
require.Equal(t, []string{"override-tag"}, eff.ExtractConfig.Tags)
}
func TestValidateProcessOverrides_NilOverrides(t *testing.T) {
t.Parallel()
err := ValidateProcessOverrides(context.Background(), &types.KnowledgeBase{}, nil, []string{"png"})
require.NoError(t, err)
}
func TestValidateProcessOverrides_ImageRequiresVLM(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
VLMConfig: types.VLMConfig{Enabled: false},
}
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"png"})
require.Error(t, err)
var badReq *werrors.AppError
require.ErrorAs(t, err, &badReq)
}
func TestValidateProcessOverrides_ImageWithEffectiveVLM(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
VLMConfig: types.VLMConfig{Enabled: false},
}
overrides := &types.KnowledgeProcessOverrides{
VLMConfig: &types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
}
err := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"jpg"})
require.NoError(t, err)
}
func TestValidateProcessOverrides_AudioRequiresASR(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ASRConfig: types.ASRConfig{Enabled: false},
}
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"mp3"})
require.Error(t, err)
}
func TestValidateProcessOverrides_AudioWithEffectiveASR(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ASRConfig: types.ASRConfig{Enabled: false},
}
overrides := &types.KnowledgeProcessOverrides{
ASRConfig: &types.ASRConfig{Enabled: true, ModelID: "asr-1"},
}
err := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"wav"})
require.NoError(t, err)
}
func TestValidateProcessOverrides_NonMediaFileTypes(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{}
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"pdf", "txt"})
require.NoError(t, err)
}
func TestValidateProcessOverrides_COSIncompleteForImage(t *testing.T) {
t.Parallel()
ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, &types.Tenant{
StorageEngineConfig: &types.StorageEngineConfig{
COS: &types.COSEngineConfig{SecretID: "id"},
},
})
kb := &types.KnowledgeBase{
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
}
kb.SetStorageProvider("cos")
err := ValidateProcessOverrides(ctx, kb, &types.KnowledgeProcessOverrides{}, []string{"png"})
require.Error(t, err)
}
+30 -9
View File
@@ -237,6 +237,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
// @Param fileName formData string false "自定义文件名"
// @Param metadata formData string false "元数据JSON"
// @Param enable_multimodel formData bool false "启用多模态处理"
// @Param process_config formData string false "处理配置JSONKnowledgeProcessOverrides"
// @Success 200 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "文件重复"
@@ -318,6 +319,23 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
enableMultimodel = &parseBool
}
var processOverrides *types.KnowledgeProcessOverrides
if raw := c.PostForm("process_config"); raw != "" {
processOverrides = &types.KnowledgeProcessOverrides{}
if err := json.Unmarshal([]byte(raw), processOverrides); err != nil {
logger.Error(ctx, "Failed to parse process_config", err)
c.Error(errors.NewBadRequestError("Invalid process_config format").WithDetails(err.Error()))
return
}
}
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
if processOverrides == nil {
processOverrides = &types.KnowledgeProcessOverrides{EnableMultimodel: enableMultimodel}
} else {
processOverrides.EnableMultimodel = enableMultimodel
}
}
// 获取分类ID(如果提供),用于知识分类管理
tagID := c.PostForm("tag_id")
// 过滤特殊值,空字符串或 "__untagged__" 表示未分类
@@ -328,7 +346,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
channel := c.PostForm("channel")
// Create knowledge entry from the file
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID, channel)
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID, channel, processOverrides)
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") {
@@ -389,13 +407,14 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
// Parse URL from request body
var req struct {
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
Channel string `json:"channel"`
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
Channel string `json:"channel"`
ProcessConfig *types.KnowledgeProcessOverrides `json:"process_config"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse URL request", err)
@@ -423,7 +442,9 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
)
// Create knowledge entry from the URL
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID, req.Channel)
knowledge, err := h.kgService.CreateKnowledgeFromURL(
ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID, req.Channel, req.ProcessConfig,
)
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") {
+1 -1
View File
@@ -2430,7 +2430,7 @@ func (s *Service) processFileToKnowledgeBase(ctx context.Context, msg *IncomingM
fh := newInMemoryFileHeader(fileName, content)
// Create knowledge entry via the knowledge service
knowledge, err := s.knowledgeService.CreateKnowledgeFromFile(kbCtx, kbID, fh, nil, nil, "", "", imPlatformToChannel(channel.Platform))
knowledge, err := s.knowledgeService.CreateKnowledgeFromFile(kbCtx, kbID, fh, nil, nil, "", "", imPlatformToChannel(channel.Platform), nil)
if err != nil {
errMsg := err.Error()
// Check for duplicate file
+2
View File
@@ -22,6 +22,7 @@ type KnowledgeService interface {
customFileName string,
tagID string,
channel string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error)
// CreateKnowledgeFromURL creates knowledge from a URL.
// When fileName or fileType is provided (or the URL path has a known file extension),
@@ -37,6 +38,7 @@ type KnowledgeService interface {
title string,
tagID string,
channel string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error)
// CreateKnowledgeFromPassage creates knowledge from text passages.
// channel identifies the ingestion channel; empty defaults to "web".
+62 -5
View File
@@ -204,11 +204,12 @@ type ManualKnowledgeMetadata struct {
// ManualKnowledgePayload represents the payload for manual knowledge operations.
type ManualKnowledgePayload struct {
Title string `json:"title"`
Content string `json:"content"`
Status string `json:"status"`
TagID string `json:"tag_id"`
Channel string `json:"channel"`
Title string `json:"title"`
Content string `json:"content"`
Status string `json:"status"`
TagID string `json:"tag_id"`
Channel string `json:"channel"`
ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
}
// KnowledgeSearchScope defines a (tenant_id, knowledge_base_id) scope for knowledge search (e.g. own KBs + shared KBs).
@@ -342,6 +343,62 @@ func (p ManualKnowledgePayload) IsDraft() bool {
return p.Status == "" || p.Status == ManualKnowledgeStatusDraft
}
const metadataKeyProcessOverrides = "process_overrides"
// ProcessOverrides parses process config overrides from knowledge metadata.
func (k *Knowledge) ProcessOverrides() (*KnowledgeProcessOverrides, error) {
if k == nil || len(k.Metadata) == 0 {
return nil, nil
}
metadataMap, err := k.Metadata.Map()
if err != nil {
return nil, err
}
raw, ok := metadataMap[metadataKeyProcessOverrides]
if !ok || raw == nil {
return nil, nil
}
bytes, err := json.Marshal(raw)
if err != nil {
return nil, err
}
var overrides KnowledgeProcessOverrides
if err := json.Unmarshal(bytes, &overrides); err != nil {
return nil, err
}
return &overrides, nil
}
// SetProcessOverrides merges process config overrides into knowledge metadata.
func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {
if k == nil {
return nil
}
metadataMap, err := k.Metadata.Map()
if err != nil {
return err
}
if o == nil {
delete(metadataMap, metadataKeyProcessOverrides)
} else {
bytes, err := json.Marshal(o)
if err != nil {
return err
}
var value interface{}
if err := json.Unmarshal(bytes, &value); err != nil {
return err
}
metadataMap[metadataKeyProcessOverrides] = value
}
bytes, err := json.Marshal(metadataMap)
if err != nil {
return err
}
k.Metadata = JSON(bytes)
return nil
}
// KnowledgeCheckParams defines parameters used to check if knowledge already exists.
type KnowledgeCheckParams struct {
// File parameters
+24
View File
@@ -0,0 +1,24 @@
package types
// KnowledgeProcessOverrides stores per-upload parse config overrides in knowledge metadata.
type KnowledgeProcessOverrides struct {
ParserEngineRules []ParserEngineRule `json:"parser_engine_rules,omitempty"`
ChunkingConfig *ChunkingConfig `json:"chunking_config,omitempty"`
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
VLMConfig *VLMConfig `json:"vlm_config,omitempty"`
ASRConfig *ASRConfig `json:"asr_config,omitempty"`
QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"`
GraphEnabled *bool `json:"graph_enabled,omitempty"`
ExtractConfig *ExtractConfig `json:"extract_config,omitempty"`
}
// EffectiveProcessConfig is the merged view used by the parse pipeline.
type EffectiveProcessConfig struct {
ChunkingConfig ChunkingConfig
EnableMultimodel bool
VLMConfig VLMConfig
ASRConfig ASRConfig
QuestionGenerationConfig QuestionGenerationConfig
GraphEnabled bool
ExtractConfig ExtractConfig
}
+48
View File
@@ -0,0 +1,48 @@
package types
import (
"testing"
"github.com/stretchr/testify/require"
)
func boolPtr(v bool) *bool {
return &v
}
func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) {
k := &Knowledge{}
overrides := &KnowledgeProcessOverrides{
EnableMultimodel: boolPtr(true),
ChunkingConfig: &ChunkingConfig{ChunkSize: 1024},
}
require.NoError(t, k.SetProcessOverrides(overrides))
got, err := k.ProcessOverrides()
require.NoError(t, err)
require.NotNil(t, got)
require.True(t, *got.EnableMultimodel)
require.Equal(t, 1024, got.ChunkingConfig.ChunkSize)
}
func TestSetProcessOverridesPreservesOtherMetadata(t *testing.T) {
k := &Knowledge{}
manualMeta := NewManualKnowledgeMetadata("# hello", ManualKnowledgeStatusDraft, 1)
require.NoError(t, k.SetManualMetadata(manualMeta))
overrides := &KnowledgeProcessOverrides{
EnableMultimodel: boolPtr(false),
}
require.NoError(t, k.SetProcessOverrides(overrides))
gotManual, err := k.ManualMetadata()
require.NoError(t, err)
require.NotNil(t, gotManual)
require.Equal(t, "# hello", gotManual.Content)
require.Equal(t, ManualKnowledgeFormatMarkdown, gotManual.Format)
require.Equal(t, ManualKnowledgeStatusDraft, gotManual.Status)
gotOverrides, err := k.ProcessOverrides()
require.NoError(t, err)
require.NotNil(t, gotOverrides)
require.False(t, *gotOverrides.EnableMultimodel)
}