mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-21 05:43:43 +08:00
feat(client): enhance knowledge file upload with process configuration support
- Updated the `CreateKnowledgeFromFile` function to accept an optional `processConfig` parameter for per-upload parse configuration overrides. - Introduced the `KnowledgeProcessOverrides` type to define various parsing settings, including parser engine rules and chunking configurations. - Modified the API client and related components to handle the new process configuration, improving flexibility in knowledge file uploads. - Updated documentation and API specifications to reflect the new `process_config` parameter in the upload process. - Added a new `UploadConfirmHost` component to facilitate user confirmation for uploads, integrating with the updated upload logic. - Enhanced user experience by providing detailed configuration options during the upload process.
This commit is contained in:
+1
-1
@@ -71,7 +71,7 @@ func ExampleUsage() {
|
||||
"source": "local",
|
||||
"type": "document",
|
||||
}
|
||||
knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), createdKB.ID, filePath, metadata, nil, "", "")
|
||||
knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), createdKB.ID, filePath, metadata, nil, "", "", nil)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to upload knowledge file: %v\n", err)
|
||||
} else {
|
||||
|
||||
@@ -83,6 +83,19 @@ var ErrDuplicateFile = errors.New("file already exists")
|
||||
// ErrDuplicateURL is returned when attempting to create a knowledge entry with a URL that already exists
|
||||
var ErrDuplicateURL = errors.New("URL already exists")
|
||||
|
||||
// KnowledgeProcessOverrides stores per-upload parse config overrides sent as process_config.
|
||||
// When nil, the server uses the knowledge base defaults only.
|
||||
type KnowledgeProcessOverrides struct {
|
||||
ParserEngineRules []ParserEngineRule `json:"parser_engine_rules,omitempty"`
|
||||
ChunkingConfig *ChunkingConfig `json:"chunking_config,omitempty"`
|
||||
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
|
||||
VLMConfig *VLMConfig `json:"vlm_config,omitempty"`
|
||||
ASRConfig *ASRConfig `json:"asr_config,omitempty"`
|
||||
QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"`
|
||||
GraphEnabled *bool `json:"graph_enabled,omitempty"`
|
||||
ExtractConfig *ExtractConfig `json:"extract_config,omitempty"`
|
||||
}
|
||||
|
||||
// CreateKnowledgeFromFile creates a knowledge entry from a local file path
|
||||
// Parameters:
|
||||
// - knowledgeBaseID: The ID of the knowledge base
|
||||
@@ -91,8 +104,10 @@ var ErrDuplicateURL = errors.New("URL already exists")
|
||||
// - enableMultimodel: Optional flag to enable multimodal processing
|
||||
// - customFileName: Optional custom file name (useful for folder uploads with path)
|
||||
// - channel: Optional ingestion channel (e.g. "web", "api", "wechat"); empty defaults to "web"
|
||||
// - processConfig: Optional parse config overrides (serialized as process_config form field)
|
||||
func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
|
||||
knowledgeBaseID string, filePath string, metadata map[string]string, enableMultimodel *bool, customFileName string, channel string,
|
||||
processConfig *KnowledgeProcessOverrides,
|
||||
) (*Knowledge, error) {
|
||||
// Open the local file
|
||||
file, err := os.Open(filePath)
|
||||
@@ -159,6 +174,16 @@ func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
if processConfig != nil {
|
||||
processConfigBytes, err := json.Marshal(processConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize process_config: %w", err)
|
||||
}
|
||||
if err := writer.WriteField("process_config", string(processConfigBytes)); err != nil {
|
||||
return nil, fmt.Errorf("failed to write process_config field: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Close the multipart writer
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
@@ -209,6 +234,8 @@ type CreateKnowledgeFromURLRequest struct {
|
||||
TagID string `json:"tag_id,omitempty"`
|
||||
// Channel identifies the ingestion channel (e.g. "web", "browser_extension", "api")
|
||||
Channel string `json:"channel,omitempty"`
|
||||
// ProcessConfig is optional per-upload parse config overrides (KnowledgeProcessOverrides).
|
||||
ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
|
||||
}
|
||||
|
||||
// CreateKnowledgeFromURL creates a knowledge entry from a URL.
|
||||
|
||||
@@ -108,6 +108,25 @@ type GraphRelation struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// ParserEngineRule maps a set of file types to a specific parser engine.
|
||||
type ParserEngineRule struct {
|
||||
FileTypes []string `json:"file_types"`
|
||||
Engine string `json:"engine"`
|
||||
}
|
||||
|
||||
// QuestionGenerationConfig controls LLM-generated questions per chunk during parsing.
|
||||
type QuestionGenerationConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
QuestionCount int `json:"question_count"`
|
||||
}
|
||||
|
||||
// ASRConfig represents automatic speech recognition settings for audio files.
|
||||
type ASRConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
ModelID string `json:"model_id"`
|
||||
Language string `json:"language,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON keeps backward compatibility for legacy responses that still
|
||||
// use `cos_config` instead of `storage_config`.
|
||||
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
|
||||
|
||||
@@ -52,9 +52,12 @@
|
||||
| `fileName` | string | 否 | 自定义文件名,用于"文件夹上传"时保留相对路径(如 `docs/intro.md`) |
|
||||
| `metadata` | string | 否 | JSON 字符串,会被反序列化为 `map[string]string` |
|
||||
| `enable_multimodel` | string | 否 | `"true"` / `"false"`,是否启用图文多模态解析 |
|
||||
| `process_config` | string | 否 | JSON 字符串,批次解析配置覆盖(`KnowledgeProcessOverrides`);写入 `knowledge.metadata.process_overrides`。未传时行为与现网一致 |
|
||||
| `tag_id` | string | 否 | 标签 ID;传 `__untagged__` 或空字符串表示未分类 |
|
||||
| `channel` | string | 否 | 来源渠道标识(写入 `channel` 字段,默认 `web`) |
|
||||
|
||||
`process_config` 可选字段包括:`parser_engine_rules`、`chunking_config`、`enable_multimodel`、`vlm_config`、`asr_config`、`question_generation_config`、`graph_enabled`、`extract_config`。若同时传 `enable_multimodel` 与 `process_config.enable_multimodel`,以 `process_config` 为准。
|
||||
|
||||
**请求**:
|
||||
|
||||
```curl
|
||||
|
||||
@@ -4775,6 +4775,12 @@ const docTemplate = `{
|
||||
"description": "启用多模态处理",
|
||||
"name": "enable_multimodel",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "处理配置JSON(KnowledgeProcessOverrides)",
|
||||
"name": "process_config",
|
||||
"in": "formData"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
||||
@@ -4768,6 +4768,12 @@
|
||||
"description": "启用多模态处理",
|
||||
"name": "enable_multimodel",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "处理配置JSON(KnowledgeProcessOverrides)",
|
||||
"name": "process_config",
|
||||
"in": "formData"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
||||
@@ -7746,6 +7746,10 @@ paths:
|
||||
in: formData
|
||||
name: enable_multimodel
|
||||
type: boolean
|
||||
- description: 处理配置JSON(KnowledgeProcessOverrides)
|
||||
in: formData
|
||||
name: process_config
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
|
||||
@@ -4,6 +4,7 @@ import { useRouter } from 'vue-router'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { MessagePlugin, NotifyPlugin } from 'tdesign-vue-next'
|
||||
import ManualKnowledgeEditor from '@/components/manual-knowledge-editor.vue'
|
||||
import UploadConfirmHost from '@/components/UploadConfirmHost.vue'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import { useSettingsStore } from '@/stores/settings'
|
||||
import { getCurrentUser, userInfoFromApi } from '@/api/auth'
|
||||
@@ -257,6 +258,7 @@ onUnmounted(() => {
|
||||
<div id="app">
|
||||
<RouterView />
|
||||
<ManualKnowledgeEditor />
|
||||
<UploadConfirmHost />
|
||||
</div>
|
||||
</t-config-provider>
|
||||
</template>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { get, post, put, del, postUpload, getDown } from "../../utils/request";
|
||||
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess';
|
||||
|
||||
// 知识库管理 API(列表、创建、获取、更新、删除、复制)
|
||||
export function listKnowledgeBases(params?: {
|
||||
@@ -160,23 +161,51 @@ export function togglePinKnowledgeBase(id: string) {
|
||||
|
||||
// 知识文件 API(基于具体知识库)
|
||||
// data.tag_id: 可选,指定知识所属的分类ID
|
||||
export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string; [key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) {
|
||||
export function uploadKnowledgeFile(
|
||||
kbId: string,
|
||||
data: {
|
||||
file: File
|
||||
tag_id?: string
|
||||
fileName?: string
|
||||
process_config?: KnowledgeProcessOverrides | string
|
||||
[key: string]: any
|
||||
} = { file: new File([], '') },
|
||||
onProgress?: (progressEvent: any) => void,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
Object.keys(data).forEach(key => {
|
||||
if (data[key] !== undefined) formData.append(key, data[key]);
|
||||
const value = data[key];
|
||||
if (value === undefined) return;
|
||||
if (key === 'process_config' && value && typeof value !== 'string') {
|
||||
formData.append(key, JSON.stringify(value));
|
||||
} else {
|
||||
formData.append(key, value);
|
||||
}
|
||||
});
|
||||
return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress);
|
||||
}
|
||||
|
||||
// 从URL创建知识
|
||||
// data.tag_id: 可选,指定知识所属的分类ID
|
||||
export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) {
|
||||
export function createKnowledgeFromURL(
|
||||
kbId: string,
|
||||
data: { url: string; enable_multimodel?: boolean; tag_id?: string; process_config?: KnowledgeProcessOverrides },
|
||||
) {
|
||||
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data);
|
||||
}
|
||||
|
||||
// 手工创建知识
|
||||
// data.tag_id: 可选,指定知识所属的分类ID
|
||||
export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) {
|
||||
export function createManualKnowledge(
|
||||
kbId: string,
|
||||
data: {
|
||||
title: string
|
||||
content: string
|
||||
status: string
|
||||
tag_id?: string
|
||||
process_config?: KnowledgeProcessOverrides
|
||||
},
|
||||
) {
|
||||
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data);
|
||||
}
|
||||
|
||||
@@ -215,7 +244,10 @@ export function getKnowledgeDetails(id: string, options?: { agent_id?: string })
|
||||
return get(qs ? `/api/v1/knowledge/${id}?${qs}` : `/api/v1/knowledge/${id}`);
|
||||
}
|
||||
|
||||
export function updateManualKnowledge(id: string, data: { title: string; content: string; status: string }) {
|
||||
export function updateManualKnowledge(
|
||||
id: string,
|
||||
data: { title: string; content: string; status: string; process_config?: KnowledgeProcessOverrides },
|
||||
) {
|
||||
return put(`/api/v1/knowledge/manual/${id}`, data);
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
:placeholder="placeholderText"
|
||||
:disabled="disabled"
|
||||
:loading="loading"
|
||||
:status="status"
|
||||
filterable
|
||||
style="width: 100%;"
|
||||
>
|
||||
@@ -51,13 +52,15 @@ interface Props {
|
||||
selectedModelId?: string
|
||||
disabled?: boolean
|
||||
placeholder?: string
|
||||
status?: 'default' | 'success' | 'warning' | 'error'
|
||||
// 可选:外部传入的所有模型列表,如果提供则不调用API
|
||||
allModels?: ModelConfig[]
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
disabled: false,
|
||||
placeholder: ''
|
||||
placeholder: '',
|
||||
status: 'default',
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
<script setup lang="ts">
|
||||
import UploadConfirmDialog from '@/views/knowledge/components/UploadConfirmDialog.vue'
|
||||
import { useUploadConfirmStore } from '@/stores/uploadConfirm'
|
||||
|
||||
const uploadConfirmStore = useUploadConfirmStore()
|
||||
|
||||
const handleConfirm = (payload: Parameters<typeof uploadConfirmStore.resolveConfirm>[0]) => {
|
||||
uploadConfirmStore.resolveConfirm(payload)
|
||||
}
|
||||
|
||||
const handleCancel = () => {
|
||||
uploadConfirmStore.rejectConfirm()
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<UploadConfirmDialog
|
||||
v-model:visible="uploadConfirmStore.visible"
|
||||
:mode="uploadConfirmStore.mode"
|
||||
:kb-info="uploadConfirmStore.kbInfo"
|
||||
:files="uploadConfirmStore.files"
|
||||
:urls="uploadConfirmStore.urls"
|
||||
:manual-preview="uploadConfirmStore.manual"
|
||||
:accept-file-types="uploadConfirmStore.acceptFileTypes"
|
||||
:supported-file-types="uploadConfirmStore.supportedFileTypes"
|
||||
@confirm="handleConfirm"
|
||||
@cancel="handleCancel"
|
||||
/>
|
||||
</template>
|
||||
@@ -3,7 +3,15 @@ import { ref, reactive, computed, watch, nextTick, onBeforeUnmount } from 'vue'
|
||||
import { marked } from 'marked'
|
||||
import { MessagePlugin } from 'tdesign-vue-next'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
import { listKnowledgeBases, getKnowledgeDetails, createManualKnowledge, updateManualKnowledge } from '@/api/knowledge-base'
|
||||
import {
|
||||
listKnowledgeBases,
|
||||
getKnowledgeDetails,
|
||||
getKnowledgeBaseById,
|
||||
createManualKnowledge,
|
||||
updateManualKnowledge,
|
||||
} from '@/api/knowledge-base'
|
||||
import { useUploadConfirmStore } from '@/stores/uploadConfirm'
|
||||
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
|
||||
import { sanitizeHTML, safeMarkdownToHTML } from '@/utils/security'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
@@ -24,6 +32,7 @@ interface KnowledgeDetailResponse {
|
||||
type ManualStatus = 'draft' | 'publish'
|
||||
|
||||
const uiStore = useUIStore()
|
||||
const uploadConfirmStore = useUploadConfirmStore()
|
||||
const { t } = useI18n()
|
||||
|
||||
const visible = computed({
|
||||
@@ -576,11 +585,53 @@ const handleSave = async (targetStatus: ManualStatus) => {
|
||||
saving.value = true
|
||||
savingAction.value = targetStatus
|
||||
try {
|
||||
const payload: { title: string; content: string; status: string; tag_id?: string } = {
|
||||
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
|
||||
const payload: {
|
||||
title: string
|
||||
content: string
|
||||
status: string
|
||||
tag_id?: string
|
||||
process_config?: KnowledgeProcessOverrides
|
||||
} = {
|
||||
title: form.title.trim(),
|
||||
content: form.content,
|
||||
status: targetStatus,
|
||||
}
|
||||
if (tagIdToUpload) {
|
||||
payload.tag_id = tagIdToUpload
|
||||
}
|
||||
|
||||
if (targetStatus === 'publish') {
|
||||
let kbInfo: any
|
||||
try {
|
||||
const kbRes: any = await getKnowledgeBaseById(form.kbId)
|
||||
kbInfo = kbRes?.data
|
||||
} catch {
|
||||
MessagePlugin.error(t('manualEditor.error.fetchDetailFailed'))
|
||||
return
|
||||
}
|
||||
if (!kbInfo) {
|
||||
MessagePlugin.error(t('manualEditor.error.fetchDetailFailed'))
|
||||
return
|
||||
}
|
||||
try {
|
||||
const confirmResult = await uploadConfirmStore.open({
|
||||
mode: 'manual',
|
||||
kbInfo,
|
||||
manual: {
|
||||
kbId: form.kbId,
|
||||
knowledgeId: knowledgeId.value || undefined,
|
||||
title: payload.title,
|
||||
content: payload.content,
|
||||
tagId: tagIdToUpload,
|
||||
},
|
||||
})
|
||||
payload.process_config = confirmResult.processConfig
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
let response: any
|
||||
let knowledgeID = knowledgeId.value
|
||||
let kbId = form.kbId
|
||||
@@ -588,11 +639,6 @@ const handleSave = async (targetStatus: ManualStatus) => {
|
||||
if (mode.value === 'edit' && knowledgeId.value) {
|
||||
response = await updateManualKnowledge(knowledgeId.value, payload)
|
||||
} else {
|
||||
// 创建新知识时,从 store 获取当前选中的分类ID
|
||||
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
|
||||
if (tagIdToUpload) {
|
||||
payload.tag_id = tagIdToUpload
|
||||
}
|
||||
response = await createManualKnowledge(form.kbId, payload)
|
||||
knowledgeID = response?.data?.id || knowledgeID
|
||||
kbId = form.kbId
|
||||
|
||||
@@ -636,6 +636,74 @@ export default {
|
||||
chunkLoadFailed: 'Failed to load chunks',
|
||||
},
|
||||
|
||||
uploadConfirm: {
|
||||
title: 'Confirm Upload',
|
||||
fileList: 'Files to upload',
|
||||
parseConfig: 'Parse settings',
|
||||
configNav: 'Parse settings navigation',
|
||||
tabOverview: 'Overview',
|
||||
overviewTitle: 'Parse settings for this batch',
|
||||
overviewDesc: 'Click any row to edit',
|
||||
backToOverview: 'Back to overview',
|
||||
summaryChunkOverlapShort: 'Overlap {overlap}',
|
||||
summaryParentChildShort: 'Parent-child',
|
||||
summaryParserMode: 'Mode',
|
||||
summaryParserBuiltin: 'Built-in engine (default)',
|
||||
summaryChunkSize: 'Chunk size',
|
||||
summaryChunkOverlap: 'Overlap',
|
||||
summaryStrategy: 'Strategy',
|
||||
summaryStrategyDefault: 'Default',
|
||||
summaryParentChild: 'Parent-child',
|
||||
summaryParentChildOn: 'On (parent {parent} / child {child})',
|
||||
summaryParentChildOff: 'Off',
|
||||
summaryStatus: 'Status',
|
||||
summaryModel: 'Model',
|
||||
summaryQuestionCount: 'Questions per chunk',
|
||||
summaryQuestionCountValue: '{count}',
|
||||
summaryGraphTags: 'Relation types',
|
||||
summaryGraphTagsValue: '{count}',
|
||||
navChunkingSummary: 'Chunk {size}',
|
||||
statusOn: 'On',
|
||||
statusOff: 'Off',
|
||||
notSet: 'Not set',
|
||||
confirm: 'Upload and parse',
|
||||
cancel: 'Cancel',
|
||||
uploading: 'Uploading {current}/{total}',
|
||||
tabParser: 'Parser',
|
||||
tabChunking: 'Chunking',
|
||||
tabMultimodal: 'Multimodal',
|
||||
tabAsr: 'Audio',
|
||||
tabQuestion: 'Question generation',
|
||||
tabGraph: 'Knowledge graph',
|
||||
noFiles: 'Keep at least one file to upload',
|
||||
noItems: 'Add at least one file or URL',
|
||||
urlItemLabel: 'URL',
|
||||
urlAdded: 'URL added',
|
||||
urlDuplicate: 'This URL is already in the list',
|
||||
addUrl: 'Add',
|
||||
vlmModelRequired: 'This batch includes images. Enable multimodal and select a VLM model.',
|
||||
multimodalRequiredForImages: 'Off (batch includes images)',
|
||||
asrModelRequired: 'This batch includes audio. Enable ASR and select a speech recognition model.',
|
||||
asrRequiredForAudio: 'Off (batch includes audio)',
|
||||
vlmModelSelectRequired: 'Multimodal is enabled. Please select a VLM model.',
|
||||
asrModelSelectRequired: 'Speech recognition is enabled. Please select an ASR model.',
|
||||
continueAdd: 'Add more',
|
||||
addMoreFiles: 'Add more files',
|
||||
addMoreFolder: 'Add more from folder',
|
||||
filesAdded: 'Added {count} file(s)',
|
||||
filesAllDuplicate: 'Selected files are already in the list',
|
||||
titleUrl: 'Confirm URL import',
|
||||
titleManual: 'Confirm online publish',
|
||||
overviewDescUrl: 'The URL will be fetched and parsed after you confirm the settings',
|
||||
overviewDescManual: 'The document will be published and indexed after you confirm the settings',
|
||||
confirmUrl: 'Import and parse',
|
||||
confirmManual: 'Publish and parse',
|
||||
urlSource: 'Import source',
|
||||
manualSource: 'Document to publish',
|
||||
editUrl: 'Edit URL',
|
||||
manualCharCount: '{count} characters',
|
||||
},
|
||||
|
||||
knowledgeStages: {
|
||||
title: 'Processing pipeline',
|
||||
root: 'Knowledge processing',
|
||||
|
||||
@@ -642,6 +642,73 @@ export default {
|
||||
deleteSuccess: "지식이 성공적으로 삭제되었습니다!",
|
||||
chunkLoadFailed: "청크 로드 실패",
|
||||
},
|
||||
uploadConfirm: {
|
||||
title: "문서 업로드 확인",
|
||||
fileList: "업로드할 파일",
|
||||
parseConfig: "파싱 설정",
|
||||
configNav: "파싱 설정 탐색",
|
||||
tabOverview: "구성 개요",
|
||||
overviewTitle: "이번 배치 파싱 설정",
|
||||
overviewDesc: "항목을 클릭하여 설정을 수정하세요",
|
||||
backToOverview: "구성 개요로 돌아가기",
|
||||
summaryChunkOverlapShort: "겹침 {overlap}",
|
||||
summaryParentChildShort: "부모-자식 청킹",
|
||||
summaryParserMode: "모드",
|
||||
summaryParserBuiltin: "내장 엔진(기본)",
|
||||
summaryChunkSize: "청크 크기",
|
||||
summaryChunkOverlap: "겹침",
|
||||
summaryStrategy: "전략",
|
||||
summaryStrategyDefault: "기본",
|
||||
summaryParentChild: "부모-자식 청킹",
|
||||
summaryParentChildOn: "켜짐(부모 {parent} / 자식 {child})",
|
||||
summaryParentChildOff: "꺼짐",
|
||||
summaryStatus: "상태",
|
||||
summaryModel: "모델",
|
||||
summaryQuestionCount: "청크당 질문 수",
|
||||
summaryQuestionCountValue: "{count}개",
|
||||
summaryGraphTags: "관계 유형",
|
||||
summaryGraphTagsValue: "{count}개",
|
||||
navChunkingSummary: "청크 {size}",
|
||||
statusOn: "켜짐",
|
||||
statusOff: "꺼짐",
|
||||
notSet: "미설정",
|
||||
confirm: "업로드 및 파싱 확인",
|
||||
cancel: "취소",
|
||||
uploading: "업로드 중 {current}/{total}",
|
||||
tabParser: "파서",
|
||||
tabChunking: "청킹",
|
||||
tabMultimodal: "멀티모달",
|
||||
tabAsr: "오디오",
|
||||
tabQuestion: "질문 생성",
|
||||
tabGraph: "지식 그래프",
|
||||
noFiles: "업로드할 파일을 하나 이상 남겨 두세요",
|
||||
noItems: "파일 또는 URL을 하나 이상 추가하세요",
|
||||
urlItemLabel: "URL",
|
||||
urlAdded: "URL이 추가되었습니다",
|
||||
urlDuplicate: "이 URL은 이미 목록에 있습니다",
|
||||
addUrl: "추가",
|
||||
vlmModelRequired: "이 배치에 이미지가 포함되어 있습니다. 멀티모달을 활성화하고 VLM 모델을 선택하세요",
|
||||
multimodalRequiredForImages: "꺼짐(이 배치에 이미지 포함)",
|
||||
asrModelRequired: "이 배치에 오디오가 포함되어 있습니다. ASR을 활성화하고 음성 인식 모델을 선택하세요",
|
||||
asrRequiredForAudio: "꺼짐(이 배치에 오디오 포함)",
|
||||
vlmModelSelectRequired: "멀티모달이 활성화되었습니다. VLM 모델을 선택하세요",
|
||||
asrModelSelectRequired: "음성 인식이 활성화되었습니다. ASR 모델을 선택하세요",
|
||||
continueAdd: "계속 추가",
|
||||
addMoreFiles: "파일 더 추가",
|
||||
addMoreFolder: "폴더에서 더 추가",
|
||||
filesAdded: "{count}개 파일이 추가되었습니다",
|
||||
filesAllDuplicate: "선택한 파일이 이미 목록에 있습니다",
|
||||
titleUrl: "URL 가져오기 확인",
|
||||
titleManual: "온라인 편집 게시 확인",
|
||||
overviewDescUrl: "설정 확인 후 URL을 가져와 파싱합니다",
|
||||
overviewDescManual: "설정 확인 후 문서를 게시하고 인덱싱합니다",
|
||||
confirmUrl: "가져오기 및 파싱",
|
||||
confirmManual: "게시 및 파싱",
|
||||
urlSource: "가져오기 소스",
|
||||
manualSource: "게시할 문서",
|
||||
editUrl: "URL 수정",
|
||||
manualCharCount: "{count}자",
|
||||
},
|
||||
knowledgeStages: {
|
||||
title: "처리 파이프라인",
|
||||
root: "지식 처리",
|
||||
|
||||
@@ -606,6 +606,73 @@ export default {
|
||||
cancelParseSubmitted: 'Разбор остановлен',
|
||||
cancelParseFailed: 'Не удалось остановить, попробуйте позже'
|
||||
},
|
||||
uploadConfirm: {
|
||||
title: 'Подтверждение загрузки',
|
||||
fileList: 'Файлы для загрузки',
|
||||
parseConfig: 'Настройки разбора',
|
||||
configNav: 'Навигация по настройкам разбора',
|
||||
tabOverview: 'Обзор',
|
||||
overviewTitle: 'Настройки разбора для этой партии',
|
||||
overviewDesc: 'Нажмите на строку, чтобы изменить',
|
||||
backToOverview: 'Вернуться к обзору',
|
||||
summaryChunkOverlapShort: 'Перекрытие {overlap}',
|
||||
summaryParentChildShort: 'Родительско-дочернее',
|
||||
summaryParserMode: 'Режим',
|
||||
summaryParserBuiltin: 'Встроенный движок (по умолчанию)',
|
||||
summaryChunkSize: 'Размер чанка',
|
||||
summaryChunkOverlap: 'Перекрытие',
|
||||
summaryStrategy: 'Стратегия',
|
||||
summaryStrategyDefault: 'По умолчанию',
|
||||
summaryParentChild: 'Родительско-дочернее',
|
||||
summaryParentChildOn: 'Вкл. (родитель {parent} / дочерний {child})',
|
||||
summaryParentChildOff: 'Выкл.',
|
||||
summaryStatus: 'Статус',
|
||||
summaryModel: 'Модель',
|
||||
summaryQuestionCount: 'Вопросов на чанк',
|
||||
summaryQuestionCountValue: '{count}',
|
||||
summaryGraphTags: 'Типы связей',
|
||||
summaryGraphTagsValue: '{count}',
|
||||
navChunkingSummary: 'Чанк {size}',
|
||||
statusOn: 'Вкл.',
|
||||
statusOff: 'Выкл.',
|
||||
notSet: 'Не задано',
|
||||
confirm: 'Загрузить и обработать',
|
||||
cancel: 'Отмена',
|
||||
uploading: 'Загрузка {current}/{total}',
|
||||
tabParser: 'Парсер',
|
||||
tabChunking: 'Разбиение',
|
||||
tabMultimodal: 'Мультимодальность',
|
||||
tabAsr: 'Аудио',
|
||||
tabQuestion: 'Генерация вопросов',
|
||||
tabGraph: 'Граф знаний',
|
||||
noFiles: 'Оставьте хотя бы один файл для загрузки',
|
||||
noItems: 'Добавьте хотя бы один файл или URL',
|
||||
urlItemLabel: 'URL',
|
||||
urlAdded: 'URL добавлен',
|
||||
urlDuplicate: 'Этот URL уже в списке',
|
||||
addUrl: 'Добавить',
|
||||
vlmModelRequired: 'В партии есть изображения. Включите мультимодальность и выберите модель VLM.',
|
||||
multimodalRequiredForImages: 'Выкл. (в партии есть изображения)',
|
||||
asrModelRequired: 'В партии есть аудио. Включите ASR и выберите модель распознавания речи.',
|
||||
asrRequiredForAudio: 'Выкл. (в партии есть аудио)',
|
||||
vlmModelSelectRequired: 'Мультимодальность включена. Выберите модель VLM.',
|
||||
asrModelSelectRequired: 'Распознавание речи включено. Выберите модель ASR.',
|
||||
continueAdd: 'Добавить ещё',
|
||||
addMoreFiles: 'Добавить файлы',
|
||||
addMoreFolder: 'Добавить из папки',
|
||||
filesAdded: 'Добавлено файлов: {count}',
|
||||
filesAllDuplicate: 'Выбранные файлы уже в списке',
|
||||
titleUrl: 'Подтверждение импорта URL',
|
||||
titleManual: 'Подтверждение публикации',
|
||||
overviewDescUrl: 'После подтверждения настроек URL будет загружен и обработан',
|
||||
overviewDescManual: 'После подтверждения настроек документ будет опубликован и проиндексирован',
|
||||
confirmUrl: 'Импортировать и обработать',
|
||||
confirmManual: 'Опубликовать и обработать',
|
||||
urlSource: 'Источник импорта',
|
||||
manualSource: 'Документ для публикации',
|
||||
editUrl: 'Изменить URL',
|
||||
manualCharCount: '{count} символов',
|
||||
},
|
||||
knowledgeStages: {
|
||||
title: 'Конвейер обработки',
|
||||
root: 'Обработка знаний',
|
||||
|
||||
@@ -639,6 +639,73 @@ export default {
|
||||
deleteSuccess: "知识删除成功!",
|
||||
chunkLoadFailed: "分块加载失败",
|
||||
},
|
||||
uploadConfirm: {
|
||||
title: "上传文档确认",
|
||||
fileList: "待上传文件",
|
||||
parseConfig: "解析配置",
|
||||
configNav: "解析配置导航",
|
||||
tabOverview: "配置总览",
|
||||
overviewTitle: "本批次解析配置",
|
||||
overviewDesc: "点击任一项可修改配置",
|
||||
backToOverview: "返回配置总览",
|
||||
summaryChunkOverlapShort: "重叠 {overlap}",
|
||||
summaryParentChildShort: "父子分块",
|
||||
summaryParserMode: "模式",
|
||||
summaryParserBuiltin: "内置引擎(默认)",
|
||||
summaryChunkSize: "分块大小",
|
||||
summaryChunkOverlap: "重叠",
|
||||
summaryStrategy: "策略",
|
||||
summaryStrategyDefault: "默认",
|
||||
summaryParentChild: "父子分块",
|
||||
summaryParentChildOn: "已开启(父 {parent} / 子 {child})",
|
||||
summaryParentChildOff: "未开启",
|
||||
summaryStatus: "状态",
|
||||
summaryModel: "模型",
|
||||
summaryQuestionCount: "每块问题数",
|
||||
summaryQuestionCountValue: "{count} 个",
|
||||
summaryGraphTags: "关系类型",
|
||||
summaryGraphTagsValue: "{count} 个",
|
||||
navChunkingSummary: "分块 {size}",
|
||||
statusOn: "已开启",
|
||||
statusOff: "未开启",
|
||||
notSet: "未设置",
|
||||
confirm: "确认上传并解析",
|
||||
cancel: "取消",
|
||||
uploading: "正在上传 {current}/{total}",
|
||||
tabParser: "解析引擎",
|
||||
tabChunking: "分块",
|
||||
tabMultimodal: "多模态",
|
||||
tabAsr: "音频",
|
||||
tabQuestion: "问题生成",
|
||||
tabGraph: "图谱",
|
||||
noFiles: "请至少保留一个待上传文件",
|
||||
noItems: "请至少添加一个文件或 URL",
|
||||
urlItemLabel: "URL",
|
||||
urlAdded: "已添加 URL",
|
||||
urlDuplicate: "该 URL 已在列表中",
|
||||
addUrl: "添加",
|
||||
vlmModelRequired: "本批次包含图片,请启用多模态并选择 VLM 模型",
|
||||
multimodalRequiredForImages: "未开启(本批次含图片)",
|
||||
asrModelRequired: "本批次包含音频,请启用 ASR 并选择语音识别模型",
|
||||
asrRequiredForAudio: "未开启(本批次含音频)",
|
||||
vlmModelSelectRequired: "已启用多模态,请选择 VLM 模型",
|
||||
asrModelSelectRequired: "已启用语音识别,请选择 ASR 模型",
|
||||
continueAdd: "继续添加",
|
||||
addMoreFiles: "继续添加文件",
|
||||
addMoreFolder: "继续添加文件夹",
|
||||
filesAdded: "已添加 {count} 个文件",
|
||||
filesAllDuplicate: "所选文件已在列表中",
|
||||
titleUrl: "URL 导入确认",
|
||||
titleManual: "在线编辑发布确认",
|
||||
overviewDescUrl: "确认解析配置后将抓取并解析该 URL",
|
||||
overviewDescManual: "确认解析配置后将发布并索引该文档",
|
||||
confirmUrl: "确认导入并解析",
|
||||
confirmManual: "确认发布并解析",
|
||||
urlSource: "导入来源",
|
||||
manualSource: "待发布文档",
|
||||
editUrl: "修改 URL",
|
||||
manualCharCount: "{count} 个字符",
|
||||
},
|
||||
knowledgeStages: {
|
||||
title: "处理流水线",
|
||||
root: "知识处理",
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
|
||||
|
||||
export type UploadConfirmMode = 'file' | 'url' | 'manual'
|
||||
|
||||
export interface UploadConfirmManualSource {
|
||||
kbId: string
|
||||
knowledgeId?: string
|
||||
title: string
|
||||
content: string
|
||||
tagId?: string
|
||||
}
|
||||
|
||||
export interface UploadConfirmResult {
|
||||
processConfig: KnowledgeProcessOverrides
|
||||
mode: UploadConfirmMode
|
||||
files?: File[]
|
||||
urls?: string[]
|
||||
manual?: UploadConfirmManualSource
|
||||
}
|
||||
|
||||
export interface OpenUploadConfirmOptions {
|
||||
mode: UploadConfirmMode
|
||||
kbInfo: any
|
||||
files?: File[]
|
||||
urls?: string[]
|
||||
manual?: UploadConfirmManualSource
|
||||
acceptFileTypes?: string
|
||||
supportedFileTypes?: string[]
|
||||
}
|
||||
|
||||
export const useUploadConfirmStore = defineStore('uploadConfirm', {
|
||||
state: () => ({
|
||||
visible: false,
|
||||
mode: 'file' as UploadConfirmMode,
|
||||
kbInfo: null as any,
|
||||
files: [] as File[],
|
||||
urls: [] as string[],
|
||||
manual: null as UploadConfirmManualSource | null,
|
||||
acceptFileTypes: '',
|
||||
supportedFileTypes: [] as string[],
|
||||
pendingResolve: null as ((value: UploadConfirmResult) => void) | null,
|
||||
pendingReject: null as (() => void) | null,
|
||||
}),
|
||||
|
||||
actions: {
|
||||
open(options: OpenUploadConfirmOptions): Promise<UploadConfirmResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.visible = true
|
||||
this.mode = options.mode
|
||||
this.kbInfo = options.kbInfo
|
||||
this.files = options.files ? [...options.files] : []
|
||||
this.urls = options.urls ? [...options.urls] : []
|
||||
this.manual = options.manual || null
|
||||
this.acceptFileTypes = options.acceptFileTypes || ''
|
||||
this.supportedFileTypes = options.supportedFileTypes ? [...options.supportedFileTypes] : []
|
||||
this.pendingResolve = resolve
|
||||
this.pendingReject = reject
|
||||
})
|
||||
},
|
||||
|
||||
resolveConfirm(payload: UploadConfirmResult) {
|
||||
this.pendingResolve?.(payload)
|
||||
this.reset()
|
||||
},
|
||||
|
||||
rejectConfirm() {
|
||||
this.pendingReject?.()
|
||||
this.reset()
|
||||
},
|
||||
|
||||
reset() {
|
||||
this.visible = false
|
||||
this.mode = 'file'
|
||||
this.kbInfo = null
|
||||
this.files = []
|
||||
this.urls = []
|
||||
this.manual = null
|
||||
this.acceptFileTypes = ''
|
||||
this.supportedFileTypes = []
|
||||
this.pendingResolve = null
|
||||
this.pendingReject = null
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,66 @@
|
||||
/** Matches backend types.KnowledgeProcessOverrides (snake_case JSON). */
|
||||
|
||||
export interface ParserEngineRule {
|
||||
file_types: string[]
|
||||
engine: string
|
||||
}
|
||||
|
||||
export interface ChunkingConfigOverride {
|
||||
chunk_size?: number
|
||||
chunk_overlap?: number
|
||||
separators?: string[]
|
||||
enable_multimodal?: boolean
|
||||
parser_engine_rules?: ParserEngineRule[]
|
||||
enable_parent_child?: boolean
|
||||
parent_chunk_size?: number
|
||||
child_chunk_size?: number
|
||||
strategy?: string
|
||||
token_limit?: number
|
||||
languages?: string[]
|
||||
}
|
||||
|
||||
export interface VLMConfigOverride {
|
||||
enabled?: boolean
|
||||
model_id?: string
|
||||
}
|
||||
|
||||
export interface ASRConfigOverride {
|
||||
enabled?: boolean
|
||||
model_id?: string
|
||||
language?: string
|
||||
}
|
||||
|
||||
export interface QuestionGenerationConfigOverride {
|
||||
enabled?: boolean
|
||||
question_count?: number
|
||||
}
|
||||
|
||||
export interface GraphNodeOverride {
|
||||
name: string
|
||||
attributes?: string[]
|
||||
}
|
||||
|
||||
export interface GraphRelationOverride {
|
||||
node1: string
|
||||
node2: string
|
||||
type: string
|
||||
}
|
||||
|
||||
export interface ExtractConfigOverride {
|
||||
enabled?: boolean
|
||||
text?: string
|
||||
tags?: string[]
|
||||
nodes?: GraphNodeOverride[]
|
||||
relations?: GraphRelationOverride[]
|
||||
}
|
||||
|
||||
export interface KnowledgeProcessOverrides {
|
||||
parser_engine_rules?: ParserEngineRule[]
|
||||
chunking_config?: ChunkingConfigOverride
|
||||
enable_multimodel?: boolean
|
||||
vlm_config?: VLMConfigOverride
|
||||
asr_config?: ASRConfigOverride
|
||||
question_generation_config?: QuestionGenerationConfigOverride
|
||||
graph_enabled?: boolean
|
||||
extract_config?: ExtractConfigOverride
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted, onUnmounted, watch, reactive, computed, nextTick, h, type ComponentPublicInstance } from "vue";
|
||||
import { MessagePlugin, Icon as TIcon } from "tdesign-vue-next";
|
||||
import { ref, onMounted, onUnmounted, watch, reactive, computed, nextTick, type ComponentPublicInstance } from "vue";
|
||||
import { MessagePlugin } from "tdesign-vue-next";
|
||||
import DocContent from "@/components/doc-content.vue";
|
||||
import KnowledgeProcessingTimeline from "@/components/knowledge-processing-timeline.vue";
|
||||
import useKnowledgeBase from '@/hooks/useKnowledgeBase';
|
||||
@@ -42,6 +42,9 @@ import { knowledgeSpansPayloadHasTrace } from '@/utils/knowledgeTrace';
|
||||
import FAQEntryManager from './components/FAQEntryManager.vue';
|
||||
import DocumentListView from './components/DocumentListView.vue';
|
||||
import DocumentBatchBar from './components/DocumentBatchBar.vue';
|
||||
import KbUploadSourceDropdown from './components/KbUploadSourceDropdown.vue';
|
||||
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess';
|
||||
import { useUploadConfirmStore, type UploadConfirmResult } from '@/stores/uploadConfirm';
|
||||
import WikiBrowser from './wiki/WikiBrowser.vue';
|
||||
import { getWikiStats } from '@/api/wiki';
|
||||
import {
|
||||
@@ -51,15 +54,14 @@ import {
|
||||
} from './wikiStatusRefresh';
|
||||
import { listMoveTargets, moveKnowledge, getKnowledgeMoveProgress } from '@/api/knowledge-base';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { formatStringDate, kbFileTypeVerification } from '@/utils';
|
||||
import { formatStringDate } from '@/utils';
|
||||
import { formatFileSize } from '@/utils/files';
|
||||
import type { ParserEngineInfo } from '@/api/system';
|
||||
const route = useRoute();
|
||||
const { t } = useI18n();
|
||||
const kbId = computed(() => (route.params as any).kbId as string || '');
|
||||
const kbInfo = ref<any>(null);
|
||||
const uploadInputRef = ref<HTMLInputElement | null>(null);
|
||||
const folderUploadInputRef = ref<HTMLInputElement | null>(null);
|
||||
const uploadSourceRef = ref<InstanceType<typeof KbUploadSourceDropdown> | null>(null);
|
||||
const uploading = ref(false);
|
||||
const kbLoading = ref(false);
|
||||
const docListLoading = ref(true);
|
||||
@@ -984,7 +986,9 @@ const handleOpenURLImportDialog = (event: CustomEvent) => {
|
||||
const eventKbId = event.detail.kbId;
|
||||
console.log('接收到URL导入对话框打开事件,知识库ID:', eventKbId, '当前知识库ID:', kbId.value);
|
||||
if (eventKbId && eventKbId === kbId.value && !isFAQ.value) {
|
||||
urlDialogVisible.value = true;
|
||||
if (ensureDocumentKbReady()) {
|
||||
uploadSourceRef.value?.openUrlDialog();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1352,32 +1356,6 @@ const documentTitle = computed(() => {
|
||||
return t('knowledgeEditor.document.title');
|
||||
});
|
||||
|
||||
// 文档操作下拉菜单选项
|
||||
const documentActionOptions = computed(() => [
|
||||
{ content: t('upload.uploadDocument'), value: 'upload', prefixIcon: () => h(TIcon, { name: 'upload', size: '16px' }) },
|
||||
{ content: t('upload.uploadFolder'), value: 'uploadFolder', prefixIcon: () => h(TIcon, { name: 'folder-add', size: '16px' }) },
|
||||
{ content: t('knowledgeBase.importURL'), value: 'importURL', prefixIcon: () => h(TIcon, { name: 'link', size: '16px' }) },
|
||||
{ content: t('upload.onlineEdit'), value: 'manualCreate', prefixIcon: () => h(TIcon, { name: 'edit', size: '16px' }) },
|
||||
]);
|
||||
|
||||
// 处理文档操作下拉菜单选择
|
||||
const handleDocumentActionSelect = (data: { value: string }) => {
|
||||
switch (data.value) {
|
||||
case 'upload':
|
||||
handleDocumentUploadClick();
|
||||
break;
|
||||
case 'uploadFolder':
|
||||
handleFolderUploadClick();
|
||||
break;
|
||||
case 'importURL':
|
||||
handleURLImportClick();
|
||||
break;
|
||||
case 'manualCreate':
|
||||
handleManualCreate();
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
const ensureDocumentKbReady = () => {
|
||||
if (isFAQ.value) {
|
||||
MessagePlugin.warning(t('knowledgeBase.operationNotSupportedForType'));
|
||||
@@ -1406,338 +1384,147 @@ const ensureDocumentKbReady = () => {
|
||||
};
|
||||
|
||||
|
||||
const handleDocumentUploadClick = () => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
uploadInputRef.value?.click();
|
||||
const IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
|
||||
const AUDIO_EXTENSIONS = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
|
||||
|
||||
const uploadConfirmStore = useUploadConfirmStore();
|
||||
|
||||
const getFolderUploadFileName = (file: File) => {
|
||||
const relativePath = (file as any).webkitRelativePath;
|
||||
if (!relativePath) return undefined;
|
||||
const pathParts = relativePath.split('/');
|
||||
if (pathParts.length <= 2) return undefined;
|
||||
const subPath = pathParts.slice(1, -1).join('/');
|
||||
return `${subPath}/${file.name}`;
|
||||
};
|
||||
|
||||
const handleFolderUploadClick = () => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
folderUploadInputRef.value?.click();
|
||||
};
|
||||
|
||||
const resetUploadInput = () => {
|
||||
if (uploadInputRef.value) {
|
||||
uploadInputRef.value.value = '';
|
||||
}
|
||||
};
|
||||
|
||||
const handleDocumentUpload = async (event: Event) => {
|
||||
const input = event.target as HTMLInputElement;
|
||||
const files = input?.files;
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
const targetKbId = kbId.value;
|
||||
if (!targetKbId) {
|
||||
MessagePlugin.error(t('error.missingKbId'));
|
||||
resetUploadInput();
|
||||
return;
|
||||
}
|
||||
|
||||
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
|
||||
const asrEnabled = kbInfo.value?.asr_config?.enabled || false;
|
||||
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
|
||||
const validFiles: File[] = [];
|
||||
let skippedCount = 0;
|
||||
let imageFilteredCount = 0;
|
||||
let videoFilteredCount = 0;
|
||||
let audioFilteredCount = 0;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
|
||||
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
|
||||
const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'];
|
||||
const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
|
||||
|
||||
if (videoTypes.includes(fileExt)) {
|
||||
videoFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!vlmEnabled) {
|
||||
if (imageTypes.includes(fileExt)) {
|
||||
imageFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!asrEnabled && audioTypes.includes(fileExt)) {
|
||||
audioFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!kbFileTypeVerification(file, files.length > 1, dynamicTypes)) {
|
||||
validFiles.push(file);
|
||||
const showUploadResultMessages = (
|
||||
successCount: number,
|
||||
failCount: number,
|
||||
totalCount: number,
|
||||
mode: 'document' | 'folder',
|
||||
) => {
|
||||
if (mode === 'folder') {
|
||||
if (failCount === 0) {
|
||||
MessagePlugin.success(t('knowledgeBase.uploadAllSuccess', { count: successCount }));
|
||||
} else if (successCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.uploadPartialSuccess', { success: successCount, fail: failCount }));
|
||||
} else {
|
||||
skippedCount++;
|
||||
MessagePlugin.error(t('knowledgeBase.uploadAllFailed'));
|
||||
}
|
||||
}
|
||||
|
||||
if (imageFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
|
||||
}
|
||||
if (videoFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
|
||||
}
|
||||
if (audioFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
|
||||
}
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.allFilesSkippedNoEngine'));
|
||||
}
|
||||
resetUploadInput();
|
||||
return;
|
||||
}
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.filesSkippedNoEngine', { count: skippedCount }));
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
const totalCount = validFiles.length;
|
||||
|
||||
// 获取当前选中的分类ID(如果不是"未分类"则传递)
|
||||
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
|
||||
|
||||
for (const file of validFiles) {
|
||||
try {
|
||||
const responseData: any = await uploadKnowledgeFile(targetKbId, { file, tag_id: tagIdToUpload });
|
||||
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
|
||||
if (isSuccess) {
|
||||
successCount++;
|
||||
} else {
|
||||
failCount++;
|
||||
let errorMessage = t('knowledgeBase.uploadFailed');
|
||||
if (responseData?.error?.message) {
|
||||
errorMessage = responseData.error.message;
|
||||
} else if (responseData?.message) {
|
||||
errorMessage = responseData.message;
|
||||
}
|
||||
if (responseData?.code === 'duplicate_file' || responseData?.error?.code === 'duplicate_file') {
|
||||
errorMessage = t('knowledgeBase.fileExists');
|
||||
}
|
||||
if (totalCount === 1) {
|
||||
MessagePlugin.error(errorMessage);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
failCount++;
|
||||
let errorMessage = error?.error?.message || error?.message || t('knowledgeBase.uploadFailed');
|
||||
if (error?.code === 'duplicate_file') {
|
||||
errorMessage = t('knowledgeBase.fileExists');
|
||||
}
|
||||
if (totalCount === 1) {
|
||||
MessagePlugin.error(errorMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 显示上传结果
|
||||
if (successCount > 0) {
|
||||
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
|
||||
detail: { kbId: targetKbId }
|
||||
}));
|
||||
}
|
||||
|
||||
if (totalCount === 1) {
|
||||
if (successCount === 1) {
|
||||
MessagePlugin.success(t('knowledgeBase.uploadSuccess'));
|
||||
}
|
||||
} else {
|
||||
if (failCount === 0) {
|
||||
MessagePlugin.success(t('knowledgeBase.allUploadSuccess', { count: successCount }));
|
||||
} else if (successCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.partialUploadSuccess', { success: successCount, fail: failCount }));
|
||||
} else {
|
||||
MessagePlugin.error(t('knowledgeBase.allUploadFailed', { count: failCount }));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
resetUploadInput();
|
||||
if (failCount === 0) {
|
||||
MessagePlugin.success(t('knowledgeBase.allUploadSuccess', { count: successCount }));
|
||||
} else if (successCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.partialUploadSuccess', { success: successCount, fail: failCount }));
|
||||
} else {
|
||||
MessagePlugin.error(t('knowledgeBase.allUploadFailed', { count: failCount }));
|
||||
}
|
||||
};
|
||||
|
||||
// 处理文件夹上传
|
||||
const handleFolderUpload = async (event: Event) => {
|
||||
const input = event.target as HTMLInputElement;
|
||||
const files = input?.files;
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
const executeUploadBatch = async (
|
||||
files: File[],
|
||||
options: { processConfig?: KnowledgeProcessOverrides } = {},
|
||||
) => {
|
||||
const targetKbId = kbId.value;
|
||||
if (!targetKbId) {
|
||||
MessagePlugin.error(t('error.missingKbId'));
|
||||
if (input) input.value = '';
|
||||
return;
|
||||
if (!targetKbId || files.length === 0) {
|
||||
return { successCount: 0, failCount: files.length };
|
||||
}
|
||||
|
||||
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
|
||||
const asrEnabled = kbInfo.value?.asr_config?.enabled || false;
|
||||
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
|
||||
|
||||
const validFiles: File[] = [];
|
||||
let hiddenFileCount = 0;
|
||||
let imageFilteredCount = 0;
|
||||
let videoFilteredCount = 0;
|
||||
let audioFilteredCount = 0;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const relativePath = (file as any).webkitRelativePath || file.name;
|
||||
|
||||
const pathParts = relativePath.split('/');
|
||||
const hasHiddenComponent = pathParts.some((part: string) => part.startsWith('.'));
|
||||
if (hasHiddenComponent) {
|
||||
hiddenFileCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
|
||||
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
|
||||
const videoTypes = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv'];
|
||||
const audioTypes = ['mp3', 'wav', 'm4a', 'flac', 'ogg'];
|
||||
|
||||
if (videoTypes.includes(fileExt)) {
|
||||
videoFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!vlmEnabled) {
|
||||
if (imageTypes.includes(fileExt)) {
|
||||
imageFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!asrEnabled && audioTypes.includes(fileExt)) {
|
||||
audioFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!kbFileTypeVerification(file, true, dynamicTypes)) {
|
||||
validFiles.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
if (imageFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.imagesFilteredNoVLM', { count: imageFilteredCount }));
|
||||
}
|
||||
if (videoFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }));
|
||||
}
|
||||
if (audioFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.audiosFilteredNoASR', { count: audioFilteredCount }));
|
||||
}
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.noValidFilesInFolder', { total: files.length }));
|
||||
if (input) input.value = '';
|
||||
return;
|
||||
}
|
||||
MessagePlugin.info(t('knowledgeBase.uploadingFolder', { total: validFiles.length }));
|
||||
|
||||
// 批量上传
|
||||
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
|
||||
|
||||
for (const file of validFiles) {
|
||||
const relativePath = (file as any).webkitRelativePath;
|
||||
let fileName = file.name;
|
||||
if (relativePath) {
|
||||
const pathParts = relativePath.split('/');
|
||||
if (pathParts.length > 2) {
|
||||
const subPath = pathParts.slice(1, -1).join('/');
|
||||
fileName = `${subPath}/${file.name}`;
|
||||
}
|
||||
}
|
||||
const totalCount = files.length;
|
||||
const hasFolderPaths = files.some((file) => {
|
||||
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
|
||||
return !!relativePath && relativePath.split('/').length > 2;
|
||||
});
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
await uploadKnowledgeFile(targetKbId, { file, fileName, tag_id: tagIdToUpload });
|
||||
successCount++;
|
||||
const uploadData: {
|
||||
file: File
|
||||
tag_id?: string
|
||||
fileName?: string
|
||||
process_config?: KnowledgeProcessOverrides
|
||||
} = { file, tag_id: tagIdToUpload };
|
||||
|
||||
const fileName = getFolderUploadFileName(file);
|
||||
if (fileName) uploadData.fileName = fileName;
|
||||
if (options.processConfig) {
|
||||
uploadData.process_config = options.processConfig;
|
||||
}
|
||||
|
||||
const responseData: any = await uploadKnowledgeFile(targetKbId, uploadData);
|
||||
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
|
||||
if (isSuccess) {
|
||||
successCount++;
|
||||
} else {
|
||||
failCount++;
|
||||
if (totalCount === 1) {
|
||||
let errorMessage = t('knowledgeBase.uploadFailed');
|
||||
if (responseData?.error?.message) {
|
||||
errorMessage = responseData.error.message;
|
||||
} else if (responseData?.message) {
|
||||
errorMessage = responseData.message;
|
||||
}
|
||||
if (responseData?.code === 'duplicate_file' || responseData?.error?.code === 'duplicate_file') {
|
||||
errorMessage = t('knowledgeBase.fileExists');
|
||||
}
|
||||
MessagePlugin.error(errorMessage);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
failCount++;
|
||||
if (totalCount === 1) {
|
||||
let errorMessage = error?.error?.message || error?.message || t('knowledgeBase.uploadFailed');
|
||||
if (error?.code === 'duplicate_file') {
|
||||
errorMessage = t('knowledgeBase.fileExists');
|
||||
}
|
||||
MessagePlugin.error(errorMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (successCount > 0) {
|
||||
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
|
||||
detail: { kbId: targetKbId }
|
||||
detail: { kbId: targetKbId },
|
||||
}));
|
||||
}
|
||||
|
||||
if (failCount === 0) {
|
||||
MessagePlugin.success(t('knowledgeBase.uploadAllSuccess', { count: successCount }));
|
||||
} else if (successCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.uploadPartialSuccess', { success: successCount, fail: failCount }));
|
||||
} else {
|
||||
MessagePlugin.error(t('knowledgeBase.uploadAllFailed'));
|
||||
}
|
||||
|
||||
if (input) input.value = '';
|
||||
showUploadResultMessages(successCount, failCount, totalCount, hasFolderPaths ? 'folder' : 'document');
|
||||
return { successCount, failCount };
|
||||
};
|
||||
|
||||
const handleManualCreate = () => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
uiStore.openManualEditor({
|
||||
mode: 'create',
|
||||
kbId: kbId.value,
|
||||
status: 'draft',
|
||||
onSuccess: manualEditorSuccess,
|
||||
});
|
||||
};
|
||||
|
||||
// URL 导入相关
|
||||
const urlDialogVisible = ref(false);
|
||||
const urlInputValue = ref('');
|
||||
const urlImporting = ref(false);
|
||||
|
||||
const handleURLImportClick = () => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
urlInputValue.value = '';
|
||||
urlDialogVisible.value = true;
|
||||
};
|
||||
|
||||
const handleURLImportCancel = () => {
|
||||
urlDialogVisible.value = false;
|
||||
urlInputValue.value = '';
|
||||
};
|
||||
|
||||
const handleURLImportConfirm = async () => {
|
||||
const url = urlInputValue.value.trim();
|
||||
if (!url) {
|
||||
MessagePlugin.warning(t('knowledgeBase.urlRequired'));
|
||||
return;
|
||||
}
|
||||
|
||||
// 简单的URL格式验证
|
||||
try {
|
||||
new URL(url);
|
||||
} catch (error) {
|
||||
MessagePlugin.warning(t('knowledgeBase.invalidURL'));
|
||||
return;
|
||||
}
|
||||
|
||||
const executeUrlImport = async (url: string, processConfig?: KnowledgeProcessOverrides) => {
|
||||
const targetKbId = kbId.value;
|
||||
if (!targetKbId) {
|
||||
MessagePlugin.error(t('error.missingKbId'));
|
||||
return;
|
||||
}
|
||||
|
||||
urlImporting.value = true;
|
||||
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
|
||||
try {
|
||||
// 获取当前选中的分类ID
|
||||
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
|
||||
const responseData: any = await createKnowledgeFromURL(targetKbId, { url, tag_id: tagIdToUpload });
|
||||
const responseData: any = await createKnowledgeFromURL(targetKbId, {
|
||||
url,
|
||||
tag_id: tagIdToUpload,
|
||||
process_config: processConfig,
|
||||
});
|
||||
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
|
||||
detail: { kbId: targetKbId }
|
||||
detail: { kbId: targetKbId },
|
||||
}));
|
||||
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
|
||||
if (isSuccess) {
|
||||
MessagePlugin.success(t('knowledgeBase.urlImportSuccess'));
|
||||
urlDialogVisible.value = false;
|
||||
urlInputValue.value = '';
|
||||
} else {
|
||||
let errorMessage = t('knowledgeBase.urlImportFailed');
|
||||
if (responseData?.error?.message) {
|
||||
@@ -1756,11 +1543,73 @@ const handleURLImportConfirm = async () => {
|
||||
errorMessage = t('knowledgeBase.urlExists');
|
||||
}
|
||||
MessagePlugin.error(errorMessage);
|
||||
} finally {
|
||||
urlImporting.value = false;
|
||||
}
|
||||
};
|
||||
|
||||
const handleUploadConfirmResult = async (result: UploadConfirmResult) => {
|
||||
if (result.mode === 'manual') {
|
||||
return;
|
||||
}
|
||||
|
||||
const files = result.files || [];
|
||||
const urls = result.urls || [];
|
||||
const processConfig = result.processConfig;
|
||||
|
||||
if (files.length > 0) {
|
||||
const hasFolderPaths = files.some((file) => {
|
||||
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
|
||||
return !!relativePath && relativePath.split('/').length > 2;
|
||||
});
|
||||
if (hasFolderPaths) {
|
||||
MessagePlugin.info(t('knowledgeBase.uploadingFolder', { total: files.length }));
|
||||
}
|
||||
await executeUploadBatch(files, { processConfig });
|
||||
}
|
||||
|
||||
for (const url of urls) {
|
||||
await executeUrlImport(url, processConfig);
|
||||
}
|
||||
};
|
||||
|
||||
const openUploadConfirmDialog = async (files: File[], urls: string[] = []) => {
|
||||
if (!kbInfo.value) return;
|
||||
if (files.length === 0 && urls.length === 0) return;
|
||||
try {
|
||||
const result = await uploadConfirmStore.open({
|
||||
mode: 'file',
|
||||
kbInfo: kbInfo.value,
|
||||
files,
|
||||
urls,
|
||||
acceptFileTypes: acceptFileTypes.value,
|
||||
supportedFileTypes: [...supportedFileTypes.value],
|
||||
});
|
||||
await handleUploadConfirmResult(result);
|
||||
} catch {
|
||||
// cancelled
|
||||
}
|
||||
};
|
||||
|
||||
const handleUploadSourceFiles = (files: File[]) => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
if (files.length === 0) return;
|
||||
openUploadConfirmDialog(files);
|
||||
};
|
||||
|
||||
const handleUploadSourceUrl = (url: string) => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
openUploadConfirmDialog([], [url]);
|
||||
};
|
||||
|
||||
const handleManualCreate = () => {
|
||||
if (!ensureDocumentKbReady()) return;
|
||||
uiStore.openManualEditor({
|
||||
mode: 'create',
|
||||
kbId: kbId.value,
|
||||
status: 'draft',
|
||||
onSuccess: manualEditorSuccess,
|
||||
});
|
||||
};
|
||||
|
||||
const handleOpenKBSettings = () => {
|
||||
if (!kbId.value) {
|
||||
MessagePlugin.warning(t('knowledgeEditor.messages.missingId'));
|
||||
@@ -2158,11 +2007,6 @@ async function createNewSession(value: string): Promise<void> {
|
||||
</div>
|
||||
|
||||
<template v-if="activeKbTab === 'documents' || !isWiki">
|
||||
<input ref="uploadInputRef" type="file" class="document-upload-input"
|
||||
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt,.mp3,.wav,.m4a,.flac,.ogg'"
|
||||
multiple @change="handleDocumentUpload" />
|
||||
<input ref="folderUploadInputRef" type="file" class="document-upload-input" webkitdirectory
|
||||
@change="handleFolderUpload" />
|
||||
<div class="knowledge-main">
|
||||
<aside class="tag-sidebar">
|
||||
<div class="sidebar-header">
|
||||
@@ -2319,15 +2163,20 @@ async function createNewSession(value: string): Promise<void> {
|
||||
</t-tooltip>
|
||||
</div>
|
||||
<div v-if="canEdit" class="doc-filter-actions">
|
||||
<t-tooltip :content="$t('knowledgeBase.addDocument')" placement="top">
|
||||
<t-dropdown :options="documentActionOptions" trigger="click" placement="bottom-right"
|
||||
@click="handleDocumentActionSelect">
|
||||
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
|
||||
data-guide="kb-detail-add-doc">
|
||||
<template #icon><t-icon name="file-add" size="16px" /></template>
|
||||
</t-button>
|
||||
</t-dropdown>
|
||||
</t-tooltip>
|
||||
<KbUploadSourceDropdown
|
||||
ref="uploadSourceRef"
|
||||
:accept-file-types="acceptFileTypes"
|
||||
:supported-file-types="[...supportedFileTypes]"
|
||||
include-manual
|
||||
trigger-icon="file-add"
|
||||
trigger-class="content-bar-icon-btn"
|
||||
data-guide="kb-detail-add-doc"
|
||||
:tooltip="t('knowledgeBase.addDocument')"
|
||||
placement="bottom-right"
|
||||
@files="handleUploadSourceFiles"
|
||||
@url="handleUploadSourceUrl"
|
||||
@manual="handleManualCreate"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="doc-scroll-container" :class="{ 'is-empty': !cardList.length && !docListLoading }" ref="knowledgeScroll"
|
||||
@@ -2687,21 +2536,6 @@ async function createNewSession(value: string): Promise<void> {
|
||||
@delete="confirmBatchDelete" />
|
||||
</div>
|
||||
</div>
|
||||
<!-- URL 导入对话框 -->
|
||||
<t-dialog v-model:visible="urlDialogVisible" :header="$t('knowledgeBase.importURLTitle')" :confirm-btn="{
|
||||
content: $t('common.confirm'),
|
||||
theme: 'primary',
|
||||
loading: urlImporting,
|
||||
}" :cancel-btn="{ content: $t('common.cancel') }" @confirm="handleURLImportConfirm"
|
||||
@cancel="handleURLImportCancel" width="500px">
|
||||
<div class="url-import-form">
|
||||
<div class="url-input-label">{{ $t('knowledgeBase.urlLabel') }}</div>
|
||||
<t-input v-model="urlInputValue" :placeholder="$t('knowledgeBase.urlPlaceholder')" clearable autofocus
|
||||
@enter="handleURLImportConfirm" />
|
||||
<div class="url-input-tip">{{ $t('knowledgeBase.urlTip') }}</div>
|
||||
</div>
|
||||
</t-dialog>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
<template>
|
||||
<div class="kb-upload-source-dropdown">
|
||||
<input
|
||||
ref="fileInputRef"
|
||||
type="file"
|
||||
class="hidden-file-input"
|
||||
multiple
|
||||
:accept="acceptFileTypes || undefined"
|
||||
@change="(e) => handleFilesChange(e, false)"
|
||||
/>
|
||||
<input
|
||||
ref="folderInputRef"
|
||||
type="file"
|
||||
class="hidden-file-input"
|
||||
webkitdirectory
|
||||
multiple
|
||||
@change="(e) => handleFilesChange(e, true)"
|
||||
/>
|
||||
|
||||
<t-tooltip :content="tooltipText" placement="top">
|
||||
<t-dropdown
|
||||
:options="dropdownOptions"
|
||||
trigger="click"
|
||||
:placement="placement"
|
||||
@click="handleActionSelect"
|
||||
>
|
||||
<t-button
|
||||
variant="text"
|
||||
theme="default"
|
||||
:class="['kb-upload-source-trigger', triggerClass]"
|
||||
:data-guide="dataGuide || undefined"
|
||||
size="small"
|
||||
>
|
||||
<template #icon><t-icon :name="triggerIcon" size="16px" /></template>
|
||||
</t-button>
|
||||
</t-dropdown>
|
||||
</t-tooltip>
|
||||
|
||||
<t-dialog
|
||||
v-model:visible="urlDialogVisible"
|
||||
:header="t('knowledgeBase.importURLTitle')"
|
||||
:confirm-btn="{ content: t('common.confirm'), theme: 'primary' }"
|
||||
:cancel-btn="{ content: t('common.cancel') }"
|
||||
width="500px"
|
||||
@confirm="handleUrlDialogConfirm"
|
||||
@cancel="handleUrlDialogCancel"
|
||||
>
|
||||
<div class="url-import-form">
|
||||
<div class="url-input-label">{{ t('knowledgeBase.urlLabel') }}</div>
|
||||
<t-input
|
||||
v-model="urlInputValue"
|
||||
:placeholder="t('knowledgeBase.urlPlaceholder')"
|
||||
clearable
|
||||
autofocus
|
||||
@enter="handleUrlDialogConfirm"
|
||||
/>
|
||||
<div class="url-input-tip">{{ t('knowledgeBase.urlTip') }}</div>
|
||||
</div>
|
||||
</t-dialog>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, h, withDefaults } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { MessagePlugin, Icon as TIcon } from 'tdesign-vue-next'
|
||||
import { filterUploadFiles } from '../utils/uploadSources'
|
||||
|
||||
const props = withDefaults(defineProps<{
|
||||
acceptFileTypes?: string
|
||||
supportedFileTypes?: string[]
|
||||
includeManual?: boolean
|
||||
triggerIcon?: string
|
||||
triggerClass?: string
|
||||
dataGuide?: string
|
||||
tooltip?: string
|
||||
placement?: 'top' | 'bottom' | 'bottom-right' | 'bottom-left'
|
||||
}>(), {
|
||||
acceptFileTypes: '',
|
||||
supportedFileTypes: () => [],
|
||||
includeManual: false,
|
||||
triggerIcon: 'file-add',
|
||||
triggerClass: '',
|
||||
dataGuide: '',
|
||||
tooltip: '',
|
||||
placement: 'bottom-right',
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
files: [files: File[]]
|
||||
url: [url: string]
|
||||
manual: []
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
const fileInputRef = ref<HTMLInputElement | null>(null)
|
||||
const folderInputRef = ref<HTMLInputElement | null>(null)
|
||||
const urlDialogVisible = ref(false)
|
||||
const urlInputValue = ref('')
|
||||
|
||||
const tooltipText = computed(() => props.tooltip || t('knowledgeBase.addDocument'))
|
||||
|
||||
const dropdownOptions = computed(() => {
|
||||
const options = [
|
||||
{
|
||||
content: t('upload.uploadDocument'),
|
||||
value: 'upload',
|
||||
prefixIcon: () => h(TIcon, { name: 'upload', size: '16px' }),
|
||||
},
|
||||
{
|
||||
content: t('upload.uploadFolder'),
|
||||
value: 'uploadFolder',
|
||||
prefixIcon: () => h(TIcon, { name: 'folder-add', size: '16px' }),
|
||||
},
|
||||
{
|
||||
content: t('knowledgeBase.importURL'),
|
||||
value: 'importURL',
|
||||
prefixIcon: () => h(TIcon, { name: 'link', size: '16px' }),
|
||||
},
|
||||
]
|
||||
if (props.includeManual) {
|
||||
options.push({
|
||||
content: t('upload.onlineEdit'),
|
||||
value: 'manualCreate',
|
||||
prefixIcon: () => h(TIcon, { name: 'edit', size: '16px' }),
|
||||
})
|
||||
}
|
||||
return options
|
||||
})
|
||||
|
||||
const handleActionSelect = (data: { value: string }) => {
|
||||
switch (data.value) {
|
||||
case 'upload':
|
||||
fileInputRef.value?.click()
|
||||
break
|
||||
case 'uploadFolder':
|
||||
folderInputRef.value?.click()
|
||||
break
|
||||
case 'importURL':
|
||||
urlInputValue.value = ''
|
||||
urlDialogVisible.value = true
|
||||
break
|
||||
case 'manualCreate':
|
||||
emit('manual')
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const notifyFilterResult = (result: ReturnType<typeof filterUploadFiles>, emptyAllSkippedKey: string) => {
|
||||
const { validFiles, skippedCount, videoFilteredCount } = result
|
||||
if (validFiles.length === 0) {
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(t(emptyAllSkippedKey))
|
||||
}
|
||||
return false
|
||||
}
|
||||
if (videoFilteredCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.videosFilteredNoVLM', { count: videoFilteredCount }))
|
||||
}
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(t('knowledgeBase.filesSkippedNoEngine', { count: skippedCount }))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const handleFilesChange = (event: Event, fromFolder: boolean) => {
|
||||
const input = event.target as HTMLInputElement
|
||||
const files = input.files
|
||||
if (!files || files.length === 0) return
|
||||
|
||||
const result = filterUploadFiles(files, {
|
||||
supportedFileTypes: props.supportedFileTypes,
|
||||
fromFolder,
|
||||
multiFile: files.length > 1,
|
||||
})
|
||||
|
||||
if (!notifyFilterResult(result, 'knowledgeBase.allFilesSkippedNoEngine')) {
|
||||
input.value = ''
|
||||
return
|
||||
}
|
||||
|
||||
emit('files', result.validFiles)
|
||||
input.value = ''
|
||||
}
|
||||
|
||||
const handleUrlDialogConfirm = () => {
|
||||
const url = urlInputValue.value.trim()
|
||||
if (!url) {
|
||||
MessagePlugin.warning(t('knowledgeBase.urlRequired'))
|
||||
return
|
||||
}
|
||||
try {
|
||||
new URL(url)
|
||||
} catch {
|
||||
MessagePlugin.warning(t('knowledgeBase.invalidURL'))
|
||||
return
|
||||
}
|
||||
urlDialogVisible.value = false
|
||||
urlInputValue.value = ''
|
||||
emit('url', url)
|
||||
}
|
||||
|
||||
const handleUrlDialogCancel = () => {
|
||||
urlDialogVisible.value = false
|
||||
urlInputValue.value = ''
|
||||
}
|
||||
|
||||
const openUrlDialog = () => {
|
||||
urlInputValue.value = ''
|
||||
urlDialogVisible.value = true
|
||||
}
|
||||
|
||||
defineExpose({ openUrlDialog })
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.hidden-file-input {
|
||||
position: absolute;
|
||||
width: 0;
|
||||
height: 0;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.kb-upload-source-trigger {
|
||||
color: var(--td-text-color-secondary);
|
||||
|
||||
&:hover {
|
||||
color: var(--td-brand-color);
|
||||
}
|
||||
}
|
||||
|
||||
.url-import-form {
|
||||
.url-input-label {
|
||||
margin-bottom: 8px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
|
||||
.url-input-tip {
|
||||
margin-top: 8px;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-placeholder);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,9 @@
|
||||
<template>
|
||||
<div class="graph-settings">
|
||||
<div class="section-header">
|
||||
<div class="graph-settings" :class="{ 'graph-settings--embedded': embedded }">
|
||||
<div v-if="!embedded" class="section-header">
|
||||
<h2>{{ t('graphSettings.title') }}</h2>
|
||||
<p class="section-description">{{ t('graphSettings.description') }}</p>
|
||||
|
||||
<!-- Warning message when graph database is not enabled -->
|
||||
|
||||
<t-alert
|
||||
v-if="!isGraphDatabaseEnabled"
|
||||
theme="warning"
|
||||
@@ -18,6 +17,15 @@
|
||||
</template>
|
||||
</t-alert>
|
||||
</div>
|
||||
<t-alert
|
||||
v-else-if="!isGraphDatabaseEnabled"
|
||||
theme="warning"
|
||||
class="embedded-graph-alert"
|
||||
>
|
||||
<template #message>
|
||||
<div>{{ t('graphSettings.disabledWarning') }}</div>
|
||||
</template>
|
||||
</t-alert>
|
||||
|
||||
<div v-if="isGraphDatabaseEnabled" class="settings-group">
|
||||
<!-- 启用实体关系提取 -->
|
||||
@@ -296,7 +304,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, onMounted, computed } from 'vue'
|
||||
import { ref, watch, onMounted, computed, withDefaults } from 'vue'
|
||||
import { MessagePlugin } from 'tdesign-vue-next'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { extractTextRelations, fabriText, fabriTag, type Node, type Relation } from '@/api/initialization'
|
||||
@@ -323,9 +331,12 @@ interface Props {
|
||||
graphExtract: GraphExtractConfig
|
||||
modelId: string
|
||||
allModels?: any[]
|
||||
embedded?: boolean
|
||||
}
|
||||
|
||||
const props = defineProps<Props>()
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
embedded: false,
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:graphExtract': [value: GraphExtractConfig]
|
||||
@@ -766,4 +777,27 @@ onMounted(async () => {
|
||||
gap: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.graph-settings--embedded {
|
||||
.embedded-graph-alert {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.setting-row:not(.vertical) {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
gap: 8px;
|
||||
padding: 12px 0;
|
||||
}
|
||||
|
||||
.setting-row:not(.vertical) .setting-info {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
padding-right: 0;
|
||||
}
|
||||
|
||||
.setting-row:not(.vertical) .setting-control {
|
||||
align-self: flex-start;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<div class="kb-advanced-settings">
|
||||
<div class="section-header">
|
||||
<div class="kb-advanced-settings" :class="{ 'kb-advanced-settings--embedded': embedded }">
|
||||
<div v-if="!embedded" class="section-header">
|
||||
<h2>{{ $t('knowledgeEditor.advanced.title') }}</h2>
|
||||
<p class="section-description">{{ $t('knowledgeEditor.advanced.description') }}</p>
|
||||
</div>
|
||||
@@ -49,7 +49,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch } from 'vue'
|
||||
import { ref, watch, withDefaults } from 'vue'
|
||||
|
||||
interface QuestionGenerationConfig {
|
||||
enabled: boolean
|
||||
@@ -60,9 +60,12 @@ interface Props {
|
||||
questionGeneration?: QuestionGenerationConfig
|
||||
ragEnabled?: boolean
|
||||
allModels?: any[]
|
||||
embedded?: boolean
|
||||
}
|
||||
|
||||
const props = defineProps<Props>()
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
embedded: false,
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:questionGeneration': [value: QuestionGenerationConfig]
|
||||
@@ -182,4 +185,51 @@ const handleQuestionGenerationChange = () => {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.kb-advanced-settings--embedded {
|
||||
.setting-row {
|
||||
padding: 12px 0;
|
||||
}
|
||||
|
||||
.setting-row:has(.t-switch) {
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
|
||||
.setting-info {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
max-width: none;
|
||||
padding-right: 0;
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex: none;
|
||||
align-self: center;
|
||||
}
|
||||
}
|
||||
|
||||
.subsection .setting-row {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
gap: 8px;
|
||||
|
||||
.setting-info {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
padding-right: 0;
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
align-self: flex-start;
|
||||
}
|
||||
}
|
||||
|
||||
.subsection {
|
||||
margin-top: 0;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: none;
|
||||
}
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<div class="kb-chunking-settings">
|
||||
<div class="section-header">
|
||||
<div class="kb-chunking-settings" :class="{ 'kb-chunking-settings--embedded': embedded }">
|
||||
<div v-if="!embedded" class="section-header">
|
||||
<div class="section-header-text">
|
||||
<h2>{{ $t('knowledgeEditor.chunking.title') }}</h2>
|
||||
<p class="section-description">{{ $t('knowledgeEditor.chunking.description') }}</p>
|
||||
@@ -21,12 +21,12 @@
|
||||
:placeholder="$t('knowledgeEditor.chunking.strategyPlaceholder')"
|
||||
:clearable="true"
|
||||
@change="handleStrategyChange"
|
||||
style="width: 280px;"
|
||||
:style="selectStyle"
|
||||
/>
|
||||
<!-- Test trigger sits right next to the strategy picker so users
|
||||
discover it exactly when they're deciding which strategy to
|
||||
use on their content. -->
|
||||
<KBChunkingDebug :config="debugConfig" />
|
||||
<KBChunkingDebug v-if="!embedded" :config="debugConfig" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -51,9 +51,9 @@
|
||||
:min="100"
|
||||
:max="4000"
|
||||
:step="50"
|
||||
:marks="{ 100: '100', 1000: '1000', 2000: '2000', 4000: '4000' }"
|
||||
:marks="embedded ? undefined : chunkSizeMarks"
|
||||
@change="handleChunkSizeChange"
|
||||
style="width: 200px;"
|
||||
:style="sliderStyle"
|
||||
/>
|
||||
<span class="value-display">{{ localChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
|
||||
</div>
|
||||
@@ -74,9 +74,9 @@
|
||||
:min="0"
|
||||
:max="500"
|
||||
:step="20"
|
||||
:marks="{ 0: '0', 250: '250', 500: '500' }"
|
||||
:marks="embedded ? undefined : chunkOverlapMarks"
|
||||
@change="handleChunkOverlapChange"
|
||||
style="width: 200px;"
|
||||
:style="sliderStyle"
|
||||
/>
|
||||
<span class="value-display">{{ localChunkOverlap }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
|
||||
</div>
|
||||
@@ -84,7 +84,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Separators -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-row setting-row--separators">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.chunking.separatorsLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.chunking.separatorsDescription') }}</p>
|
||||
@@ -98,13 +98,13 @@
|
||||
filterable
|
||||
:placeholder="$t('knowledgeEditor.chunking.separatorsPlaceholder')"
|
||||
@change="handleSeparatorsChange"
|
||||
style="width: 280px;"
|
||||
:style="selectStyle"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Parent-Child Chunking -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-row setting-row--toggle">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.chunking.parentChildLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.chunking.parentChildDescription') }}</p>
|
||||
@@ -130,9 +130,9 @@
|
||||
:min="512"
|
||||
:max="8192"
|
||||
:step="64"
|
||||
:marks="{ 512: '512', 2048: '2048', 4096: '4096', 8192: '8192' }"
|
||||
:marks="embedded ? undefined : parentChunkSizeMarks"
|
||||
@change="handleParentChunkSizeChange"
|
||||
style="width: 200px;"
|
||||
:style="sliderStyle"
|
||||
/>
|
||||
<span class="value-display">{{ localParentChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
|
||||
</div>
|
||||
@@ -152,9 +152,9 @@
|
||||
:min="64"
|
||||
:max="2048"
|
||||
:step="32"
|
||||
:marks="{ 64: '64', 384: '384', 1024: '1024', 2048: '2048' }"
|
||||
:marks="embedded ? undefined : childChunkSizeMarks"
|
||||
@change="handleChildChunkSizeChange"
|
||||
style="width: 200px;"
|
||||
:style="sliderStyle"
|
||||
/>
|
||||
<span class="value-display">{{ localChildChunkSize }} {{ $t('knowledgeEditor.chunking.characters') }}</span>
|
||||
</div>
|
||||
@@ -201,7 +201,7 @@
|
||||
:disabled="advancedDisabled"
|
||||
:placeholder="$t('knowledgeEditor.chunking.languagesPlaceholder')"
|
||||
@change="handleLanguagesChange"
|
||||
style="width: 280px;"
|
||||
:style="selectStyle"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -212,7 +212,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, computed } from 'vue'
|
||||
import { ref, watch, computed, withDefaults } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { ChevronRightIcon } from 'tdesign-icons-vue-next'
|
||||
import KBChunkingDebug from './KBChunkingDebug.vue'
|
||||
@@ -253,9 +253,20 @@ interface ChunkingConfig {
|
||||
|
||||
interface Props {
|
||||
config: ChunkingConfig
|
||||
embedded?: boolean
|
||||
}
|
||||
|
||||
const props = defineProps<Props>()
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
embedded: false,
|
||||
})
|
||||
|
||||
const selectStyle = computed(() => (props.embedded ? { width: '100%' } : { width: '280px' }))
|
||||
const sliderStyle = computed(() => (props.embedded ? { width: '100%' } : { width: '200px' }))
|
||||
|
||||
const chunkSizeMarks = { 100: '100', 1000: '1000', 2000: '2000', 4000: '4000' }
|
||||
const chunkOverlapMarks = { 0: '0', 250: '250', 500: '500' }
|
||||
const parentChunkSizeMarks = { 512: '512', 2048: '2048', 4096: '4096', 8192: '8192' }
|
||||
const childChunkSizeMarks = { 64: '64', 384: '384', 1024: '1024', 2048: '2048' }
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:config': [value: ChunkingConfig]
|
||||
@@ -563,4 +574,102 @@ const emitUpdate = () => {
|
||||
// panel edge and looks detached from the rest of the form.
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.kb-chunking-settings--embedded {
|
||||
.setting-row {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
gap: 10px;
|
||||
padding: 14px 0;
|
||||
}
|
||||
|
||||
.setting-row--toggle {
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
|
||||
.setting-info {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex: none;
|
||||
width: auto;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-row--separators {
|
||||
padding-bottom: 18px;
|
||||
}
|
||||
|
||||
.setting-info {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
padding-right: 0;
|
||||
|
||||
label {
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.desc {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
justify-content: flex-start;
|
||||
align-items: stretch;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.strategy-control {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.strategy-info-panel {
|
||||
margin: -4px 0 10px;
|
||||
padding: 8px 12px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.slider-container {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
gap: 8px;
|
||||
width: 100%;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
.value-display {
|
||||
order: -1;
|
||||
text-align: left;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
:deep(.t-slider) {
|
||||
padding-bottom: 0;
|
||||
}
|
||||
|
||||
:deep(.t-select__wrap) {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
:deep(.t-tag) {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.advanced-toggle {
|
||||
padding-top: 10px;
|
||||
}
|
||||
|
||||
.advanced-section .setting-row {
|
||||
padding: 14px 0;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<div class="kb-parser-settings">
|
||||
<div class="section-header">
|
||||
<div class="kb-parser-settings" :class="{ 'kb-parser-settings--embedded': embedded }">
|
||||
<div v-if="!embedded" class="section-header">
|
||||
<h2>{{ $t('kbSettings.parser.title') }}</h2>
|
||||
<p class="section-description">{{ $t('kbSettings.parser.description') }}</p>
|
||||
</div>
|
||||
@@ -33,7 +33,7 @@
|
||||
<t-select
|
||||
:value="getEngineForGroup(group.extensions) || undefined"
|
||||
@change="(val: string) => handleEngineChange(group.extensions, val)"
|
||||
style="width: 280px;"
|
||||
:style="embedded ? { width: '100%' } : { width: '280px' }"
|
||||
:status="hasAvailableEngine(group.extensions) ? 'default' : 'warning'"
|
||||
:placeholder="$t('kbSettings.parser.noEngine')"
|
||||
>
|
||||
@@ -123,10 +123,16 @@ interface EngineOption {
|
||||
|
||||
interface Props {
|
||||
parserEngineRules?: ParserEngineRule[]
|
||||
/** Compact layout for upload-confirm dialog */
|
||||
embedded?: boolean
|
||||
/** When set, only show file-type groups matching these extensions */
|
||||
relevantExtensions?: string[]
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
parserEngineRules: () => []
|
||||
parserEngineRules: () => [],
|
||||
embedded: false,
|
||||
relevantExtensions: () => [],
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -182,7 +188,11 @@ const fileTypeGroups = computed(() => {
|
||||
})
|
||||
}
|
||||
|
||||
return groups
|
||||
const rel = props.relevantExtensions
|
||||
if (!rel?.length) return groups
|
||||
const relSet = new Set(rel)
|
||||
const filtered = groups.filter(g => g.extensions.some(e => relSet.has(e)))
|
||||
return filtered.length > 0 ? filtered : groups
|
||||
})
|
||||
|
||||
function getEngineOptions(extensions: string[]): EngineOption[] {
|
||||
@@ -468,6 +478,46 @@ watch(() => props.parserEngineRules, (v) => {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.kb-parser-settings--embedded {
|
||||
.setting-row {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
gap: 10px;
|
||||
padding: 12px 0;
|
||||
}
|
||||
|
||||
.setting-info {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
padding-right: 0;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 6px 10px;
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex: none;
|
||||
max-width: none;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.group-label {
|
||||
font-size: 14px;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.ext-tags {
|
||||
margin-top: 0;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.ext-tag {
|
||||
font-size: 11px;
|
||||
padding: 2px 6px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<style lang="less">
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { kbFileTypeVerification } from '@/utils'
|
||||
|
||||
export const UPLOAD_VIDEO_EXTENSIONS = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv']
|
||||
|
||||
export function getUploadFileExt(file: File): string {
|
||||
const dot = file.name.lastIndexOf('.')
|
||||
if (dot < 0) return ''
|
||||
return file.name.substring(dot + 1).toLowerCase()
|
||||
}
|
||||
|
||||
export function getUploadFileKey(file: File): string {
|
||||
const path = (file as File & { webkitRelativePath?: string }).webkitRelativePath || ''
|
||||
return `${path || file.name}\0${file.size}`
|
||||
}
|
||||
|
||||
export interface FilterUploadFilesOptions {
|
||||
supportedFileTypes?: Set<string> | string[]
|
||||
fromFolder?: boolean
|
||||
multiFile?: boolean
|
||||
}
|
||||
|
||||
export interface FilterUploadFilesResult {
|
||||
validFiles: File[]
|
||||
skippedCount: number
|
||||
videoFilteredCount: number
|
||||
hiddenFileCount: number
|
||||
}
|
||||
|
||||
export function filterUploadFiles(
|
||||
files: FileList | File[],
|
||||
options: FilterUploadFilesOptions = {},
|
||||
): FilterUploadFilesResult {
|
||||
const list = Array.from(files)
|
||||
const dynamicTypes = options.supportedFileTypes
|
||||
? options.supportedFileTypes instanceof Set
|
||||
? options.supportedFileTypes
|
||||
: new Set(options.supportedFileTypes)
|
||||
: undefined
|
||||
|
||||
const validFiles: File[] = []
|
||||
let skippedCount = 0
|
||||
let videoFilteredCount = 0
|
||||
let hiddenFileCount = 0
|
||||
const multiFile = options.multiFile ?? list.length > 1
|
||||
|
||||
for (const file of list) {
|
||||
if (options.fromFolder) {
|
||||
const relativePath = (file as File & { webkitRelativePath?: string }).webkitRelativePath || file.name
|
||||
if (relativePath.split('/').some(part => part.startsWith('.'))) {
|
||||
hiddenFileCount++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const fileExt = getUploadFileExt(file)
|
||||
if (UPLOAD_VIDEO_EXTENSIONS.includes(fileExt)) {
|
||||
videoFilteredCount++
|
||||
continue
|
||||
}
|
||||
|
||||
if (kbFileTypeVerification(file, multiFile, dynamicTypes)) {
|
||||
skippedCount++
|
||||
continue
|
||||
}
|
||||
|
||||
validFiles.push(file)
|
||||
}
|
||||
|
||||
return { validFiles, skippedCount, videoFilteredCount, hiddenFileCount }
|
||||
}
|
||||
@@ -839,6 +839,7 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
|
||||
item.FileName, // customFileName — must include extension for file-type validation
|
||||
tagID, // auto-tag from data source
|
||||
channel,
|
||||
nil,
|
||||
)
|
||||
return isUpdate, err
|
||||
}
|
||||
@@ -855,6 +856,7 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
|
||||
item.Title,
|
||||
tagID, // auto-tag from data source
|
||||
channel,
|
||||
nil,
|
||||
)
|
||||
return isUpdate, err
|
||||
}
|
||||
|
||||
@@ -290,10 +290,22 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
if kb.ExtractConfig == nil {
|
||||
logger.Warnf(ctx, "failed to get extract config")
|
||||
graphOut["skipped"] = "no_extract_config"
|
||||
return err
|
||||
|
||||
var processOverrides *types.KnowledgeProcessOverrides
|
||||
knowledgeID := p.KnowledgeID
|
||||
if knowledgeID == "" {
|
||||
knowledgeID = chunk.KnowledgeID
|
||||
}
|
||||
if knowledgeID != "" && s.knowledgeRepo != nil {
|
||||
if k, kerr := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, knowledgeID); kerr == nil && k != nil {
|
||||
processOverrides, _ = k.ProcessOverrides()
|
||||
}
|
||||
}
|
||||
extractCfg := ResolveProcessConfig(kb, processOverrides).ExtractConfig
|
||||
if !extractCfg.Enabled {
|
||||
logger.Warnf(ctx, "extract config not enabled")
|
||||
graphOut["skipped"] = "extract_disabled"
|
||||
return nil
|
||||
}
|
||||
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID)
|
||||
@@ -305,12 +317,12 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: s.template.Description,
|
||||
Tags: kb.ExtractConfig.Tags,
|
||||
Tags: extractCfg.Tags,
|
||||
Examples: []types.GraphData{
|
||||
{
|
||||
Text: kb.ExtractConfig.Text,
|
||||
Node: kb.ExtractConfig.Nodes,
|
||||
Relation: kb.ExtractConfig.Relations,
|
||||
Text: extractCfg.Text,
|
||||
Node: extractCfg.Nodes,
|
||||
Relation: extractCfg.Relations,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
}()
|
||||
|
||||
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
vlmModel, vlmCfg, err := s.resolveVLM(ctx, payload.KnowledgeBaseID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
handleErr = fmt.Errorf("resolve VLM: %w", err)
|
||||
return handleErr
|
||||
@@ -210,12 +210,10 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
// legacy inline-config path) so the trace shows WHICH model handled
|
||||
// this image. Without this, debugging "VLM is slow" requires a
|
||||
// separate hop to the KB config.
|
||||
if kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil {
|
||||
if id := strings.TrimSpace(kb.VLMConfig.ModelID); id != "" {
|
||||
imgOut["vlm_model_id"] = id
|
||||
} else {
|
||||
imgOut["vlm_model_id"] = "legacy_inline"
|
||||
}
|
||||
if id := strings.TrimSpace(vlmCfg.ModelID); id != "" {
|
||||
imgOut["vlm_model_id"] = id
|
||||
} else {
|
||||
imgOut["vlm_model_id"] = "legacy_inline"
|
||||
}
|
||||
|
||||
// Read image bytes. A provider:// URL must be resolved via FileService —
|
||||
@@ -456,27 +454,36 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.
|
||||
|
||||
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
||||
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
||||
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
||||
// Per-upload process_overrides on the knowledge entry take precedence over KB defaults.
|
||||
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID, knowledgeID string) (vlm.VLM, types.VLMConfig, error) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
return nil, types.VLMConfig{}, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
}
|
||||
if kb == nil {
|
||||
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
return nil, types.VLMConfig{}, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
}
|
||||
|
||||
vlmCfg := kb.VLMConfig
|
||||
var processOverrides *types.KnowledgeProcessOverrides
|
||||
if knowledgeID != "" && s.knowledgeRepo != nil {
|
||||
if k, kerr := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, knowledgeID); kerr == nil && k != nil {
|
||||
processOverrides, _ = k.ProcessOverrides()
|
||||
}
|
||||
}
|
||||
vlmCfg := ResolveProcessConfig(kb, processOverrides).VLMConfig
|
||||
if !vlmCfg.IsEnabled() {
|
||||
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
||||
return nil, types.VLMConfig{}, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
||||
}
|
||||
|
||||
// New-style: resolve model through ModelService
|
||||
if vlmCfg.ModelID != "" {
|
||||
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
||||
model, err := s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
||||
return model, vlmCfg, err
|
||||
}
|
||||
|
||||
// Legacy: create VLM from inline config
|
||||
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
||||
model, err := vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
||||
return model, vlmCfg, err
|
||||
}
|
||||
|
||||
// resolveFileServiceForPayload resolves tenant/KB scoped file service for reading provider:// URLs.
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
// CreateKnowledgeFromFile creates a knowledge entry from an uploaded file
|
||||
func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagID string, channel string,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
) (*types.Knowledge, error) {
|
||||
logger.Info(ctx, "Start creating knowledge from file")
|
||||
|
||||
@@ -60,62 +61,6 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查多模态配置完整性 - 只在图片文件时校验
|
||||
if !IsImageType(getFileType(fileName)) {
|
||||
logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation")
|
||||
} else {
|
||||
// 解析有效 provider:优先 KB 级别(新字段 > 旧字段),其次租户默认
|
||||
provider := kb.GetStorageProvider()
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
|
||||
// 根据 provider 校验租户级存储引擎配置
|
||||
switch provider {
|
||||
case "cos":
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
|
||||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
|
||||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
|
||||
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
case "minio":
|
||||
ok := false
|
||||
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
||||
m := tenant.StorageEngineConfig.MinIO
|
||||
if m.Mode == "remote" {
|
||||
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
|
||||
} else {
|
||||
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
|
||||
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
|
||||
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
}
|
||||
|
||||
// 检查VLM配置
|
||||
if !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" {
|
||||
logger.Error(ctx, "VLM model is not configured")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Image multimodal configuration validation passed")
|
||||
}
|
||||
|
||||
// 检查音频ASR配置完整性 - 只在音频文件时校验
|
||||
if IsAudioType(getFileType(fileName)) {
|
||||
if !kb.ASRConfig.IsASREnabled() {
|
||||
logger.Error(ctx, "ASR model is not configured")
|
||||
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
|
||||
}
|
||||
logger.Info(ctx, "Audio ASR configuration validation passed")
|
||||
}
|
||||
|
||||
// Validate file type
|
||||
logger.Infof(ctx, "Checking file type: %s", fileName)
|
||||
if !isValidFileType(fileName) {
|
||||
@@ -179,6 +124,64 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
return nil, werrors.NewValidationError("文件名包含非法字符")
|
||||
}
|
||||
|
||||
eff := ResolveProcessConfig(kb, processOverrides)
|
||||
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
|
||||
eff.EnableMultimodel = *enableMultimodel
|
||||
}
|
||||
|
||||
if processOverrides != nil {
|
||||
if err := ValidateProcessOverrides(ctx, kb, processOverrides, []string{getFileType(safeFilename)}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// 检查多模态配置完整性 - 只在图片文件时校验
|
||||
if IsImageType(getFileType(safeFilename)) {
|
||||
provider := kb.GetStorageProvider()
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case "cos":
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
|
||||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
|
||||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
|
||||
logger.Error(ctx, "COS configuration incomplete for image multimodal processing")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
case "minio":
|
||||
ok := false
|
||||
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
||||
m := tenant.StorageEngineConfig.MinIO
|
||||
if m.Mode == "remote" {
|
||||
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
|
||||
} else {
|
||||
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
|
||||
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
|
||||
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
}
|
||||
|
||||
if !kb.VLMConfig.Enabled || kb.VLMConfig.ModelID == "" {
|
||||
logger.Error(ctx, "VLM model is not configured")
|
||||
return nil, werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
|
||||
}
|
||||
}
|
||||
|
||||
if IsAudioType(getFileType(safeFilename)) {
|
||||
if !kb.ASRConfig.IsASREnabled() {
|
||||
logger.Error(ctx, "ASR model is not configured")
|
||||
return nil, werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare knowledge record
|
||||
logger.Info(ctx, "Preparing knowledge record")
|
||||
knowledge := &types.Knowledge{
|
||||
@@ -201,6 +204,13 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
Metadata: metadataJSON,
|
||||
}
|
||||
|
||||
if processOverrides != nil {
|
||||
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
|
||||
logger.Errorf(ctx, "Failed to set process overrides: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Save the file to storage (use KB-level storage engine if configured)
|
||||
logger.Infof(ctx, "Saving file, knowledge ID: %s", knowledge.ID)
|
||||
fileSvc := s.resolveFileService(ctx, kb)
|
||||
@@ -223,21 +233,12 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
|
||||
// Enqueue document processing task to Asynq
|
||||
logger.Info(ctx, "Enqueuing document processing task to Asynq")
|
||||
enableMultimodelValue := false
|
||||
if enableMultimodel != nil {
|
||||
enableMultimodelValue = *enableMultimodel
|
||||
} else {
|
||||
enableMultimodelValue = kb.IsMultimodalEnabled()
|
||||
}
|
||||
enableMultimodelValue := eff.EnableMultimodel
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
|
||||
questionCount := eff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -307,13 +308,16 @@ func isFileURL(rawURL, fileName, fileType string) bool {
|
||||
|
||||
func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagID string, channel string,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
) (*types.Knowledge, error) {
|
||||
logger.Info(ctx, "Start creating knowledge from URL")
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL)
|
||||
|
||||
// Route to file_url logic when the URL points to a downloadable file
|
||||
if isFileURL(rawURL, fileName, fileType) {
|
||||
return s.createKnowledgeFromFileURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID, channel)
|
||||
return s.createKnowledgeFromFileURL(
|
||||
ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID, channel, processOverrides,
|
||||
)
|
||||
}
|
||||
|
||||
url := rawURL
|
||||
@@ -397,6 +401,11 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
|
||||
// Save knowledge record
|
||||
logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID)
|
||||
eff, err := ApplyKnowledgeProcessOverrides(ctx, kb, knowledge, processOverrides, []string{"html"}, enableMultimodel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
|
||||
return nil, err
|
||||
@@ -404,21 +413,11 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
|
||||
// Enqueue URL processing task to Asynq
|
||||
logger.Info(ctx, "Enqueuing URL processing task to Asynq")
|
||||
enableMultimodelValue := false
|
||||
if enableMultimodel != nil {
|
||||
enableMultimodelValue = *enableMultimodel
|
||||
} else {
|
||||
enableMultimodelValue = kb.IsMultimodalEnabled()
|
||||
}
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableMultimodelValue := eff.EnableMultimodel
|
||||
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
|
||||
questionCount := eff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -515,6 +514,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
|
||||
title string,
|
||||
tagID string,
|
||||
channel string,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
) (*types.Knowledge, error) {
|
||||
logger.Info(ctx, "Start creating knowledge from file URL")
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, file URL: %s", kbID, fileURL)
|
||||
@@ -624,26 +624,32 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
|
||||
knowledge.Title = displayName
|
||||
}
|
||||
|
||||
resolvedFileType := fileType
|
||||
if resolvedFileType == "" && fileName != "" {
|
||||
resolvedFileType = getFileType(fileName)
|
||||
}
|
||||
if resolvedFileType == "" {
|
||||
resolvedFileType = getFileType(extractFileNameFromURL(fileURL))
|
||||
}
|
||||
|
||||
eff, err := ApplyKnowledgeProcessOverrides(
|
||||
ctx, kb, knowledge, processOverrides, []string{resolvedFileType}, enableMultimodel,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Build async task payload
|
||||
enableMultimodelValue := false
|
||||
if enableMultimodel != nil {
|
||||
enableMultimodelValue = *enableMultimodel
|
||||
} else {
|
||||
enableMultimodelValue = kb.IsMultimodalEnabled()
|
||||
}
|
||||
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableMultimodelValue := eff.EnableMultimodel
|
||||
enableQuestionGeneration := eff.QuestionGenerationConfig.Enabled
|
||||
questionCount := eff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -775,6 +781,12 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context,
|
||||
knowledge.ParseStatus = "pending"
|
||||
}
|
||||
|
||||
if status == types.ManualKnowledgeStatusPublish {
|
||||
if _, err := ApplyKnowledgeProcessOverrides(ctx, kb, knowledge, payload.ProcessConfig, nil, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create manual knowledge record: %v", err)
|
||||
return nil, err
|
||||
@@ -998,6 +1010,10 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
existing.Description = ""
|
||||
existing.ProcessedAt = nil
|
||||
|
||||
if _, err := ApplyKnowledgeProcessOverrides(ctx, kb, existing, payload.ProcessConfig, nil, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
|
||||
logger.Errorf(ctx, "Failed to persist manual knowledge before indexing: %v", err)
|
||||
return nil, err
|
||||
@@ -1101,26 +1117,27 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
processOverrides, _ := knowledge.ProcessOverrides()
|
||||
eff := ResolveProcessConfig(kb, processOverrides)
|
||||
|
||||
// Manual content is markdown - chunk directly with Go chunker
|
||||
chunkCfg := buildSplitterConfig(kb)
|
||||
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
|
||||
|
||||
var parsed []types.ParsedChunk
|
||||
opts := ProcessChunksOptions{
|
||||
// When the KB has VLM enabled and we resolved remote images, pass them
|
||||
// through so processChunks will enqueue image:multimodal tasks (OCR + caption).
|
||||
EnableMultimodel: kb.IsMultimodalEnabled() && len(resolvedImages) > 0,
|
||||
EnableMultimodel: eff.EnableMultimodel && len(resolvedImages) > 0,
|
||||
StoredImages: resolvedImages,
|
||||
}
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
if eff.QuestionGenerationConfig.Enabled {
|
||||
opts.EnableQuestionGeneration = true
|
||||
opts.QuestionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
opts.QuestionCount = eff.QuestionGenerationConfig.QuestionCount
|
||||
if opts.QuestionCount <= 0 {
|
||||
opts.QuestionCount = 3
|
||||
}
|
||||
}
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
if eff.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(eff.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg)
|
||||
parsed = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
|
||||
@@ -138,6 +138,7 @@ func TestCreateKnowledgeFromFileDoesNotPersistWhenStorageSaveFails(t *testing.T)
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
nil,
|
||||
)
|
||||
|
||||
require.Error(t, err)
|
||||
@@ -168,6 +169,7 @@ func TestCreateKnowledgeFromFilePersistsStoredFilePathOnCreate(t *testing.T) {
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
nil,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
@@ -201,6 +203,7 @@ func TestCreateKnowledgeFromFileDeletesStoredFileWhenCreateFails(t *testing.T) {
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
nil,
|
||||
)
|
||||
|
||||
require.EqualError(t, err, "database unavailable")
|
||||
@@ -211,6 +214,52 @@ func TestCreateKnowledgeFromFileDeletesStoredFileWhenCreateFails(t *testing.T) {
|
||||
require.Equal(t, "stored/"+fileSvc.savedWithKnowledgeID, fileSvc.deletedPath)
|
||||
}
|
||||
|
||||
func TestCreateKnowledgeFromFile_PersistsProcessOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
repo := &createKnowledgeFileRepoStub{}
|
||||
fileSvc := &createKnowledgeFileServiceStub{}
|
||||
task := &createKnowledgeTaskEnqueuerStub{}
|
||||
svc := &knowledgeService{
|
||||
repo: repo,
|
||||
kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}},
|
||||
fileSvc: fileSvc,
|
||||
task: task,
|
||||
}
|
||||
|
||||
chunkSize := 512
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ChunkingConfig: &types.ChunkingConfig{ChunkSize: chunkSize},
|
||||
}
|
||||
|
||||
knowledge, err := svc.CreateKnowledgeFromFile(
|
||||
newCreateKnowledgeFileContext(),
|
||||
"kb-1",
|
||||
newMultipartFileHeader(t, "doc.txt", "hello"),
|
||||
map[string]string{"source": "test"},
|
||||
nil,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
overrides,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, knowledge)
|
||||
require.Equal(t, 1, repo.createCalls)
|
||||
require.NotNil(t, repo.createdKnowledge)
|
||||
|
||||
parsed, err := repo.createdKnowledge.ProcessOverrides()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, parsed)
|
||||
require.NotNil(t, parsed.ChunkingConfig)
|
||||
require.Equal(t, chunkSize, parsed.ChunkingConfig.ChunkSize)
|
||||
|
||||
metadataMap, err := repo.createdKnowledge.Metadata.Map()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "test", metadataMap["source"])
|
||||
}
|
||||
|
||||
func newCreateKnowledgeFileContext() context.Context {
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, &types.Tenant{})
|
||||
|
||||
@@ -124,6 +124,9 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err)
|
||||
}
|
||||
|
||||
processOverrides, _ := knowledge.ProcessOverrides()
|
||||
eff := ResolveProcessConfig(kb, processOverrides)
|
||||
|
||||
// 2. Fetch all chunks
|
||||
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
@@ -155,7 +158,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
// drains is bounded by the housekeeping finalizing sweep.
|
||||
willSpawnSummary := len(textChunks) > 0
|
||||
willSpawnQuestion := willSpawnSummary && kb.NeedsEmbeddingModel() &&
|
||||
kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled
|
||||
eff.QuestionGenerationConfig.Enabled
|
||||
willSpawnWiki := kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0
|
||||
|
||||
// Question generation now fans out one subtask per plain text chunk
|
||||
@@ -184,7 +187,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
questionBatchCount := (len(questionChunks) + questionGenChunkBatchSize - 1) / questionGenChunkBatchSize
|
||||
|
||||
graphChunkCount := 0
|
||||
if kb.IsGraphEnabled() {
|
||||
if eff.GraphEnabled {
|
||||
graphChunkCount = len(textChunks)
|
||||
}
|
||||
expectedSubtasks := 0
|
||||
@@ -302,7 +305,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
"chunk_count": len(questionChunks),
|
||||
})
|
||||
}
|
||||
enqueuedQuestionCount = s.enqueueQuestionGenerationTasks(ctx, payload, kb, attempt, questionChunks)
|
||||
enqueuedQuestionCount = s.enqueueQuestionGenerationTasks(ctx, payload, eff.QuestionGenerationConfig, attempt, questionChunks)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -466,18 +469,18 @@ const postprocessQuestionGroupSpanName = "postprocess.question"
|
||||
func (s *KnowledgePostProcessService) enqueueQuestionGenerationTasks(
|
||||
ctx context.Context,
|
||||
payload types.KnowledgePostProcessPayload,
|
||||
kb *types.KnowledgeBase,
|
||||
qg types.QuestionGenerationConfig,
|
||||
attempt int,
|
||||
questionChunks []*types.Chunk,
|
||||
) int {
|
||||
if s.taskEnqueuer == nil || len(questionChunks) == 0 {
|
||||
return 0
|
||||
}
|
||||
if kb.QuestionGenerationConfig == nil || !kb.QuestionGenerationConfig.Enabled {
|
||||
if !qg.Enabled {
|
||||
return 0
|
||||
}
|
||||
|
||||
questionCount := kb.QuestionGenerationConfig.QuestionCount
|
||||
questionCount := qg.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
@@ -201,13 +201,17 @@ func finalizeIndexedKnowledgeState(
|
||||
// identical whether callers come through this path or invoke the chunker
|
||||
// directly with a zero-value config.
|
||||
func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig {
|
||||
return buildSplitterConfigFromChunking(kb.ChunkingConfig)
|
||||
}
|
||||
|
||||
func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig {
|
||||
chunkCfg := chunker.SplitterConfig{
|
||||
ChunkSize: kb.ChunkingConfig.ChunkSize,
|
||||
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
|
||||
Separators: kb.ChunkingConfig.Separators,
|
||||
Strategy: kb.ChunkingConfig.Strategy,
|
||||
TokenLimit: kb.ChunkingConfig.TokenLimit,
|
||||
Languages: kb.ChunkingConfig.Languages,
|
||||
ChunkSize: cc.ChunkSize,
|
||||
ChunkOverlap: cc.ChunkOverlap,
|
||||
Separators: cc.Separators,
|
||||
Strategy: cc.Strategy,
|
||||
TokenLimit: cc.TokenLimit,
|
||||
Languages: cc.Languages,
|
||||
}
|
||||
if chunkCfg.ChunkSize <= 0 {
|
||||
chunkCfg.ChunkSize = chunker.DefaultChunkSize
|
||||
@@ -1973,6 +1977,9 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
return nil, err
|
||||
}
|
||||
|
||||
processOverrides, _ := existing.ProcessOverrides()
|
||||
reparseEff := ResolveProcessConfig(kb, processOverrides)
|
||||
|
||||
// Keep wiki's pending queue consistent across both manual and non-manual
|
||||
// paths. The destructive work (swapping old wiki contributions for new)
|
||||
// happens asynchronously inside mapOneDocument — see its oldPageSlugs
|
||||
@@ -2060,17 +2067,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
if existing.FilePath != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
// Determine multimodal setting
|
||||
enableMultimodel := kb.IsMultimodalEnabled()
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableMultimodel := reparseEff.EnableMultimodel
|
||||
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
|
||||
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -2119,16 +2120,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
if existing.Type == "file_url" && existing.Source != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
enableMultimodel := kb.IsMultimodalEnabled()
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableMultimodel := reparseEff.EnableMultimodel
|
||||
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
|
||||
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -2172,16 +2168,11 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
|
||||
if existing.Type == "url" && existing.Source != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
enableMultimodel := kb.IsMultimodalEnabled()
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
enableMultimodel := reparseEff.EnableMultimodel
|
||||
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
|
||||
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
@@ -2734,6 +2725,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
processOverrides, _ := knowledge.ProcessOverrides()
|
||||
eff := ResolveProcessConfig(kb, processOverrides)
|
||||
|
||||
// Re-check abort status right before flipping to "processing" — closes
|
||||
// the race where the user cancels between the entry guard above and
|
||||
// this write (otherwise the worker would overwrite cancelled→processing
|
||||
@@ -2775,7 +2769,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
}
|
||||
|
||||
// 检查音频ASR配置(仅对文件导入)
|
||||
if payload.FilePath != "" && IsAudioType(payload.FileType) && !kb.ASRConfig.IsASREnabled() {
|
||||
if payload.FilePath != "" && IsAudioType(payload.FileType) && !eff.ASRConfig.IsASREnabled() {
|
||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
||||
Errorf("processDocument audio without ASR model configured")
|
||||
knowledge.ParseStatus = "failed"
|
||||
@@ -2857,7 +2851,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
payload.FilePath = filePath
|
||||
payload.FileName = resolvedFileName
|
||||
payload.FileType = resolvedFileType
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2866,7 +2860,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
}
|
||||
} else if payload.URL != "" {
|
||||
// URL import
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2910,7 +2904,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
return nil
|
||||
} else {
|
||||
// File import
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, isLastRetry)
|
||||
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2921,7 +2915,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
|
||||
// Step 1.5: ASR transcription for audio files
|
||||
if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 {
|
||||
if !kb.ASRConfig.IsASREnabled() {
|
||||
if !eff.ASRConfig.IsASREnabled() {
|
||||
logger.Error(ctx, "Audio file detected but ASR is not configured")
|
||||
knowledge.ParseStatus = "failed"
|
||||
knowledge.ErrorMessage = "ASR model is not configured for audio transcription"
|
||||
@@ -2933,7 +2927,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes",
|
||||
knowledge.ID, len(convertResult.AudioData))
|
||||
|
||||
asrModel, err := s.modelService.GetASRModel(ctx, kb.ASRConfig.ModelID)
|
||||
asrModel, err := s.modelService.GetASRModel(ctx, eff.ASRConfig.ModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err)
|
||||
knowledge.ParseStatus = "failed"
|
||||
@@ -3003,7 +2997,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
}
|
||||
|
||||
// Step 3: Split into chunks using Go chunker
|
||||
chunkCfg := buildSplitterConfig(kb)
|
||||
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
|
||||
|
||||
processOpts := ProcessChunksOptions{
|
||||
EnableQuestionGeneration: payload.EnableQuestionGeneration,
|
||||
@@ -3016,8 +3010,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
processOpts.Metadata = convertResult.Metadata
|
||||
}
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
if eff.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(eff.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
|
||||
chunks = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
@@ -3064,6 +3058,7 @@ func (s *knowledgeService) convert(
|
||||
payload types.DocumentProcessPayload,
|
||||
kb *types.KnowledgeBase,
|
||||
knowledge *types.Knowledge,
|
||||
eff types.EffectiveProcessConfig,
|
||||
isLastRetry bool,
|
||||
) (*types.ReadResult, error) {
|
||||
// Stage tracking: docreader. Mark the stage as running here so the
|
||||
@@ -3097,13 +3092,13 @@ func (s *knowledgeService) convert(
|
||||
}
|
||||
}
|
||||
|
||||
parserEngine := kb.ChunkingConfig.ResolveParserEngine(fileType)
|
||||
parserEngine := eff.ChunkingConfig.ResolveParserEngine(fileType)
|
||||
if isURL {
|
||||
parserEngine = kb.ChunkingConfig.ResolveParserEngine("url")
|
||||
parserEngine = eff.ChunkingConfig.ResolveParserEngine("url")
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v",
|
||||
kb.ID, fileType, isURL, parserEngine, kb.ChunkingConfig.ParserEngineRules)
|
||||
kb.ID, fileType, isURL, parserEngine, eff.ChunkingConfig.ParserEngineRules)
|
||||
|
||||
var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides)
|
||||
if reader == nil {
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ResolveProcessConfig merges KB defaults with per-upload overrides for the parse pipeline.
|
||||
func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgeProcessOverrides) types.EffectiveProcessConfig {
|
||||
eff := types.EffectiveProcessConfig{
|
||||
ChunkingConfig: kb.ChunkingConfig,
|
||||
EnableMultimodel: kb.IsMultimodalEnabled(),
|
||||
VLMConfig: kb.VLMConfig,
|
||||
ASRConfig: kb.ASRConfig,
|
||||
QuestionGenerationConfig: defaultQuestionGenerationConfig(kb),
|
||||
GraphEnabled: kb.IsGraphEnabled(),
|
||||
ExtractConfig: derefExtractConfig(kb.ExtractConfig),
|
||||
}
|
||||
if overrides == nil {
|
||||
return eff
|
||||
}
|
||||
|
||||
if overrides.ChunkingConfig != nil {
|
||||
eff.ChunkingConfig = mergeChunkingConfig(eff.ChunkingConfig, overrides.ChunkingConfig)
|
||||
}
|
||||
if len(overrides.ParserEngineRules) > 0 {
|
||||
eff.ChunkingConfig.ParserEngineRules = overrides.ParserEngineRules
|
||||
}
|
||||
if overrides.EnableMultimodel != nil {
|
||||
eff.EnableMultimodel = *overrides.EnableMultimodel
|
||||
}
|
||||
if overrides.VLMConfig != nil {
|
||||
eff.VLMConfig = *overrides.VLMConfig
|
||||
}
|
||||
if overrides.ASRConfig != nil {
|
||||
eff.ASRConfig = *overrides.ASRConfig
|
||||
}
|
||||
if overrides.QuestionGenerationConfig != nil {
|
||||
eff.QuestionGenerationConfig = *overrides.QuestionGenerationConfig
|
||||
}
|
||||
if overrides.GraphEnabled != nil {
|
||||
eff.GraphEnabled = *overrides.GraphEnabled
|
||||
}
|
||||
if overrides.ExtractConfig != nil {
|
||||
eff.ExtractConfig = mergeExtractConfig(eff.ExtractConfig, overrides.ExtractConfig)
|
||||
}
|
||||
|
||||
return eff
|
||||
}
|
||||
|
||||
// ValidateProcessOverrides validates batch overrides against file types in the upload.
|
||||
func ValidateProcessOverrides(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
overrides *types.KnowledgeProcessOverrides,
|
||||
fileTypes []string,
|
||||
) error {
|
||||
if overrides == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
hasImage := false
|
||||
hasAudio := false
|
||||
for _, ft := range fileTypes {
|
||||
if IsImageType(ft) {
|
||||
hasImage = true
|
||||
}
|
||||
if IsAudioType(ft) {
|
||||
hasAudio = true
|
||||
}
|
||||
}
|
||||
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
|
||||
if hasImage {
|
||||
if err := validateImageMultimodalConfig(ctx, kb); err != nil {
|
||||
return err
|
||||
}
|
||||
if !eff.VLMConfig.IsEnabled() {
|
||||
return werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
|
||||
}
|
||||
}
|
||||
|
||||
if hasAudio && !eff.ASRConfig.IsASREnabled() {
|
||||
return werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ApplyKnowledgeProcessOverrides validates optional overrides, persists them on the
|
||||
// knowledge record, and returns the effective config for task enqueue.
|
||||
func ApplyKnowledgeProcessOverrides(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
knowledge *types.Knowledge,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
fileTypes []string,
|
||||
enableMultimodel *bool,
|
||||
) (types.EffectiveProcessConfig, error) {
|
||||
eff := ResolveProcessConfig(kb, processOverrides)
|
||||
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
|
||||
eff.EnableMultimodel = *enableMultimodel
|
||||
}
|
||||
if processOverrides == nil {
|
||||
return eff, nil
|
||||
}
|
||||
if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil {
|
||||
return eff, err
|
||||
}
|
||||
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
|
||||
return eff, err
|
||||
}
|
||||
return eff, nil
|
||||
}
|
||||
|
||||
func defaultQuestionGenerationConfig(kb *types.KnowledgeBase) types.QuestionGenerationConfig {
|
||||
if kb == nil || kb.QuestionGenerationConfig == nil {
|
||||
return types.QuestionGenerationConfig{}
|
||||
}
|
||||
return *kb.QuestionGenerationConfig
|
||||
}
|
||||
|
||||
func derefExtractConfig(cfg *types.ExtractConfig) types.ExtractConfig {
|
||||
if cfg == nil {
|
||||
return types.ExtractConfig{}
|
||||
}
|
||||
return *cfg
|
||||
}
|
||||
|
||||
func mergeChunkingConfig(base types.ChunkingConfig, override *types.ChunkingConfig) types.ChunkingConfig {
|
||||
if override == nil {
|
||||
return base
|
||||
}
|
||||
result := base
|
||||
if override.ChunkSize != 0 {
|
||||
result.ChunkSize = override.ChunkSize
|
||||
}
|
||||
if override.ChunkOverlap != 0 {
|
||||
result.ChunkOverlap = override.ChunkOverlap
|
||||
}
|
||||
if len(override.Separators) > 0 {
|
||||
result.Separators = override.Separators
|
||||
}
|
||||
if override.EnableMultimodal {
|
||||
result.EnableMultimodal = override.EnableMultimodal
|
||||
}
|
||||
if len(override.ParserEngineRules) > 0 {
|
||||
result.ParserEngineRules = override.ParserEngineRules
|
||||
}
|
||||
if override.EnableParentChild {
|
||||
result.EnableParentChild = override.EnableParentChild
|
||||
}
|
||||
if override.ParentChunkSize != 0 {
|
||||
result.ParentChunkSize = override.ParentChunkSize
|
||||
}
|
||||
if override.ChildChunkSize != 0 {
|
||||
result.ChildChunkSize = override.ChildChunkSize
|
||||
}
|
||||
if override.Strategy != "" {
|
||||
result.Strategy = override.Strategy
|
||||
}
|
||||
if override.TokenLimit != 0 {
|
||||
result.TokenLimit = override.TokenLimit
|
||||
}
|
||||
if len(override.Languages) > 0 {
|
||||
result.Languages = override.Languages
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig) types.ExtractConfig {
|
||||
if override == nil {
|
||||
return base
|
||||
}
|
||||
result := base
|
||||
if override.Enabled {
|
||||
result.Enabled = true
|
||||
}
|
||||
if override.Text != "" {
|
||||
result.Text = override.Text
|
||||
}
|
||||
if len(override.Tags) > 0 {
|
||||
result.Tags = override.Tags
|
||||
}
|
||||
if len(override.Nodes) > 0 {
|
||||
result.Nodes = override.Nodes
|
||||
}
|
||||
if len(override.Relations) > 0 {
|
||||
result.Relations = override.Relations
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) error {
|
||||
provider := kb.GetStorageProvider()
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case "cos":
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil || tenant.StorageEngineConfig.COS == nil ||
|
||||
tenant.StorageEngineConfig.COS.SecretID == "" || tenant.StorageEngineConfig.COS.SecretKey == "" ||
|
||||
tenant.StorageEngineConfig.COS.Region == "" || tenant.StorageEngineConfig.COS.BucketName == "" {
|
||||
return werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
case "minio":
|
||||
ok := false
|
||||
if tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
|
||||
m := tenant.StorageEngineConfig.MinIO
|
||||
if m.Mode == "remote" {
|
||||
ok = m.Endpoint != "" && m.AccessKeyID != "" && m.SecretAccessKey != "" && m.BucketName != ""
|
||||
} else {
|
||||
ok = os.Getenv("MINIO_ENDPOINT") != "" && os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
|
||||
os.Getenv("MINIO_SECRET_ACCESS_KEY") != "" &&
|
||||
(m.BucketName != "" || os.Getenv("MINIO_BUCKET_NAME") != "")
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往知识库存储设置或系统设置页面进行补全")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func processConfigBoolPtr(v bool) *bool {
|
||||
return &v
|
||||
}
|
||||
|
||||
func testKBWithGraphEnabled(enabled bool) *types.KnowledgeBase {
|
||||
return &types.KnowledgeBase{
|
||||
IndexingStrategy: types.IndexingStrategy{GraphEnabled: enabled},
|
||||
ExtractConfig: &types.ExtractConfig{Enabled: enabled},
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_OverridesChunkSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 2048},
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.Equal(t, 2048, eff.ChunkingConfig.ChunkSize)
|
||||
require.Equal(t, 50, eff.ChunkingConfig.ChunkOverlap)
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_GraphDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := testKBWithGraphEnabled(true)
|
||||
overrides := &types.KnowledgeProcessOverrides{GraphEnabled: processConfigBoolPtr(false)}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.False(t, eff.GraphEnabled)
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_NilOverridesUsesKBDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50},
|
||||
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
|
||||
ASRConfig: types.ASRConfig{Enabled: true, ModelID: "asr-1"},
|
||||
QuestionGenerationConfig: &types.QuestionGenerationConfig{
|
||||
Enabled: true,
|
||||
QuestionCount: 3,
|
||||
},
|
||||
IndexingStrategy: types.IndexingStrategy{GraphEnabled: true},
|
||||
ExtractConfig: &types.ExtractConfig{Enabled: true, Tags: []string{"tag-a"}},
|
||||
}
|
||||
kb.ChunkingConfig.EnableMultimodal = true
|
||||
|
||||
eff := ResolveProcessConfig(kb, nil)
|
||||
|
||||
require.Equal(t, 512, eff.ChunkingConfig.ChunkSize)
|
||||
require.Equal(t, 50, eff.ChunkingConfig.ChunkOverlap)
|
||||
require.True(t, eff.EnableMultimodel)
|
||||
require.Equal(t, "vlm-1", eff.VLMConfig.ModelID)
|
||||
require.Equal(t, "asr-1", eff.ASRConfig.ModelID)
|
||||
require.True(t, eff.QuestionGenerationConfig.Enabled)
|
||||
require.Equal(t, 3, eff.QuestionGenerationConfig.QuestionCount)
|
||||
require.True(t, eff.GraphEnabled)
|
||||
require.True(t, eff.ExtractConfig.Enabled)
|
||||
require.Equal(t, []string{"tag-a"}, eff.ExtractConfig.Tags)
|
||||
}
|
||||
|
||||
func TestBuildSplitterConfigFromChunking_UsesEffectiveChunkingConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50, Strategy: "token"},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 1500, ChunkOverlap: 120, Strategy: "character"},
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
cfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
|
||||
|
||||
require.Equal(t, 1500, cfg.ChunkSize)
|
||||
require.Equal(t, 120, cfg.ChunkOverlap)
|
||||
require.Equal(t, "character", cfg.Strategy)
|
||||
}
|
||||
|
||||
func TestEffectiveChunkingConfig_ResolveParserEngineFromOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{
|
||||
ParserEngineRules: []types.ParserEngineRule{
|
||||
{FileTypes: []string{"pdf"}, Engine: "builtin"},
|
||||
},
|
||||
},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ParserEngineRules: []types.ParserEngineRule{
|
||||
{FileTypes: []string{"pdf"}, Engine: "mineru"},
|
||||
},
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.Equal(t, "mineru", eff.ChunkingConfig.ResolveParserEngine("pdf"))
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_ParserEngineRulesReplaced(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{
|
||||
ParserEngineRules: []types.ParserEngineRule{
|
||||
{FileTypes: []string{"pdf"}, Engine: "builtin"},
|
||||
},
|
||||
},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ParserEngineRules: []types.ParserEngineRule{
|
||||
{FileTypes: []string{"docx"}, Engine: "custom"},
|
||||
},
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.Len(t, eff.ChunkingConfig.ParserEngineRules, 1)
|
||||
require.Equal(t, []string{"docx"}, eff.ChunkingConfig.ParserEngineRules[0].FileTypes)
|
||||
require.Equal(t, "custom", eff.ChunkingConfig.ParserEngineRules[0].Engine)
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_EnableMultimodelOverride(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
EnableMultimodel: processConfigBoolPtr(false),
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.False(t, eff.EnableMultimodel)
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_ExtractConfigFieldMerge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ExtractConfig: &types.ExtractConfig{
|
||||
Enabled: true,
|
||||
Text: "base text",
|
||||
Tags: []string{"base-tag"},
|
||||
},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ExtractConfig: &types.ExtractConfig{
|
||||
Tags: []string{"override-tag"},
|
||||
},
|
||||
}
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.True(t, eff.ExtractConfig.Enabled)
|
||||
require.Equal(t, "base text", eff.ExtractConfig.Text)
|
||||
require.Equal(t, []string{"override-tag"}, eff.ExtractConfig.Tags)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_NilOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := ValidateProcessOverrides(context.Background(), &types.KnowledgeBase{}, nil, []string{"png"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_ImageRequiresVLM(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
VLMConfig: types.VLMConfig{Enabled: false},
|
||||
}
|
||||
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"png"})
|
||||
require.Error(t, err)
|
||||
var badReq *werrors.AppError
|
||||
require.ErrorAs(t, err, &badReq)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_ImageWithEffectiveVLM(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
VLMConfig: types.VLMConfig{Enabled: false},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
VLMConfig: &types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
|
||||
}
|
||||
err := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"jpg"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_AudioRequiresASR(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ASRConfig: types.ASRConfig{Enabled: false},
|
||||
}
|
||||
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"mp3"})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_AudioWithEffectiveASR(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ASRConfig: types.ASRConfig{Enabled: false},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ASRConfig: &types.ASRConfig{Enabled: true, ModelID: "asr-1"},
|
||||
}
|
||||
err := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"wav"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_NonMediaFileTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{}
|
||||
err := ValidateProcessOverrides(context.Background(), kb, &types.KnowledgeProcessOverrides{}, []string{"pdf", "txt"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_COSIncompleteForImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, &types.Tenant{
|
||||
StorageEngineConfig: &types.StorageEngineConfig{
|
||||
COS: &types.COSEngineConfig{SecretID: "id"},
|
||||
},
|
||||
})
|
||||
kb := &types.KnowledgeBase{
|
||||
VLMConfig: types.VLMConfig{Enabled: true, ModelID: "vlm-1"},
|
||||
}
|
||||
kb.SetStorageProvider("cos")
|
||||
|
||||
err := ValidateProcessOverrides(ctx, kb, &types.KnowledgeProcessOverrides{}, []string{"png"})
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -237,6 +237,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
|
||||
// @Param fileName formData string false "自定义文件名"
|
||||
// @Param metadata formData string false "元数据JSON"
|
||||
// @Param enable_multimodel formData bool false "启用多模态处理"
|
||||
// @Param process_config formData string false "处理配置JSON(KnowledgeProcessOverrides)"
|
||||
// @Success 200 {object} map[string]interface{} "创建的知识"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Failure 409 {object} map[string]interface{} "文件重复"
|
||||
@@ -318,6 +319,23 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
|
||||
enableMultimodel = &parseBool
|
||||
}
|
||||
|
||||
var processOverrides *types.KnowledgeProcessOverrides
|
||||
if raw := c.PostForm("process_config"); raw != "" {
|
||||
processOverrides = &types.KnowledgeProcessOverrides{}
|
||||
if err := json.Unmarshal([]byte(raw), processOverrides); err != nil {
|
||||
logger.Error(ctx, "Failed to parse process_config", err)
|
||||
c.Error(errors.NewBadRequestError("Invalid process_config format").WithDetails(err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
|
||||
if processOverrides == nil {
|
||||
processOverrides = &types.KnowledgeProcessOverrides{EnableMultimodel: enableMultimodel}
|
||||
} else {
|
||||
processOverrides.EnableMultimodel = enableMultimodel
|
||||
}
|
||||
}
|
||||
|
||||
// 获取分类ID(如果提供),用于知识分类管理
|
||||
tagID := c.PostForm("tag_id")
|
||||
// 过滤特殊值,空字符串或 "__untagged__" 表示未分类
|
||||
@@ -328,7 +346,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
|
||||
channel := c.PostForm("channel")
|
||||
|
||||
// Create knowledge entry from the file
|
||||
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID, channel)
|
||||
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID, channel, processOverrides)
|
||||
// Check for duplicate knowledge error
|
||||
if err != nil {
|
||||
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") {
|
||||
@@ -389,13 +407,14 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
|
||||
|
||||
// Parse URL from request body
|
||||
var req struct {
|
||||
URL string `json:"url" binding:"required"`
|
||||
FileName string `json:"file_name"`
|
||||
FileType string `json:"file_type"`
|
||||
EnableMultimodel *bool `json:"enable_multimodel"`
|
||||
Title string `json:"title"`
|
||||
TagID string `json:"tag_id"`
|
||||
Channel string `json:"channel"`
|
||||
URL string `json:"url" binding:"required"`
|
||||
FileName string `json:"file_name"`
|
||||
FileType string `json:"file_type"`
|
||||
EnableMultimodel *bool `json:"enable_multimodel"`
|
||||
Title string `json:"title"`
|
||||
TagID string `json:"tag_id"`
|
||||
Channel string `json:"channel"`
|
||||
ProcessConfig *types.KnowledgeProcessOverrides `json:"process_config"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
logger.Error(ctx, "Failed to parse URL request", err)
|
||||
@@ -423,7 +442,9 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
|
||||
)
|
||||
|
||||
// Create knowledge entry from the URL
|
||||
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID, req.Channel)
|
||||
knowledge, err := h.kgService.CreateKnowledgeFromURL(
|
||||
ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID, req.Channel, req.ProcessConfig,
|
||||
)
|
||||
// Check for duplicate knowledge error
|
||||
if err != nil {
|
||||
if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") {
|
||||
|
||||
@@ -2430,7 +2430,7 @@ func (s *Service) processFileToKnowledgeBase(ctx context.Context, msg *IncomingM
|
||||
fh := newInMemoryFileHeader(fileName, content)
|
||||
|
||||
// Create knowledge entry via the knowledge service
|
||||
knowledge, err := s.knowledgeService.CreateKnowledgeFromFile(kbCtx, kbID, fh, nil, nil, "", "", imPlatformToChannel(channel.Platform))
|
||||
knowledge, err := s.knowledgeService.CreateKnowledgeFromFile(kbCtx, kbID, fh, nil, nil, "", "", imPlatformToChannel(channel.Platform), nil)
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
// Check for duplicate file
|
||||
|
||||
@@ -22,6 +22,7 @@ type KnowledgeService interface {
|
||||
customFileName string,
|
||||
tagID string,
|
||||
channel string,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromURL creates knowledge from a URL.
|
||||
// When fileName or fileType is provided (or the URL path has a known file extension),
|
||||
@@ -37,6 +38,7 @@ type KnowledgeService interface {
|
||||
title string,
|
||||
tagID string,
|
||||
channel string,
|
||||
processOverrides *types.KnowledgeProcessOverrides,
|
||||
) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromPassage creates knowledge from text passages.
|
||||
// channel identifies the ingestion channel; empty defaults to "web".
|
||||
|
||||
@@ -204,11 +204,12 @@ type ManualKnowledgeMetadata struct {
|
||||
|
||||
// ManualKnowledgePayload represents the payload for manual knowledge operations.
|
||||
type ManualKnowledgePayload struct {
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Status string `json:"status"`
|
||||
TagID string `json:"tag_id"`
|
||||
Channel string `json:"channel"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Status string `json:"status"`
|
||||
TagID string `json:"tag_id"`
|
||||
Channel string `json:"channel"`
|
||||
ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
|
||||
}
|
||||
|
||||
// KnowledgeSearchScope defines a (tenant_id, knowledge_base_id) scope for knowledge search (e.g. own KBs + shared KBs).
|
||||
@@ -342,6 +343,62 @@ func (p ManualKnowledgePayload) IsDraft() bool {
|
||||
return p.Status == "" || p.Status == ManualKnowledgeStatusDraft
|
||||
}
|
||||
|
||||
const metadataKeyProcessOverrides = "process_overrides"
|
||||
|
||||
// ProcessOverrides parses process config overrides from knowledge metadata.
|
||||
func (k *Knowledge) ProcessOverrides() (*KnowledgeProcessOverrides, error) {
|
||||
if k == nil || len(k.Metadata) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
metadataMap, err := k.Metadata.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, ok := metadataMap[metadataKeyProcessOverrides]
|
||||
if !ok || raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
bytes, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var overrides KnowledgeProcessOverrides
|
||||
if err := json.Unmarshal(bytes, &overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &overrides, nil
|
||||
}
|
||||
|
||||
// SetProcessOverrides merges process config overrides into knowledge metadata.
|
||||
func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
metadataMap, err := k.Metadata.Map()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o == nil {
|
||||
delete(metadataMap, metadataKeyProcessOverrides)
|
||||
} else {
|
||||
bytes, err := json.Marshal(o)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var value interface{}
|
||||
if err := json.Unmarshal(bytes, &value); err != nil {
|
||||
return err
|
||||
}
|
||||
metadataMap[metadataKeyProcessOverrides] = value
|
||||
}
|
||||
bytes, err := json.Marshal(metadataMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
k.Metadata = JSON(bytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
// KnowledgeCheckParams defines parameters used to check if knowledge already exists.
|
||||
type KnowledgeCheckParams struct {
|
||||
// File parameters
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package types
|
||||
|
||||
// KnowledgeProcessOverrides stores per-upload parse config overrides in knowledge metadata.
|
||||
type KnowledgeProcessOverrides struct {
|
||||
ParserEngineRules []ParserEngineRule `json:"parser_engine_rules,omitempty"`
|
||||
ChunkingConfig *ChunkingConfig `json:"chunking_config,omitempty"`
|
||||
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
|
||||
VLMConfig *VLMConfig `json:"vlm_config,omitempty"`
|
||||
ASRConfig *ASRConfig `json:"asr_config,omitempty"`
|
||||
QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"`
|
||||
GraphEnabled *bool `json:"graph_enabled,omitempty"`
|
||||
ExtractConfig *ExtractConfig `json:"extract_config,omitempty"`
|
||||
}
|
||||
|
||||
// EffectiveProcessConfig is the merged view used by the parse pipeline.
|
||||
type EffectiveProcessConfig struct {
|
||||
ChunkingConfig ChunkingConfig
|
||||
EnableMultimodel bool
|
||||
VLMConfig VLMConfig
|
||||
ASRConfig ASRConfig
|
||||
QuestionGenerationConfig QuestionGenerationConfig
|
||||
GraphEnabled bool
|
||||
ExtractConfig ExtractConfig
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func boolPtr(v bool) *bool {
|
||||
return &v
|
||||
}
|
||||
|
||||
func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) {
|
||||
k := &Knowledge{}
|
||||
overrides := &KnowledgeProcessOverrides{
|
||||
EnableMultimodel: boolPtr(true),
|
||||
ChunkingConfig: &ChunkingConfig{ChunkSize: 1024},
|
||||
}
|
||||
require.NoError(t, k.SetProcessOverrides(overrides))
|
||||
got, err := k.ProcessOverrides()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
require.True(t, *got.EnableMultimodel)
|
||||
require.Equal(t, 1024, got.ChunkingConfig.ChunkSize)
|
||||
}
|
||||
|
||||
func TestSetProcessOverridesPreservesOtherMetadata(t *testing.T) {
|
||||
k := &Knowledge{}
|
||||
manualMeta := NewManualKnowledgeMetadata("# hello", ManualKnowledgeStatusDraft, 1)
|
||||
require.NoError(t, k.SetManualMetadata(manualMeta))
|
||||
|
||||
overrides := &KnowledgeProcessOverrides{
|
||||
EnableMultimodel: boolPtr(false),
|
||||
}
|
||||
require.NoError(t, k.SetProcessOverrides(overrides))
|
||||
|
||||
gotManual, err := k.ManualMetadata()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, gotManual)
|
||||
require.Equal(t, "# hello", gotManual.Content)
|
||||
require.Equal(t, ManualKnowledgeFormatMarkdown, gotManual.Format)
|
||||
require.Equal(t, ManualKnowledgeStatusDraft, gotManual.Status)
|
||||
|
||||
gotOverrides, err := k.ProcessOverrides()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, gotOverrides)
|
||||
require.False(t, *gotOverrides.EnableMultimodel)
|
||||
}
|
||||
Reference in New Issue
Block a user