opensource/WeKnora
3
0
Fork 0
mirror of https://github.com/Tencent/WeKnora.git synced 2026-09-21 13:52:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Enhance FAQ import functionality and UI improvements

Browse Source 
- Added support for tracking the status of FAQ import tasks, including progress indicators for ongoing imports.
- Introduced new localization strings for processing tasks and batch operations in multiple languages.
- Updated the FAQ entry manager to display import progress and improved user feedback during the import process.
- Enhanced the KnowledgeBase component to reflect the processing state of knowledge items, improving user experience during document uploads.
- Implemented batch deletion functionality for chunks, streamlining the management of knowledge base content.
This commit is contained in:
wizardchen
2025-11-23 17:32:50 +08:00
parent e853605cea
commit 057cfd9eb0
46 changed files with 3672 additions and 1400 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/package-lock.json
Generated
+17
View File
@@ -10,10 +10,12 @@
"dependencies": {
"@microsoft/fetch-event-source": "^2.0.1",
"@types/dompurify": "^3.0.5",
"@types/papaparse": "^5.5.0",
"axios": "^1.8.4",
"dompurify": "^3.2.6",
"marked": "^5.1.2",
"pagefind": "^1.1.1",
"papaparse": "^5.5.3",
"pinia": "^3.0.1",
"swiper": "^12.0.3",
"tdesign-icons-vue-next": "^0.4.1",
@@ -1394,6 +1396,15 @@
"undici-types": "~6.21.0"
}
},
"node_modules/@types/papaparse": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.5.0.tgz",
"integrity": "sha512-GVs5iMQmUr54BAZYYkByv8zPofFxmyxUpISPb2oh8sayR3+1zbxasrOvoKiHJ/nnoq/uULuPsu1Lze1EkagVFg==",
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/sortablejs": {
"version": "1.15.8",
"resolved": "https://mirrors.tencent.com/npm/@types/sortablejs/-/sortablejs-1.15.8.tgz",
@@ -3046,6 +3057,12 @@
"@pagefind/windows-x64": "1.3.0"
}
},
"node_modules/papaparse": {
"version": "5.5.3",
"resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.3.tgz",
"integrity": "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==",
"license": "MIT"
},
"node_modules/parse-node-version": {
"version": "1.0.1",
"resolved": "https://mirrors.tencent.com/npm/parse-node-version/-/parse-node-version-1.0.1.tgz",
frontend/package.json
+2
View File
@@ -14,10 +14,12 @@
"dependencies": {
"@microsoft/fetch-event-source": "^2.0.1",
"@types/dompurify": "^3.0.5",
"@types/papaparse": "^5.5.0",
"axios": "^1.8.4",
"dompurify": "^3.2.6",
"marked": "^5.1.2",
"pagefind": "^1.1.1",
"papaparse": "^5.5.3",
"pinia": "^3.0.1",
"swiper": "^12.0.3",
"tdesign-icons-vue-next": "^0.4.1",
frontend/src/components/menu.vue
+141 -12
View File
@@ -85,6 +85,37 @@
<span class="kb-action-title">{{ t('knowledgeEditor.faqImport.importButton') }}</span>
</div>
</div>
<div class="menu_item kb-action-item" @click.stop="handleFaqSearchTestFromMenu">
<div class="kb-action-icon-wrapper">
<svg class="kb-action-icon" width="18" height="18" viewBox="0 0 18 18" fill="none">
<path d="M8.25 15C11.9779 15 15 11.9779 15 8.25C15 4.52208 11.9779 1.5 8.25 1.5C4.52208 1.5 1.5 4.52208 1.5 8.25C1.5 11.9779 4.52208 15 8.25 15Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M16.5 16.5L12.4875 12.4875" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
</div>
<div class="kb-action-content">
<span class="kb-action-title">{{ t('knowledgeEditor.faq.searchTest') }}</span>
</div>
</div>
<t-dropdown
v-if="selectedFaqCount > 0"
:options="faqBatchActionOptions"
trigger="hover"
placement="right"
@click="handleFaqBatchActionFromMenu"
>
<div class="menu_item kb-action-item">
<div class="kb-action-icon-wrapper">
<svg class="kb-action-icon" width="18" height="18" viewBox="0 0 18 18" fill="none">
<path d="M3.75 9H14.25M9 3.75V14.25" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M3.75 3.75H14.25V14.25H3.75V3.75Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
</div>
<div class="kb-action-content">
<span class="kb-action-title">{{ t('knowledgeEditor.faq.batchOperations') }}</span>
<span class="kb-action-count">({{ selectedFaqCount }})</span>
</div>
</div>
</t-dropdown>
</template>
</div>
</div>
@@ -153,7 +184,7 @@
<script setup lang="ts">
import { storeToRefs } from 'pinia';
import { onMounted, watch, computed, ref, reactive } from 'vue';
import { onMounted, onUnmounted, watch, computed, ref, reactive } from 'vue';
import { useRoute, useRouter } from 'vue-router';
import { getSessionsList, delSession } from "@/api/chat/index";
import { getKnowledgeBaseById, uploadKnowledgeFile, createKnowledgeFromURL } from '@/api/knowledge-base';
@@ -197,6 +228,11 @@ const isInKnowledgeBaseList = computed<boolean>(() => {
return route.name === 'knowledgeBaseList';
});
// 是否在创建聊天页面
const isInCreatChat = computed<boolean>(() => {
return route.name === 'globalCreatChat' || route.name === 'kbCreatChat';
});
// 统一的菜单项激活状态判断
const isMenuItemActive = (itemPath: string): boolean => {
const currentRoute = route.name;
@@ -234,7 +270,7 @@ const getIconActiveState = (itemPath: string) => {
// 分离上下两部分菜单
const topMenuItems = computed<MenuItem[]>(() => {
return (menuArr.value as unknown as MenuItem[]).filter((item: MenuItem) =>
item.path === 'creatChat'
item.path === 'knowledge-bases' || item.path === 'creatChat'
);
});
@@ -253,12 +289,23 @@ const currentKbInfo = ref<any>(null)
const docUploadInput = ref<HTMLInputElement | null>(null)
const docFolderInput = ref<HTMLInputElement | null>(null)
const pendingUploadKbId = ref<string | null>(null)
const selectedFaqCount = ref<number>(0)
const selectedFaqEnabledCount = ref<number>(0)
const selectedFaqDisabledCount = ref<number>(0)
const showKbActions = computed(() => (isInKnowledgeBase.value && !!currentKbInfo.value) || isInKnowledgeBaseList.value)
// 监听FAQ选中数量变化
const handleFaqSelectionChanged = ((event: CustomEvent<{ count: number; enabledCount?: number; disabledCount?: number }>) => {
const count = event.detail?.count || 0
selectedFaqCount.value = count
selectedFaqEnabledCount.value = event.detail?.enabledCount || 0
selectedFaqDisabledCount.value = event.detail?.disabledCount || 0
}) as EventListener
const showKbActions = computed(() => (isInKnowledgeBase.value && !!currentKbInfo.value) || isInKnowledgeBaseList.value || isInCreatChat.value)
const currentKbType = computed(() => currentKbInfo.value?.type || 'document')
const showDocActions = computed(() => showKbActions.value && isInKnowledgeBase.value && currentKbType.value !== 'faq')
const showFaqActions = computed(() => showKbActions.value && isInKnowledgeBase.value && currentKbType.value === 'faq')
const showCreateKbAction = computed(() => showKbActions.value && isInKnowledgeBaseList.value)
const showCreateKbAction = computed(() => showKbActions.value && (isInKnowledgeBaseList.value || isInCreatChat.value))
// 时间分组函数
const getTimeCategory = (dateStr: string): string => {
@@ -438,9 +485,20 @@ onMounted(async () => {
// 加载对话列表
getMessageList();
// 监听FAQ选中数量变化
window.addEventListener('faqSelectionChanged', handleFaqSelectionChanged)
});
onUnmounted(() => {
window.removeEventListener('faqSelectionChanged', handleFaqSelectionChanged)
})
watch([() => route.name, () => route.params], (newvalue, oldvalue) => {
// 切换知识库时重置选中数量
if (newvalue[1].kbId !== oldvalue?.[1]?.kbId) {
selectedFaqCount.value = 0
}
const nameStr = typeof newvalue[0] === 'string' ? (newvalue[0] as string) : (newvalue[0] ? String(newvalue[0]) : '')
currentpath.value = nameStr;
if (newvalue[1].chatid) {
@@ -558,13 +616,11 @@ const gotopage = async (path: string) => {
return;
} else {
if (path === 'creatChat') {
// 尝试获取当前知识库ID
const kbId = await getCurrentKbId()
if (kbId) {
// 如果在知识库内部，进入该知识库的对话页
router.push(`/platform/knowledge-bases/${kbId}/creatChat`)
// 如果在知识库详情页，跳转到全局对话创建页
if (isInKnowledgeBase.value) {
router.push('/platform/creatChat')
} else {
// 如果不在知识库内，也进入对话创建页，让用户通过 @ 按钮选择知识库
// 如果不在知识库内，进入对话创建页
router.push(`/platform/creatChat`)
}
} else {
@@ -938,7 +994,7 @@ const handleDocURLImport = async () => {
}))
}
const dispatchFaqMenuAction = (action: 'create' | 'import', kbId: string) => {
const dispatchFaqMenuAction = (action: 'create' | 'import' | 'search' | 'batch' | 'batchTag' | 'batchEnable' | 'batchDisable' | 'batchDelete', kbId: string) => {
window.dispatchEvent(new CustomEvent('faqMenuAction', {
detail: { action, kbId }
}))
@@ -962,6 +1018,65 @@ const handleFaqImportFromMenu = async () => {
dispatchFaqMenuAction('import', kbId)
}
const handleFaqSearchTestFromMenu = async () => {
const kbId = await getCurrentKbId()
if (!kbId) {
MessagePlugin.warning(t('knowledgeEditor.messages.missingId'))
return
}
dispatchFaqMenuAction('search', kbId)
}
const faqBatchActionOptions = computed(() => {
if (selectedFaqCount.value === 0) {
return []
}
const options = [
{
content: `${t('knowledgeEditor.faq.batchUpdateTag')} (${selectedFaqCount.value})`,
value: 'batchTag',
icon: 'folder'
}
]
// 根据选中条目的状态显示批量启用或禁用
if (selectedFaqDisabledCount.value > 0) {
options.push({
content: `${t('knowledgeEditor.faq.batchEnable')} (${selectedFaqDisabledCount.value})`,
value: 'batchEnable',
icon: 'check-circle',
})
}
if (selectedFaqEnabledCount.value > 0) {
options.push({
content: `${t('knowledgeEditor.faq.batchDisable')} (${selectedFaqEnabledCount.value})`,
value: 'batchDisable',
icon: 'close-circle',
})
}
options.push({
content: `${t('knowledgeEditor.faqImport.deleteSelected')} (${selectedFaqCount.value})`,
value: 'batchDelete',
icon: 'delete',
})
return options
})
const handleFaqBatchActionFromMenu = async (data: { value: string }) => {
const kbId = await getCurrentKbId()
if (!kbId) {
MessagePlugin.warning(t('knowledgeEditor.messages.missingId'))
return
}
if (selectedFaqCount.value === 0) {
MessagePlugin.warning(t('knowledgeEditor.faq.selectEntriesFirst') || '请先选中要操作的FAQ条目')
return
}
dispatchFaqMenuAction(data.value as 'batchTag' | 'batchEnable' | 'batchDisable' | 'batchDelete', kbId)
}
const handleCreateKnowledgeBase = () => {
uiStore.openCreateKB()
}
@@ -1097,6 +1212,9 @@ const handleCreateKnowledgeBase = () => {
.kb-action-content {
flex: 1;
min-width: 0;
display: flex;
align-items: center;
white-space: nowrap;
}
.kb-action-title {
@@ -1104,10 +1222,21 @@ const handleCreateKnowledgeBase = () => {
font-weight: 500;
color: #0f172a;
transition: color 0.08s ease;
display: block;
display: inline;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
flex-shrink: 1;
min-width: 0;
}
.kb-action-count {
font-size: 12px;
color: #10b981;
font-weight: 600;
margin-left: 4px;
flex-shrink: 0;
white-space: nowrap;
}
.menu_box {
frontend/src/i18n/locales/en-US.ts
+4 -1
View File
@@ -754,6 +754,8 @@ export default {
knowledgeGraph: 'Knowledge Graph Enabled',
multimodal: 'Multimodal Enabled'
},
processing: 'Processing import task',
processingDocuments: 'Processing {count} documents',
stats: {
documents: 'Document Count',
faqEntries: 'FAQ Entries',
@@ -863,6 +865,7 @@ export default {
statusEnableSuccess: 'FAQ entry enabled',
statusDisableSuccess: 'FAQ entry disabled',
statusUpdateFailed: 'Failed to update status',
batchOperations: 'Batch Operations',
batchUpdateTag: 'Batch Update Category',
batchUpdateTagTip: 'Set category for {count} selected entries',
batchEnable: 'Batch Enable',
@@ -874,7 +877,7 @@ export default {
appendMode: 'Append',
replaceMode: 'Replace existing entries',
fileLabel: 'Select File',
fileTip: 'Supports JSON / CSV / Excel with fields standard_question, answers, similar_questions, negative_questions',
fileTip: 'Supports JSON / CSV / Excel. CSV/Excel headers: 分类(必填), 问题(必填), 相似问题(选填-多个用##分隔), 反例问题(选填-多个用##分隔), 机器人回答(必填-多个用##分隔), 是否全部回复(选填-默认FALSE), 是否停用(选填-默认FALSE), 是否禁止被推荐(选填-默认False 可被推荐). Also supports old format: standard_question, answers, similar_questions, negative_questions',
clickToUpload: 'Click to upload file',
dragDropTip: 'or drag and drop file here',
importButton: 'Import FAQ',
frontend/src/i18n/locales/ru-RU.ts
+4 -1
View File
@@ -842,6 +842,8 @@ export default {
knowledgeGraph: 'Граф знаний включен',
multimodal: 'Мультимодальность включена'
},
processing: 'Обработка задачи импорта',
processingDocuments: 'Обработка {count} документов',
stats: {
documents: 'Количество документов',
faqEntries: 'FAQ записи',
@@ -951,6 +953,7 @@ export default {
statusEnableSuccess: 'Запись FAQ включена',
statusDisableSuccess: 'Запись FAQ отключена',
statusUpdateFailed: 'Не удалось обновить статус',
batchOperations: 'Пакетные операции',
batchUpdateTag: 'Пакетное обновление категории',
batchUpdateTagTip: 'Установить категорию для {count} выбранных записей',
batchEnable: 'Пакетное включение',
@@ -962,7 +965,7 @@ export default {
appendMode: 'Добавить',
replaceMode: 'Заменить существующие записи',
fileLabel: 'Выберите файл',
fileTip: 'Поддерживаются JSON / CSV / Excel. Поля: standard_question, answers, similar_questions, negative_questions',
fileTip: 'Поддерживаются JSON / CSV / Excel. Заголовки CSV/Excel: 分类(必填), 问题(必填), 相似问题(选填-多个用##分隔), 反例问题(选填-多个用##分隔), 机器人回答(必填-多个用##分隔), 是否全部回复(选填-默认FALSE), 是否停用(选填-默认FALSE), 是否禁止被推荐(选填-默认False 可被推荐). Также поддерживается старый формат: standard_question, answers, similar_questions, negative_questions',
clickToUpload: 'Нажмите для загрузки файла',
dragDropTip: 'или перетащите файл сюда',
importButton: 'Импортировать FAQ',
frontend/src/i18n/locales/zh-CN.ts
+4 -1
View File
@@ -1077,6 +1077,8 @@ export default {
knowledgeGraph: "已启用知识图谱",
multimodal: "已启用多模态",
},
processing: "正在处理导入任务",
processingDocuments: "正在处理 {count} 个文档",
stats: {
documents: "文档数量",
faqEntries: "问答条目",
@@ -1190,6 +1192,7 @@ export default {
statusEnableSuccess: "FAQ 条目已启用",
statusDisableSuccess: "FAQ 条目已禁用",
statusUpdateFailed: "更新状态失败",
batchOperations: "批量操作",
batchUpdateTag: "批量分类",
batchUpdateTagTip: "将为 {count} 个选中的条目设置分类",
batchEnable: "批量启用",
@@ -1201,7 +1204,7 @@ export default {
appendMode: "追加导入",
replaceMode: "替换现有条目",
fileLabel: "选择文件",
fileTip: "支持 JSON / CSV / Excel，字段包含 standard_question、answers、similar_questions、negative_questions",
fileTip: "支持 JSON / CSV / Excel。CSV/Excel 表头：分类(必填)、问题(必填)、相似问题(选填-多个用##分隔)、反例问题(选填-多个用##分隔)、机器人回答(必填-多个用##分隔)、是否全部回复(选填-默认FALSE)、是否停用(选填-默认FALSE)、是否禁止被推荐(选填-默认False 可被推荐)。也支持旧格式：standard_question、answers、similar_questions、negative_questions",
clickToUpload: "点击上传文件",
dragDropTip: "或拖拽文件到此处",
importButton: "导入 FAQ",
frontend/src/types/tool-results.ts
+1 -1
View File
@@ -207,7 +207,7 @@ export interface GrepChunkItem {
// Grep results data
export interface GrepResultsData {
display_type: 'grep_results';
pattern: string;
patterns: string[];
case_sensitive: boolean;
use_regex: boolean;
results: GrepChunkItem[];
frontend/src/views/chat/components/AgentStreamDisplay.vue
+146 -69
View File
@@ -76,7 +76,7 @@
'action-error': event.success === false
}"
>
<div class="action-header" @click="toggleEvent(event.tool_call_id)">
<div class="action-header" @click="handleActionHeaderClick(event)" :class="{ 'no-results': !hasResults(event) }">
<div class="action-title">
<img v-if="event.tool_name && !isBookIcon(event.tool_name)" class="action-title-icon" :src="getToolIcon(event.tool_name)" alt="" />
<t-icon v-if="event.tool_name && isBookIcon(event.tool_name)" class="action-title-icon" name="book" />
@@ -91,7 +91,7 @@
<span class="action-name">{{ getToolTitle(event) }}</span>
</t-tooltip>
</div>
<div v-if="!event.pending" class="action-show-icon">
<div v-if="!event.pending && hasResults(event)" class="action-show-icon">
<t-icon :name="isEventExpanded(event.tool_call_id) ? 'chevron-up' : 'chevron-down'" />
</div>
</div>
@@ -109,7 +109,7 @@
<!-- Search Results Summary (Fixed, always visible, outside action-details) -->
<div v-if="!event.pending && (event.tool_name === 'search_knowledge' || event.tool_name === 'knowledge_search') && event.tool_data" class="search-results-summary-fixed">
<div class="results-summary-text" v-html="getSearchResultsSummary(event.tool_data)"></div>
<div class="results-summary-text" v-html="getSearchResultsSummary(event)"></div>
</div>
<!-- Web Search Results Summary (Fixed, always visible, outside action-details) -->
@@ -122,7 +122,7 @@
<div class="results-summary-text" v-html="getGrepResultsSummary(event.tool_data)"></div>
</div>
<div v-if="isEventExpanded(event.tool_call_id) && !event.pending" class="action-details">
<div v-if="isEventExpanded(event.tool_call_id) && !event.pending && hasResults(event)" class="action-details">
<!-- Thinking tool: only render markdown thought content -->
<template v-if="event.tool_name === 'thinking' && event.tool_data?.thought">
<div class="thinking-thought-content">
@@ -676,10 +676,45 @@ const toggleEvent = (eventId: string) => {
}
};
const handleActionHeaderClick = (event: any) => {
if (hasResults(event) && event.tool_call_id) {
toggleEvent(event.tool_call_id);
}
};
const isEventExpanded = (eventId: string): boolean => {
return expandedEvents.value.has(eventId);
};
// Check if search/grep tools have results
const hasResults = (event: any): boolean => {
if (!event || !event.tool_data) return true; // Default to true for other tools
const toolName = event.tool_name;
// For knowledge search tools
if (toolName === 'search_knowledge' || toolName === 'knowledge_search') {
const count = event.tool_data.results?.length || event.tool_data.count || 0;
return count > 0;
}
// For web search tools
if (toolName === 'web_search') {
const count = event.tool_data.results?.length || event.tool_data.count || 0;
return count > 0;
}
// For grep tools
if (toolName === 'grep_chunks') {
const totalMatches = event.tool_data.total_matches || 0;
const resultCount = event.tool_data.result_count || 0;
return totalMatches > 0 || resultCount > 0;
}
// For other tools, always allow expansion
return true;
};
// Delegated handlers for span-based citation clicks/keyboard
const handleCitationActivate = (el: HTMLElement) => {
const url = el.getAttribute('data-url');
@@ -1215,17 +1250,22 @@ const getToolIcon = (toolName: string): string => {
};
// Get search results summary text (returns HTML with colored numbers)
const getSearchResultsSummary = (toolData: any): string => {
if (!toolData) return '';
const getSearchResultsSummary = (event: any): string => {
if (!event || !event.tool_data) return '';
const toolData = event.tool_data;
const count = toolData.results?.length || toolData.count || 0;
if (count === 0) return '';
if (count === 0) return `未找到匹配的内容`;
// Build summary text
let summary = '';
const kbCount = toolData.kb_counts ? Object.keys(toolData.kb_counts).length : 0;
if (kbCount > 0) {
return `找到 <strong>${count}</strong> 个结果，来自 <strong>${kbCount}</strong> 个知识库`;
summary = `找到 <strong>${count}</strong> 个结果，来自 <strong>${kbCount}</strong> 个文件`;
} else {
summary = `找到 <strong>${count}</strong> 个结果`;
}
return `找到 <strong>${count}</strong> 个结果`;
return summary;
};
// Get web search results summary text
@@ -1248,12 +1288,11 @@ const getResultsCount = (toolData: any): number => {
const getGrepResultsSummary = (toolData: any): string => {
if (!toolData) return '';
const pattern = toolData.pattern || '';
const totalMatches = toolData.total_matches || 0;
const resultCount = toolData.result_count || 0;
if (totalMatches === 0) {
return `未找到匹配 <code>${pattern}</code> 的内容`;
return '未找到匹配的内容';
}
let summary = `找到 <strong>${totalMatches}</strong> 处匹配`;
@@ -1288,28 +1327,15 @@ const getQueryText = (args: any): string => {
}
// Add vector_queries if exists
if (Array.isArray(parsedArgs.vector_queries) && parsedArgs.vector_queries.length > 0) {
const vectorQueries = parsedArgs.vector_queries
if (Array.isArray(parsedArgs.queries) && parsedArgs.queries.length > 0) {
queries.push(...parsedArgs.queries
.filter((q: any) => q && typeof q === 'string')
.join(' ');
if (vectorQueries) {
queries.push(vectorQueries);
}
);
}
// Add keyword_queries if exists
if (Array.isArray(parsedArgs.keyword_queries) && parsedArgs.keyword_queries.length > 0) {
const keywordQueries = parsedArgs.keyword_queries
.filter((q: any) => q && typeof q === 'string')
.join(' ');
if (keywordQueries) {
queries.push(keywordQueries);
}
}
// Join all queries with space and remove duplicates
// Join all queries with comma and remove duplicates
const uniqueQueries = Array.from(new Set(queries));
return uniqueQueries.join(' ');
return uniqueQueries.join('，');
};
// Get tool title - prefer summary over description, add query for search tools
@@ -1322,6 +1348,7 @@ const getToolTitle = (event: any): string => {
const toolName = event.tool_name;
const isSearchTool = toolName === 'search_knowledge' || toolName === 'knowledge_search';
const isWebSearchTool = toolName === 'web_search';
const isGrepTool = toolName === 'grep_chunks';
// For search tools, use description with query text
if (isSearchTool) {
@@ -1341,9 +1368,21 @@ const getToolTitle = (event: any): string => {
// Try to get query from arguments or tool_data
let queryText = '';
if (event.arguments && typeof event.arguments === 'object' && event.arguments.query) {
queryText = event.arguments.query;
const query = event.arguments.query;
// Handle both string and array formats
if (Array.isArray(query)) {
queryText = query.filter((q: any) => q && typeof q === 'string').join('，');
} else if (typeof query === 'string') {
queryText = query;
}
} else if (event.tool_data && event.tool_data.query) {
queryText = event.tool_data.query;
const query = event.tool_data.query;
// Handle both string and array formats
if (Array.isArray(query)) {
queryText = query.filter((q: any) => q && typeof q === 'string').join('，');
} else if (typeof query === 'string') {
queryText = query;
}
}
if (queryText) {
return `${baseTitle}：「${queryText}」`;
@@ -1351,6 +1390,34 @@ const getToolTitle = (event: any): string => {
return baseTitle;
}
// For grep tools, use description with patterns
if (isGrepTool) {
const baseTitle = getToolDescription(event);
// Try to get patterns from arguments or tool_data
let patterns: string[] = [];
if (event.arguments && typeof event.arguments === 'object') {
if (Array.isArray(event.arguments.patterns)) {
patterns = event.arguments.patterns;
} else if (event.arguments.pattern) {
patterns = [event.arguments.pattern];
}
} else if (event.tool_data) {
if (Array.isArray(event.tool_data.patterns)) {
patterns = event.tool_data.patterns;
} else if (event.tool_data.pattern) {
patterns = [event.tool_data.pattern];
}
}
if (patterns.length > 0) {
// Show up to 2 patterns in title
const displayPatterns = patterns.slice(0, 2);
const patternText = displayPatterns.join('、');
const moreText = patterns.length > 2 ? ` +${patterns.length - 2}` : '';
return `${baseTitle}：「${patternText}${moreText}」`;
}
return baseTitle;
}
// Use tool summary if available
const summary = getToolSummary(event);
return summary || getToolDescription(event);
@@ -1514,80 +1581,93 @@ const handleAddToKnowledge = (answerEvent: any) => {
// 时间轴连线容器
.event-item {
position: relative;
padding-left: 28px;
padding-left: 32px;
margin-bottom: 12px;
// 时间轴垂直线
&::before {
content: '';
position: absolute;
left: 8px;
left: 10px;
top: 0;
bottom: -12px;
width: 2px;
background: linear-gradient(to bottom, #e5e7eb 0%, #e5e7eb 100%);
width: 1.5px;
background: linear-gradient(
to bottom,
rgba(7, 192, 95, 0.1) 0%,
rgba(7, 192, 95, 0.15) 50%,
rgba(7, 192, 95, 0.1) 100%
);
z-index: 0;
}
// 第一个事件的连线从节点开始
&:first-child::before {
top: 12px;
top: 14px;
}
// 最后一个事件不显示底部连线
&.event-last::before {
bottom: auto;
height: 20px;
height: 22px;
}
// 时间轴节点（圆点）
&::after {
content: '';
position: absolute;
left: 4px;
top: 12px;
width: 10px;
height: 10px;
left: 6.25px; // 线条中心 10.75px - 圆点半径 4.5px = 6.25px (box-sizing: border-box)
top: 14px;
width: 9px;
height: 9px;
border-radius: 50%;
background: #ffffff;
border: 2px solid #e5e7eb;
border: 2px solid rgba(7, 192, 95, 0.3);
z-index: 1;
transition: all 0.2s ease;
transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1);
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
box-sizing: border-box; // 确保 border 包含在尺寸内
}
// 不同事件类型的节点颜色
&:has(.thinking-event)::after {
border-color: #9ca3af;
border-color: rgba(156, 163, 175, 0.4);
background: #f9fafb;
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
}
&:has(.answer-event)::after {
border-color: #07c05f;
background: #07c05f;
box-shadow: 0 0 0 3px rgba(7, 192, 95, 0.15);
box-shadow: 0 0 0 2px rgba(7, 192, 95, 0.12), 0 2px 4px rgba(7, 192, 95, 0.2);
transform: scale(1.1);
}
&:has(.tool-event)::after {
border-color: #07c05f;
background: #ffffff;
box-shadow: 0 1px 3px rgba(7, 192, 95, 0.15);
}
&:has(.tool-event .action-pending)::after {
border-color: #07c05f;
background: rgba(7, 192, 95, 0.2);
background: rgba(7, 192, 95, 0.15);
box-shadow: 0 0 0 2px rgba(7, 192, 95, 0.1);
animation: pulseNode 2s ease-in-out infinite;
}
&:has(.tool-event .action-error)::after {
border-color: #e34d59;
background: #e34d59;
box-shadow: 0 0 0 2px rgba(227, 77, 89, 0.15), 0 2px 4px rgba(227, 77, 89, 0.2);
}
&:has(.plan-task-change-event)::after {
border-color: #07c05f;
background: #07c05f;
transform: rotate(45deg);
transform: rotate(45deg) scale(0.9);
border-radius: 2px;
box-shadow: 0 1px 3px rgba(7, 192, 95, 0.2);
}
}
@@ -1606,21 +1686,6 @@ const handleAddToKnowledge = (answerEvent: any) => {
margin-bottom: 16px;
position: relative;
// 添加时间轴起点标记
&::before {
content: '';
position: absolute;
left: -20px;
top: 12px;
width: 10px;
height: 10px;
border-radius: 50%;
background: #07c05f;
border: 2px solid #ffffff;
box-shadow: 0 0 0 2px #07c05f, 0 0 0 4px rgba(7, 192, 95, 0.1);
z-index: 2;
}
.intermediate-steps-header {
display: flex;
justify-content: space-between;
@@ -1630,6 +1695,10 @@ const handleAddToKnowledge = (answerEvent: any) => {
font-weight: 500;
cursor: pointer;
background: linear-gradient(to right, rgba(7, 192, 95, 0.03), transparent);
&:hover {
background: linear-gradient(to right, rgba(7, 192, 95, 0.05), rgba(7, 192, 95, 0.01));
}
}
.intermediate-steps-title {
@@ -1658,10 +1727,6 @@ const handleAddToKnowledge = (answerEvent: any) => {
padding: 0 2px 1px 2px;
color: #07c05f;
}
.intermediate-steps-header:hover {
background: linear-gradient(to right, rgba(7, 192, 95, 0.05), rgba(7, 192, 95, 0.01));
}
}
// Thinking Event
@@ -1700,7 +1765,7 @@ const handleAddToKnowledge = (answerEvent: any) => {
&.markdown-content {
:deep(p) {
margin: 6px 0;
margin: 0 0;
line-height: 1.6;
}
@@ -2079,6 +2144,14 @@ const handleAddToKnowledge = (answerEvent: any) => {
&:hover {
background-color: rgba(7, 192, 95, 0.03);
}
&.no-results {
cursor: default;
&:hover {
background-color: transparent;
}
}
}
.action-title {
@@ -2151,11 +2224,15 @@ const handleAddToKnowledge = (answerEvent: any) => {
@keyframes pulseNode {
0%, 100% {
border-color: #07c05f;
box-shadow: 0 0 0 0 rgba(7, 192, 95, 0.4);
background: rgba(7, 192, 95, 0.15);
box-shadow: 0 0 0 2px rgba(7, 192, 95, 0.1);
transform: scale(1);
}
50% {
border-color: #0ae06f;
box-shadow: 0 0 0 4px rgba(7, 192, 95, 0.1);
background: rgba(7, 192, 95, 0.25);
box-shadow: 0 0 0 3px rgba(7, 192, 95, 0.15);
transform: scale(1.05);
}
}
frontend/src/views/chat/components/tool-results/GrepResults.vue
+185
View File
@@ -0,0 +1,185 @@
<template>
<div class="grep-results">
<!-- Results List -->
<div v-if="results && results.length > 0" class="results-list">
<div
v-for="(result, index) in results"
:key="result.id"
class="result-item"
>
<t-popup
:overlayClassName="`grep-popup-${result.id}`"
placement="bottom-left"
width="400"
:showArrow="false"
trigger="click"
destroy-on-close
>
<template #content>
<ContentPopup
:content="result.content"
:chunk-id="result.id"
:knowledge-id="result.knowledge_id"
/>
</template>
<div class="result-header">
<div class="result-title">
<span class="result-index">#{{ index + 1 }}</span>
<span class="knowledge-title">{{ result.knowledge_title || 'Untitled' }}</span>
</div>
</div>
</t-popup>
</div>
</div>
<!-- Empty State -->
<div v-else class="empty-state">
未找到匹配的内容
</div>
</div>
</template>
<script setup lang="ts">
import { computed } from 'vue';
import type { GrepResultsData } from '@/types/tool-results';
import ContentPopup from './ContentPopup.vue';
const props = defineProps<{
data: GrepResultsData;
}>();
const maxDisplayPatterns = 2;
const displayPatterns = computed(() => {
if (!props.data.patterns || props.data.patterns.length === 0) {
return [];
}
return props.data.patterns.slice(0, maxDisplayPatterns);
});
const results = computed(() => props.data.results || []);
</script>
<style lang="less" scoped>
@import './tool-results.less';
.grep-results {
display: flex;
flex-direction: column;
gap: 3px;
padding: 0 0 0 12px;
}
.results-list {
display: flex;
flex-direction: column;
gap: 3px;
}
.result-item {
background: transparent;
border: none;
border-radius: 0;
overflow: visible;
}
.result-header {
padding: 2px 0;
cursor: pointer;
user-select: none;
display: flex;
align-items: center;
gap: 6px;
transition: color 0.15s ease;
&:hover {
color: #07c05f;
}
}
.result-title {
display: flex;
align-items: center;
gap: 6px;
flex: 1;
min-width: 0;
font-size: 12px;
line-height: 1.4;
}
.result-index {
font-size: 11px;
color: #9ca3af;
font-weight: 600;
flex-shrink: 0;
}
.pattern-badge {
display: inline-flex;
align-items: center;
gap: 3px;
flex-shrink: 0;
.pattern-text {
font-family: 'Monaco', 'Menlo', 'Courier New', monospace;
font-size: 10px;
background: #f3f4f6;
color: #6b7280;
padding: 2px 5px;
border-radius: 3px;
white-space: nowrap;
font-weight: 500;
}
.more-patterns {
font-size: 10px;
color: #9ca3af;
padding: 2px 4px;
background: #f3f4f6;
border-radius: 3px;
}
}
.knowledge-title {
font-size: 12px;
color: #374151;
flex: 1;
font-weight: 500;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
min-width: 0;
}
.chunk-info {
font-size: 10px;
color: #9ca3af;
background: #f3f4f6;
padding: 2px 5px;
border-radius: 3px;
flex-shrink: 0;
}
.empty-state {
padding: 20px;
text-align: center;
color: #9ca3af;
font-size: 12px;
font-style: italic;
}
// Popup overlay styles
:deep([class*="grep-popup-"]) {
.t-popup__content {
max-height: 400px;
max-width: 500px;
overflow-y: auto;
overflow-x: hidden;
padding: 0;
border-radius: 6px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.12);
word-wrap: break-word;
word-break: break-word;
}
}
</style>
frontend/src/views/chat/components/tool-results/SearchResults.vue
-3
View File
@@ -25,9 +25,6 @@
<div class="result-header">
<div class="result-title">
<span class="result-index">#{{ result.result_index }}</span>
<span class="relevance-badge" :class="getRelevanceClass(result.relevance_level)">
{{ getRelevanceLabel(result.relevance_level) }}
</span>
<span class="knowledge-title">{{ result.knowledge_title }}</span>
</div>
</div>
frontend/src/views/creatChat/creatChat.vue
+22
View File
@@ -7,6 +7,16 @@
<InputField @send-msg="sendMsg"></InputField>
</div>
</div>
<!-- 知识库编辑器（创建/编辑统一组件） -->
<KnowledgeBaseEditorModal
:visible="uiStore.showKBEditorModal"
:mode="uiStore.kbEditorMode"
:kb-id="uiStore.currentKBId || undefined"
:initial-type="uiStore.kbEditorType"
@update:visible="(val) => val ? null : uiStore.closeKBEditor()"
@success="handleKBEditorSuccess"
/>
</template>
<script setup lang="ts">
import { ref } from 'vue';
@@ -14,14 +24,17 @@ import InputField from '@/components/Input-field.vue';
import { createSessions } from "@/api/chat/index";
import { useMenuStore } from '@/stores/menu';
import { useSettingsStore } from '@/stores/settings';
import { useUIStore } from '@/stores/ui';
import { useRoute, useRouter } from 'vue-router';
import { MessagePlugin } from 'tdesign-vue-next';
import { useI18n } from 'vue-i18n';
import KnowledgeBaseEditorModal from '@/views/knowledge/KnowledgeBaseEditorModal.vue';
const router = useRouter();
const route = useRoute();
const usemenuStore = useMenuStore();
const settingsStore = useSettingsStore();
const uiStore = useUIStore();
const { t } = useI18n();
const sendMsg = (value: string) => {
@@ -79,6 +92,15 @@ const navigateToSession = async (sessionId: string, value: string) => {
router.push(`/platform/chat/${sessionId}`);
}
const handleKBEditorSuccess = (kbId: string) => {
console.log('[creatChat] knowledge base created successfully:', kbId)
// 创建成功后跳转到知识库列表页面，并高亮新创建的知识库
router.push({
path: '/platform/knowledge-bases',
query: { highlightKbId: kbId }
})
}
</script>
<style lang="less" scoped>
.dialogue-wrap {
frontend/src/views/knowledge/KnowledgeBase.vue
+304
View File
@@ -45,6 +45,15 @@ let knowledgeIndex = ref(-1)
let knowledgeScroll = ref()
let page = 1;
let pageSize = 35;
// 文档处理进度条状态
const documentProcessingState = reactive({
processingIds: [] as string[],
total: 0,
completed: 0,
failed: 0,
pollingInterval: null as ReturnType<typeof setInterval> | null,
})
const selectedTagId = ref<string>("");
const tagList = ref<any[]>([]);
const tagLoading = ref(false);
@@ -396,6 +405,15 @@ const handleFileUploaded = (event: CustomEvent) => {
// 如果上传的文件属于当前知识库，使用 loadKnowledgeFiles 刷新文件列表
loadKnowledgeFiles(uploadedKbId);
loadTags(uploadedKbId);
// 延迟一下，等待文件列表加载完成后再检查处理状态
setTimeout(() => {
const processingList = cardList.value.filter(item =>
item.parse_status === 'pending' || item.parse_status === 'processing'
)
if (processingList.length > 0) {
updateDocumentProcessingState(processingList)
}
}, 500)
}
};
@@ -416,11 +434,17 @@ onMounted(() => {
window.addEventListener('knowledgeFileUploaded', handleFileUploaded as EventListener);
// 监听URL导入对话框打开事件
window.addEventListener('openURLImportDialog', handleOpenURLImportDialog as EventListener);
// 恢复文档处理状态
if (!isFAQ.value) {
restoreDocumentProcessingState()
}
});
onUnmounted(() => {
window.removeEventListener('knowledgeFileUploaded', handleFileUploaded as EventListener);
window.removeEventListener('openURLImportDialog', handleOpenURLImportDialog as EventListener);
stopDocumentProcessingPolling()
});
watch(() => cardList.value, (newValue) => {
if (isFAQ.value) return;
@@ -435,6 +459,9 @@ watch(() => cardList.value, (newValue) => {
if (analyzeList.length) {
updateStatus(analyzeList)
}
// 更新文档处理进度条状态
updateDocumentProcessingState(analyzeList)
}, { deep: true })
type KnowledgeCard = {
id: string;
@@ -477,6 +504,168 @@ const updateStatus = (analyzeList: KnowledgeCard[]) => {
}, 1500);
};
// 更新文档处理进度条状态
const updateDocumentProcessingState = (processingList: KnowledgeCard[]) => {
const processingIds = processingList.map(item => item.id)
const hasChanged = JSON.stringify(processingIds.sort()) !== JSON.stringify(documentProcessingState.processingIds.sort())
if (hasChanged) {
documentProcessingState.processingIds = processingIds
documentProcessingState.total = processingIds.length
documentProcessingState.completed = 0
documentProcessingState.failed = 0
// 保存到 localStorage
saveProcessingIdsToStorage(processingIds)
// 开始轮询
if (processingIds.length > 0) {
startDocumentProcessingPolling()
} else {
stopDocumentProcessingPolling()
}
}
}
// 开始轮询文档处理状态
const startDocumentProcessingPolling = () => {
stopDocumentProcessingPolling()
if (documentProcessingState.processingIds.length === 0) return
documentProcessingState.pollingInterval = setInterval(() => {
if (documentProcessingState.processingIds.length === 0) {
stopDocumentProcessingPolling()
return
}
let query = ``
documentProcessingState.processingIds.forEach(id => {
query += `ids=${id}&`
})
batchQueryKnowledge(query).then((result: any) => {
if (result.success && result.data) {
const completedIds: string[] = []
const failedIds: string[] = []
;(result.data as KnowledgeCard[]).forEach((item: KnowledgeCard) => {
if (item.parse_status === 'completed') {
completedIds.push(item.id)
documentProcessingState.completed++
} else if (item.parse_status === 'failed') {
failedIds.push(item.id)
documentProcessingState.failed++
}
})
// 从处理列表中移除已完成的文档
documentProcessingState.processingIds = documentProcessingState.processingIds.filter(
id => !completedIds.includes(id) && !failedIds.includes(id)
)
// 如果所有文档都处理完成，停止轮询
if (documentProcessingState.processingIds.length === 0) {
stopDocumentProcessingPolling()
clearProcessingIdsFromStorage()
// 刷新文件列表
if (kbId.value) {
loadKnowledgeFiles(kbId.value)
}
} else {
// 更新 localStorage
saveProcessingIdsToStorage(documentProcessingState.processingIds)
}
}
}).catch((_err) => {
// 错误处理
})
}, 2000)
}
// 停止轮询文档处理状态
const stopDocumentProcessingPolling = () => {
if (documentProcessingState.pollingInterval) {
clearInterval(documentProcessingState.pollingInterval)
documentProcessingState.pollingInterval = null
}
}
// localStorage 相关函数
const getProcessingIdsStorageKey = () => {
return `document_processing_ids_${kbId.value}`
}
const saveProcessingIdsToStorage = (ids: string[]) => {
if (!kbId.value) return
try {
localStorage.setItem(getProcessingIdsStorageKey(), JSON.stringify(ids))
} catch (error) {
console.error('Failed to save processing IDs to localStorage:', error)
}
}
const getProcessingIdsFromStorage = (): string[] => {
if (!kbId.value) return []
try {
const data = localStorage.getItem(getProcessingIdsStorageKey())
return data ? JSON.parse(data) : []
} catch (error) {
console.error('Failed to get processing IDs from localStorage:', error)
return []
}
}
const clearProcessingIdsFromStorage = () => {
if (!kbId.value) return
try {
localStorage.removeItem(getProcessingIdsStorageKey())
} catch (error) {
console.error('Failed to clear processing IDs from localStorage:', error)
}
}
// 恢复文档处理状态（用于刷新后恢复）
const restoreDocumentProcessingState = async () => {
if (!kbId.value || isFAQ.value) return
const savedIds = getProcessingIdsFromStorage()
if (savedIds.length === 0) return
// 检查这些文档是否还在处理中
let query = ``
savedIds.forEach(id => {
query += `ids=${id}&`
})
try {
const result: any = await batchQueryKnowledge(query)
if (result.success && result.data) {
const stillProcessing: string[] = []
;(result.data as KnowledgeCard[]).forEach((item: KnowledgeCard) => {
if (item.parse_status === 'pending' || item.parse_status === 'processing') {
stillProcessing.push(item.id)
}
})
if (stillProcessing.length > 0) {
documentProcessingState.processingIds = stillProcessing
documentProcessingState.total = stillProcessing.length
documentProcessingState.completed = 0
documentProcessingState.failed = 0
saveProcessingIdsToStorage(stillProcessing)
startDocumentProcessingPolling()
} else {
clearProcessingIdsFromStorage()
}
}
} catch (error) {
console.error('Failed to restore document processing state:', error)
clearProcessingIdsFromStorage()
}
}
const closeDoc = () => {
isCardDetails.value = false;
};
@@ -520,6 +709,16 @@ const ensureDocumentKbReady = () => {
return true;
};
// 关闭文档处理进度条
const handleCloseDocumentProgress = () => {
stopDocumentProcessingPolling()
documentProcessingState.processingIds = []
documentProcessingState.total = 0
documentProcessingState.completed = 0
documentProcessingState.failed = 0
clearProcessingIdsFromStorage()
}
const handleDocumentUploadClick = () => {
if (!ensureDocumentKbReady()) return;
uploadInputRef.value?.click();
@@ -818,6 +1017,41 @@ async function createNewSession(value: string): Promise<void> {
<p class="document-subtitle">{{ $t('knowledgeEditor.document.subtitle') }}</p>
</div>
</div>
<!-- 文档处理进度条 -->
<div v-if="!isFAQ && documentProcessingState.processingIds.length > 0" class="document-processing-progress-bar">
<div class="progress-bar-content">
<div class="progress-bar-header">
<t-icon
name="loading"
size="16px"
class="progress-icon icon-loading"
/>
<span class="progress-title">
{{ $t('knowledgeList.processingDocuments', { count: documentProcessingState.processingIds.length }) }}
</span>
<span class="progress-count">
{{ documentProcessingState.completed + documentProcessingState.failed }}/{{ documentProcessingState.total }}
</span>
<t-button
variant="text"
theme="default"
size="small"
class="progress-close-btn"
@click="handleCloseDocumentProgress"
>
<t-icon name="close" size="14px" />
</t-button>
</div>
<t-progress
:percentage="Math.round(((documentProcessingState.completed + documentProcessingState.failed) / documentProcessingState.total) * 100)"
:status="documentProcessingState.failed > 0 ? 'error' : 'active'"
:label="false"
class="progress-bar"
/>
</div>
</div>
<input
ref="uploadInputRef"
type="file"
@@ -1872,6 +2106,76 @@ async function createNewSession(value: string): Promise<void> {
min-height: 100%;
}
.document-processing-progress-bar {
margin-bottom: 16px;
background: #fff;
border: 1px solid #e7ebf0;
border-radius: 8px;
padding: 12px 16px;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.04);
.progress-bar-content {
display: flex;
flex-direction: column;
gap: 8px;
}
.progress-bar-header {
display: flex;
align-items: center;
gap: 8px;
font-size: 14px;
color: #000000e6;
.progress-icon {
flex-shrink: 0;
&.icon-loading {
animation: rotate 1s linear infinite;
color: #07c05f;
}
}
.progress-title {
font-weight: 500;
flex: 1;
}
.progress-count {
color: #86909c;
font-size: 13px;
}
.progress-close-btn {
flex-shrink: 0;
padding: 4px;
margin-left: 8px;
}
}
.progress-bar {
margin: 0;
width: 100%;
:deep(.t-progress) {
width: 100%;
}
:deep(.t-progress__bar) {
width: 100%;
}
}
}
@keyframes rotate {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
:deep(.del-knowledge) {
padding: 0px !important;
border-radius: 6px !important;
frontend/src/views/knowledge/KnowledgeBaseList.vue
+116 -9
View File
@@ -24,8 +24,10 @@
:class="{
'uninitialized': !isInitialized(kb),
'kb-type-document': (kb.type || 'document') === 'document',
'kb-type-faq': kb.type === 'faq'
'kb-type-faq': kb.type === 'faq',
'highlight-flash': highlightedKbId !== null && highlightedKbId === kb.id
}"
:ref="el => { if (highlightedKbId !== null && highlightedKbId === kb.id && el) highlightedCardRef = el as HTMLElement }"
@click="handleCardClick(kb)"
>
<!-- 卡片头部 -->
@@ -78,6 +80,15 @@
{{ kb.type === 'faq' ? $t('knowledgeEditor.basic.typeFAQ') : $t('knowledgeEditor.basic.typeDocument') }}
({{ kb.type === 'faq' ? (kb.chunk_count || 0) : (kb.knowledge_count || 0) }})
</span>
<t-tooltip
v-if="kb.isProcessing"
:content="kb.type === 'document' && (kb.processing_count || 0) > 0
? $t('knowledgeList.processingDocuments', { count: kb.processing_count || 0 })
: $t('knowledgeList.processing')"
placement="top"
>
<t-icon name="loading" size="14px" class="processing-icon" />
</t-tooltip>
</div>
<div class="feature-badges">
<t-tooltip v-if="kb.extract_config?.enabled" :content="$t('knowledgeList.features.knowledgeGraph')" placement="top">
@@ -144,9 +155,9 @@
</template>
<script setup lang="ts">
import { onMounted, ref, computed, watch } from 'vue'
import { useRouter } from 'vue-router'
import { MessagePlugin } from 'tdesign-vue-next'
import { onMounted, ref, computed, watch, nextTick } from 'vue'
import { useRouter, useRoute } from 'vue-router'
import { MessagePlugin, Icon as TIcon } from 'tdesign-vue-next'
import { listKnowledgeBases, deleteKnowledgeBase } from '@/api/knowledge-base'
import { formatStringDate } from '@/utils/index'
import { useUIStore } from '@/stores/ui'
@@ -155,6 +166,7 @@ import Settings from '@/views/settings/Settings.vue'
import { useI18n } from 'vue-i18n'
const router = useRouter()
const route = useRoute()
const uiStore = useUIStore()
const { t } = useI18n()
@@ -172,6 +184,8 @@ interface KB {
cos_config?: { provider?: string; bucket_name?: string };
knowledge_count?: number;
chunk_count?: number;
isProcessing?: boolean; // 是否有正在处理的导入任务
processing_count?: number; // 正在处理的文档数量（仅文档类型）
}
const kbs = ref<KB[]>([])
@@ -179,22 +193,43 @@ const loading = ref(false)
const deleteVisible = ref(false)
const deletingKb = ref<KB | null>(null)
const currentMoreIndex = ref<number>(-1)
const highlightedKbId = ref<string | null>(null)
const highlightedCardRef = ref<HTMLElement | null>(null)
const fetchList = () => {
loading.value = true
listKnowledgeBases().then((res: any) => {
return listKnowledgeBases().then((res: any) => {
const data = res.data || []
// 格式化时间，并初始化 showMore 状态
kbs.value = data.map((kb: KB) => ({
// is_processing 字段由后端返回
kbs.value = data.map((kb: any) => ({
...kb,
updated_at: kb.updated_at ? formatStringDate(new Date(kb.updated_at)) : '',
showMore: false
showMore: false,
isProcessing: kb.is_processing || false,
processing_count: kb.processing_count || 0
}))
}).finally(() => loading.value = false)
}
onMounted(() => {
fetchList()
fetchList().then(() => {
// 检查路由参数中是否有需要高亮的知识库ID
const highlightKbId = route.query.highlightKbId as string
if (highlightKbId) {
triggerHighlightFlash(highlightKbId)
// 清除 URL 中的查询参数
router.replace({ query: {} })
}
})
})
// 监听路由变化，处理从其他页面跳转过来的高亮需求
watch(() => route.query.highlightKbId, (newKbId) => {
if (newKbId && typeof newKbId === 'string' && kbs.value.length > 0) {
triggerHighlightFlash(newKbId)
router.replace({ query: {} })
}
})
const openMore = (index: number) => {
@@ -270,7 +305,32 @@ const goSettings = (id: string) => {
// 知识库编辑器成功回调（创建或编辑成功）
const handleKBEditorSuccess = (kbId: string) => {
console.log('[KnowledgeBaseList] knowledge operation success:', kbId)
fetchList()
fetchList().then(() => {
// 如果是从路由参数中获取的高亮ID，触发闪烁效果
if (route.query.highlightKbId === kbId) {
triggerHighlightFlash(kbId)
// 清除 URL 中的查询参数
router.replace({ query: {} })
}
})
}
// 触发高亮闪烁效果
const triggerHighlightFlash = (kbId: string) => {
highlightedKbId.value = kbId
nextTick(() => {
if (highlightedCardRef.value) {
// 滚动到高亮的卡片
highlightedCardRef.value.scrollIntoView({
behavior: 'smooth',
block: 'center'
})
}
// 3秒后清除高亮
setTimeout(() => {
highlightedKbId.value = null
}, 3000)
})
}
</script>
@@ -533,6 +593,19 @@ const handleKBEditorSuccess = (kbId: string) => {
color: #0052d9;
border: 1px solid rgba(0, 82, 217, 0.2);
}
.processing-icon {
animation: spin 1s linear infinite;
margin-left: 4px;
}
&.document .processing-icon {
color: #059669;
}
&.faq .processing-icon {
color: #0052d9;
}
}
.feature-badges {
@@ -569,6 +642,40 @@ const handleKBEditorSuccess = (kbId: string) => {
background: rgba(255, 152, 0, 0.15);
}
}
}
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
@keyframes highlightFlash {
0% {
border-color: #07c05f;
box-shadow: 0 0 0 0 rgba(7, 192, 95, 0.4);
transform: scale(1);
}
50% {
border-color: #07c05f;
box-shadow: 0 0 0 8px rgba(7, 192, 95, 0);
transform: scale(1.02);
}
100% {
border-color: #07c05f;
box-shadow: 0 0 0 0 rgba(7, 192, 95, 0);
transform: scale(1);
}
}
.kb-card.highlight-flash {
animation: highlightFlash 0.6s ease-in-out 3;
border-color: #07c05f !important;
box-shadow: 0 0 12px rgba(7, 192, 95, 0.3) !important;
}
.card-time {
frontend/src/views/knowledge/components/FAQEntryManager.vue
+768 -122
View File
File diff suppressed because it is too large Load Diff
internal/agent/prompts.go
+401 -740
View File
File diff suppressed because it is too large Load Diff
internal/agent/tools/grep_chunks.go
+479
View File
@@ -0,0 +1,479 @@
package tools
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"gorm.io/gorm"
)
// GrepChunksTool performs text pattern matching in knowledge base chunks
// Similar to grep command in Unix-like systems, but operates on knowledge base content
type GrepChunksTool struct {
BaseTool
db *gorm.DB
tenantID uint
knowledgeBaseIDs []string
}
// NewGrepChunksTool creates a new grep chunks tool
func NewGrepChunksTool(db *gorm.DB, tenantID uint, knowledgeBaseIDs []string) *GrepChunksTool {
description := `Unix-style text pattern matching tool for knowledge base chunks.
Searches for text patterns in chunk content, similar to the Unix grep command.
## Core Function
Performs **exact text pattern matching** (NOT semantic search). Finds chunks containing any of the specified patterns using literal text matching (fixed string). Supports multiple patterns with OR logic.
## CRITICAL - Keyword Granularity
**MUST use SHORT keywords (1-3 words), NOT long phrases.** Break down long phrases into smaller keywords for better match rate.
**Why**: Long phrases like "中国饮食文化" may not match if the document contains "中国" and "饮食" and "文化" separately but not the exact phrase.
**Guidelines**:
- ❌ **Bad**: ["中国饮食文化", "日本饮食文化"] - too long, may not match
- ✅ **Good**: ["中国", "饮食", "文化", "日本", "料理", "和食"] - short keywords, higher match rate
- For comparisons: Extract keywords for each entity separately
- Use single words or 2-word phrases, avoid 3+ word phrases
**Examples**:
- "中国饮食文化" → ["中国", "饮食", "文化", "中餐", "中华"]
- "日本饮食文化" → ["日本", "饮食", "文化", "料理", "和食", "日式"]
- "对比中国和日本" → Search separately: ["中国", "中华"] then ["日本", "日式"]
## Usage
grep_chunks searches through all enabled chunks in the knowledge base(s) and displays matching chunks with context. When multiple patterns are provided, results match any pattern (OR logic).
Default behavior: case-insensitive matching, shows chunk indices, displays context around matches.
## When to Use
- Finding specific entities: "FAISS", "Redis", "404", "RAG"
- Exact keyword lookup
- Quick text search before semantic search
## Examples
- Single pattern: pattern=["FAISS"]
- Multiple patterns: pattern=["向量", "vector", "embedding"]
- Search synonyms: pattern=["RAG", "检索增强生成"]
- Short keywords: pattern=["中国", "饮食", "文化"] (NOT ["中国饮食文化"])
`
return &GrepChunksTool{
BaseTool: NewBaseTool("grep_chunks", description),
db: db,
tenantID: tenantID,
knowledgeBaseIDs: knowledgeBaseIDs,
}
}
// Parameters returns the JSON schema for the tool's parameters
func (t *GrepChunksTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"pattern": map[string]interface{}{
"type": "array",
"description": "REQUIRED: Text patterns to search for. Can be a single pattern or multiple patterns. Treated as literal text (fixed string matching). Results match any of the patterns (OR logic).",
"items": map[string]interface{}{
"type": "string",
},
"minItems": 1,
},
"knowledge_base_ids": map[string]interface{}{
"type": "array",
"description": "Filter by knowledge base IDs. If empty, searches all allowed KBs.",
"items": map[string]interface{}{
"type": "string",
},
},
"knowledge_ids": map[string]interface{}{
"type": "array",
"description": "Filter by document/knowledge IDs. If empty, searches all documents.",
"items": map[string]interface{}{
"type": "string",
},
},
"max_results": map[string]interface{}{
"type": "integer",
"description": "Maximum number of matching chunks to return (default: 50, max: 200)",
"default": 50,
"minimum": 1,
"maximum": 200,
},
},
"required": []string{"pattern"},
}
}
// Execute executes the grep chunks tool
func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
logger.Infof(ctx, "[Tool][GrepChunks] Execute started")
// Parse pattern parameter (required) - support multiple patterns
var patterns []string
if patternsRaw, ok := args["pattern"].([]interface{}); ok && len(patternsRaw) > 0 {
for _, p := range patternsRaw {
if pStr, ok := p.(string); ok && strings.TrimSpace(pStr) != "" {
patterns = append(patterns, strings.TrimSpace(pStr))
}
}
}
// Also support single string for backward compatibility
if len(patterns) == 0 {
if patternStr, ok := args["pattern"].(string); ok && strings.TrimSpace(patternStr) != "" {
patterns = append(patterns, strings.TrimSpace(patternStr))
}
}
if len(patterns) == 0 {
logger.Errorf(ctx, "[Tool][GrepChunks] Missing or invalid pattern parameter")
return &types.ToolResult{
Success: false,
Error: "pattern parameter is required and must contain at least one non-empty pattern",
}, fmt.Errorf("missing pattern parameter")
}
// Use default values for all options
contextLines := 50 // default: 50 context characters
countOnly := false // default: show results
showLineNumbers := true // default: show chunk indices
maxResults := 50
if mr, ok := args["max_results"].(float64); ok {
maxResults = int(mr)
if maxResults < 1 {
maxResults = 1
} else if maxResults > 200 {
maxResults = 200
}
}
// Parse knowledge_base_ids filter
var kbIDs []string
if kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}); ok {
for _, id := range kbIDsRaw {
if idStr, ok := id.(string); ok && idStr != "" {
kbIDs = append(kbIDs, idStr)
}
}
}
if len(kbIDs) == 0 {
kbIDs = t.knowledgeBaseIDs
}
// Parse knowledge_ids filter
var knowledgeIDs []string
if knowledgeIDsRaw, ok := args["knowledge_ids"].([]interface{}); ok {
for _, id := range knowledgeIDsRaw {
if idStr, ok := id.(string); ok && idStr != "" {
knowledgeIDs = append(knowledgeIDs, idStr)
}
}
}
logger.Infof(ctx, "[Tool][GrepChunks] Patterns: %v, MaxResults: %d",
patterns, maxResults)
// Build and execute query
results, totalCount, err := t.searchChunks(ctx, patterns, kbIDs, knowledgeIDs, maxResults)
if err != nil {
logger.Errorf(ctx, "[Tool][GrepChunks] Search failed: %v", err)
return &types.ToolResult{
Success: false,
Error: fmt.Sprintf("Search failed: %v", err),
}, err
}
logger.Infof(ctx, "[Tool][GrepChunks] Found %d matching chunks", len(results))
// Format output
output := t.formatOutput(ctx, results, totalCount, patterns, contextLines, countOnly, showLineNumbers, kbIDs, knowledgeIDs)
return &types.ToolResult{
Success: true,
Output: output,
Data: map[string]interface{}{
"patterns": patterns,
"results": results,
"result_count": len(results),
"total_matches": totalCount,
"knowledge_base_ids": kbIDs,
"knowledge_ids": knowledgeIDs,
"max_results": maxResults,
"display_type": "grep_results",
},
}, nil
}
// searchChunks performs the database search with pattern matching
func (t *GrepChunksTool) searchChunks(
ctx context.Context,
patterns []string,
kbIDs []string,
knowledgeIDs []string,
maxResults int,
) ([]map[string]interface{}, int64, error) {
// Build base query
query := t.db.WithContext(ctx).Table("chunks").
Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, chunks.knowledge_base_id, chunks.chunk_type, chunks.created_at, knowledges.title as knowledge_title").
Joins("LEFT JOIN knowledges ON chunks.knowledge_id = knowledges.id").
Where("chunks.tenant_id = ?", t.tenantID).
Where("chunks.is_enabled = ?", true).
Where("chunks.deleted_at IS NULL").
Where("knowledges.deleted_at IS NULL")
// Apply knowledge base filter
if len(kbIDs) > 0 {
query = query.Where("chunks.knowledge_base_id IN ?", kbIDs)
}
// Apply knowledge filter
if len(knowledgeIDs) > 0 {
query = query.Where("chunks.knowledge_id IN ?", knowledgeIDs)
}
// Apply pattern matching (case-insensitive fixed string matching, OR logic for multiple patterns)
if len(patterns) == 1 {
query = query.Where("chunks.content ILIKE ?", "%"+patterns[0]+"%")
} else {
// Multiple patterns: use OR logic
var conditions []string
var args []interface{}
for _, pattern := range patterns {
conditions = append(conditions, "chunks.content ILIKE ?")
args = append(args, "%"+pattern+"%")
}
query = query.Where("("+strings.Join(conditions, " OR ")+")", args...)
}
// Count total matches first (for count_only mode)
var totalCount int64
if err := query.Count(&totalCount).Error; err != nil {
logger.Warnf(ctx, "[Tool][GrepChunks] Failed to count matches: %v", err)
}
// Fetch results
var results []map[string]interface{}
if err := query.Limit(maxResults).Order("chunks.created_at DESC").Find(&results).Error; err != nil {
logger.Errorf(ctx, "[Tool][GrepChunks] Failed to fetch results: %v", err)
return nil, 0, err
}
return results, totalCount, nil
}
// formatOutput formats the search results for display (grep-style output)
func (t *GrepChunksTool) formatOutput(
ctx context.Context,
results []map[string]interface{},
totalCount int64,
patterns []string,
contextLines int,
countOnly bool,
showLineNumbers bool,
kbIDs []string,
knowledgeIDs []string,
) string {
var output strings.Builder
// If count_only mode, just return the count
if countOnly {
output.WriteString(fmt.Sprintf("%d\n", totalCount))
return output.String()
}
// Show search info
if len(patterns) == 1 {
output.WriteString(fmt.Sprintf("Pattern: '%s' (case-insensitive)\n", patterns[0]))
} else {
output.WriteString(fmt.Sprintf("Patterns (%d): %v (case-insensitive, OR logic)\n", len(patterns), patterns))
}
output.WriteString(fmt.Sprintf("Matches: %d chunk(s)\n\n", len(results)))
if len(results) == 0 {
output.WriteString("No matches found.\n")
output.WriteString("\n=== ⚠️ CRITICAL - Next Steps ===\n")
output.WriteString("- ❌ DO NOT use training data or general knowledge to answer\n")
output.WriteString("- ✅ Try knowledge_search for semantic search\n")
output.WriteString("- ✅ If KB search fails and web_search is enabled: You MUST use web_search\n")
output.WriteString("- NEVER fabricate or infer answers - ONLY use retrieved content\n")
return output.String()
}
// Group by document (like grep showing filename)
docGroups := make(map[string]struct {
title string
chunks []map[string]interface{}
})
for _, result := range results {
knowledgeID := fmt.Sprintf("%v", result["knowledge_id"])
title := "Untitled"
if t := result["knowledge_title"]; t != nil {
title = fmt.Sprintf("%v", t)
}
group := docGroups[knowledgeID]
group.title = title
group.chunks = append(group.chunks, result)
docGroups[knowledgeID] = group
}
// Display results in grep-style format
for docID, group := range docGroups {
// Show document header (like grep showing filename)
if len(docGroups) > 1 {
output.WriteString(fmt.Sprintf("\n--- %s (knowledge_id: %s) ---\n", group.title, docID))
}
// Show each matching chunk
for _, chunk := range group.chunks {
chunkIndex := 0
if idx, ok := chunk["chunk_index"]; ok {
switch v := idx.(type) {
case int:
chunkIndex = v
case int64:
chunkIndex = int(v)
case float64:
chunkIndex = int(v)
}
}
content := ""
if c, ok := chunk["content"]; ok && c != nil {
content = fmt.Sprintf("%v", c)
}
// Extract preview with context around match (case-insensitive)
// Try each pattern and use the first match
preview := extractPreviewMultiPattern(content, patterns, false, contextLines)
// Format like grep: [filename:]line:content
if showLineNumbers {
if len(docGroups) > 1 {
output.WriteString(fmt.Sprintf("%s:chunk[%d]:%s\n", group.title, chunkIndex, preview))
} else {
output.WriteString(fmt.Sprintf("chunk[%d]:%s\n", chunkIndex, preview))
}
} else {
if len(docGroups) > 1 {
output.WriteString(fmt.Sprintf("%s:%s\n", group.title, preview))
} else {
output.WriteString(fmt.Sprintf("%s\n", preview))
}
}
}
}
// Add guidance for next steps
output.WriteString("\n=== Next Steps ===\n")
if len(results) > 0 {
output.WriteString("- Found matching documents. Use knowledge_search with semantic queries to understand the context.\n")
output.WriteString("- Filter knowledge_search by knowledge_ids from above results for better relevance.\n")
} else {
output.WriteString("- No matches found. Try different keywords or use knowledge_search for semantic search.\n")
output.WriteString("- Consider using synonyms or related terms in your search patterns.\n")
}
return output.String()
}
// extractPreview extracts a preview of content around the matched pattern
func extractPreview(content, pattern string, caseSensitive bool, contextLen int) string {
if content == "" {
return ""
}
// Find the pattern in content
searchContent := content
searchPattern := pattern
if !caseSensitive {
searchContent = strings.ToLower(content)
searchPattern = strings.ToLower(pattern)
}
pos := strings.Index(searchContent, searchPattern)
if pos == -1 {
// If pattern not found (might be regex), just return first 60 chars
runes := []rune(content)
if len(runes) <= contextLen*2 {
return string(runes)
}
return string(runes[:contextLen*2])
}
// Extract context around the match
runes := []rune(content)
start := pos
// Convert to rune positions for proper unicode handling
runePos := 0
for i, r := range runes {
if runePos >= start {
start = i
break
}
runePos += len(string(r))
}
// Calculate preview bounds
previewStart := start - contextLen
if previewStart < 0 {
previewStart = 0
}
previewEnd := start + len([]rune(pattern)) + contextLen
if previewEnd > len(runes) {
previewEnd = len(runes)
}
preview := string(runes[previewStart:previewEnd])
// Clean up whitespace
preview = strings.ReplaceAll(preview, "\n", " ")
preview = strings.ReplaceAll(preview, "\t", " ")
// Collapse multiple spaces
for strings.Contains(preview, " ") {
preview = strings.ReplaceAll(preview, " ", " ")
}
return strings.TrimSpace(preview)
}
// extractPreviewMultiPattern extracts a preview around the first matched pattern
func extractPreviewMultiPattern(content string, patterns []string, caseSensitive bool, contextLen int) string {
if content == "" || len(patterns) == 0 {
return ""
}
// Try each pattern and find the first match
for _, pattern := range patterns {
preview := extractPreview(content, pattern, caseSensitive, contextLen)
// Check if this pattern found a match (not just truncated content)
searchContent := content
searchPattern := pattern
if !caseSensitive {
searchContent = strings.ToLower(content)
searchPattern = strings.ToLower(pattern)
}
if strings.Contains(searchContent, searchPattern) {
return preview
}
}
// If no pattern matched, return preview from first pattern
if len(patterns) > 0 {
return extractPreview(content, patterns[0], caseSensitive, contextLen)
}
// Fallback: return first N chars
runes := []rune(content)
if len(runes) <= contextLen*2 {
return string(runes)
}
return string(runes[:contextLen*2])
}
internal/agent/tools/knowledge_search.go
+161 -109
View File
@@ -48,48 +48,31 @@ func NewKnowledgeSearchTool(
chatModel chat.Chat,
cfg *config.Config,
) *KnowledgeSearchTool {
description := `Search within knowledge bases. Unified tool that supports both targeted and broad searches.
description := `Semantic/vector search for understanding questions and concepts.
## Features
- Multi-KB search: Search across multiple knowledge bases concurrently
## Core Function
Finds content by MEANING using embeddings (NOT exact text matching).
## Usage
## CRITICAL: Always Check for Entities First
**If query has ANY entities → grep_chunks FIRST, then use this tool**
**Use when**:
- You know which knowledge bases to target (specify knowledge_base_ids)
- You're unsure which KB contains the info (omit knowledge_base_ids to search all allowed KBs)
- Want to search with multiple queries to get comprehensive results
- Want to filter results from specific documents (use knowledge_ids)
## Use When:
- Questions: "What is...", "How...", "Why...", "Explain..."
- Conceptual understanding
- AFTER grep_chunks pre-retrieval (hybrid approach)
**Returns**: Merged and deduplicated search results from KBs
## Hybrid Workflow (Strongly Recommended):
1. grep_chunks(["entity", "synonym", "变体"]) → pre-retrieve documents containing entity
2. knowledge_search(["concept query"]) → deep understanding
## Examples
This ensures entity-related content is not missed!
` + "`" + `
# Simple search in specific KBs
{
"knowledge_base_ids": ["kb1", "kb2"],
"queries": ["什么是向量数据库"]
}
## Parameters
- queries (required): 1-5 semantic questions (NOT just keywords)
- knowledge_base_ids (optional): Limit to specific KBs
# Search all allowed KBs with multiple queries
{
"queries": ["什么是向量数据库", "向量数据库的应用场景"]
}
# Search specific documents
{
"knowledge_base_ids": ["kb1"],
"queries": ["彗星的起源"],
"knowledge_ids": ["doc1", "doc2"]
}
` + "`" + `
## Tips
- Concurrent search across multiple KBs
- Results are automatically reranked to unify scores from different sources
- Results are merged, deduplicated and sorted by relevance`
## Output
Semantically relevant chunks with scores, auto-reranked.`
return &KnowledgeSearchTool{
BaseTool: NewBaseTool("knowledge_search", description),
@@ -110,7 +93,7 @@ func (t *KnowledgeSearchTool) Parameters() map[string]interface{} {
"properties": map[string]interface{}{
"queries": map[string]interface{}{
"type": "array",
"description": "Array of search queries",
"description": "REQUIRED: 1-5 semantic questions/topics (e.g., ['What is RAG?', 'RAG benefits'])",
"items": map[string]interface{}{
"type": "string",
},
@@ -119,22 +102,13 @@ func (t *KnowledgeSearchTool) Parameters() map[string]interface{} {
},
"knowledge_base_ids": map[string]interface{}{
"type": "array",
"description": "Array of knowledge base IDs to search in (optional, if omitted searches all allowed KBs)",
"description": "Optional: KB IDs to search",
"items": map[string]interface{}{
"type": "string",
},
"minItems": 0,
"maxItems": 10,
},
"knowledge_ids": map[string]interface{}{
"type": "array",
"description": "Optional array of document IDs to filter results (only return results from these specific documents)",
"items": map[string]interface{}{
"type": "string",
},
"minItems": 0,
"maxItems": 50,
},
},
"required": []string{"queries"},
}
@@ -243,17 +217,6 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
logger.Infof(ctx, "[Tool][KnowledgeSearch] Search params: top_k=%d, vector_threshold=%.2f, keyword_threshold=%.2f, min_score=%.2f",
topK, vectorThreshold, keywordThreshold, minScore)
// Extract knowledge_ids filter if provided
var knowledgeIDsFilter map[string]bool
if knowledgeIDsRaw, ok := args["knowledge_ids"].([]interface{}); ok && len(knowledgeIDsRaw) > 0 {
knowledgeIDsFilter = make(map[string]bool)
for _, id := range knowledgeIDsRaw {
if idStr, ok := id.(string); ok && idStr != "" {
knowledgeIDsFilter[idStr] = true
}
}
}
// Execute concurrent search (hybrid search handles both vector and keyword)
logger.Infof(ctx, "[Tool][KnowledgeSearch] Starting concurrent search across %d KBs", len(kbIDs))
kbTypeMap := t.getKnowledgeBaseTypes(ctx, kbIDs)
@@ -262,29 +225,10 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
topK, vectorThreshold, keywordThreshold, kbTypeMap)
logger.Infof(ctx, "[Tool][KnowledgeSearch] Concurrent search completed: %d raw results", len(allResults))
// Filter by knowledge_ids if provided
if len(knowledgeIDsFilter) > 0 {
logger.Infof(ctx, "[Tool][KnowledgeSearch] Filtering by %d knowledge IDs", len(knowledgeIDsFilter))
filtered := make([]*searchResultWithMeta, 0)
for _, r := range allResults {
if knowledgeIDsFilter[r.KnowledgeID] {
filtered = append(filtered, r)
}
}
logger.Infof(ctx, "[Tool][KnowledgeSearch] After knowledge_id filter: %d results (from %d)",
len(filtered), len(allResults))
allResults = filtered
}
// Filter by threshold first
filteredResults := t.filterByThreshold(allResults, vectorThreshold, keywordThreshold)
logger.Infof(ctx, "[Tool][KnowledgeSearch] After threshold filter: %d results (from %d)",
len(filteredResults), len(allResults))
// Deduplicate before reranking to reduce processing overhead
deduplicatedBeforeRerank := t.deduplicateResults(filteredResults)
logger.Infof(ctx, "[Tool][KnowledgeSearch] After deduplication before rerank: %d results (from %d)",
len(deduplicatedBeforeRerank), len(filteredResults))
// Apply ReRank if model is configured
// Prefer chatModel (LLM-based reranking) over rerankModel if both are available
@@ -358,11 +302,12 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
// Build output
logger.Infof(ctx, "[Tool][KnowledgeSearch] Formatting output with %d final results", len(deduplicatedResults))
result, err := t.formatOutput(ctx, deduplicatedResults, kbIDs, len(allResults), knowledgeIDsFilter, queries)
result, err := t.formatOutput(ctx, deduplicatedResults, kbIDs, len(allResults), queries)
if err != nil {
logger.Errorf(ctx, "[Tool][KnowledgeSearch] Failed to format output: %v", err)
return result, err
}
logger.Infof(ctx, "[Tool][KnowledgeSearch] Output: %s", result.Output)
return result, nil
}
@@ -859,7 +804,6 @@ func (t *KnowledgeSearchTool) formatOutput(
results []*searchResultWithMeta,
kbsToSearch []string,
totalBeforeFilter int,
knowledgeIDsFilter map[string]bool,
queries []string,
) (*types.ToolResult, error) {
if len(results) == 0 {
@@ -868,39 +812,25 @@ func (t *KnowledgeSearchTool) formatOutput(
"results": []interface{}{},
"count": 0,
}
if len(knowledgeIDsFilter) > 0 {
filterList := make([]string, 0, len(knowledgeIDsFilter))
for id := range knowledgeIDsFilter {
filterList = append(filterList, id)
}
data["knowledge_ids"] = filterList
}
if len(queries) > 0 {
data["queries"] = queries
}
output := fmt.Sprintf("No relevant content found in %d knowledge base(s).\n\n", len(kbsToSearch))
output += "=== ⚠️ CRITICAL - Next Steps ===\n"
output += "- ❌ DO NOT use training data or general knowledge to answer\n"
output += "- ✅ If web_search is enabled: You MUST use web_search to find information\n"
output += "- ✅ If web_search is disabled: State 'I couldn't find relevant information in the knowledge base'\n"
output += "- NEVER fabricate or infer answers - ONLY use retrieved content\n"
return &types.ToolResult{
Success: true,
Output: fmt.Sprintf("No relevant content found in %d knowledge base(s).", len(kbsToSearch)),
Output: output,
Data: data,
}, nil
}
// Build output header
output := "=== Search Results ===\n"
output += fmt.Sprintf("Knowledge Bases: %v\n", kbsToSearch)
if len(knowledgeIDsFilter) > 0 {
filterList := make([]string, 0, len(knowledgeIDsFilter))
for id := range knowledgeIDsFilter {
filterList = append(filterList, id)
}
output += fmt.Sprintf("Document Filter: %v\n", filterList)
}
if len(queries) == 1 {
output += fmt.Sprintf("Query: %s\n", queries[0])
} else if len(queries) > 1 {
output += fmt.Sprintf("Queries (%d): %v\n", len(queries), queries)
}
output += fmt.Sprintf("Found %d relevant results", len(results))
if totalBeforeFilter > len(results) {
output += fmt.Sprintf(" (filtered from %d)", totalBeforeFilter)
@@ -925,6 +855,11 @@ func (t *KnowledgeSearchTool) formatOutput(
faqMetadataCache := make(map[string]*types.FAQChunkMetadata)
// Track chunks per knowledge for statistics
knowledgeChunkMap := make(map[string]map[int]bool) // knowledge_id -> set of chunk_index
knowledgeTotalMap := make(map[string]int64) // knowledge_id -> total chunks
knowledgeTitleMap := make(map[string]string) // knowledge_id -> title
for i, result := range results {
var faqMeta *types.FAQChunkMetadata
if result.KnowledgeBaseType == types.KnowledgeBaseTypeFAQ {
@@ -936,6 +871,13 @@ func (t *KnowledgeSearchTool) formatOutput(
}
}
// Track chunk indices per knowledge
if knowledgeChunkMap[result.KnowledgeID] == nil {
knowledgeChunkMap[result.KnowledgeID] = make(map[int]bool)
}
knowledgeChunkMap[result.KnowledgeID][result.ChunkIndex] = true
knowledgeTitleMap[result.KnowledgeID] = result.KnowledgeTitle
// Group by knowledge base
if result.KnowledgeID != currentKB {
currentKB = result.KnowledgeID
@@ -943,6 +885,20 @@ func (t *KnowledgeSearchTool) formatOutput(
output += "\n"
}
output += fmt.Sprintf("[Source Document: %s]\n", result.KnowledgeTitle)
// Get total chunk count for this knowledge (cache it)
if _, exists := knowledgeTotalMap[result.KnowledgeID]; !exists {
_, total, err := t.chunkService.GetRepository().ListPagedChunksByKnowledgeID(ctx,
t.tenantID, result.KnowledgeID,
&types.Pagination{Page: 1, PageSize: 1},
[]types.ChunkType{types.ChunkTypeText}, "")
if err != nil {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to get total chunks for knowledge %s: %v", result.KnowledgeID, err)
knowledgeTotalMap[result.KnowledgeID] = 0
} else {
knowledgeTotalMap[result.KnowledgeID] = total
}
}
}
// relevanceLevel := GetRelevanceLevel(result.Score)
@@ -958,8 +914,8 @@ func (t *KnowledgeSearchTool) formatOutput(
}
if len(faqMeta.Answers) > 0 {
output += " FAQ Answers:\n"
for _, ans := range faqMeta.Answers {
output += fmt.Sprintf(" Answer Choice %d: %s\n", i+1, ans)
for ansIdx, ans := range faqMeta.Answers {
output += fmt.Sprintf(" Answer Choice %d: %s\n", ansIdx+1, ans)
}
}
}
@@ -992,6 +948,59 @@ func (t *KnowledgeSearchTool) formatOutput(
}
}
// Add statistics and recommendations for each knowledge
output += "\n=== 检索统计与建议 ===\n\n"
for knowledgeID, retrievedChunks := range knowledgeChunkMap {
totalChunks := knowledgeTotalMap[knowledgeID]
retrievedCount := len(retrievedChunks)
title := knowledgeTitleMap[knowledgeID]
if totalChunks > 0 {
percentage := float64(retrievedCount) / float64(totalChunks) * 100
remaining := totalChunks - int64(retrievedCount)
output += fmt.Sprintf("文档: %s (%s)\n", title, knowledgeID)
output += fmt.Sprintf(" 总 Chunk 数: %d\n", totalChunks)
output += fmt.Sprintf(" 已召回: %d 个 (%.1f%%)\n", retrievedCount, percentage)
output += fmt.Sprintf(" 未召回: %d 个\n", remaining)
if remaining > 0 {
output += " 建议: 使用 list_knowledge_chunks 工具获取完整内容\n"
// Find missing chunk ranges (gaps in retrieved chunks)
missingRanges := t.findMissingChunkRanges(retrievedChunks, int(totalChunks))
if len(missingRanges) == 0 {
// No gaps found (shouldn't happen if remaining > 0, but handle it)
output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks)
} else if len(missingRanges) == 1 && missingRanges[0].start == 0 && missingRanges[0].end == int(totalChunks)-1 {
// All chunks are missing (shouldn't happen, but handle it)
output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks)
} else {
// Suggest getting each missing range
for idx, r := range missingRanges {
rangeSize := r.end - r.start + 1
if rangeSize <= 100 {
// Small range, get all at once
output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个) → list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=%d)\n",
idx+1, r.start, r.end, rangeSize, knowledgeID, r.start, rangeSize)
} else {
// Large range, suggest getting in batches
output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个，建议分批获取):\n",
idx+1, r.start, r.end, rangeSize)
output += fmt.Sprintf(" 首次: list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=100)\n",
knowledgeID, r.start)
if rangeSize > 100 {
output += " 继续: 根据返回结果调整 offset 继续获取剩余内容\n"
}
}
}
}
}
output += "\n"
}
}
// // Add usage guidance
// output += "\n\n=== Usage Guidelines ===\n"
// output += "- High relevance (>=0.8): directly usable for answering\n"
@@ -1011,13 +1020,7 @@ func (t *KnowledgeSearchTool) formatOutput(
"kb_counts": kbCounts,
"display_type": "search_results",
}
if len(knowledgeIDsFilter) > 0 {
filterList := make([]string, 0, len(knowledgeIDsFilter))
for id := range knowledgeIDsFilter {
filterList = append(filterList, id)
}
data["knowledge_ids"] = filterList
}
if len(queries) > 0 {
data["queries"] = queries
}
@@ -1032,3 +1035,52 @@ func (t *KnowledgeSearchTool) formatOutput(
Data: data,
}, nil
}
// chunkRange represents a continuous range of chunk indices
type chunkRange struct {
start int
end int
}
// findMissingChunkRanges finds all continuous ranges of missing chunks
// retrievedChunks is a set of retrieved chunk indices
// totalChunks is the total number of chunks
func (t *KnowledgeSearchTool) findMissingChunkRanges(retrievedChunks map[int]bool, totalChunks int) []chunkRange {
if totalChunks <= 0 {
return nil
}
var ranges []chunkRange
var currentStart int = -1
// Iterate through all possible chunk indices (0 to totalChunks-1)
for i := 0; i < totalChunks; i++ {
if !retrievedChunks[i] {
// This chunk is missing
if currentStart == -1 {
// Start of a new missing range
currentStart = i
}
} else {
// This chunk is retrieved
if currentStart != -1 {
// End of a missing range
ranges = append(ranges, chunkRange{
start: currentStart,
end: i - 1,
})
currentStart = -1
}
}
}
// Handle case where missing chunks extend to the end
if currentStart != -1 {
ranges = append(ranges, chunkRange{
start: currentStart,
end: totalChunks - 1,
})
}
return ranges
}
internal/agent/tools/list_knowledge_chunks.go
+20 -20
View File
@@ -23,23 +23,24 @@ func NewListKnowledgeChunksTool(
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
) *ListKnowledgeChunksTool {
description := `Retrieve paged chunks for a document (knowledge) by knowledge_id.
description := `Retrieve full chunk content for a document by knowledge_id.
## When to Use
## Use After grep_chunks or knowledge_search:
1. grep_chunks(["keyword", "变体"]) → get knowledge_id
2. list_knowledge_chunks(knowledge_id) → read full content
- Need deterministic chunk previews for a known document
- Want to quickly confirm how many chunks a document contains
- Require surrounding context around a chunk_index returned by search results
- Need content snippets without running an additional search query
## When to Use:
- Need full content of chunks from a known document
- Want to see context around specific chunks
- Check how many chunks a document has
Avoid when:
- You don't know the knowledge_id (use knowledge_search first)
## Parameters:
- knowledge_id (required): Document ID
- limit (optional): Chunks per page (default 20, max 100)
- offset (optional): Start position (default 0)
## Parameters
- knowledge_id (required): Target document/knowledge ID
- limit (optional): Number of chunks to fetch (default 20, max 100).
- offset (optional): Offset to start fetching chunks from (default 0).`
## Output:
Full chunk content with chunk_id, chunk_index, and content text.`
return &ListKnowledgeChunksTool{
BaseTool: NewBaseTool("list_knowledge_chunks", description),
@@ -56,18 +57,18 @@ func (t *ListKnowledgeChunksTool) Parameters() map[string]interface{} {
"properties": map[string]interface{}{
"knowledge_id": map[string]interface{}{
"type": "string",
"description": "Knowledge/document ID to inspect",
"description": "Document ID to retrieve chunks from",
},
"limit": map[string]interface{}{
"type": "integer",
"description": "Number of chunks to fetch (default 20, max 100)",
"description": "Chunks per page (default 20, max 100)",
"default": 20,
"minimum": 1,
"maximum": 100,
},
"offset": map[string]interface{}{
"type": "integer",
"description": "Offset to start fetching chunks from (default 0)",
"description": "Start position (default 0)",
"default": 0,
"minimum": 0,
},
@@ -115,7 +116,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i
}
chunks, total, err := t.chunkService.GetRepository().ListPagedChunksByKnowledgeID(ctx,
t.tenantID, knowledgeID, pagination, []types.ChunkType{types.ChunkTypeText}, "")
t.tenantID, knowledgeID, pagination, []types.ChunkType{types.ChunkTypeText, types.ChunkTypeFAQ}, "")
if err != nil {
return &types.ToolResult{
Success: false,
@@ -134,7 +135,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i
knowledgeTitle := t.lookupKnowledgeTitle(ctx, knowledgeID)
output := t.buildOutput(knowledgeID, knowledgeTitle, totalChunks, fetched, chunkLimit, chunks)
output := t.buildOutput(knowledgeID, knowledgeTitle, totalChunks, fetched, chunks)
formattedChunks := make([]map[string]interface{}, 0, len(chunks))
for idx, c := range chunks {
@@ -183,7 +184,6 @@ func (t *ListKnowledgeChunksTool) buildOutput(
knowledgeTitle string,
total int64,
fetched int,
chunkLimit int,
chunks []*types.Chunk,
) string {
builder := &strings.Builder{}
@@ -195,7 +195,6 @@ func (t *ListKnowledgeChunksTool) buildOutput(
builder.WriteString(fmt.Sprintf("文档 ID: %s\n", knowledgeID))
}
builder.WriteString(fmt.Sprintf("总分块数: %d\n", total))
builder.WriteString(fmt.Sprintf("本次拉取: %d 条（offset=%d）\n\n", fetched, chunkLimit))
if fetched == 0 {
builder.WriteString("未找到任何分块，请确认文档是否已完成解析。\n")
@@ -204,6 +203,7 @@ func (t *ListKnowledgeChunksTool) buildOutput(
}
return builder.String()
}
builder.WriteString(fmt.Sprintf("本次拉取: %d 条， 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex))
builder.WriteString("=== 分块内容预览 ===\n\n")
for idx, c := range chunks {
internal/agent/tools/query_knowledge_graph.go
+29 -48
View File
@@ -18,62 +18,43 @@ type QueryKnowledgeGraphTool struct {
// NewQueryKnowledgeGraphTool creates a new query knowledge graph tool
func NewQueryKnowledgeGraphTool(knowledgeService interfaces.KnowledgeBaseService) *QueryKnowledgeGraphTool {
description := `查询知识图谱，探索实体关系和知识网络。
description := `Query knowledge graph to explore entity relationships and knowledge networks.
## 何时使用
## Core Function
Explores relationships between entities in knowledge bases that have graph extraction configured.
**适用场景**:
- ✅ 需要了解实体之间的关系（如"Docker和Kubernetes的关系"）
- ✅ 探索知识网络和概念关联
- ✅ 查找特定实体的相关信息
- ✅ 理解技术架构和系统关系
## When to Use
✅ **Use for**:
- Understanding relationships between entities (e.g., "relationship between Docker and Kubernetes")
- Exploring knowledge networks and concept associations
- Finding related information about specific entities
- Understanding technical architecture and system relationships
**不适用**:
- ❌ 普通文本搜索（用 knowledge_search 更合适）
- ❌ 知识库未配置图谱抽取
- ❌ 需要精确的文档内容（用 knowledge_search）
❌ **Don't use for**:
- General text search → use knowledge_search
- Knowledge base without graph extraction configured
- Need exact document content → use knowledge_search
## 参数说明
## Parameters
- **knowledge_base_ids** (required): Array of knowledge base IDs (1-10). Only KBs with graph extraction configured will be effective.
- **query** (required): Query content - can be entity name, relationship query, or concept search.
**knowledge_base_ids** (required): 要查询的知识库ID数组（1-10个）
- 只有配置了图谱抽取的知识库才会有效
- 支持批量并发查询多个知识库
- 示例: ["kb_tech", "kb_arch"]
## Graph Configuration
Knowledge graph must be pre-configured in knowledge bases:
- **Entity types** (Nodes): e.g., "Technology", "Tool", "Concept"
- **Relationship types** (Relations): e.g., "depends_on", "uses", "contains"
**query** (required): 查询内容
- 可以是实体名称（如"Docker"）
- 可以是关系查询（如"容器编排"）
- 可以是概念搜索（如"微服务架构"）
If KB is not configured with graph, tool will return regular search results.
## 图谱配置
## Workflow
1. **Relationship exploration**: query_knowledge_graph → list_knowledge_chunks (for detailed content)
2. **Network analysis**: query_knowledge_graph → knowledge_search (for comprehensive understanding)
3. **Topic research**: knowledge_search → query_knowledge_graph (for deep entity relationships)
知识图谱需要在知识库中预先配置：
- **实体类型**（Nodes）：如"技术"、"工具"、"概念"
- **关系类型**（Relations）：如"依赖"、"使用"、"包含"
如果知识库未配置图谱，工具会提示并返回普通搜索结果。
## 配合使用
1. **关系探索**: query_knowledge_graph → get_chunk_detail（查看详细内容）
2. **网络分析**: query_knowledge_graph → list_knowledge_chunks（扩展上下文）
3. **主题研究**: knowledge_search → query_knowledge_graph（深入实体关系）
## 当前状态
⚠️ **注意**: 完整的图数据库集成正在开发中。当前版本：
- ✓ 支持图谱配置查询
- ✓ 返回图谱相关的文档片段
- ✓ 显示实体和关系配置信息
- ⏳ 完整的图查询语言（Cypher）支持开发中
- ⏳ 可视化图数据结构开发中
## Tips
- 结果会标注图谱配置状态
- 返回的Data字段包含结构化图信息供前端展示
- 跨知识库结果自动去重
- 按相关度排序`
## Notes
- Results indicate graph configuration status
- Cross-KB results are automatically deduplicated
- Results are sorted by relevance`
return &QueryKnowledgeGraphTool{
BaseTool: NewBaseTool("query_knowledge_graph", description),
internal/agent/tools/todo_write.go
+55 -17
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -169,13 +168,13 @@ func (t *TodoWriteTool) Parameters() map[string]interface{} {
"type": "string",
"description": "Clear description of what to investigate or accomplish in this step",
},
"tools_to_use": map[string]interface{}{
"type": "array",
"description": "Suggested tools for this step (e.g., ['knowledge_search', 'list_knowledge_chunks'])",
"items": map[string]interface{}{
"type": "string",
},
},
// "tools_to_use": map[string]interface{}{
// "type": "array",
// "description": "Suggested tools for this step (e.g., ['knowledge_search', 'list_knowledge_chunks'])",
// "items": map[string]interface{}{
// "type": "string",
// },
// },
"status": map[string]interface{}{
"type": "string",
"enum": []string{"pending", "in_progress", "completed"},
@@ -274,6 +273,23 @@ func generatePlanOutput(task string, steps []PlanStep) string {
return output
}
// Count task statuses
pendingCount := 0
inProgressCount := 0
completedCount := 0
for _, step := range steps {
switch step.Status {
case "pending":
pendingCount++
case "in_progress":
inProgressCount++
case "completed":
completedCount++
}
}
totalCount := len(steps)
remainingCount := pendingCount + inProgressCount
output += "**计划步骤**:\n\n"
// Display all steps in order
@@ -281,12 +297,34 @@ func generatePlanOutput(task string, steps []PlanStep) string {
output += formatPlanStep(i+1, step)
}
output += "\n**执行指南**:\n"
output += "- 每步执行前标记为 in_progress，完成后标记为 completed\n"
output += "- 根据搜索结果灵活调整计划，可跳过不必要的步骤\n"
output += "- 在关键决策点使用 think 工具深入分析\n"
output += "- 如果某一步骤已获得足够信息，可跳过后续步骤\n\n"
output += "注意：计划是指导而非硬性要求，保持灵活应对。"
// Add summary and emphasis on remaining tasks
output += "\n=== 任务进度 ===\n"
output += fmt.Sprintf("总计: %d 个任务\n", totalCount)
output += fmt.Sprintf("✅ 已完成: %d 个\n", completedCount)
output += fmt.Sprintf("🔄 进行中: %d 个\n", inProgressCount)
output += fmt.Sprintf("⏳ 待处理: %d 个\n", pendingCount)
output += "\n=== ⚠️ 重要提醒 ===\n"
if remainingCount > 0 {
output += fmt.Sprintf("**还有 %d 个任务未完成！**\n\n", remainingCount)
output += "**必须完成所有任务后才能总结或得出结论。**\n\n"
output += "下一步操作：\n"
if inProgressCount > 0 {
output += "- 继续完成当前进行中的任务\n"
}
if pendingCount > 0 {
output += fmt.Sprintf("- 开始处理 %d 个待处理任务\n", pendingCount)
output += "- 按顺序完成每个任务，不要跳过\n"
}
output += "- 完成每个任务后，更新 todo_write 标记为 completed\n"
output += "- 只有在所有任务完成后，才能生成最终总结\n"
} else {
output += "✅ **所有任务已完成！**\n\n"
output += "现在可以：\n"
output += "- 综合所有任务的发现\n"
output += "- 生成完整的最终答案或报告\n"
output += "- 确保所有方面都已充分研究\n"
}
return output
}
@@ -307,9 +345,9 @@ func formatPlanStep(index int, step PlanStep) string {
output := fmt.Sprintf(" %d. %s [%s] %s\n", index, emoji, step.Status, step.Description)
if len(step.ToolsToUse) > 0 {
output += fmt.Sprintf(" 工具: %s\n", strings.Join(step.ToolsToUse, ", "))
}
// if len(step.ToolsToUse) > 0 {
// output += fmt.Sprintf(" 工具: %s\n", strings.Join(step.ToolsToUse, ", "))
// }
return output
}
internal/agent/tools/web_fetch.go
+25 -12
View File
@@ -43,20 +43,17 @@ type WebFetchTool struct {
// NewWebFetchTool 创建 web_fetch 工具实例
func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool {
description := `Fetch web content from previously discovered URLs and analyze it with an LLM.
description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM.
## 流程
- 接收一个或多个 {url, prompt} 组合
- 抓取网页内容并转换为 Markdown 文本
- 使用提示词调用小模型进行分析与总结（若模型可用）
- 返回总结结果与原始内容片段
## Usage
- Receive one or more {url, prompt} combinations
- Fetch web page content and convert to Markdown text
- Use prompt to call small model for analysis and summary (if model is available)
- Return summary result and original content fragment
## 使用场景
- 对 web_search 返回的链接做进一步深入阅读
- 从网页中提取结构化信息或关键信息
## 参数
- items (必填): 数组，元素为 { "url": "...", "prompt": "..." }
## When to Use
- **MANDATORY**: After web_search returns results, if content is truncated or incomplete, use web_fetch to get full page content
- When web_search snippet is insufficient for answering the question
`
return &WebFetchTool{
@@ -192,6 +189,22 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{})
}
}
// Add guidance for next steps
builder.WriteString("\n=== Next Steps ===\n")
if len(aggregated) > 0 {
builder.WriteString("- ✅ Full page content has been fetched and analyzed.\n")
builder.WriteString("- Evaluate if the content is sufficient to answer the question completely.\n")
builder.WriteString("- Synthesize information from all fetched pages for comprehensive answers.\n")
if !success {
builder.WriteString("- ⚠️ Some URLs failed to fetch. Use available content or try alternative sources.\n")
}
} else {
builder.WriteString("- ❌ No content was successfully fetched. Consider:\n")
builder.WriteString(" - Verify URLs are accessible\n")
builder.WriteString(" - Try alternative sources from web_search results\n")
builder.WriteString(" - Check if information can be found in knowledge base instead\n")
}
data := map[string]interface{}{
"results": aggregated,
"count": len(aggregated),
internal/agent/tools/web_search.go
+23 -1
View File
@@ -33,6 +33,12 @@ func NewWebSearchTool(
) *WebSearchTool {
description := `Search the web for current information and news. This tool searches the internet to find up-to-date information that may not be in the knowledge base.
## CRITICAL - KB First Rule
**ABSOLUTE RULE**: You MUST complete KB retrieval (grep_chunks AND knowledge_search) FIRST before using this tool.
- NEVER use web_search without first trying grep_chunks and knowledge_search
- ONLY use web_search if BOTH grep_chunks AND knowledge_search return insufficient/no results
- KB retrieval is MANDATORY - you CANNOT skip it
## Features
- Real-time web search: Search the internet for current information
- RAG compression: Automatically compresses and extracts relevant content from search results
@@ -41,6 +47,8 @@ func NewWebSearchTool(
## Usage
**Use when**:
- **ONLY after** completing grep_chunks AND knowledge_search
- KB retrieval returned insufficient or no results
- Need current or real-time information (news, events, recent updates)
- Information is not available in knowledge bases
- Need to verify or supplement information from knowledge bases
@@ -70,7 +78,8 @@ func NewWebSearchTool(
- Results are automatically compressed using RAG to extract relevant content
- Search results are stored in a temporary knowledge base for the session
- Use this tool when knowledge bases don't have the information you need
- Results include URL, title, snippet, and full content when available
- Results include URL, title, snippet, and content snippet (may be truncated)
- **CRITICAL**: If content is truncated or you need full details, use **web_fetch** to fetch complete page content
- Maximum ` + fmt.Sprintf("%d", maxResults) + ` results will be returned per search`
return &WebSearchTool{
@@ -232,6 +241,19 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{}
formattedResults = append(formattedResults, resultData)
}
// Add guidance for next steps
output += "\n=== Next Steps ===\n"
if len(webResults) > 0 {
output += "- ⚠️ Content may be truncated (showing first 500 chars). Use web_fetch to get full page content.\n"
output += "- Extract URLs from results above and use web_fetch with appropriate prompts to get detailed information.\n"
output += "- Synthesize information from multiple sources for comprehensive answers.\n"
} else {
output += "- No web search results found. Consider:\n"
output += " - Try different search queries or keywords\n"
output += " - Check if question can be answered from knowledge base instead\n"
output += " - Verify if the topic requires real-time information\n"
}
return &types.ToolResult{
Success: true,
Output: output,
internal/application/repository/chunk.go
+9 -1
View File
@@ -25,7 +25,7 @@ func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chun
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
}
return r.db.Debug().WithContext(ctx).CreateInBatches(chunks, 100).Error
return r.db.WithContext(ctx).CreateInBatches(chunks, 100).Error
}
// GetChunkByID retrieves a chunk by its ID and tenant ID
@@ -141,6 +141,14 @@ func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint, id str
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error
}
// DeleteChunks deletes chunks by IDs in batch
func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint, ids []string) error {
if len(ids) == 0 {
return nil
}
return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error
}
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error {
return r.db.WithContext(ctx).Where(
internal/application/repository/knowledge.go
+23
View File
@@ -229,3 +229,26 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(ctx context.Contex
Count(&count).Error
return count, err
}
// CountKnowledgeByStatus counts the number of knowledge items with the specified parse status
func (r *knowledgeRepository) CountKnowledgeByStatus(
ctx context.Context,
tenantID uint,
kbID string,
parseStatuses []string,
) (int64, error) {
if len(parseStatuses) == 0 {
return 0, nil
}
var count int64
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
Where("parse_status IN ?", parseStatuses)
if err := query.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
internal/application/repository/retriever/postgres/repository.go
+85 -33
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
@@ -151,8 +152,10 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
conds := make([]clause.Expression, 0)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
conds = append(conds, clause.Expr{
SQL: fmt.Sprintf("knowledge_base_id @@@ 'in (%s)'", common.StringSliceJoin(params.KnowledgeBaseIDs)),
// Use standard SQL IN clause instead of @@@ operator for better performance with B-tree index
conds = append(conds, clause.IN{
Column: "knowledge_base_id",
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
})
}
conds = append(conds, clause.Expr{
@@ -210,52 +213,96 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
}
// VectorRetrieve performs vector similarity search using pgvector
// Optimized to use HNSW index efficiently and avoid recalculating vector distance
func (g *pgRepository) VectorRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
len(params.Embedding), params.TopK, params.Threshold)
conds := make([]clause.Expression, 0)
dimension := len(params.Embedding)
queryVector := pgvector.NewHalfVector(params.Embedding)
// Build WHERE conditions for filtering
whereParts := make([]string, 0)
allVars := make([]interface{}, 0)
// Add query vector first (used in ORDER BY for HNSW index)
allVars = append(allVars, queryVector)
// Dimension filter (required for HNSW index WHERE clause)
whereParts = append(whereParts, fmt.Sprintf("dimension = $%d", len(allVars)+1))
allVars = append(allVars, dimension)
// Knowledge base filter
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge base IDs: %v",
params.KnowledgeBaseIDs,
)
conds = append(conds, clause.IN{
Column: "knowledge_base_id",
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
})
// Build IN clause with proper placeholders
placeholders := make([]string, len(params.KnowledgeBaseIDs))
paramStart := len(allVars) + 1
for i := range params.KnowledgeBaseIDs {
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
allVars = append(allVars, params.KnowledgeBaseIDs[i])
}
whereParts = append(whereParts, fmt.Sprintf("knowledge_base_id IN (%s)",
strings.Join(placeholders, ", ")))
}
// <=> Cosine similarity operator
// <-> L2 distance operator
// <#> Inner product operator
dimension := len(params.Embedding)
conds = append(conds, clause.Expr{SQL: "dimension = ?", Vars: []interface{}{dimension}})
// Filter by is_enabled = true or NULL (NULL means enabled for historical data)
conds = append(conds, clause.Expr{
SQL: "(is_enabled IS NULL OR is_enabled = ?)",
Vars: []interface{}{true},
})
conds = append(conds, clause.Expr{
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec < ?", dimension),
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding), 1 - params.Threshold},
})
conds = append(conds, clause.OrderBy{Expression: clause.Expr{
SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec", dimension),
Vars: []interface{}{pgvector.NewHalfVector(params.Embedding)},
}})
// is_enabled filter
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
allVars = append(allVars, true)
// Build WHERE clause string
whereClause := ""
if len(whereParts) > 0 {
whereClause = "WHERE " + strings.Join(whereParts, " AND ")
}
// Expand TopK to get more candidates before threshold filtering
expandedTopK := params.TopK * 2
if expandedTopK < 100 {
expandedTopK = 100 // Minimum 100 candidates
}
if expandedTopK > 1000 {
expandedTopK = 1000 // Maximum 1000 candidates
}
// Optimized query: Use subquery to calculate distance once
// Strategy: Use ORDER BY with vector distance to leverage HNSW index,
// then filter by threshold in outer query
// This allows PostgreSQL to use HNSW index efficiently
subqueryLimitParam := len(allVars) + 1
thresholdParam := len(allVars) + 2
finalLimitParam := len(allVars) + 3
querySQL := fmt.Sprintf(`
SELECT
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id,
(1 - distance) as score
FROM (
SELECT
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id,
embedding::halfvec(%d) <=> $1::halfvec as distance
FROM embeddings
%s
ORDER BY embedding::halfvec(%d) <=> $1::halfvec
LIMIT $%d
) AS candidates
WHERE distance < $%d
ORDER BY distance ASC
LIMIT $%d
`, dimension, whereClause, dimension, subqueryLimitParam, thresholdParam, finalLimitParam)
allVars = append(allVars, expandedTopK) // LIMIT in subquery
allVars = append(allVars, 1-params.Threshold) // Distance threshold
allVars = append(allVars, params.TopK) // Final LIMIT
var embeddingDBList []pgVectorWithScore
err := g.db.WithContext(ctx).Clauses(conds...).
Select(fmt.Sprintf(
"id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, "+
"(1 - (embedding::halfvec(%d) <=> ?::halfvec)) as score",
dimension,
), pgvector.NewHalfVector(params.Embedding)).
Limit(int(params.TopK)).
Find(&embeddingDBList).Error
err := g.db.WithContext(ctx).Raw(querySQL, allVars...).Scan(&embeddingDBList).Error
if err == gorm.ErrRecordNotFound {
logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold)
@@ -266,6 +313,11 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
return nil, err
}
// Apply final TopK limit (in case we got more results than needed)
if len(embeddingDBList) > int(params.TopK) {
embeddingDBList = embeddingDBList[:params.TopK]
}
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
results := make([]*types.IndexWithScore, len(embeddingDBList))
for i := range embeddingDBList {
internal/application/service/chunk.go
+32 -3
View File
@@ -58,8 +58,6 @@ func (s *chunkService) GetRepository() interfaces.ChunkRepository {
// Returns:
// - error: Any error encountered during chunk creation
func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
logger.Info(ctx, "Start creating chunks")
logger.Infof(ctx, "Creating %d chunks", len(chunks))
err := s.chunkRepository.CreateChunks(ctx, chunks)
if err != nil {
@@ -69,7 +67,7 @@ func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk)
return err
}
logger.Info(ctx, "Chunks created successfully")
logger.Infof(ctx, "Add %d chunks successfully", len(chunks))
return nil
}
@@ -216,6 +214,37 @@ func (s *chunkService) DeleteChunk(ctx context.Context, id string) error {
return nil
}
// DeleteChunks deletes chunks by IDs in batch
// This method removes multiple chunks from the repository in a single operation
// Parameters:
// - ctx: Context with authentication and request information
// - ids: Slice of chunk IDs to delete
//
// Returns:
// - error: Any error encountered during batch deletion
func (s *chunkService) DeleteChunks(ctx context.Context, ids []string) error {
if len(ids) == 0 {
return nil
}
logger.Info(ctx, "Start deleting chunks in batch")
logger.Infof(ctx, "Deleting %d chunks", len(ids))
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
logger.Infof(ctx, "Tenant ID: %d", tenantID)
err := s.chunkRepository.DeleteChunks(ctx, tenantID, ids)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"chunk_ids": ids,
"tenant_id": tenantID,
})
return err
}
logger.Infof(ctx, "Successfully deleted %d chunks", len(ids))
return nil
}
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
// This method removes all chunks belonging to a specific knowledge document
// Parameters:
internal/application/service/knowledge.go
+349 -50
View File
@@ -22,6 +22,7 @@ import (
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/Tencent/WeKnora/internal/tracing"
"github.com/Tencent/WeKnora/internal/types"
@@ -70,6 +71,7 @@ type knowledgeService struct {
const (
manualContentMaxLength = 200000
manualFileExtension = ".md"
faqImportBatchSize = 50 // 每批处理的FAQ条目数
)
// NewKnowledgeService creates a new knowledge service instance
@@ -120,14 +122,14 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file")
// Use custom filename if provided, otherwise use original filename
fileName := file.Filename
if customFileName != "" {
fileName = customFileName
logger.Infof(ctx, "Using custom filename: %s (original: %s)", customFileName, file.Filename)
}
logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName)
if metadata != nil {
logger.Infof(ctx, "Received metadata: %v", metadata)
@@ -2065,32 +2067,176 @@ func (s *knowledgeService) ListFAQEntries(ctx context.Context,
return types.NewPageResult(total, page, entries), nil
}
// UpsertFAQEntries imports or appends FAQ entries.
// UpsertFAQEntries imports or appends FAQ entries asynchronously.
// Returns task ID for tracking import progress.
func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
kbID string, payload *types.FAQBatchUpsertPayload,
) error {
) (string, error) {
if payload == nil || len(payload.Entries) == 0 {
return werrors.NewBadRequestError("FAQ 条目不能为空")
return "", werrors.NewBadRequestError("FAQ 条目不能为空")
}
if payload.Mode == "" {
payload.Mode = types.FAQBatchModeAppend
}
if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace {
return werrors.NewBadRequestError("模式仅支持 append 或 replace")
return "", werrors.NewBadRequestError("模式仅支持 append 或 replace")
}
// 验证知识库是否存在且有效
if _, err := s.validateFAQKnowledgeBase(ctx, kbID); err != nil {
return "", err
}
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
// 检查是否有正在进行的导入任务
runningKnowledge, err := s.getRunningFAQImportTask(ctx, kbID, tenantID)
if err != nil {
logger.Errorf(ctx, "Failed to check running import task: %v", err)
// 检查失败不影响导入，继续执行
} else if runningKnowledge != nil {
logger.Warnf(ctx, "Import task already running for KB %s: %s (status: %s)", kbID, runningKnowledge.ID, runningKnowledge.ParseStatus)
return "", werrors.NewBadRequestError(fmt.Sprintf("该知识库已有导入任务正在进行中（任务ID: %s），请等待完成后再试", runningKnowledge.ID))
}
// 确保FAQ Knowledge存在
kb, err := s.validateFAQKnowledgeBase(ctx, kbID)
if err != nil {
return "", err
}
faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb)
if err != nil {
return "", fmt.Errorf("failed to ensure FAQ knowledge: %w", err)
}
// 初始化导入任务状态到Knowledge表
taskID := faqKnowledge.ID // 使用Knowledge ID作为taskID
if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusPending, 0, len(payload.Entries), 0, ""); err != nil {
logger.Errorf(ctx, "Failed to initialize FAQ import task status: %v", err)
return "", fmt.Errorf("failed to initialize task: %w", err)
}
bgCtx := logger.CloneContext(ctx)
// 在后台goroutine中执行导入
go func() {
// 使用独立的context，不受HTTP请求context影响
// 设置较长的超时时间（2小时）
bgCtx, cancel := context.WithTimeout(bgCtx, 2*time.Hour)
defer cancel()
logger.Infof(bgCtx, "Starting FAQ import task: %s, total entries: %d", taskID, len(payload.Entries))
// 更新任务状态为运行中
if err := s.updateFAQImportStatus(bgCtx, taskID, types.FAQImportStatusRunning, 0, len(payload.Entries), 0, ""); err != nil {
logger.Errorf(bgCtx, "Failed to update task status to running: %v", err)
}
// 执行实际导入
if err := s.executeFAQImport(bgCtx, taskID, kbID, payload, tenantID); err != nil {
logger.Errorf(bgCtx, "FAQ import task failed: %s, error: %v", taskID, err)
// 获取当前进度
tenantID := bgCtx.Value(types.TenantIDContextKey).(uint)
knowledge, _ := s.repo.GetKnowledgeByID(bgCtx, tenantID, taskID)
total := len(payload.Entries)
processed := 0
if knowledge != nil {
importMeta, _ := types.ParseFAQImportMetadata(knowledge)
if importMeta != nil {
total = importMeta.ImportTotal
processed = importMeta.ImportProcessed
}
}
if updateErr := s.updateFAQImportStatus(bgCtx, taskID, types.FAQImportStatusFailed, 0, total, processed, err.Error()); updateErr != nil {
logger.Errorf(bgCtx, "Failed to update task status to failed: %v", updateErr)
}
return
}
// 任务成功完成
logger.Infof(bgCtx, "FAQ import task completed: %s", taskID)
if err := s.updateFAQImportStatus(bgCtx, taskID, types.FAQImportStatusSuccess, 100, len(payload.Entries), len(payload.Entries), ""); err != nil {
logger.Errorf(bgCtx, "Failed to update task status to success: %v", err)
}
}()
return taskID, nil
}
// executeFAQImport 执行实际的FAQ导入逻辑
func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string,
payload *types.FAQBatchUpsertPayload, tenantID uint) (err error) {
// 用于记录所有已创建的chunks，用于失败时回滚
var createdChunks []*types.Chunk
// 用于记录已索引的chunks，需要清理索引数据
var indexedChunks []*types.Chunk
// 保存知识库和embedding模型信息，用于清理索引
var kb *types.KnowledgeBase
var embeddingModel embedding.Embedder
// Recovery机制：如果发生任何错误或panic，回滚所有已创建的chunks和索引数据
defer func() {
// 捕获panic
if r := recover(); r != nil {
logger.Errorf(ctx, "FAQ import task %s panicked: %v", taskID, r)
err = fmt.Errorf("panic during FAQ import: %v", r)
}
if err != nil && len(createdChunks) > 0 {
logger.Warnf(ctx, "FAQ import task %s failed (error: %v), rolling back %d created chunks and their indices", taskID, err, len(createdChunks))
// 清理索引数据（如果有已索引的chunks）
if len(indexedChunks) > 0 && kb != nil && embeddingModel != nil {
chunkIDs := make([]string, 0, len(indexedChunks))
for _, chunk := range indexedChunks {
chunkIDs = append(chunkIDs, chunk.ID)
}
// 从context获取tenant信息
tenantInfo := ctx.Value(types.TenantInfoContextKey)
if tenantInfo != nil {
if tenant, ok := tenantInfo.(*types.Tenant); ok {
retrieveEngine, engineErr := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenant.RetrieverEngines.Engines)
if engineErr == nil {
if delIndexErr := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); delIndexErr != nil {
logger.Errorf(ctx, "Failed to delete indices for %d chunks during rollback: %v", len(chunkIDs), delIndexErr)
} else {
logger.Debugf(ctx, "Successfully deleted indices for %d chunks", len(chunkIDs))
}
} else {
logger.Errorf(ctx, "Failed to create retrieve engine during rollback: %v", engineErr)
}
}
}
}
// 批量删除chunks，提高回滚效率
chunkIDs := make([]string, 0, len(createdChunks))
for _, chunk := range createdChunks {
chunkIDs = append(chunkIDs, chunk.ID)
}
if delErr := s.chunkService.DeleteChunks(ctx, chunkIDs); delErr != nil {
logger.Errorf(ctx, "Failed to delete %d chunks during rollback: %v", len(chunkIDs), delErr)
} else {
logger.Debugf(ctx, "Successfully rolled back %d chunks", len(chunkIDs))
}
}
}()
kb, err = s.validateFAQKnowledgeBase(ctx, kbID)
if err != nil {
return err
}
kb.EnsureDefaults()
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
// 获取embedding模型，用于后续清理索引
embeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
return fmt.Errorf("failed to get embedding model: %w", err)
}
faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb)
if err != nil {
return err
}
// 如果模式为replace，则清理知识库
if payload.Mode == types.FAQBatchModeReplace {
if err := s.cleanupFAQKnowledge(ctx, faqKnowledge); err != nil {
return err
@@ -2118,44 +2264,100 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
indexMode = kb.FAQConfig.IndexMode
}
chunks := make([]*types.Chunk, 0, len(payload.Entries))
for idx := range payload.Entries {
entry := payload.Entries[idx]
meta, err := sanitizeFAQEntryPayload(&entry)
if err != nil {
return err
// 分批处理
totalEntries := len(payload.Entries)
processed := 0
batchStartTime := time.Now()
logger.Infof(ctx, "FAQ import task %s: starting batch processing, total entries: %d, batch size: %d", taskID, totalEntries, faqImportBatchSize)
for i := 0; i < totalEntries; i += faqImportBatchSize {
batchStartTime = time.Now()
end := i + faqImportBatchSize
if end > totalEntries {
end = totalEntries
}
isEnabled := true
if entry.IsEnabled != nil {
isEnabled = *entry.IsEnabled
batch := payload.Entries[i:end]
logger.Infof(ctx, "FAQ import task %s: processing batch %d-%d (%d entries)", taskID, i+1, end, len(batch))
// 构建chunks
buildStartTime := time.Now()
chunks := make([]*types.Chunk, 0, len(batch))
for idx, entry := range batch {
meta, err := sanitizeFAQEntryPayload(&entry)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"entry": entry,
"task_id": taskID,
})
return fmt.Errorf("failed to sanitize entry at index %d: %w", i+idx, err)
}
isEnabled := true
if entry.IsEnabled != nil {
isEnabled = *entry.IsEnabled
}
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: faqKnowledge.ID,
KnowledgeBaseID: kb.ID,
Content: buildFAQChunkContent(meta, indexMode),
ChunkIndex: startIndex + i + idx + 1,
IsEnabled: isEnabled,
ChunkType: types.ChunkTypeFAQ,
TagID: entry.TagID,
}
if err := chunk.SetFAQMetadata(meta); err != nil {
return fmt.Errorf("failed to set FAQ metadata: %w", err)
}
chunks = append(chunks, chunk)
}
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: faqKnowledge.ID,
KnowledgeBaseID: kb.ID,
Content: buildFAQChunkContent(meta, indexMode),
ChunkIndex: startIndex + idx + 1,
IsEnabled: isEnabled,
ChunkType: types.ChunkTypeFAQ,
TagID: entry.TagID,
buildDuration := time.Since(buildStartTime)
logger.Debugf(ctx, "FAQ import task %s: batch %d-%d built %d chunks in %v", taskID, i+1, end, len(chunks), buildDuration)
// 创建chunks
createStartTime := time.Now()
if err := s.chunkService.CreateChunks(ctx, chunks); err != nil {
return fmt.Errorf("failed to create chunks: %w", err)
}
if err := chunk.SetFAQMetadata(meta); err != nil {
return err
createDuration := time.Since(createStartTime)
logger.Infof(ctx, "FAQ import task %s: batch %d-%d created %d chunks in %v", taskID, i+1, end, len(chunks), createDuration)
// 记录已创建的chunks（用于失败时回滚）
createdChunks = append(createdChunks, chunks...)
// 索引chunks
indexStartTime := time.Now()
// 注意：如果索引失败，defer中的recovery机制会自动回滚已创建的chunks和索引数据
if err := s.indexFAQChunks(ctx, kb, faqKnowledge, chunks, embeddingModel, true, false); err != nil {
return fmt.Errorf("failed to index chunks: %w", err)
}
chunks = append(chunks, chunk)
indexDuration := time.Since(indexStartTime)
logger.Infof(ctx, "FAQ import task %s: batch %d-%d indexed %d chunks in %v", taskID, i+1, end, len(chunks), indexDuration)
// 记录已成功索引的chunks（用于失败时清理索引数据）
indexedChunks = append(indexedChunks, chunks...)
processed += len(batch)
// 更新任务进度
progressUpdateStartTime := time.Now()
progress := int(float64(processed) / float64(totalEntries) * 100)
if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusRunning, progress, totalEntries, processed, ""); err != nil {
logger.Errorf(ctx, "Failed to update task progress: %v", err)
}
progressUpdateDuration := time.Since(progressUpdateStartTime)
if progressUpdateDuration > 100*time.Millisecond {
logger.Warnf(ctx, "FAQ import task %s: progress update took %v (may be slow)", taskID, progressUpdateDuration)
}
batchDuration := time.Since(batchStartTime)
logger.Infof(ctx, "FAQ import task %s: batch %d-%d completed in %v (build: %v, create: %v, index: %v, progress: %v), total progress: %d/%d (%d%%)",
taskID, i+1, end, batchDuration, buildDuration, createDuration, indexDuration, progressUpdateDuration, processed, totalEntries, progress)
}
if err := s.chunkService.CreateChunks(ctx, chunks); err != nil {
return err
}
if err := s.indexFAQChunks(ctx, kb, faqKnowledge, chunks, true); err != nil {
for _, chunk := range chunks {
_ = s.chunkService.DeleteChunk(ctx, chunk.ID)
}
return err
}
totalDuration := time.Since(batchStartTime)
logger.Infof(ctx, "FAQ import task %s: all batches completed, total: %d entries in %v, avg: %v per entry",
taskID, totalEntries, totalDuration, totalDuration/time.Duration(totalEntries))
return nil
}
@@ -2228,7 +2430,11 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
if err != nil {
return err
}
return s.indexFAQChunks(ctx, kb, faqKnowledge, []*types.Chunk{chunk}, false)
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
return err
}
return s.indexFAQChunks(ctx, kb, faqKnowledge, []*types.Chunk{chunk}, embeddingModel, false, true)
}
// UpdateFAQEntryStatus updates enable status for a FAQ entry.
@@ -2732,6 +2938,63 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint
return knowledge, nil
}
// updateFAQImportStatus 更新FAQ Knowledge的导入任务状态
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus, progress, total, processed int, errorMsg string) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
return err
}
// 更新ParseStatus：将FAQImportTaskStatus映射到ParseStatus
// pending -> "pending", running -> "processing", success -> "completed", failed -> "failed"
parseStatus := string(status)
if status == types.FAQImportStatusRunning {
parseStatus = "processing" // 使用"processing"以兼容文档类型的ParseStatus
} else if status == types.FAQImportStatusSuccess {
parseStatus = "completed"
}
knowledge.ParseStatus = parseStatus
knowledge.UpdatedAt = time.Now()
// 更新ErrorMessage
if errorMsg != "" {
knowledge.ErrorMessage = errorMsg
} else if status == types.FAQImportStatusSuccess {
knowledge.ErrorMessage = "" // 成功时清空错误信息
}
// 更新Metadata中的导入进度信息
importMeta := &types.FAQImportMetadata{
ImportProgress: progress,
ImportTotal: total,
ImportProcessed: processed,
}
metaJSON, err := importMeta.ToJSON()
if err != nil {
return fmt.Errorf("failed to marshal import metadata: %w", err)
}
knowledge.Metadata = metaJSON
return s.repo.UpdateKnowledge(ctx, knowledge)
}
// getRunningFAQImportTask 获取指定知识库的进行中导入任务
func (s *knowledgeService) getRunningFAQImportTask(ctx context.Context, kbID string, tenantID uint) (*types.Knowledge, error) {
faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kbID)
if err != nil {
return nil, err
}
if faqKnowledge == nil {
return nil, errors.New("FAQ knowledge not found")
}
// 检查ParseStatus是否为pending或processing（进行中状态）
if faqKnowledge.ParseStatus == "pending" || faqKnowledge.ParseStatus == "processing" {
return faqKnowledge, nil
}
return nil, nil
}
func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase) (*types.FAQEntry, error) {
meta, err := chunk.FAQMetadata()
if err != nil {
@@ -2826,7 +3089,6 @@ func (s *knowledgeService) buildFAQIndexInfoList(ctx context.Context, kb *types.
questionIndexMode = kb.FAQConfig.QuestionIndexMode
}
}
logger.Infof(ctx, "buildFAQIndexInfoList: indexMode: %s, questionIndexMode: %s", indexMode, questionIndexMode)
meta, err := chunk.FAQMetadata()
if err != nil {
@@ -2896,27 +3158,29 @@ func (s *knowledgeService) buildFAQIndexInfoList(ctx context.Context, kb *types.
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
logger.Infof(ctx, "buildFAQIndexInfoList: indexInfoList: %v", indexInfoList)
return indexInfoList, nil
}
func (s *knowledgeService) indexFAQChunks(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*types.Chunk, adjustStorage bool,
kb *types.KnowledgeBase, knowledge *types.Knowledge,
chunks []*types.Chunk, embeddingModel embedding.Embedder,
adjustStorage bool, needDelete bool,
) error {
if len(chunks) == 0 {
return nil
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
return err
}
indexStartTime := time.Now()
logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks))
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
if err != nil {
return err
}
// 构建索引信息
buildIndexInfoStartTime := time.Now()
indexInfo := make([]*types.IndexInfo, 0)
chunkIDs := make([]string, 0, len(chunks))
for _, chunk := range chunks {
@@ -2927,33 +3191,68 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
indexInfo = append(indexInfo, infoList...)
chunkIDs = append(chunkIDs, chunk.ID)
}
buildIndexInfoDuration := time.Since(buildIndexInfoStartTime)
logger.Debugf(ctx, "indexFAQChunks: built %d index info entries for %d chunks in %v", len(indexInfo), len(chunks), buildIndexInfoDuration)
var size int64
if adjustStorage {
estimateStartTime := time.Now()
size = retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo)
estimateDuration := time.Since(estimateStartTime)
logger.Debugf(ctx, "indexFAQChunks: estimated storage size %d bytes in %v", size, estimateDuration)
if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed+size > tenantInfo.StorageQuota {
return types.NewStorageQuotaExceededError()
}
}
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err)
// 删除旧向量
var deleteDuration time.Duration
if needDelete {
deleteStartTime := time.Now()
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err)
}
deleteDuration = time.Since(deleteStartTime)
if deleteDuration > 100*time.Millisecond {
logger.Debugf(ctx, "indexFAQChunks: deleted old vectors for %d chunks in %v", len(chunkIDs), deleteDuration)
}
}
// 批量索引（这里可能是性能瓶颈）
batchIndexStartTime := time.Now()
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
return err
}
batchIndexDuration := time.Since(batchIndexStartTime)
logger.Debugf(ctx, "indexFAQChunks: batch indexed %d index info entries in %v (avg: %v per entry)", len(indexInfo), batchIndexDuration, batchIndexDuration/time.Duration(len(indexInfo)))
if adjustStorage && size > 0 {
adjustStartTime := time.Now()
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, size); err == nil {
tenantInfo.StorageUsed += size
}
knowledge.StorageSize += size
adjustDuration := time.Since(adjustStartTime)
if adjustDuration > 50*time.Millisecond {
logger.Debugf(ctx, "indexFAQChunks: adjusted storage in %v", adjustDuration)
}
}
updateStartTime := time.Now()
now := time.Now()
knowledge.UpdatedAt = now
knowledge.ProcessedAt = &now
return s.repo.UpdateKnowledge(ctx, knowledge)
err = s.repo.UpdateKnowledge(ctx, knowledge)
updateDuration := time.Since(updateStartTime)
if updateDuration > 50*time.Millisecond {
logger.Debugf(ctx, "indexFAQChunks: updated knowledge in %v", updateDuration)
}
totalDuration := time.Since(indexStartTime)
logger.Debugf(ctx, "indexFAQChunks: completed indexing %d chunks in %v (build: %v, delete: %v, batchIndex: %v, update: %v)",
len(chunks), totalDuration, buildIndexInfoDuration, deleteDuration, batchIndexDuration, updateDuration)
return err
}
func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context,
internal/application/service/knowledgebase.go
+16 -23
View File
@@ -98,8 +98,6 @@ func (s *knowledgeBaseService) GetKnowledgeBaseByID(ctx context.Context, id stri
return nil, errors.New("knowledge base ID cannot be empty")
}
logger.Infof(ctx, "Retrieving knowledge base, ID: %s", id)
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -109,15 +107,12 @@ func (s *knowledgeBaseService) GetKnowledgeBaseByID(ctx context.Context, id stri
}
kb.EnsureDefaults()
logger.Infof(ctx, "Knowledge base retrieved successfully, ID: %s, name: %s", kb.ID, kb.Name)
return kb, nil
}
// ListKnowledgeBases returns all knowledge bases for a tenant
func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
logger.Infof(ctx, "Retrieving knowledge base list for tenant, tenant ID: %d", tenantID)
kbs, err := s.repo.ListKnowledgeBasesByTenantID(ctx, tenantID)
if err != nil {
@@ -134,7 +129,7 @@ func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types
// Query knowledge count and chunk count for each knowledge base
for _, kb := range kbs {
kb.EnsureDefaults()
// Get knowledge count
if kb.Type == types.KnowledgeBaseTypeDocument {
knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID)
@@ -152,14 +147,21 @@ func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types
kb.ChunkCount = chunkCount
}
}
}
logger.Infof(
ctx,
"Knowledge base list retrieved successfully, tenant ID: %d, knowledge base count: %d",
tenantID,
len(kbs),
)
// Check if there is a processing import task
processingCount, err := s.kgRepo.CountKnowledgeByStatus(
ctx,
tenantID,
kb.ID,
[]string{"pending", "processing"},
)
if err != nil {
logger.Warnf(ctx, "Failed to check processing status for knowledge base %s: %v", kb.ID, err)
} else {
kb.IsProcessing = processingCount > 0
kb.ProcessingCount = processingCount
}
}
return kbs, nil
}
@@ -241,7 +243,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
}
logger.Infof(ctx, "Deleting all knowledge entries and their resources")
// Delete embeddings from vector store
logger.Infof(ctx, "Deleting embeddings from vector store")
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
@@ -532,16 +534,9 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
}
logger.Infof(ctx, "Result count before deduplication: %d", len(matchResults))
for _, chunk := range matchResults {
logger.Infof(ctx, "Before deduplication: Chunk: %s, Content: %s, Score: %f, MatchType: %d", chunk.ChunkID, chunk.Content, chunk.Score, chunk.MatchType)
}
// First, try standard deduplication
deduplicatedChunks := common.DeduplicateWithScore(func(r *types.IndexWithScore) string { return r.ChunkID }, matchResults...)
logger.Infof(ctx, "Result count after deduplication: %d", len(deduplicatedChunks))
for _, chunk := range deduplicatedChunks {
logger.Infof(ctx, "After deduplication: Chunk: %s, Content: %s, Score: %f, MatchType: %d", chunk.ChunkID, chunk.Content, chunk.Score, chunk.MatchType)
}
kb.EnsureDefaults()
@@ -582,8 +577,6 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
uniqueChunks := make(map[string]*types.IndexWithScore)
for i := 0; i < maxIterations; i++ {
logger.Infof(ctx, "Iterative retrieval iteration %d, TopK: %d", i+1, currentTopK)
// Update TopK in retrieve params
updatedParams := make([]types.RetrieveParams, len(retrieveParams))
for j := range retrieveParams {
internal/application/service/retriever/keywords_vector_hybrid_indexer.go
+4 -2
View File
@@ -77,13 +77,15 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
embeddings, err = embedder.BatchEmbedWithPool(ctx, embedder, contentList)
if err == nil {
break
} else {
logger.Errorf(ctx, "BatchEmbedWithPool failed: %v", err)
time.Sleep(100 * time.Millisecond)
}
time.Sleep(100 * time.Millisecond)
}
if err != nil {
return err
}
batchSize := 20
batchSize := 40
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
internal/application/service/session.go
-31
View File
@@ -125,8 +125,6 @@ func (s *sessionService) GetSession(ctx context.Context, id string) (*types.Sess
// GetSessionsByTenant retrieves all sessions for the current tenant
func (s *sessionService) GetSessionsByTenant(ctx context.Context) ([]*types.Session, error) {
logger.Info(ctx, "Start retrieving all sessions for tenant")
// Get tenant ID from context
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
logger.Infof(ctx, "Retrieving all sessions for tenant, tenant ID: %d", tenantID)
@@ -150,13 +148,8 @@ func (s *sessionService) GetSessionsByTenant(ctx context.Context) ([]*types.Sess
func (s *sessionService) GetPagedSessionsByTenant(ctx context.Context,
pagination *types.Pagination,
) (*types.PageResult, error) {
logger.Info(ctx, "Start retrieving paged sessions for tenant")
// Get tenant ID from context
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
logger.Infof(ctx, "Retrieving paged sessions for tenant, tenant ID: %d, page: %d, page size: %d",
tenantID, pagination.Page, pagination.PageSize)
// Get paged sessions from repository
sessions, total, err := s.sessionRepo.GetPagedByTenantID(ctx, tenantID, pagination)
if err != nil {
@@ -168,22 +161,17 @@ func (s *sessionService) GetPagedSessionsByTenant(ctx context.Context,
return nil, err
}
logger.Infof(ctx, "Tenant paged sessions retrieved successfully, tenant ID: %d, total: %d", tenantID, total)
return types.NewPageResult(total, pagination, sessions), nil
}
// UpdateSession updates an existing session's properties
func (s *sessionService) UpdateSession(ctx context.Context, session *types.Session) error {
logger.Info(ctx, "Start updating session")
// Validate session ID
if session.ID == "" {
logger.Error(ctx, "Failed to update session: session ID cannot be empty")
return errors.New("session id is required")
}
logger.Infof(ctx, "Updating session, ID: %s, tenant ID: %d", session.ID, session.TenantID)
// Update session in repository
err := s.sessionRepo.Update(ctx, session)
if err != nil {
@@ -200,8 +188,6 @@ func (s *sessionService) UpdateSession(ctx context.Context, session *types.Sessi
// DeleteSession removes a session by its ID
func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
logger.Info(ctx, "Start deleting session")
// Validate session ID
if id == "" {
logger.Error(ctx, "Failed to delete session: session ID cannot be empty")
@@ -210,7 +196,6 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
// Get tenant ID from context
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
logger.Infof(ctx, "Deleting session, ID: %s, tenant ID: %d", id, tenantID)
// Cleanup temporary KB stored in Redis for this session
if err := s.DeleteWebSearchTempKBState(ctx, id); err != nil {
@@ -227,7 +212,6 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
return err
}
logger.Infof(ctx, "Session deleted successfully, ID: %s", id)
return nil
}
@@ -235,8 +219,6 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
func (s *sessionService) GenerateTitle(ctx context.Context,
session *types.Session, messages []types.Message,
) (string, error) {
logger.Info(ctx, "Start generating session title")
if session == nil {
logger.Error(ctx, "Failed to generate title: session cannot be empty")
return "", errors.New("session cannot be empty")
@@ -244,14 +226,12 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
// Skip if title already exists
if session.Title != "" {
logger.Infof(ctx, "Session already has a title, session ID: %s, title: %s", session.ID, session.Title)
return session.Title, nil
}
var err error
// Get the first user message, either from provided messages or repository
var message *types.Message
if len(messages) == 0 {
logger.Info(ctx, "Message list is empty, getting the first user message")
message, err = s.messageRepo.GetFirstMessageOfUser(ctx, session.ID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -260,7 +240,6 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
return "", err
}
} else {
logger.Info(ctx, "Searching for user message in message list")
for _, m := range messages {
if m.Role == "user" {
message = &m
@@ -278,7 +257,6 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
// Get chat model, use default if SummaryModelID is empty
modelID := session.SummaryModelID
if modelID == "" {
logger.Info(ctx, "Session SummaryModelID is empty, trying to get default chat model")
// Try to get an available KnowledgeQA model
models, err := s.modelService.ListModels(ctx)
if err != nil {
@@ -299,7 +277,6 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
}
}
logger.Infof(ctx, "Getting chat model, model ID: %s", modelID)
chatModel, err := s.modelService.GetChatModel(ctx, modelID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -309,7 +286,6 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
}
// Prepare messages for title generation
logger.Info(ctx, "Preparing to generate session title")
var chatMessages []chat.Message
chatMessages = append(chatMessages,
chat.Message{Role: "system", Content: s.cfg.Conversation.GenerateSessionTitlePrompt},
@@ -320,7 +296,6 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
// Call model to generate title
thinking := false
logger.Info(ctx, "Calling model to generate title")
response, err := chatModel.Chat(ctx, chatMessages, &chat.ChatOptions{
Temperature: 0.3,
Thinking: &thinking,
@@ -332,17 +307,14 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
// Process and store the generated title
session.Title = strings.TrimPrefix(response.Content, "<think>\n\n</think>")
logger.Infof(ctx, "Title generated successfully: %s", session.Title)
// Update session with new title
logger.Info(ctx, "Updating session title")
err = s.sessionRepo.Update(ctx, session)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
return "", err
}
logger.Infof(ctx, "Session title updated successfully, ID: %s, title: %s", session.ID, session.Title)
return session.Title, nil
}
@@ -363,11 +335,8 @@ func (s *sessionService) GenerateTitleAsync(ctx context.Context, session *types.
bgCtx = context.WithValue(bgCtx, types.RequestIDContextKey, requestID)
}
logger.Info(bgCtx, "Starting async title generation")
// Skip if title already exists
if session.Title != "" {
logger.Infof(bgCtx, "Session already has a title, skipping generation, session ID: %s", session.ID)
return
}
internal/application/service/tenant.go
-9
View File
@@ -81,15 +81,11 @@ func (s *tenantService) CreateTenant(ctx context.Context, tenant *types.Tenant)
// GetTenantByID retrieves a tenant by their ID
func (s *tenantService) GetTenantByID(ctx context.Context, id uint) (*types.Tenant, error) {
logger.Info(ctx, "Start retrieving tenant")
if id == 0 {
logger.Error(ctx, "Tenant ID cannot be 0")
return nil, errors.New("tenant ID cannot be 0")
}
logger.Infof(ctx, "Retrieving tenant, ID: %d", id)
tenant, err := s.repo.GetTenantByID(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -98,14 +94,11 @@ func (s *tenantService) GetTenantByID(ctx context.Context, id uint) (*types.Tena
return nil, err
}
logger.Infof(ctx, "Tenant retrieved successfully, ID: %d, name: %s", tenant.ID, tenant.Name)
return tenant, nil
}
// ListTenants retrieves a list of all tenants
func (s *tenantService) ListTenants(ctx context.Context) ([]*types.Tenant, error) {
logger.Info(ctx, "Start retrieving tenant list")
tenants, err := s.repo.ListTenants(ctx)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
@@ -192,8 +185,6 @@ func (s *tenantService) UpdateAPIKey(ctx context.Context, id uint) (string, erro
return "", errors.New("tenant ID cannot be 0")
}
logger.Infof(ctx, "Retrieving tenant information, ID: %d", id)
tenant, err := s.repo.GetTenantByID(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
internal/handler/auth.go
-5
View File
@@ -217,8 +217,6 @@ func (h *AuthHandler) RefreshToken(c *gin.Context) {
func (h *AuthHandler) GetCurrentUser(c *gin.Context) {
ctx := c.Request.Context()
logger.Debugf(ctx, "Get current user info")
// Get current user from service (which extracts from context)
user, err := h.userService.GetCurrentUser(ctx)
if err != nil {
@@ -235,12 +233,9 @@ func (h *AuthHandler) GetCurrentUser(c *gin.Context) {
if err != nil {
logger.Warnf(ctx, "Failed to get tenant info for user %s, tenant ID %d: %v", user.Email, user.TenantID, err)
// Don't fail the request if tenant info is not available
} else {
logger.Debugf(ctx, "Retrieved tenant info for user %s: %s", user.Email, tenant.Name)
}
}
logger.Debugf(ctx, "Retrieved current user info: %s", user.Email)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
internal/handler/faq.go
+6 -2
View File
@@ -46,7 +46,7 @@ func (h *FAQHandler) ListEntries(c *gin.Context) {
})
}
// UpsertEntries appends or replaces FAQ entries in batch.
// UpsertEntries appends or replaces FAQ entries in batch asynchronously.
func (h *FAQHandler) UpsertEntries(c *gin.Context) {
ctx := c.Request.Context()
var req types.FAQBatchUpsertPayload
@@ -56,7 +56,8 @@ func (h *FAQHandler) UpsertEntries(c *gin.Context) {
return
}
if err := h.knowledgeService.UpsertFAQEntries(ctx, c.Param("id"), &req); err != nil {
taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, c.Param("id"), &req)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
@@ -64,6 +65,9 @@ func (h *FAQHandler) UpsertEntries(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
"task_id": taskID,
},
})
}
internal/handler/knowledgebase.go
+1 -30
View File
@@ -113,8 +113,6 @@ func (h *KnowledgeBaseHandler) validateAndGetKnowledgeBase(c *gin.Context) (*typ
return nil, "", errors.NewBadRequestError("Knowledge base ID cannot be empty")
}
logger.Infof(ctx, "Retrieving knowledge base, ID: %s", id)
// Verify tenant has permission to access this knowledge base
kb, err := h.service.GetKnowledgeBaseByID(ctx, id)
if err != nil {
@@ -138,17 +136,12 @@ func (h *KnowledgeBaseHandler) validateAndGetKnowledgeBase(c *gin.Context) (*typ
// GetKnowledgeBase handles requests to retrieve a knowledge base by ID
func (h *KnowledgeBaseHandler) GetKnowledgeBase(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving knowledge base")
// Validate and get the knowledge base
kb, id, err := h.validateAndGetKnowledgeBase(c)
kb, _, err := h.validateAndGetKnowledgeBase(c)
if err != nil {
c.Error(err)
return
}
logger.Infof(ctx, "Retrieved knowledge base successfully, ID: %s, name: %s", id, kb.Name)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": kb,
@@ -159,18 +152,6 @@ func (h *KnowledgeBaseHandler) GetKnowledgeBase(c *gin.Context) {
func (h *KnowledgeBaseHandler) ListKnowledgeBases(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving knowledge base list")
// Get tenant ID from context
tenantID, exists := c.Get(types.TenantIDContextKey.String())
if !exists {
logger.Error(ctx, "Failed to get tenant ID")
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
logger.Infof(ctx, "Retrieving knowledge base list for tenant, tenant ID: %d", tenantID.(uint))
// Get all knowledge bases for this tenant
kbs, err := h.service.ListKnowledgeBases(ctx)
if err != nil {
@@ -179,11 +160,6 @@ func (h *KnowledgeBaseHandler) ListKnowledgeBases(c *gin.Context) {
return
}
logger.Infof(
ctx,
"Retrieved knowledge base list successfully, tenant ID: %d, total: %d knowledge bases",
tenantID.(uint), len(kbs),
)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": kbs,
@@ -269,8 +245,6 @@ type CopyKnowledgeBaseRequest struct {
func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start copy knowledge base")
var req CopyKnowledgeBaseRequest
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
@@ -278,8 +252,6 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
return
}
logger.Infof(ctx, "Copy knowledge base, ID: %s to ID: %s", req.SourceID, req.TargetID)
go func(ctx context.Context) {
err := h.knowledgeService.CloneKnowledgeBase(ctx, req.SourceID, req.TargetID)
if err != nil {
@@ -289,7 +261,6 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
logger.Infof(ctx, "Knowledge base copy from ID: %s to ID: %s successfully", req.SourceID, req.TargetID)
}(logger.CloneContext(ctx))
logger.Infof(ctx, "Knowledge base start copy from ID: %s to ID: %s", req.SourceID, req.TargetID)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge base copy successfully",
internal/handler/session/handler.go
-13
View File
@@ -226,8 +226,6 @@ func (h *Handler) GetSession(c *gin.Context) {
func (h *Handler) GetSessionsByTenant(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving all sessions for tenant")
// Parse pagination parameters from query
var pagination types.Pagination
if err := c.ShouldBindQuery(&pagination); err != nil {
@@ -236,8 +234,6 @@ func (h *Handler) GetSessionsByTenant(c *gin.Context) {
return
}
logger.Debugf(ctx, "Using pagination parameters: page=%d, page_size=%d", pagination.Page, pagination.PageSize)
// Use paginated query to get sessions
result, err := h.sessionService.GetPagedSessionsByTenant(ctx, &pagination)
if err != nil {
@@ -247,7 +243,6 @@ func (h *Handler) GetSessionsByTenant(c *gin.Context) {
}
// Return sessions with pagination data
logger.Infof(ctx, "Successfully retrieved tenant sessions, total: %d", result.Total)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": result.Data,
@@ -261,8 +256,6 @@ func (h *Handler) GetSessionsByTenant(c *gin.Context) {
func (h *Handler) UpdateSession(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start updating session")
// Get session ID from URL parameter
id := c.Param("id")
if id == "" {
@@ -287,8 +280,6 @@ func (h *Handler) UpdateSession(c *gin.Context) {
return
}
// Set session ID and tenant ID
logger.Infof(ctx, "Updating session, ID: %s, tenant ID: %d", id, tenantID.(uint))
session.ID = id
session.TenantID = tenantID.(uint)
@@ -316,8 +307,6 @@ func (h *Handler) UpdateSession(c *gin.Context) {
func (h *Handler) DeleteSession(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start deleting session")
// Get session ID from URL parameter
id := c.Param("id")
if id == "" {
@@ -327,7 +316,6 @@ func (h *Handler) DeleteSession(c *gin.Context) {
}
// Call service to delete session
logger.Infof(ctx, "Deleting session, ID: %s", id)
if err := h.sessionService.DeleteSession(ctx, id); err != nil {
if err == errors.ErrSessionNotFound {
logger.Warnf(ctx, "Session not found, ID: %s", id)
@@ -340,7 +328,6 @@ func (h *Handler) DeleteSession(c *gin.Context) {
}
// Return success message
logger.Infof(ctx, "Session deleted successfully, ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Session deleted successfully",
internal/handler/tenant.go
-12
View File
@@ -84,8 +84,6 @@ func (h *TenantHandler) CreateTenant(c *gin.Context) {
func (h *TenantHandler) GetTenant(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving tenant")
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
logger.Errorf(ctx, "Invalid tenant ID: %s", c.Param("id"))
@@ -93,8 +91,6 @@ func (h *TenantHandler) GetTenant(c *gin.Context) {
return
}
logger.Infof(ctx, "Retrieving tenant, ID: %d", id)
tenant, err := h.service.GetTenantByID(ctx, uint(id))
if err != nil {
// Check if this is an application-specific error
@@ -108,7 +104,6 @@ func (h *TenantHandler) GetTenant(c *gin.Context) {
return
}
logger.Infof(ctx, "Retrieved tenant successfully, ID: %d, Name: %s", tenant.ID, tenant.Name)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": tenant,
@@ -207,8 +202,6 @@ func (h *TenantHandler) DeleteTenant(c *gin.Context) {
func (h *TenantHandler) ListTenants(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving tenant list")
tenants, err := h.service.ListTenants(ctx)
if err != nil {
// Check if this is an application-specific error
@@ -222,7 +215,6 @@ func (h *TenantHandler) ListTenants(c *gin.Context) {
return
}
logger.Infof(ctx, "Retrieved tenant list successfully, Total: %d tenants", len(tenants))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
@@ -246,8 +238,6 @@ type AgentConfigRequest struct {
// This is the global agent configuration that applies to all sessions by default
func (h *TenantHandler) GetTenantAgentConfig(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving tenant agent config")
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
@@ -558,8 +548,6 @@ func validateConversationConfig(req *types.ConversationConfig) error {
// This is the global conversation configuration that applies to normal mode sessions by default
func (h *TenantHandler) GetTenantConversationConfig(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving tenant conversation config")
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
internal/middleware/logger.go
+142 -11
View File
@@ -1,7 +1,11 @@
package middleware
import (
"bytes"
"context"
"io"
"regexp"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
@@ -10,6 +14,88 @@ import (
"github.com/google/uuid"
)
const (
maxBodySize = 1024 * 10 // 最大记录10KB的body内容
)
// loggerResponseBodyWriter 自定义ResponseWriter用于捕获响应内容（用于logger中间件）
type loggerResponseBodyWriter struct {
gin.ResponseWriter
body *bytes.Buffer
}
// Write 重写Write方法，同时写入buffer和原始writer
func (r loggerResponseBodyWriter) Write(b []byte) (int, error) {
r.body.Write(b)
return r.ResponseWriter.Write(b)
}
// sanitizeBody 清理敏感信息
func sanitizeBody(body string) string {
result := body
// 替换常见的敏感字段（JSON格式）
sensitivePatterns := []struct {
pattern string
replacement string
}{
{`"password"\s*:\s*"[^"]*"`, `"password":"***"`},
{`"token"\s*:\s*"[^"]*"`, `"token":"***"`},
{`"access_token"\s*:\s*"[^"]*"`, `"access_token":"***"`},
{`"refresh_token"\s*:\s*"[^"]*"`, `"refresh_token":"***"`},
{`"authorization"\s*:\s*"[^"]*"`, `"authorization":"***"`},
{`"api_key"\s*:\s*"[^"]*"`, `"api_key":"***"`},
{`"secret"\s*:\s*"[^"]*"`, `"secret":"***"`},
{`"apikey"\s*:\s*"[^"]*"`, `"apikey":"***"`},
{`"apisecret"\s*:\s*"[^"]*"`, `"apisecret":"***"`},
}
for _, p := range sensitivePatterns {
re := regexp.MustCompile(p.pattern)
result = re.ReplaceAllString(result, p.replacement)
}
return result
}
// readRequestBody 读取请求体（限制大小用于日志，但完整读取用于重置）
func readRequestBody(c *gin.Context) string {
if c.Request.Body == nil {
return ""
}
// 检查Content-Type，只记录JSON类型
contentType := c.GetHeader("Content-Type")
if !strings.Contains(contentType, "application/json") &&
!strings.Contains(contentType, "application/x-www-form-urlencoded") &&
!strings.Contains(contentType, "text/") {
return "[非文本类型，已跳过]"
}
// 完整读取body内容（不限制大小），因为需要完整重置给后续handler使用
bodyBytes, err := io.ReadAll(c.Request.Body)
if err != nil {
return "[读取请求体失败]"
}
// 重置request body，使用完整内容，确保后续handler能读取到完整数据
c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
// 用于日志的body（限制大小）
var logBodyBytes []byte
if len(bodyBytes) > maxBodySize {
logBodyBytes = bodyBytes[:maxBodySize]
} else {
logBodyBytes = bodyBytes
}
bodyStr := string(logBodyBytes)
if len(bodyBytes) > maxBodySize {
bodyStr += "... [内容过长，已截断]"
}
return sanitizeBody(bodyStr)
}
// RequestID middleware adds a unique request ID to the context
func RequestID() gin.HandlerFunc {
return func(c *gin.Context) {
@@ -41,13 +127,27 @@ func RequestID() gin.HandlerFunc {
}
}
// Logger middleware logs request details with request ID
// Logger middleware logs request details with request ID, input and output
func Logger() gin.HandlerFunc {
return func(c *gin.Context) {
start := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
// 读取请求体（在Next之前读取，因为Next会消费body）
var requestBody string
if c.Request.Method == "POST" || c.Request.Method == "PUT" || c.Request.Method == "PATCH" {
requestBody = readRequestBody(c)
}
// 创建响应体捕获器
responseBody := &bytes.Buffer{}
responseWriter := &loggerResponseBodyWriter{
ResponseWriter: c.Writer,
body: responseBody,
}
c.Writer = responseWriter
// Process request
c.Next()
@@ -69,15 +169,46 @@ func Logger() gin.HandlerFunc {
path = path + "?" + raw
}
// Log with request ID
logger.GetLogger(c).Infof("[%s] %d | %3d | %13v | %15s | %s %s",
requestID,
statusCode,
c.Writer.Size(),
latency,
clientIP,
method,
path,
)
// 读取响应体
responseBodyStr := ""
if responseBody.Len() > 0 {
// 检查Content-Type，只记录JSON类型
contentType := c.Writer.Header().Get("Content-Type")
if strings.Contains(contentType, "application/json") ||
strings.Contains(contentType, "text/") {
bodyBytes := responseBody.Bytes()
if len(bodyBytes) > maxBodySize {
responseBodyStr = string(bodyBytes[:maxBodySize]) + "... [内容过长，已截断]"
} else {
responseBodyStr = string(bodyBytes)
}
responseBodyStr = sanitizeBody(responseBodyStr)
} else {
responseBodyStr = "[非文本类型，已跳过]"
}
}
// 构建日志消息
logMsg := logger.GetLogger(c)
logMsg = logMsg.WithFields(map[string]interface{}{
"request_id": requestID,
"method": method,
"path": path,
"status_code": statusCode,
"size": c.Writer.Size(),
"latency": latency.String(),
"client_ip": clientIP,
})
// 添加请求体（如果有）
if requestBody != "" {
logMsg = logMsg.WithField("request_body", requestBody)
}
// 添加响应体（如果有）
if responseBodyStr != "" {
logMsg = logMsg.WithField("response_body", responseBodyStr)
}
logMsg.Info()
}
}
internal/models/chat/remote_api.go
+5 -5
View File
@@ -200,11 +200,11 @@ func (c *RemoteAPIChat) buildChatCompletionRequest(messages []Message,
}
// print req
jsonData, err := json.Marshal(req)
if err != nil {
logger.Error(context.Background(), "marshal request: %w", err)
}
logger.Infof(context.Background(), "llm request: %s", string(jsonData))
// jsonData, err := json.Marshal(req)
// if err != nil {
// logger.Error(context.Background(), "marshal request: %w", err)
// }
// logger.Infof(context.Background(), "llm request: %s", string(jsonData))
return req
}
internal/models/embedding/batch.go
+10 -1
View File
@@ -2,6 +2,8 @@ package embedding
import (
"context"
"os"
"strconv"
"sync"
"github.com/Tencent/WeKnora/internal/models/utils"
@@ -26,7 +28,14 @@ func (e *batchEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder,
var wg sync.WaitGroup
var mu sync.Mutex // For synchronizing access to error
var firstErr error // Record the first error that occurs
batchSize := 5
batchSizeStr := os.Getenv("BATCH_EMBED_SIZE")
if batchSizeStr == "" {
batchSizeStr = "5"
}
batchSize, err := strconv.Atoi(batchSizeStr)
if err != nil {
return nil, err
}
textEmbeddings := utils.MapSlice(texts, func(text string) *textEmbedding {
return &textEmbedding{text: text}
})
internal/stream/redis_manager.go
+1 -1
View File
@@ -111,7 +111,7 @@ func (r *RedisStreamManager) GetEvents(ctx context.Context, sessionID, messageID
continue
}
events = append(events, event)
}
}
// Calculate next offset
nextOffset := fromOffset + len(results)
internal/types/faq.go
+41
View File
@@ -110,6 +110,47 @@ type FAQSearchRequest struct {
MatchCount int `json:"match_count"`
}
// FAQImportTaskStatus 导入任务状态
type FAQImportTaskStatus string
const (
FAQImportStatusPending FAQImportTaskStatus = "pending"
FAQImportStatusRunning FAQImportTaskStatus = "running"
FAQImportStatusSuccess FAQImportTaskStatus = "success"
FAQImportStatusFailed FAQImportTaskStatus = "failed"
)
// FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息
type FAQImportMetadata struct {
ImportProgress int `json:"import_progress"` // 0-100
ImportTotal int `json:"import_total"`
ImportProcessed int `json:"import_processed"`
}
// ToJSON converts the metadata to JSON type.
func (m *FAQImportMetadata) ToJSON() (JSON, error) {
if m == nil {
return nil, nil
}
bytes, err := json.Marshal(m)
if err != nil {
return nil, err
}
return JSON(bytes), nil
}
// ParseFAQImportMetadata parses FAQ import metadata from Knowledge.
func ParseFAQImportMetadata(k *Knowledge) (*FAQImportMetadata, error) {
if k == nil || len(k.Metadata) == 0 {
return nil, nil
}
var metadata FAQImportMetadata
if err := json.Unmarshal(k.Metadata, &metadata); err != nil {
return nil, err
}
return &metadata, nil
}
func normalizeStrings(values []string) []string {
if len(values) == 0 {
return nil
internal/types/interfaces/chunk.go
+4
View File
@@ -33,6 +33,8 @@ type ChunkRepository interface {
UpdateChunks(ctx context.Context, chunks []*types.Chunk) error
// DeleteChunk deletes a chunk
DeleteChunk(ctx context.Context, tenantID uint, id string) error
// DeleteChunks deletes chunks by IDs in batch
DeleteChunks(ctx context.Context, tenantID uint, ids []string) error
// DeleteChunksByKnowledgeID deletes chunks by knowledge id
DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error
// DeleteByKnowledgeList deletes all chunks for a knowledge list
@@ -60,6 +62,8 @@ type ChunkService interface {
UpdateChunk(ctx context.Context, chunk *types.Chunk) error
// DeleteChunk deletes a chunk
DeleteChunk(ctx context.Context, id string) error
// DeleteChunks deletes chunks by IDs in batch
DeleteChunks(ctx context.Context, ids []string) error
// DeleteChunksByKnowledgeID deletes chunks by knowledge id
DeleteChunksByKnowledgeID(ctx context.Context, knowledgeID string) error
// DeleteByKnowledgeList deletes all chunks for a knowledge list
internal/types/interfaces/knowledge.go
+5 -2
View File
@@ -56,8 +56,9 @@ type KnowledgeService interface {
// ListFAQEntries lists FAQ entries under a FAQ knowledge base.
// When tagID is non-empty, results are filtered by tag_id on FAQ chunks.
ListFAQEntries(ctx context.Context, kbID string, page *types.Pagination, tagID string) (*types.PageResult, error)
// UpsertFAQEntries imports or appends FAQ entries.
UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) error
// UpsertFAQEntries imports or appends FAQ entries asynchronously.
// Returns task ID (Knowledge ID) for tracking import progress.
UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error)
// UpdateFAQEntry updates a single FAQ entry.
UpdateFAQEntry(ctx context.Context, kbID string, entryID string, payload *types.FAQEntryPayload) error
// UpdateFAQEntryStatusBatch updates enable status for FAQ entries in batch.
@@ -105,4 +106,6 @@ type KnowledgeRepository interface {
UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base.
CountKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint, kbID string) (int64, error)
// CountKnowledgeByStatus counts the number of knowledge items with the specified parse status.
CountKnowledgeByStatus(ctx context.Context, tenantID uint, kbID string, parseStatuses []string) (int64, error)
}
internal/types/knowledgebase.go
+4
View File
@@ -73,6 +73,10 @@ type KnowledgeBase struct {
KnowledgeCount int64 `yaml:"knowledge_count" json:"knowledge_count" gorm:"-"`
// Chunk count (not stored in database, calculated on query)
ChunkCount int64 `yaml:"chunk_count" json:"chunk_count" gorm:"-"`
// IsProcessing indicates if there is a processing import task (for FAQ type knowledge bases)
IsProcessing bool `yaml:"is_processing" json:"is_processing" gorm:"-"`
// ProcessingCount indicates the number of knowledge items being processed (for document type knowledge bases)
ProcessingCount int64 `yaml:"processing_count" json:"processing_count" gorm:"-"`
}
// KnowledgeBaseConfig represents the knowledge base configuration
migrations/versioned/000012_add_embeddings_kb_id_index.down.sql
+13
View File
@@ -0,0 +1,13 @@
-- 000012_add_embeddings_kb_id_index.down.sql
-- Remove B-tree index on knowledge_base_id from embeddings table
BEGIN;
DROP INDEX IF EXISTS idx_embeddings_knowledge_base_id;
COMMIT;
migrations/versioned/000012_add_embeddings_kb_id_index.up.sql
+15
View File
@@ -0,0 +1,15 @@
-- 000012_add_embeddings_kb_id_index.up.sql
-- Add B-tree index on knowledge_base_id for embeddings table to improve query performance
BEGIN;
-- Create index for knowledge_base_id to optimize filtering queries
CREATE INDEX IF NOT EXISTS idx_embeddings_knowledge_base_id
ON embeddings(knowledge_base_id);
COMMIT;