mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-19 02:18:25 +08:00
Merge branch 'main' into login_page
This commit is contained in:
Binary file not shown.
|
After Width: | Height: | Size: 504 KiB |
@@ -3,6 +3,13 @@ server {
|
||||
server_name localhost;
|
||||
client_max_body_size 50M;
|
||||
|
||||
# 安全头配置
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https: http:; font-src 'self' data:; connect-src 'self' http: https: ws: wss:; frame-ancestors 'self';" always;
|
||||
|
||||
# 错误日志配置
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
access_log /var/log/nginx/access.log;
|
||||
|
||||
Generated
+26
@@ -9,7 +9,9 @@
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@microsoft/fetch-event-source": "^2.0.1",
|
||||
"@types/dompurify": "^3.0.5",
|
||||
"axios": "^1.8.4",
|
||||
"dompurify": "^3.2.6",
|
||||
"marked": "^5.1.2",
|
||||
"pagefind": "^1.1.1",
|
||||
"pinia": "^3.0.1",
|
||||
@@ -1274,6 +1276,15 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/dompurify": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://mirrors.tencent.com/npm/@types/dompurify/-/dompurify-3.0.5.tgz",
|
||||
"integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/trusted-types": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/eslint": {
|
||||
"version": "9.6.1",
|
||||
"resolved": "https://mirrors.tencent.com/npm/@types/eslint/-/eslint-9.6.1.tgz",
|
||||
@@ -1346,6 +1357,12 @@
|
||||
"resolved": "https://mirrors.tencent.com/npm/@types/tinycolor2/-/tinycolor2-1.4.6.tgz",
|
||||
"integrity": "sha512-iEN8J0BoMnsWBqjVbWH/c0G0Hh7O21lpR2/+PrvAVgWdzL7eexIFm4JN/Wn10PTcmNdtS6U67r499mlWMXOxNw=="
|
||||
},
|
||||
"node_modules/@types/trusted-types": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://mirrors.tencent.com/npm/@types/trusted-types/-/trusted-types-2.0.7.tgz",
|
||||
"integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/validator": {
|
||||
"version": "13.15.2",
|
||||
"resolved": "https://mirrors.tencent.com/npm/@types/validator/-/validator-13.15.2.tgz",
|
||||
@@ -2121,6 +2138,15 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
"version": "3.2.6",
|
||||
"resolved": "https://mirrors.tencent.com/npm/dompurify/-/dompurify-3.2.6.tgz",
|
||||
"integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==",
|
||||
"license": "(MPL-2.0 OR Apache-2.0)",
|
||||
"optionalDependencies": {
|
||||
"@types/trusted-types": "^2.0.7"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://mirrors.tencent.com/npm/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@microsoft/fetch-event-source": "^2.0.1",
|
||||
"@types/dompurify": "^3.0.5",
|
||||
"axios": "^1.8.4",
|
||||
"dompurify": "^3.2.6",
|
||||
"marked": "^5.1.2",
|
||||
"pagefind": "^1.1.1",
|
||||
"pinia": "^3.0.1",
|
||||
|
||||
@@ -4,6 +4,8 @@ import { onMounted, ref, nextTick, onUnmounted, onUpdated, watch } from "vue";
|
||||
import { downKnowledgeDetails } from "@/api/knowledge-base/index";
|
||||
import { MessagePlugin } from "tdesign-vue-next";
|
||||
import picturePreview from '@/components/picture-preview.vue';
|
||||
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL } from '@/utils/security';
|
||||
|
||||
marked.use({
|
||||
mangle: false,
|
||||
headerIds: false,
|
||||
@@ -37,10 +39,16 @@ const checkImage = (url) => {
|
||||
});
|
||||
};
|
||||
renderer.image = function (href, title, text) {
|
||||
// 自定义HTML结构,图片展示带标题
|
||||
// 安全地处理图片链接
|
||||
if (!isValidImageURL(href)) {
|
||||
return `<p>无效的图片链接</p>`;
|
||||
}
|
||||
|
||||
// 使用安全的图片创建函数
|
||||
const safeImage = createSafeImage(href, text || '', title || '');
|
||||
return `<figure>
|
||||
<img class="markdown-image" src="${href}" alt="${title}" title="${text}">
|
||||
<figcaption style="text-align: left;">${text}</figcaption>
|
||||
${safeImage}
|
||||
<figcaption style="text-align: left;">${text || ''}</figcaption>
|
||||
</figure>`;
|
||||
};
|
||||
const props = defineProps(["visible", "details"]);
|
||||
@@ -66,14 +74,23 @@ watch(() => props.details.md, (newVal) => {
|
||||
deep: true
|
||||
})
|
||||
|
||||
// 处理 Markdown 中的图片
|
||||
// 安全地处理 Markdown 内容
|
||||
const processMarkdown = (markdownText) => {
|
||||
// 自定义渲染器处理图片
|
||||
if (!markdownText || typeof markdownText !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
// 首先对 Markdown 内容进行安全处理
|
||||
const safeMarkdown = safeMarkdownToHTML(markdownText);
|
||||
|
||||
// 使用安全的渲染器
|
||||
marked.use({ renderer });
|
||||
let html = marked.parse(markdownText);
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(html, 'text/html');
|
||||
return doc.body.innerHTML;
|
||||
let html = marked.parse(safeMarkdown);
|
||||
|
||||
// 使用 DOMPurify 进行最终的安全清理
|
||||
const sanitizedHTML = sanitizeHTML(html);
|
||||
|
||||
return sanitizedHTML;
|
||||
};
|
||||
const closePreImg = () => {
|
||||
reviewImg.value = false
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* 安全工具类 - 防止 XSS 攻击
|
||||
*/
|
||||
|
||||
import DOMPurify from 'dompurify';
|
||||
|
||||
// 配置 DOMPurify 的安全策略
|
||||
const DOMPurifyConfig = {
|
||||
// 允许的标签
|
||||
ALLOWED_TAGS: [
|
||||
'p', 'br', 'strong', 'em', 'u', 's', 'del', 'ins',
|
||||
'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
|
||||
'ul', 'ol', 'li', 'blockquote', 'pre', 'code',
|
||||
'a', 'img', 'table', 'thead', 'tbody', 'tr', 'th', 'td',
|
||||
'div', 'span', 'figure', 'figcaption'
|
||||
],
|
||||
// 允许的属性
|
||||
ALLOWED_ATTR: [
|
||||
'href', 'title', 'alt', 'src', 'class', 'id', 'style',
|
||||
'target', 'rel', 'width', 'height'
|
||||
],
|
||||
// 允许的协议
|
||||
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
|
||||
// 禁止的标签和属性
|
||||
FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'],
|
||||
FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'],
|
||||
// 其他安全配置
|
||||
KEEP_CONTENT: true,
|
||||
RETURN_DOM: false,
|
||||
RETURN_DOM_FRAGMENT: false,
|
||||
RETURN_DOM_IMPORT: false,
|
||||
SANITIZE_DOM: true,
|
||||
SANITIZE_NAMED_PROPS: true,
|
||||
WHOLE_DOCUMENT: false,
|
||||
// 自定义钩子函数
|
||||
HOOKS: {
|
||||
// 在清理前处理
|
||||
beforeSanitizeElements: (currentNode: Element) => {
|
||||
// 移除所有 script 标签
|
||||
if (currentNode.tagName === 'SCRIPT') {
|
||||
currentNode.remove();
|
||||
return null;
|
||||
}
|
||||
// 移除所有事件处理器
|
||||
const eventAttrs = ['onclick', 'onload', 'onerror', 'onmouseover', 'onfocus', 'onblur'];
|
||||
eventAttrs.forEach(attr => {
|
||||
if (currentNode.hasAttribute(attr)) {
|
||||
currentNode.removeAttribute(attr);
|
||||
}
|
||||
});
|
||||
},
|
||||
// 在清理后处理
|
||||
afterSanitizeElements: (currentNode: Element) => {
|
||||
// 确保所有链接都有 rel="noopener noreferrer"
|
||||
if (currentNode.tagName === 'A') {
|
||||
const href = currentNode.getAttribute('href');
|
||||
if (href && href.startsWith('http')) {
|
||||
currentNode.setAttribute('rel', 'noopener noreferrer');
|
||||
currentNode.setAttribute('target', '_blank');
|
||||
}
|
||||
}
|
||||
// 确保所有图片都有 alt 属性
|
||||
if (currentNode.tagName === 'IMG') {
|
||||
if (!currentNode.getAttribute('alt')) {
|
||||
currentNode.setAttribute('alt', '');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 安全地清理 HTML 内容
|
||||
* @param html 需要清理的 HTML 字符串
|
||||
* @returns 清理后的安全 HTML 字符串
|
||||
*/
|
||||
export function sanitizeHTML(html: string): string {
|
||||
if (!html || typeof html !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
try {
|
||||
return DOMPurify.sanitize(html, DOMPurifyConfig);
|
||||
} catch (error) {
|
||||
console.error('HTML sanitization failed:', error);
|
||||
// 如果清理失败,返回转义的纯文本
|
||||
return escapeHTML(html);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义 HTML 特殊字符
|
||||
* @param text 需要转义的文本
|
||||
* @returns 转义后的文本
|
||||
*/
|
||||
export function escapeHTML(text: string): string {
|
||||
if (!text || typeof text !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
const map: { [key: string]: string } = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": ''',
|
||||
'/': '/',
|
||||
'`': '`',
|
||||
'=': '='
|
||||
};
|
||||
|
||||
return text.replace(/[&<>"'`=\/]/g, (s) => map[s]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 URL 是否安全
|
||||
* @param url 需要验证的 URL
|
||||
* @returns 是否为安全 URL
|
||||
*/
|
||||
export function isValidURL(url: string): boolean {
|
||||
if (!url || typeof url !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
// 只允许 http 和 https 协议
|
||||
return ['http:', 'https:'].includes(urlObj.protocol);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 安全地处理 Markdown 内容
|
||||
* @param markdown Markdown 文本
|
||||
* @returns 安全的 HTML 字符串
|
||||
*/
|
||||
export function safeMarkdownToHTML(markdown: string): string {
|
||||
if (!markdown || typeof markdown !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
// 首先转义可能的 HTML 标签
|
||||
const escapedMarkdown = markdown
|
||||
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
|
||||
.replace(/<iframe\b[^<]*(?:(?!<\/iframe>)<[^<]*)*<\/iframe>/gi, '')
|
||||
.replace(/<object\b[^<]*(?:(?!<\/object>)<[^<]*)*<\/object>/gi, '')
|
||||
.replace(/<embed\b[^<]*(?:(?!<\/embed>)<[^<]*)*<\/embed>/gi, '');
|
||||
|
||||
return escapedMarkdown;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理用户输入
|
||||
* @param input 用户输入
|
||||
* @returns 清理后的安全输入
|
||||
*/
|
||||
export function sanitizeUserInput(input: string): string {
|
||||
if (!input || typeof input !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
// 移除控制字符
|
||||
let cleaned = input.replace(/[\x00-\x1F\x7F-\x9F]/g, '');
|
||||
|
||||
// 限制长度
|
||||
if (cleaned.length > 10000) {
|
||||
cleaned = cleaned.substring(0, 10000);
|
||||
}
|
||||
|
||||
return cleaned.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证图片 URL 是否安全
|
||||
* @param url 图片 URL
|
||||
* @returns 是否为安全的图片 URL
|
||||
*/
|
||||
export function isValidImageURL(url: string): boolean {
|
||||
if (!isValidURL(url)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否为图片文件
|
||||
const imageExtensions = /\.(jpg|jpeg|png|gif|webp|svg|bmp|ico)(\?.*)?$/i;
|
||||
return imageExtensions.test(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建安全的图片元素
|
||||
* @param src 图片源
|
||||
* @param alt 替代文本
|
||||
* @param title 标题
|
||||
* @returns 安全的图片 HTML
|
||||
*/
|
||||
export function createSafeImage(src: string, alt: string = '', title: string = ''): string {
|
||||
if (!isValidImageURL(src)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const safeSrc = escapeHTML(src);
|
||||
const safeAlt = escapeHTML(alt);
|
||||
const safeTitle = escapeHTML(title);
|
||||
|
||||
return `<img src="${safeSrc}" alt="${safeAlt}" title="${safeTitle}" class="markdown-image" style="max-width: 100%; height: auto;">`;
|
||||
}
|
||||
@@ -23,6 +23,8 @@ import { marked } from 'marked';
|
||||
import docInfo from './docInfo.vue';
|
||||
import deepThink from './deepThink.vue';
|
||||
import picturePreview from '@/components/picture-preview.vue';
|
||||
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL } from '@/utils/security';
|
||||
|
||||
marked.use({
|
||||
mangle: false,
|
||||
headerIds: false,
|
||||
@@ -89,36 +91,36 @@ const checkImage = (url) => {
|
||||
img.src = url;
|
||||
});
|
||||
};
|
||||
// 处理 Markdown 中的图片
|
||||
// 安全地处理 Markdown 内容
|
||||
const processMarkdown = (markdownText) => {
|
||||
// 自定义渲染器处理图片
|
||||
if (!markdownText || typeof markdownText !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
// 首先对 Markdown 内容进行安全处理
|
||||
const safeMarkdown = safeMarkdownToHTML(markdownText);
|
||||
|
||||
// 自定义安全的渲染器处理图片
|
||||
const renderer = {
|
||||
image(href, title, text) {
|
||||
return `<img src="${href}" alt="${text}" title="${title || ''}" class="markdown-image" style="max-width: 708px;height: 230px;">`;
|
||||
// 验证图片 URL 是否安全
|
||||
if (!isValidImageURL(href)) {
|
||||
return `<p>无效的图片链接</p>`;
|
||||
}
|
||||
// 使用安全的图片创建函数
|
||||
return createSafeImage(href, text || '', title || '');
|
||||
}
|
||||
};
|
||||
|
||||
marked.use({ renderer });
|
||||
|
||||
// 第一次渲染
|
||||
let html = marked.parse(markdownText);
|
||||
// 安全地渲染 Markdown
|
||||
let html = marked.parse(safeMarkdown);
|
||||
|
||||
// 创建虚拟 DOM 来操作
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(html, 'text/html');
|
||||
|
||||
// 检查所有图片
|
||||
// const images = doc.querySelectorAll('img');
|
||||
// images.forEach(async item => {
|
||||
// const isValid = await checkImage(item.src);
|
||||
// if (!isValid) {
|
||||
// item.remove();
|
||||
// }
|
||||
// });
|
||||
// if (props.isFirstEnter) {
|
||||
// emit('scroll-bottom')
|
||||
// }
|
||||
return doc.body.innerHTML;
|
||||
// 使用 DOMPurify 进行最终的安全清理
|
||||
const sanitizedHTML = sanitizeHTML(html);
|
||||
|
||||
return sanitizedHTML;
|
||||
};
|
||||
const handleImg = async (newVal) => {
|
||||
let index = newVal.lastIndexOf('![');
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
</template>
|
||||
<script setup>
|
||||
import { onMounted, watch, computed, ref, reactive, defineProps } from 'vue';
|
||||
import { sanitizeHTML } from '@/utils/security';
|
||||
|
||||
const isFold = ref(true)
|
||||
const props = defineProps({
|
||||
@@ -51,7 +52,6 @@ const showHide = () => {
|
||||
}
|
||||
const handlePanelChange = (val) => {
|
||||
isFold.value = !val.length ? true : false;
|
||||
|
||||
}
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
trigger="click">
|
||||
<template #content>
|
||||
<div class="doc_content">
|
||||
<div v-html="item.content.replace(/\n/g, '<br/>')"></div>
|
||||
<div v-html="safeProcessContent(item.content)"></div>
|
||||
</div>
|
||||
</template>
|
||||
<span class="doc">
|
||||
@@ -28,6 +28,7 @@
|
||||
</template>
|
||||
<script setup>
|
||||
import { onMounted, defineProps, computed, ref, reactive } from "vue";
|
||||
import { sanitizeHTML } from '@/utils/security';
|
||||
const props = defineProps({
|
||||
// 必填项
|
||||
content: {
|
||||
@@ -44,6 +45,14 @@ const referBoxSwitch = () => {
|
||||
showReferBox.value = !showReferBox.value;
|
||||
};
|
||||
|
||||
// 安全地处理内容
|
||||
const safeProcessContent = (content) => {
|
||||
if (!content) return '';
|
||||
// 先进行安全清理,然后处理换行
|
||||
const sanitized = sanitizeHTML(content);
|
||||
return sanitized.replace(/\n/g, '<br/>');
|
||||
};
|
||||
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.refer {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// PluginIntoChatMessage handles the transformation of search results into chat messages
|
||||
@@ -50,9 +51,16 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
weekdayName := []string{"星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"}
|
||||
var userContent bytes.Buffer
|
||||
|
||||
// 验证用户查询的安全性
|
||||
safeQuery, isValid := secutils.ValidateInput(chatManage.Query)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query)
|
||||
return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容"))
|
||||
}
|
||||
|
||||
// Execute template with context data
|
||||
err = tmpl.Execute(&userContent, map[string]interface{}{
|
||||
"Query": chatManage.Query, // User's original query
|
||||
"Query": safeQuery, // User's original query
|
||||
"Contexts": passages, // Extracted passages from search results
|
||||
"CurrentTime": time.Now().Format("2006-01-02 15:04:05"), // Formatted current time
|
||||
"CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese
|
||||
|
||||
@@ -23,8 +23,8 @@ type cosFileService struct {
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(appId, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com", appId, region)
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
@@ -59,7 +59,7 @@ func (s *cosFileService) SaveFile(ctx context.Context,
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to COS: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("https://%s/%s", s.bucketURL, objectName), nil
|
||||
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from COS storage by its path URL
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/tracing"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/client"
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
"github.com/google/uuid"
|
||||
@@ -191,15 +192,22 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
metadataJSON = types.JSON(metadataBytes)
|
||||
}
|
||||
|
||||
// 验证文件名安全性
|
||||
safeFilename, isValid := secutils.ValidateInput(file.Filename)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid filename: %s", file.Filename)
|
||||
return nil, werrors.NewValidationError("文件名包含非法字符")
|
||||
}
|
||||
|
||||
// Create knowledge record
|
||||
logger.Info(ctx, "Creating knowledge record")
|
||||
knowledge := &types.Knowledge{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Type: "file",
|
||||
Title: file.Filename,
|
||||
FileName: file.Filename,
|
||||
FileType: getFileType(file.Filename),
|
||||
Title: safeFilename,
|
||||
FileName: safeFilename,
|
||||
FileType: getFileType(safeFilename),
|
||||
FileSize: file.Size,
|
||||
FileHash: hash,
|
||||
ParseStatus: "pending",
|
||||
@@ -258,10 +266,10 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate URL format
|
||||
// Validate URL format and security
|
||||
logger.Info(ctx, "Validating URL")
|
||||
if !isValidURL(url) {
|
||||
logger.Error(ctx, "Invalid URL format")
|
||||
if !isValidURL(url) || !secutils.IsValidURL(url) {
|
||||
logger.Error(ctx, "Invalid or unsafe URL format")
|
||||
return nil, ErrInvalidURL
|
||||
}
|
||||
|
||||
@@ -339,6 +347,17 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
logger.Info(ctx, "Start creating knowledge from passage")
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage))
|
||||
|
||||
// 验证段落内容安全性
|
||||
safePassages := make([]string, 0, len(passage))
|
||||
for i, p := range passage {
|
||||
safePassage, isValid := secutils.ValidateInput(p)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid passage content at index %d", i)
|
||||
return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1))
|
||||
}
|
||||
safePassages = append(safePassages, safePassage)
|
||||
}
|
||||
|
||||
// Get knowledge base configuration
|
||||
logger.Info(ctx, "Getting knowledge base configuration")
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
|
||||
@@ -370,7 +389,7 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
|
||||
// Process passages asynchronously
|
||||
logger.Info(ctx, "Starting asynchronous passage processing")
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, passage)
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
|
||||
|
||||
logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID)
|
||||
return knowledge, nil
|
||||
|
||||
@@ -229,7 +229,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
false,
|
||||
)
|
||||
case "cos":
|
||||
if os.Getenv("COS_APP_ID") == "" ||
|
||||
if os.Getenv("COS_BUCKET_NAME") == "" ||
|
||||
os.Getenv("COS_REGION") == "" ||
|
||||
os.Getenv("COS_SECRET_ID") == "" ||
|
||||
os.Getenv("COS_SECRET_KEY") == "" ||
|
||||
@@ -237,7 +237,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
return nil, fmt.Errorf("missing COS configuration")
|
||||
}
|
||||
return file.NewCosFileService(
|
||||
os.Getenv("COS_APP_ID"),
|
||||
os.Getenv("COS_BUCKET_NAME"),
|
||||
os.Getenv("COS_REGION"),
|
||||
os.Getenv("COS_SECRET_ID"),
|
||||
os.Getenv("COS_SECRET_KEY"),
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -52,6 +53,13 @@ func (h *ChunkHandler) ListKnowledgeChunks(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// 对 chunk 内容进行安全清理
|
||||
for _, chunk := range result.Data.([]*types.Chunk) {
|
||||
if chunk.Content != "" {
|
||||
chunk.Content = secutils.SanitizeForDisplay(chunk.Content)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx, "Successfully retrieved knowledge chunks list, knowledge ID: %s, total: %d",
|
||||
knowledgeID, result.Total,
|
||||
|
||||
@@ -180,6 +180,10 @@ func (h *InitializationHandler) CheckStatus(c *gin.Context) {
|
||||
})
|
||||
return
|
||||
}
|
||||
// ignore api key in response for security
|
||||
for _, model := range models {
|
||||
model.Parameters.APIKey = ""
|
||||
}
|
||||
|
||||
logger.Info(ctx, "System is already initialized")
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
|
||||
@@ -87,7 +87,7 @@ type CreateSessionRequest struct {
|
||||
func (h *SessionHandler) CreateSession(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
logger.Info(ctx, "Start creating session")
|
||||
logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation)
|
||||
|
||||
// Parse and validate the request body
|
||||
var request CreateSessionRequest
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// XSS 防护相关正则表达式
|
||||
var (
|
||||
// 匹配潜在的 XSS 攻击模式
|
||||
xssPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)<script[^>]*>.*?</script>`),
|
||||
regexp.MustCompile(`(?i)<iframe[^>]*>.*?</iframe>`),
|
||||
regexp.MustCompile(`(?i)<object[^>]*>.*?</object>`),
|
||||
regexp.MustCompile(`(?i)<embed[^>]*>.*?</embed>`),
|
||||
regexp.MustCompile(`(?i)<form[^>]*>.*?</form>`),
|
||||
regexp.MustCompile(`(?i)<input[^>]*>`),
|
||||
regexp.MustCompile(`(?i)<button[^>]*>.*?</button>`),
|
||||
regexp.MustCompile(`(?i)javascript:`),
|
||||
regexp.MustCompile(`(?i)vbscript:`),
|
||||
regexp.MustCompile(`(?i)onload\s*=`),
|
||||
regexp.MustCompile(`(?i)onerror\s*=`),
|
||||
regexp.MustCompile(`(?i)onclick\s*=`),
|
||||
regexp.MustCompile(`(?i)onmouseover\s*=`),
|
||||
regexp.MustCompile(`(?i)onfocus\s*=`),
|
||||
regexp.MustCompile(`(?i)onblur\s*=`),
|
||||
}
|
||||
)
|
||||
|
||||
// SanitizeHTML 清理 HTML 内容,防止 XSS 攻击
|
||||
func SanitizeHTML(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 检查输入长度
|
||||
if len(input) > 10000 {
|
||||
input = input[:10000]
|
||||
}
|
||||
|
||||
// 检查是否包含潜在的 XSS 攻击
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(input) {
|
||||
// 如果包含恶意内容,进行 HTML 转义
|
||||
return html.EscapeString(input)
|
||||
}
|
||||
}
|
||||
|
||||
// 如果内容相对安全,返回原内容
|
||||
return input
|
||||
}
|
||||
|
||||
// EscapeHTML 转义 HTML 特殊字符
|
||||
func EscapeHTML(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
return html.EscapeString(input)
|
||||
}
|
||||
|
||||
// ValidateInput 验证用户输入
|
||||
func ValidateInput(input string) (string, bool) {
|
||||
if input == "" {
|
||||
return "", true
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if len(input) > 10000 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// 检查是否包含控制字符
|
||||
for _, r := range input {
|
||||
if r < 32 && r != 9 && r != 10 && r != 13 {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// 检查 UTF-8 有效性
|
||||
if !utf8.ValidString(input) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// 检查是否包含潜在的 XSS 攻击
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(input) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(input), true
|
||||
}
|
||||
|
||||
// IsValidURL 验证 URL 是否安全
|
||||
func IsValidURL(url string) bool {
|
||||
if url == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if len(url) > 2048 {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查协议
|
||||
if !strings.HasPrefix(strings.ToLower(url), "http://") &&
|
||||
!strings.HasPrefix(strings.ToLower(url), "https://") {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否包含恶意内容
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(url) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// IsValidImageURL 验证图片 URL 是否安全
|
||||
func IsValidImageURL(url string) bool {
|
||||
if !IsValidURL(url) {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否为图片文件
|
||||
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"}
|
||||
lowerURL := strings.ToLower(url)
|
||||
|
||||
for _, ext := range imageExtensions {
|
||||
if strings.Contains(lowerURL, ext) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// CleanMarkdown 清理 Markdown 内容
|
||||
func CleanMarkdown(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 移除潜在的恶意脚本
|
||||
cleaned := input
|
||||
for _, pattern := range xssPatterns {
|
||||
cleaned = pattern.ReplaceAllString(cleaned, "")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(cleaned)
|
||||
}
|
||||
|
||||
// SanitizeForDisplay 为显示清理内容
|
||||
func SanitizeForDisplay(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 首先清理 Markdown
|
||||
cleaned := CleanMarkdown(input)
|
||||
|
||||
// 然后进行 HTML 转义
|
||||
escaped := html.EscapeString(cleaned)
|
||||
|
||||
return escaped
|
||||
}
|
||||
@@ -123,6 +123,7 @@ build_docreader_image() {
|
||||
|
||||
docker build \
|
||||
--platform $PLATFORM \
|
||||
--build-arg PLATFORM=$PLATFORM \
|
||||
--build-arg TARGETARCH=$TARGETARCH \
|
||||
-f docker/Dockerfile.docreader \
|
||||
-t wechatopenai/weknora-docreader:latest \
|
||||
|
||||
Reference in New Issue
Block a user