mirror of
https://github.com/dataelement/bisheng.git
synced 2026-09-01 15:32:50 +08:00
feat: support web link and media parsing for knowledge files
This commit is contained in:
@@ -57,7 +57,7 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 200m;
|
||||
client_max_body_size 1024m;
|
||||
add_header Access-Control-Allow-Origin $host;
|
||||
add_header X-Frame-Options SAMEORIGIN;
|
||||
}
|
||||
@@ -66,4 +66,4 @@ server {
|
||||
rewrite ^/workspace(/.*)$ $1 break;
|
||||
proxy_pass http://minio:9000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,6 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 50m;
|
||||
client_max_body_size 1024m;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,9 +173,11 @@ def delete_minio_files(file: KnowledgeFile):
|
||||
# Delete ConvertedpdfDoc.
|
||||
minio_client.remove_object_sync(bucket_name=minio_client.bucket, object_name=f"{file.id}")
|
||||
|
||||
# Delete preview file
|
||||
preview_object_name = KnowledgeUtils.get_knowledge_preview_file_object_name(
|
||||
file.id, file.file_name
|
||||
# Delete preview file. Prefer the persisted object name because generated
|
||||
# previews such as media transcripts and web pages are not always derivable
|
||||
# from the user-facing filename.
|
||||
preview_object_name = KnowledgeUtils.resolve_preview_object_name(
|
||||
file.id, file.file_name, file.preview_file_object_name
|
||||
)
|
||||
if preview_object_name:
|
||||
minio_client.remove_object_sync(bucket_name=minio_client.bucket, object_name=preview_object_name)
|
||||
|
||||
@@ -100,6 +100,16 @@ class KnowledgeFileFailedError(BaseErrorCode):
|
||||
Msg: str = "File parsing failed: {exception}"
|
||||
|
||||
|
||||
class KnowledgeMediaTranscriptionError(BaseErrorCode):
|
||||
Code: int = 10954
|
||||
Msg: str = "Media transcription failed"
|
||||
|
||||
|
||||
class KnowledgeWebLinkImportError(BaseErrorCode):
|
||||
Code: int = 10955
|
||||
Msg: str = "Web link import failed"
|
||||
|
||||
|
||||
# Is notQAThe knowledge base upon
|
||||
class KnowledgeNotQAError(BaseErrorCode):
|
||||
Code: int = 10960
|
||||
|
||||
@@ -30,6 +30,7 @@ from bisheng.knowledge.domain.schemas.knowledge_space_schema import (
|
||||
KnowledgeSpaceUpdateReq,
|
||||
RemoveSpaceMemberRequest,
|
||||
UpdateSpaceMemberRoleRequest,
|
||||
WebLinkCreateReq,
|
||||
)
|
||||
from bisheng.knowledge.domain.services.department_knowledge_space_service import (
|
||||
DepartmentKnowledgeSpaceService,
|
||||
@@ -480,6 +481,22 @@ async def add_file(
|
||||
return resp_200(file_record)
|
||||
|
||||
|
||||
@router.post("/{space_id}/web-links")
|
||||
async def import_web_link(
|
||||
space_id: int,
|
||||
req: WebLinkCreateReq,
|
||||
svc: KnowledgeSpaceService = Depends(get_knowledge_space_service),
|
||||
) -> Any:
|
||||
file_record = await svc.import_web_link(
|
||||
knowledge_id=space_id,
|
||||
url=req.url,
|
||||
title=req.title,
|
||||
parent_id=req.parent_id,
|
||||
overwrite=req.overwrite,
|
||||
)
|
||||
return resp_200(file_record)
|
||||
|
||||
|
||||
@router.put("/{space_id}/files/{file_id}")
|
||||
async def rename_file(
|
||||
space_id: int,
|
||||
|
||||
@@ -46,6 +46,16 @@ class FileSource(Enum):
|
||||
UPLOAD = "upload" # user upload
|
||||
CHANNEL = "channel"
|
||||
SPACE_UPLOAD = "space_upload"
|
||||
AUDIO_TRANSCRIPT = "audio_transcript"
|
||||
VIDEO_TRANSCRIPT = "video_transcript"
|
||||
WEB_LINK = "web_link"
|
||||
|
||||
|
||||
PORTAL_USER_UPLOAD_FILE_SOURCES = (
|
||||
FileSource.SPACE_UPLOAD.value,
|
||||
FileSource.AUDIO_TRANSCRIPT.value,
|
||||
FileSource.VIDEO_TRANSCRIPT.value,
|
||||
)
|
||||
|
||||
|
||||
class FileType(int, Enum):
|
||||
|
||||
@@ -10,6 +10,7 @@ from bisheng.knowledge.domain.models.knowledge_file import (
|
||||
KnowledgeFile,
|
||||
KnowledgeFileDao,
|
||||
KnowledgeFileStatus,
|
||||
PORTAL_USER_UPLOAD_FILE_SOURCES,
|
||||
)
|
||||
|
||||
# F027 AD-14: file extension priority for "file_type" sort order.
|
||||
@@ -344,7 +345,7 @@ class SpaceFileDao(KnowledgeFileDao):
|
||||
statement = select(func.sum(KnowledgeFile.file_size)).where(
|
||||
KnowledgeFile.user_id == user_id,
|
||||
KnowledgeFile.file_type == 1,
|
||||
col(KnowledgeFile.file_source).in_([FileSource.SPACE_UPLOAD.value, FileSource.CHANNEL.value]),
|
||||
col(KnowledgeFile.file_source).in_([*PORTAL_USER_UPLOAD_FILE_SOURCES, FileSource.CHANNEL.value]),
|
||||
)
|
||||
async with get_async_db_session() as session:
|
||||
return await session.scalar(statement) or 0
|
||||
|
||||
@@ -133,6 +133,13 @@ class FileCreateReq(BaseModel):
|
||||
parent_id: int | None = Field(None, description="Parent Folder ID")
|
||||
|
||||
|
||||
class WebLinkCreateReq(BaseModel):
|
||||
url: str = Field(..., min_length=1, max_length=2048, description="Web link URL")
|
||||
title: str | None = Field(None, max_length=200, description="Optional display title")
|
||||
parent_id: int | None = Field(None, description="Parent Folder ID")
|
||||
overwrite: bool = Field(default=False, description="Overwrite existing duplicate web link")
|
||||
|
||||
|
||||
class FileRenameReq(BaseModel):
|
||||
name: str = Field(..., description="New File Name")
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import Request
|
||||
from loguru import logger
|
||||
@@ -20,6 +23,7 @@ from bisheng.common.dependencies.user_deps import UserPayload
|
||||
from bisheng.common.errcode.knowledge import KnowledgeSpaceTagLibraryInvalidError
|
||||
from bisheng.common.errcode.knowledge_space import (
|
||||
SpaceAdminLimitExceededError,
|
||||
SpaceFileDuplicateError,
|
||||
SpaceFileExtensionError,
|
||||
SpaceFileNameDuplicateError,
|
||||
SpaceFileNotFoundError,
|
||||
@@ -47,6 +51,7 @@ from bisheng.common.models.space_channel_member import (
|
||||
)
|
||||
from bisheng.core.context.tenant import get_current_tenant_id
|
||||
from bisheng.core.database import get_async_db_session
|
||||
from bisheng.core.storage.minio.minio_manager import get_minio_storage_sync
|
||||
from bisheng.database.models.department import DepartmentDao, UserDepartmentDao
|
||||
from bisheng.database.models.group_resource import ResourceTypeEnum
|
||||
from bisheng.database.models.tag import Tag, TagBusinessTypeEnum, TagDao
|
||||
@@ -95,6 +100,7 @@ from bisheng.knowledge.domain.services.knowledge_service import KnowledgeService
|
||||
from bisheng.knowledge.domain.services.knowledge_space_tag_library_service import (
|
||||
KnowledgeSpaceTagLibraryService,
|
||||
)
|
||||
from bisheng.knowledge.domain.services.web_link_import_service import KnowledgeWebLinkImportService
|
||||
from bisheng.knowledge.domain.services.knowledge_utils import KnowledgeUtils
|
||||
from bisheng.llm.domain import LLMService
|
||||
from bisheng.message.domain.services.notification_content import build_notify_content
|
||||
@@ -169,6 +175,10 @@ _CHILD_PERMISSION_CHECK_CONCURRENCY = 8
|
||||
# filtering is refilled from the next OFFSET window, so this only bounds per-round
|
||||
# DB fetch + visibility evaluation, not the page size.
|
||||
_SEARCH_SCAN_BATCH_SIZE = 100
|
||||
_WEB_LINK_SEPARATORS = ["\n\n", "\n", "。", "\\.", ",", ",", ";", ";", "、", "\\s+", ""]
|
||||
_WEB_LINK_SEPARATOR_RULES = ["after"] * len(_WEB_LINK_SEPARATORS)
|
||||
_AUDIO_FILE_EXTENSIONS = {"mp3", "wav", "m4a", "aac", "flac", "ogg"}
|
||||
_VIDEO_FILE_EXTENSIONS = {"mp4", "mov", "avi", "mkv", "webm"}
|
||||
|
||||
|
||||
class KnowledgeSpaceService(KnowledgeUtils):
|
||||
@@ -3260,6 +3270,291 @@ class KnowledgeSpaceService(KnowledgeUtils):
|
||||
|
||||
# ──────────────────────────── Files ───────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _resolve_upload_file_source(file_name: str, default_source: str) -> str:
|
||||
suffix = file_name.rsplit(".", 1)[-1].lower() if "." in file_name else ""
|
||||
if suffix in _AUDIO_FILE_EXTENSIONS:
|
||||
return FileSource.AUDIO_TRANSCRIPT.value
|
||||
if suffix in _VIDEO_FILE_EXTENSIONS:
|
||||
return FileSource.VIDEO_TRANSCRIPT.value
|
||||
return default_source
|
||||
|
||||
@staticmethod
|
||||
def _normalize_web_link_display_name(title: str | None, fallback_url: str | None = None) -> str:
|
||||
candidate = (title or "").strip()
|
||||
if not candidate and fallback_url:
|
||||
parsed = urlparse(fallback_url)
|
||||
candidate = parsed.netloc or parsed.path.rsplit("/", 1)[-1]
|
||||
candidate = re.sub(r'[\\/:*?"<>|\r\n\t]+', " ", candidate).strip(" .")
|
||||
if not candidate:
|
||||
candidate = "web_link"
|
||||
if len(candidate) > 180:
|
||||
candidate = candidate[:180].rstrip(" .")
|
||||
if not candidate.lower().endswith(".html"):
|
||||
candidate = f"{candidate}.html"
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def _get_web_link_markdown_object_name(file_id: int) -> str:
|
||||
return f"original/{file_id}.md"
|
||||
|
||||
@staticmethod
|
||||
def _build_web_link_split_rule(knowledge_id: int) -> dict:
|
||||
split_rule = FileProcessBase(knowledge_id=knowledge_id).model_dump()
|
||||
split_rule["separator"] = _WEB_LINK_SEPARATORS
|
||||
split_rule["separator_rule"] = _WEB_LINK_SEPARATOR_RULES
|
||||
split_rule["chunk_overlap"] = 0
|
||||
return split_rule
|
||||
|
||||
async def _create_primary_document_for_file(self, db_file: KnowledgeFile) -> None:
|
||||
async with get_async_db_session() as v_session:
|
||||
v_doc = KnowledgeDocument(
|
||||
knowledge_id=db_file.knowledge_id,
|
||||
file_level_path=db_file.file_level_path,
|
||||
level=db_file.level or 0,
|
||||
)
|
||||
v_session.add(v_doc)
|
||||
await v_session.flush()
|
||||
v_version = KnowledgeDocumentVersion(
|
||||
document_id=v_doc.id,
|
||||
knowledge_file_id=db_file.id,
|
||||
version_no=1,
|
||||
is_primary=True,
|
||||
)
|
||||
v_session.add(v_version)
|
||||
await v_session.flush()
|
||||
v_doc.primary_version_id = v_version.id
|
||||
v_session.add(v_doc)
|
||||
await v_session.commit()
|
||||
|
||||
async def _cleanup_created_web_link_file(self, db_file: KnowledgeFile | None) -> None:
|
||||
if not db_file or not getattr(db_file, "id", None):
|
||||
return
|
||||
expanded_ids = await self._cascade_version_links_on_delete([db_file.id])
|
||||
await self._cleanup_resource_tuples([("knowledge_file", file_id) for file_id in expanded_ids])
|
||||
await KnowledgeFileDao.adelete_batch(expanded_ids)
|
||||
minio_client = get_minio_storage_sync()
|
||||
metadata = db_file.user_metadata or {}
|
||||
for object_name in {db_file.object_name, metadata.get("html_snapshot_object_name")}:
|
||||
if object_name:
|
||||
try:
|
||||
minio_client.remove_object_sync(bucket_name=minio_client.bucket, object_name=object_name)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to cleanup web link object {object_name}: {exc}")
|
||||
|
||||
async def import_web_link(
|
||||
self,
|
||||
knowledge_id: int,
|
||||
url: str,
|
||||
title: str | None = None,
|
||||
parent_id: int | None = None,
|
||||
overwrite: bool = False,
|
||||
) -> KnowledgeFile:
|
||||
if parent_id:
|
||||
await self._require_permission_id("folder", parent_id, "upload_file", space_id=knowledge_id)
|
||||
else:
|
||||
await self._require_permission_id("knowledge_space", knowledge_id, "upload_file")
|
||||
|
||||
db_knowledge = await KnowledgeDao.aquery_by_id(knowledge_id)
|
||||
if not db_knowledge:
|
||||
raise SpaceNotFoundError()
|
||||
self._ensure_space_async_task_tenant_consistency(db_knowledge, "import_web_link")
|
||||
|
||||
level = 0
|
||||
file_level_path = ""
|
||||
parent_type = "knowledge_space"
|
||||
parent_resource_id = knowledge_id
|
||||
if parent_id:
|
||||
parent_folder = await self._get_folder_for_action(knowledge_id, parent_id)
|
||||
level = parent_folder.level + 1
|
||||
file_level_path = f"{parent_folder.file_level_path}/{parent_id}"
|
||||
parent_type = "folder"
|
||||
parent_resource_id = parent_id
|
||||
|
||||
import_result = await KnowledgeWebLinkImportService.fetch(url)
|
||||
display_title = title or import_result.title
|
||||
file_name = self._normalize_web_link_display_name(display_title, import_result.final_url)
|
||||
markdown_bytes = import_result.markdown.encode("utf-8")
|
||||
html_snapshot_bytes = (import_result.html_snapshot or "").encode("utf-8")
|
||||
|
||||
content_repeat = KnowledgeFileDao.get_file_by_condition(knowledge_id=knowledge_id, md5_=import_result.content_hash)
|
||||
name_repeat = KnowledgeFileDao.get_file_by_condition(knowledge_id=knowledge_id, file_name=file_name)
|
||||
duplicate_file = content_repeat[0] if content_repeat else (name_repeat[0] if name_repeat else None)
|
||||
if duplicate_file and not overwrite:
|
||||
if content_repeat:
|
||||
raise SpaceFileDuplicateError()
|
||||
raise SpaceFileNameDuplicateError()
|
||||
|
||||
role_user_limit_bytes = await QuotaService.get_knowledge_space_upload_limit_bytes(self.login_user)
|
||||
current_user_total = int(await SpaceFileDao.get_user_total_file_size(self.login_user.user_id))
|
||||
if role_user_limit_bytes is not None and current_user_total + len(markdown_bytes) > role_user_limit_bytes:
|
||||
raise SpaceFileSizeLimitError()
|
||||
|
||||
tenant_remaining_bytes = await QuotaService.get_tenant_storage_remaining_bytes(db_knowledge.tenant_id)
|
||||
if tenant_remaining_bytes is not None and len(markdown_bytes) > tenant_remaining_bytes:
|
||||
tenant_used_bytes = await QuotaService.get_tenant_storage_used_bytes(db_knowledge.tenant_id)
|
||||
blocker = (
|
||||
db_knowledge.tenant_id,
|
||||
"tenant_limit",
|
||||
round((tenant_used_bytes + len(markdown_bytes)) / (1024**3), 2),
|
||||
round((tenant_used_bytes + tenant_remaining_bytes) / (1024**3), 2),
|
||||
"",
|
||||
)
|
||||
raise QuotaService._make_storage_quota_error(blocker, "storage_gb")
|
||||
|
||||
split_rule = self._build_web_link_split_rule(knowledge_id)
|
||||
user_metadata = {
|
||||
"source_type": FileSource.WEB_LINK.value,
|
||||
"source_url": url,
|
||||
"final_url": import_result.final_url,
|
||||
"web_title": file_name,
|
||||
"imported_at": datetime.utcnow().isoformat(),
|
||||
"html_snapshot_object_name": "",
|
||||
}
|
||||
|
||||
if duplicate_file:
|
||||
return await self._overwrite_web_link_file(
|
||||
duplicate_file,
|
||||
file_name=file_name,
|
||||
file_size=len(markdown_bytes),
|
||||
content_hash=import_result.content_hash,
|
||||
split_rule=split_rule,
|
||||
markdown_bytes=markdown_bytes,
|
||||
html_snapshot_bytes=html_snapshot_bytes,
|
||||
user_metadata=user_metadata,
|
||||
level=level,
|
||||
file_level_path=file_level_path,
|
||||
parent_type=parent_type,
|
||||
parent_resource_id=parent_resource_id,
|
||||
)
|
||||
|
||||
db_file: KnowledgeFile | None = None
|
||||
try:
|
||||
db_file = KnowledgeFile(
|
||||
knowledge_id=knowledge_id,
|
||||
tenant_id=db_knowledge.tenant_id,
|
||||
file_name=file_name,
|
||||
file_size=len(markdown_bytes),
|
||||
md5=import_result.content_hash,
|
||||
split_rule=json.dumps(split_rule, ensure_ascii=False),
|
||||
user_id=self.login_user.user_id,
|
||||
user_name=self.login_user.user_name,
|
||||
updater_id=self.login_user.user_id,
|
||||
updater_name=self.login_user.user_name,
|
||||
level=level,
|
||||
file_level_path=file_level_path,
|
||||
file_source=FileSource.WEB_LINK.value,
|
||||
user_metadata=user_metadata,
|
||||
)
|
||||
db_file = KnowledgeFileDao.add_file(db_file)
|
||||
db_file.object_name = self._get_web_link_markdown_object_name(db_file.id)
|
||||
html_snapshot_object_name = f"preview/{db_file.id}.html"
|
||||
db_file.user_metadata = {**user_metadata, "html_snapshot_object_name": html_snapshot_object_name}
|
||||
minio_client = get_minio_storage_sync()
|
||||
minio_client.put_object_sync(
|
||||
bucket_name=minio_client.bucket,
|
||||
object_name=db_file.object_name,
|
||||
file=markdown_bytes,
|
||||
content_type="text/markdown; charset=utf-8",
|
||||
)
|
||||
if html_snapshot_bytes:
|
||||
minio_client.put_object_sync(
|
||||
bucket_name=minio_client.bucket,
|
||||
object_name=html_snapshot_object_name,
|
||||
file=html_snapshot_bytes,
|
||||
content_type="text/html; charset=utf-8",
|
||||
)
|
||||
db_file = KnowledgeFileDao.update(db_file)
|
||||
await self._create_primary_document_for_file(db_file)
|
||||
await self._initialize_child_resource_permissions(
|
||||
"knowledge_file",
|
||||
db_file.id,
|
||||
parent_type,
|
||||
parent_resource_id,
|
||||
)
|
||||
except Exception:
|
||||
await self._cleanup_created_web_link_file(db_file)
|
||||
raise
|
||||
|
||||
KnowledgeService.audit_telemetry_service.telemetry_new_knowledge_file(self.login_user)
|
||||
from bisheng.worker.knowledge import scheduler as file_scheduler
|
||||
|
||||
file_scheduler.enqueue_or_dispatch(
|
||||
user_id=db_file.user_id,
|
||||
file_id=db_file.id,
|
||||
file_name=db_file.file_name,
|
||||
preview_cache_key=self.get_preview_cache_key(knowledge_id, import_result.final_url, import_result.content_hash),
|
||||
callback_url=None,
|
||||
)
|
||||
await self.update_folder_update_time(file_level_path)
|
||||
await KnowledgeDao.async_update_knowledge_update_time_by_id(knowledge_id)
|
||||
return db_file
|
||||
|
||||
async def _overwrite_web_link_file(
|
||||
self,
|
||||
db_file: KnowledgeFile,
|
||||
*,
|
||||
file_name: str,
|
||||
file_size: int,
|
||||
content_hash: str,
|
||||
split_rule: dict,
|
||||
markdown_bytes: bytes,
|
||||
html_snapshot_bytes: bytes,
|
||||
user_metadata: dict,
|
||||
level: int,
|
||||
file_level_path: str,
|
||||
parent_type: str,
|
||||
parent_resource_id: int,
|
||||
) -> KnowledgeFile:
|
||||
old_parent = self._parent_ref_from_level_path(db_file.file_level_path, db_file.knowledge_id)
|
||||
new_parent = (parent_type, parent_resource_id)
|
||||
db_file.file_name = file_name
|
||||
db_file.file_size = file_size
|
||||
db_file.md5 = content_hash
|
||||
db_file.split_rule = json.dumps(split_rule, ensure_ascii=False)
|
||||
db_file.level = level
|
||||
db_file.file_level_path = file_level_path
|
||||
db_file.file_source = FileSource.WEB_LINK.value
|
||||
db_file.status = KnowledgeFileStatus.WAITING.value
|
||||
db_file.remark = ""
|
||||
db_file.preview_file_object_name = ""
|
||||
db_file.updater_id = self.login_user.user_id
|
||||
db_file.updater_name = self.login_user.user_name
|
||||
db_file.object_name = self._get_web_link_markdown_object_name(db_file.id)
|
||||
html_snapshot_object_name = f"preview/{db_file.id}.html"
|
||||
db_file.user_metadata = {**user_metadata, "html_snapshot_object_name": html_snapshot_object_name}
|
||||
|
||||
minio_client = get_minio_storage_sync()
|
||||
minio_client.put_object_sync(
|
||||
bucket_name=minio_client.bucket,
|
||||
object_name=db_file.object_name,
|
||||
file=markdown_bytes,
|
||||
content_type="text/markdown; charset=utf-8",
|
||||
)
|
||||
if html_snapshot_bytes:
|
||||
minio_client.put_object_sync(
|
||||
bucket_name=minio_client.bucket,
|
||||
object_name=html_snapshot_object_name,
|
||||
file=html_snapshot_bytes,
|
||||
content_type="text/html; charset=utf-8",
|
||||
)
|
||||
updated_file = await KnowledgeFileDao.async_update(db_file)
|
||||
if old_parent != new_parent:
|
||||
await self._replace_resource_parent_tuple("knowledge_file", db_file.id, old_parent, new_parent)
|
||||
|
||||
from bisheng.worker.knowledge import scheduler as file_scheduler
|
||||
|
||||
file_scheduler.enqueue_or_dispatch(
|
||||
user_id=updated_file.user_id,
|
||||
file_id=updated_file.id,
|
||||
file_name=updated_file.file_name,
|
||||
preview_cache_key=self.get_preview_cache_key(updated_file.knowledge_id, user_metadata.get("final_url", ""), content_hash),
|
||||
callback_url=None,
|
||||
)
|
||||
await self.update_folder_update_time(file_level_path)
|
||||
await KnowledgeDao.async_update_knowledge_update_time_by_id(updated_file.knowledge_id)
|
||||
return updated_file
|
||||
|
||||
async def add_file(
|
||||
self,
|
||||
knowledge_id: int,
|
||||
@@ -3364,6 +3659,10 @@ class KnowledgeSpaceService(KnowledgeUtils):
|
||||
skip_dedup=skip_dedup,
|
||||
)
|
||||
if db_file.status != KnowledgeFileStatus.FAILED.value:
|
||||
resolved_file_source = self._resolve_upload_file_source(db_file.file_name, file_source.value)
|
||||
if db_file.file_source != resolved_file_source:
|
||||
db_file.file_source = resolved_file_source
|
||||
db_file = KnowledgeFileDao.update(db_file)
|
||||
if getattr(db_file, "id", None):
|
||||
created_files.append(db_file)
|
||||
# Plan 2 Task 9: also create a logical document + V1 (primary) for the uploaded file.
|
||||
@@ -3590,15 +3889,22 @@ class KnowledgeSpaceService(KnowledgeUtils):
|
||||
raise SpaceNotFoundError()
|
||||
self._ensure_space_async_task_tenant_consistency(space, "rename_file")
|
||||
|
||||
old_suffix = file_record.file_name.rsplit(".", 1)[-1] if "." in file_record.file_name else ""
|
||||
new_suffix = new_name.rsplit(".", 1)[-1] if "." in new_name else ""
|
||||
if old_suffix != new_suffix:
|
||||
raise SpaceFileExtensionError()
|
||||
if file_record.file_source == FileSource.WEB_LINK.value:
|
||||
new_name = self._normalize_web_link_display_name(new_name)
|
||||
else:
|
||||
old_suffix = file_record.file_name.rsplit(".", 1)[-1] if "." in file_record.file_name else ""
|
||||
new_suffix = new_name.rsplit(".", 1)[-1] if "." in new_name else ""
|
||||
if old_suffix != new_suffix:
|
||||
raise SpaceFileExtensionError()
|
||||
|
||||
if await SpaceFileDao.count_file_by_name(file_record.knowledge_id, new_name, exclude_id=file_id) > 0:
|
||||
raise SpaceFileNameDuplicateError()
|
||||
|
||||
file_record.file_name = new_name
|
||||
if file_record.file_source == FileSource.WEB_LINK.value:
|
||||
metadata = file_record.user_metadata or {}
|
||||
metadata["web_title"] = new_name
|
||||
file_record.user_metadata = metadata
|
||||
updated_file = await KnowledgeFileDao.async_update(file_record)
|
||||
|
||||
if updated_file.status == KnowledgeFileStatus.SUCCESS.value:
|
||||
@@ -4053,10 +4359,21 @@ class KnowledgeSpaceService(KnowledgeUtils):
|
||||
await self._require_permission_id("knowledge_file", file_id, "view_file", space_id=file_record.knowledge_id)
|
||||
|
||||
original_url, preview_url = KnowledgeService.get_file_share_url(file_id)
|
||||
metadata = file_record.user_metadata or {}
|
||||
html_preview_url = ""
|
||||
html_snapshot_object_name = metadata.get("html_snapshot_object_name")
|
||||
if html_snapshot_object_name:
|
||||
html_preview_url = KnowledgeService.get_file_share_url_with_empty(html_snapshot_object_name)
|
||||
|
||||
return {
|
||||
"original_url": original_url,
|
||||
"preview_url": preview_url,
|
||||
"file_source": file_record.file_source,
|
||||
"source_url": metadata.get("source_url", ""),
|
||||
"final_url": metadata.get("final_url", ""),
|
||||
"web_title": metadata.get("web_title", ""),
|
||||
"media_kind": metadata.get("media_kind", ""),
|
||||
"html_preview_url": html_preview_url,
|
||||
}
|
||||
|
||||
async def get_file_download(self, file_id: int, *, space_id: int | None = None) -> dict:
|
||||
|
||||
@@ -178,12 +178,15 @@ class KnowledgeUtils(BaseService):
|
||||
"""Get the preview file corresponding to the knowledge base file atminioStorage Path for This path is stored in the officialbucketand within"""
|
||||
if file_name:
|
||||
file_ext = file_name.split(".")[-1]
|
||||
file_ext = file_ext.lower() if file_ext else file_ext
|
||||
if file_ext in ["doc", "docx", "wps"]:
|
||||
return f"preview/{file_id}.docx"
|
||||
elif file_ext in ["xls", "xlsx", "et"]:
|
||||
return f"preview/{file_id}.xlsx"
|
||||
elif file_ext in ["ppt", "pptx", "dps", "ofd"]:
|
||||
return f"preview/{file_id}.pdf"
|
||||
elif file_ext in ["mp3", "wav", "m4a", "aac", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm"]:
|
||||
return f"preview/{file_id}.md"
|
||||
# No preview required for other file types
|
||||
return None
|
||||
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from dashscope.audio.asr import Recognition, RecognitionResult
|
||||
from loguru import logger
|
||||
|
||||
from bisheng.common.errcode.knowledge import KnowledgeMediaTranscriptionError
|
||||
from bisheng.common.errcode.server import (
|
||||
AsrModelConfigDeletedError,
|
||||
AsrModelOfflineError,
|
||||
AsrModelTypeError,
|
||||
AsrProviderDeletedError,
|
||||
NoAsrModelConfigError,
|
||||
)
|
||||
from bisheng.llm.domain.const import LLMModelType, LLMServerType
|
||||
from bisheng.llm.domain.models import LLMDao, LLMModel, LLMServer
|
||||
from bisheng.llm.domain.services.llm import LLMService
|
||||
|
||||
|
||||
@dataclass
|
||||
class TranscriptSegment:
|
||||
text: str
|
||||
begin_time: float | None = None
|
||||
end_time: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TranscriptResult:
|
||||
text: str
|
||||
markdown: str
|
||||
segments: list[TranscriptSegment]
|
||||
model_id: int
|
||||
model_name: str
|
||||
|
||||
|
||||
class KnowledgeMediaTranscriptionService:
|
||||
"""Knowledge-base specific media transcription service.
|
||||
|
||||
This service intentionally does not use the existing BaseASRClient path
|
||||
because that path preserves legacy workbench behavior.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def transcribe_media(
|
||||
cls,
|
||||
media_path: str,
|
||||
*,
|
||||
source_file_name: str,
|
||||
tenant_id: int | None = None,
|
||||
) -> TranscriptResult:
|
||||
if not os.path.exists(media_path):
|
||||
raise KnowledgeMediaTranscriptionError(msg="Media file does not exist")
|
||||
|
||||
model_info, server_info = cls._resolve_asr_model(tenant_id)
|
||||
api_key = cls._resolve_api_key(server_info, model_info)
|
||||
duration_ms = cls._probe_media_duration_ms(media_path)
|
||||
wav_path = cls._convert_to_wav(media_path)
|
||||
try:
|
||||
segments = cls._call_aliyun_asr(
|
||||
wav_path,
|
||||
api_key=api_key,
|
||||
model_name=model_info.model_name,
|
||||
media_duration_ms=duration_ms,
|
||||
)
|
||||
finally:
|
||||
if os.path.exists(wav_path):
|
||||
os.remove(wav_path)
|
||||
|
||||
text = "\n".join(segment.text for segment in segments if segment.text).strip()
|
||||
if not text:
|
||||
raise KnowledgeMediaTranscriptionError(msg="ASR returned empty text")
|
||||
|
||||
markdown = cls._build_markdown(
|
||||
source_file_name=source_file_name,
|
||||
model_name=model_info.model_name,
|
||||
segments=segments,
|
||||
)
|
||||
return TranscriptResult(
|
||||
text=text,
|
||||
markdown=markdown,
|
||||
segments=segments,
|
||||
model_id=model_info.id,
|
||||
model_name=model_info.model_name,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _resolve_asr_model(cls, tenant_id: int | None) -> tuple[LLMModel, LLMServer]:
|
||||
workbench_llm = LLMService.get_workbench_llm_sync(tenant_id=tenant_id)
|
||||
if not workbench_llm.asr_model or not workbench_llm.asr_model.id:
|
||||
raise NoAsrModelConfigError()
|
||||
|
||||
model_info = LLMDao.get_model_by_id(int(workbench_llm.asr_model.id))
|
||||
if not model_info:
|
||||
raise AsrModelConfigDeletedError()
|
||||
if model_info.model_type != LLMModelType.ASR.value:
|
||||
raise AsrModelTypeError(model_type=model_info.model_type)
|
||||
|
||||
server_info = LLMDao.get_server_by_id(model_info.server_id)
|
||||
if not server_info:
|
||||
raise AsrProviderDeletedError()
|
||||
if not model_info.online:
|
||||
raise AsrModelOfflineError(server_name=server_info.name, model_name=model_info.model_name)
|
||||
if server_info.type != LLMServerType.QWEN.value:
|
||||
raise KnowledgeMediaTranscriptionError(
|
||||
msg=f"Knowledge media transcription only supports Aliyun/Qwen ASR, got {server_info.type}"
|
||||
)
|
||||
return model_info, server_info
|
||||
|
||||
@classmethod
|
||||
def _resolve_api_key(cls, server_info: LLMServer, model_info: LLMModel) -> str:
|
||||
params: dict[str, Any] = {}
|
||||
if server_info.config:
|
||||
params.update(server_info.config)
|
||||
if model_info.config:
|
||||
params.update(model_info.config)
|
||||
api_key = params.get("openai_api_key") or params.get("api_key")
|
||||
if not api_key:
|
||||
raise KnowledgeMediaTranscriptionError(msg="ASR api key is missing")
|
||||
return api_key
|
||||
|
||||
@classmethod
|
||||
def _convert_to_wav(cls, media_path: str) -> str:
|
||||
fd, wav_path = tempfile.mkstemp(suffix="_16k_mono.wav")
|
||||
os.close(fd)
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
media_path,
|
||||
"-ar",
|
||||
"16000",
|
||||
"-ac",
|
||||
"1",
|
||||
wav_path,
|
||||
]
|
||||
try:
|
||||
subprocess.run(command, capture_output=True, check=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise KnowledgeMediaTranscriptionError(msg="ffmpeg is not installed") from exc
|
||||
except subprocess.CalledProcessError as exc:
|
||||
stderr = exc.stderr.decode("utf-8", errors="ignore") if exc.stderr else ""
|
||||
logger.warning("ffmpeg media conversion failed: {}", stderr[-1000:])
|
||||
raise KnowledgeMediaTranscriptionError(msg="Media audio extraction failed") from exc
|
||||
return wav_path
|
||||
|
||||
@classmethod
|
||||
def _call_aliyun_asr(
|
||||
cls,
|
||||
wav_path: str,
|
||||
*,
|
||||
api_key: str,
|
||||
model_name: str,
|
||||
media_duration_ms: int | None = None,
|
||||
) -> list[TranscriptSegment]:
|
||||
recognition = Recognition(
|
||||
model=model_name,
|
||||
format="wav",
|
||||
sample_rate=16000,
|
||||
callback=None,
|
||||
)
|
||||
result: RecognitionResult = recognition.call(wav_path, api_key=api_key)
|
||||
if result.status_code != 200:
|
||||
raise KnowledgeMediaTranscriptionError(
|
||||
msg=f"ASR request failed: {result.code} {result.message}"
|
||||
)
|
||||
|
||||
raw_sentences = result.get_sentence() or []
|
||||
segments: list[TranscriptSegment] = []
|
||||
for sentence in raw_sentences:
|
||||
if not isinstance(sentence, dict):
|
||||
continue
|
||||
text = str(sentence.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
segments.append(
|
||||
TranscriptSegment(
|
||||
text=text,
|
||||
begin_time=cls._coerce_time_value(sentence.get("begin_time")),
|
||||
end_time=cls._coerce_time_value(sentence.get("end_time")),
|
||||
)
|
||||
)
|
||||
if segments:
|
||||
return cls._normalize_segments(segments, media_duration_ms=media_duration_ms)
|
||||
|
||||
text = str(getattr(result, "output", {}) or result).strip()
|
||||
return [TranscriptSegment(text=text)] if text else []
|
||||
|
||||
@staticmethod
|
||||
def _coerce_time_value(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if number <= 0:
|
||||
return 0
|
||||
return number
|
||||
|
||||
@classmethod
|
||||
def _normalize_segments(
|
||||
cls,
|
||||
segments: list[TranscriptSegment],
|
||||
*,
|
||||
media_duration_ms: int | None = None,
|
||||
) -> list[TranscriptSegment]:
|
||||
raw_times = [
|
||||
time_value
|
||||
for segment in segments
|
||||
for time_value in (segment.begin_time, segment.end_time)
|
||||
if time_value is not None
|
||||
]
|
||||
multiplier = cls._resolve_timestamp_multiplier(
|
||||
raw_times,
|
||||
media_duration_ms=media_duration_ms,
|
||||
)
|
||||
|
||||
normalized_segments: list[tuple[int, TranscriptSegment]] = []
|
||||
for index, segment in enumerate(segments):
|
||||
begin_time = cls._scale_timestamp(segment.begin_time, multiplier)
|
||||
end_time = cls._scale_timestamp(segment.end_time, multiplier)
|
||||
if begin_time is not None and end_time is not None and end_time < begin_time:
|
||||
logger.warning(
|
||||
"ASR segment end_time precedes begin_time; clamping end_time. "
|
||||
"begin_time={} end_time={} text={}",
|
||||
begin_time,
|
||||
end_time,
|
||||
segment.text[:80],
|
||||
)
|
||||
end_time = begin_time
|
||||
normalized_segments.append(
|
||||
(
|
||||
index,
|
||||
TranscriptSegment(
|
||||
text=segment.text,
|
||||
begin_time=begin_time,
|
||||
end_time=end_time,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
normalized_segments.sort(
|
||||
key=lambda item: (
|
||||
item[1].begin_time is None,
|
||||
item[1].begin_time if item[1].begin_time is not None else 0,
|
||||
item[0],
|
||||
)
|
||||
)
|
||||
return [segment for _, segment in normalized_segments]
|
||||
|
||||
@staticmethod
|
||||
def _resolve_timestamp_multiplier(
|
||||
raw_times: list[float],
|
||||
*,
|
||||
media_duration_ms: int | None = None,
|
||||
) -> int:
|
||||
positive_times = [time_value for time_value in raw_times if time_value > 0]
|
||||
if not positive_times:
|
||||
return 1
|
||||
|
||||
max_time = max(positive_times)
|
||||
if media_duration_ms and media_duration_ms > 0:
|
||||
tolerance_ms = max(5000, int(media_duration_ms * 0.2))
|
||||
candidates = (1, 10, 1000)
|
||||
valid_candidates = []
|
||||
for multiplier in candidates:
|
||||
scaled_max = max_time * multiplier
|
||||
overflow = max(0, scaled_max - media_duration_ms - tolerance_ms)
|
||||
distance = abs(media_duration_ms - scaled_max)
|
||||
valid_candidates.append((overflow, distance, multiplier))
|
||||
valid_candidates.sort()
|
||||
return valid_candidates[0][2]
|
||||
|
||||
if max_time >= 1000:
|
||||
return 1
|
||||
return 1000
|
||||
|
||||
@staticmethod
|
||||
def _scale_timestamp(value: float | None, multiplier: int) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
return max(0, int(round(value * multiplier)))
|
||||
|
||||
@staticmethod
|
||||
def _probe_media_duration_ms(media_path: str) -> int | None:
|
||||
command = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
media_path,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, check=True)
|
||||
except FileNotFoundError:
|
||||
logger.warning(
|
||||
"ffprobe is not installed; ASR timestamp normalization will not use media duration"
|
||||
)
|
||||
return None
|
||||
except subprocess.CalledProcessError as exc:
|
||||
stderr = exc.stderr.decode("utf-8", errors="ignore") if exc.stderr else ""
|
||||
logger.warning("ffprobe media duration probe failed: {}", stderr[-1000:])
|
||||
return None
|
||||
|
||||
duration_text = result.stdout.decode("utf-8", errors="ignore").strip()
|
||||
try:
|
||||
duration_seconds = float(duration_text)
|
||||
except ValueError:
|
||||
logger.warning("ffprobe returned invalid media duration: {}", duration_text[:120])
|
||||
return None
|
||||
if duration_seconds <= 0:
|
||||
return None
|
||||
return int(duration_seconds * 1000)
|
||||
|
||||
@classmethod
|
||||
def _build_markdown(
|
||||
cls,
|
||||
*,
|
||||
source_file_name: str,
|
||||
model_name: str,
|
||||
segments: list[TranscriptSegment],
|
||||
) -> str:
|
||||
entry_text = "\n".join(segment.text for segment in segments if segment.text).strip()
|
||||
lines = [
|
||||
"## 入库文本",
|
||||
"",
|
||||
entry_text,
|
||||
"",
|
||||
"## 识别文本",
|
||||
"",
|
||||
]
|
||||
for segment in segments:
|
||||
if segment.begin_time is not None or segment.end_time is not None:
|
||||
begin = cls._format_timestamp(segment.begin_time)
|
||||
end = cls._format_timestamp(segment.end_time)
|
||||
lines.append(f"[{begin} - {end}] {segment.text}")
|
||||
else:
|
||||
lines.append(segment.text)
|
||||
lines.append("")
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
@staticmethod
|
||||
def _format_timestamp(milliseconds: int | None) -> str:
|
||||
if milliseconds is None:
|
||||
return "--:--:--"
|
||||
seconds = max(0, int(milliseconds / 1000))
|
||||
h, rest = divmod(seconds, 3600)
|
||||
m, s = divmod(rest, 60)
|
||||
return f"{h:02d}:{m:02d}:{s:02d}"
|
||||
@@ -0,0 +1,777 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import html as html_lib
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup, Comment
|
||||
from markdownify import markdownify
|
||||
|
||||
from bisheng.common.errcode.knowledge import KnowledgeWebLinkImportError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebLinkImportResult:
|
||||
title: str
|
||||
markdown: str
|
||||
final_url: str
|
||||
content_hash: str
|
||||
content_length: int
|
||||
html_snapshot: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RenderedContent:
|
||||
html: str
|
||||
final_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MarkdownExtraction:
|
||||
title: str
|
||||
body: str
|
||||
markdown: str
|
||||
needs_rendered_fallback: bool
|
||||
low_value_reason: str = ""
|
||||
|
||||
|
||||
class KnowledgeWebLinkImportService:
|
||||
MAX_REDIRECTS = 5
|
||||
MAX_CONTENT_BYTES = 10 * 1024 * 1024
|
||||
RENDER_TIMEOUT_MS = 20_000
|
||||
MIN_MEANINGFUL_BODY_LENGTH = 80
|
||||
BLOCKED_HOSTS = {"localhost", "localhost.localdomain"}
|
||||
PLACEHOLDER_TITLES = {"loading", "loading...", "loading…"}
|
||||
PLACEHOLDER_BODY_MARKERS = (
|
||||
"you need to enable javascript to run this app",
|
||||
"please enable javascript",
|
||||
"enable javascript to continue",
|
||||
)
|
||||
LOW_VALUE_PAGE_MARKERS = (
|
||||
"don't have an account? register",
|
||||
"do not have an account? register",
|
||||
"forgot password",
|
||||
"remember me",
|
||||
"sign in",
|
||||
"sign up",
|
||||
"login",
|
||||
"register",
|
||||
"请输入用户名",
|
||||
"请输入密码",
|
||||
"忘记密码",
|
||||
"立即登录",
|
||||
"注册账号",
|
||||
)
|
||||
DECORATIVE_IMAGE_MARKERS = (
|
||||
"logo",
|
||||
"icon",
|
||||
"avatar",
|
||||
"qrcode",
|
||||
"qr_code",
|
||||
"二维码",
|
||||
"头像",
|
||||
"图标",
|
||||
)
|
||||
CONTENT_SELECTORS = (
|
||||
{"id": "js_content"},
|
||||
{"class": "rich_media_content"},
|
||||
{"class": "TRS_Editor"},
|
||||
{"class": "TRS_AUTOADD"},
|
||||
{"class": "article-content"},
|
||||
{"class": "article-body"},
|
||||
{"class": "article_content"},
|
||||
{"class": "news_content"},
|
||||
{"class": "content_area"},
|
||||
{"class": "wp_articlecontent"},
|
||||
{"id": "UCAP-CONTENT"},
|
||||
{"id": "zoom"},
|
||||
{"id": "article_content"},
|
||||
{"id": "content"},
|
||||
{"class": "content"},
|
||||
{"role": "main"},
|
||||
{"tag": "article"},
|
||||
{"tag": "main"},
|
||||
)
|
||||
TITLE_SELECTORS = (
|
||||
{"id": "activity-name"},
|
||||
{"class": "article-title"},
|
||||
{"class": "news_title"},
|
||||
{"id": "article_title"},
|
||||
{"tag": "h1"},
|
||||
)
|
||||
SKIP_TAGS = (
|
||||
"script",
|
||||
"style",
|
||||
"link",
|
||||
"noscript",
|
||||
"svg",
|
||||
"iframe",
|
||||
"canvas",
|
||||
"template",
|
||||
"form",
|
||||
"input",
|
||||
"select",
|
||||
"textarea",
|
||||
"button",
|
||||
"nav",
|
||||
"footer",
|
||||
"aside",
|
||||
)
|
||||
REQUEST_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
async def fetch(cls, url: str) -> WebLinkImportResult:
|
||||
current_url = cls._normalize_url(url)
|
||||
content = b""
|
||||
content_type = ""
|
||||
response_url = current_url
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0), follow_redirects=False) as client:
|
||||
for _ in range(cls.MAX_REDIRECTS + 1):
|
||||
await cls._validate_url(current_url)
|
||||
async with client.stream("GET", current_url, headers=cls.REQUEST_HEADERS) as response:
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link redirect location is missing")
|
||||
current_url = str(response.url.join(location))
|
||||
continue
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise KnowledgeWebLinkImportError(
|
||||
msg=f"Web link request failed with status {response.status_code}"
|
||||
)
|
||||
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if not cls._is_supported_content_type(content_type):
|
||||
raise KnowledgeWebLinkImportError(msg="Web link content type is not supported")
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in response.aiter_bytes():
|
||||
total += len(chunk)
|
||||
if total > cls.MAX_CONTENT_BYTES:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link content is too large")
|
||||
chunks.append(chunk)
|
||||
content = b"".join(chunks)
|
||||
response_url = str(response.url)
|
||||
break
|
||||
else:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link redirects too many times")
|
||||
except KnowledgeWebLinkImportError:
|
||||
raise
|
||||
except httpx.HTTPError as exc:
|
||||
raise KnowledgeWebLinkImportError(msg=f"Web link request failed: {exc}") from exc
|
||||
|
||||
if not content:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link content is empty")
|
||||
|
||||
text = cls._decode_content(content, content_type)
|
||||
extraction = cls._extract_markdown(text, response_url, content_type)
|
||||
if extraction.needs_rendered_fallback and "text/plain" not in content_type:
|
||||
rendered_content = await cls._fetch_rendered_content(response_url)
|
||||
if rendered_content:
|
||||
rendered_extraction = cls._extract_markdown(
|
||||
rendered_content.html,
|
||||
rendered_content.final_url,
|
||||
"text/html",
|
||||
)
|
||||
if cls._should_use_rendered_extraction(extraction, rendered_extraction):
|
||||
response_url = rendered_content.final_url
|
||||
extraction = rendered_extraction
|
||||
|
||||
title = extraction.title
|
||||
markdown = extraction.markdown
|
||||
html_snapshot = cls._build_readable_html_snapshot(title, extraction.body, response_url)
|
||||
markdown_bytes = markdown.encode("utf-8")
|
||||
return WebLinkImportResult(
|
||||
title=title,
|
||||
markdown=markdown,
|
||||
final_url=response_url,
|
||||
content_hash=hashlib.sha256(markdown_bytes).hexdigest(),
|
||||
content_length=len(markdown_bytes),
|
||||
html_snapshot=html_snapshot,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _normalize_url(cls, url: str) -> str:
|
||||
normalized = (url or "").strip()
|
||||
if not normalized:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link URL is empty")
|
||||
return normalized
|
||||
|
||||
@classmethod
|
||||
async def _validate_url(cls, url: str) -> None:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise KnowledgeWebLinkImportError(msg="Only http/https web links are supported")
|
||||
if not parsed.hostname:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link host is missing")
|
||||
|
||||
hostname = parsed.hostname.rstrip(".").lower()
|
||||
if hostname in cls.BLOCKED_HOSTS:
|
||||
raise KnowledgeWebLinkImportError(msg="This web link host is not allowed")
|
||||
|
||||
for ip_address in await cls._resolve_host(hostname, parsed.port):
|
||||
if cls._is_blocked_ip(ip_address):
|
||||
raise KnowledgeWebLinkImportError(msg="This web link address is not allowed")
|
||||
|
||||
@classmethod
|
||||
async def _resolve_host(
|
||||
cls,
|
||||
hostname: str,
|
||||
port: int | None,
|
||||
) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:
|
||||
try:
|
||||
return [ipaddress.ip_address(hostname)]
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def resolve() -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:
|
||||
try:
|
||||
infos = socket.getaddrinfo(hostname, port or 443, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror as exc:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link host cannot be resolved") from exc
|
||||
resolved = []
|
||||
for info in infos:
|
||||
ip_value = info[4][0]
|
||||
try:
|
||||
resolved.append(ipaddress.ip_address(ip_value))
|
||||
except ValueError:
|
||||
continue
|
||||
return resolved
|
||||
|
||||
addresses = await asyncio.to_thread(resolve)
|
||||
if not addresses:
|
||||
raise KnowledgeWebLinkImportError(msg="Web link host cannot be resolved")
|
||||
return addresses
|
||||
|
||||
@staticmethod
|
||||
def _is_blocked_ip(ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
return (
|
||||
ip_address.is_loopback
|
||||
or ip_address.is_link_local
|
||||
or ip_address.is_multicast
|
||||
or ip_address.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_supported_content_type(content_type: str) -> bool:
|
||||
if not content_type:
|
||||
return True
|
||||
return any(
|
||||
allowed in content_type
|
||||
for allowed in ("text/html", "application/xhtml+xml", "text/plain")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_content(content: bytes, content_type: str) -> str:
|
||||
encoding = "utf-8"
|
||||
if "charset=" in content_type:
|
||||
encoding = content_type.rsplit("charset=", 1)[-1].split(";", 1)[0].strip() or encoding
|
||||
return content.decode(encoding, errors="replace")
|
||||
|
||||
@classmethod
|
||||
def _build_html_snapshot(cls, content: str, source_url: str, content_type: str) -> str:
|
||||
if "text/plain" in content_type:
|
||||
title = cls._title_from_url(source_url)
|
||||
escaped_title = html_lib.escape(title)
|
||||
escaped_body = html_lib.escape(content)
|
||||
return "\n".join(
|
||||
[
|
||||
"<!doctype html>",
|
||||
'<html lang="zh-CN">',
|
||||
"<head>",
|
||||
'<meta charset="utf-8">',
|
||||
f"<title>{escaped_title}</title>",
|
||||
"</head>",
|
||||
"<body>",
|
||||
f"<pre>{escaped_body}</pre>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
)
|
||||
|
||||
soup = cls._make_soup(content)
|
||||
if not soup.find("base"):
|
||||
head = soup.head
|
||||
if head is None:
|
||||
html_tag = soup.html
|
||||
head = soup.new_tag("head")
|
||||
if html_tag:
|
||||
html_tag.insert(0, head)
|
||||
else:
|
||||
soup.insert(0, head)
|
||||
base_tag = soup.new_tag("base", href=source_url)
|
||||
head.insert(0, base_tag)
|
||||
if not soup.find("meta", attrs={"charset": True}):
|
||||
head = soup.head
|
||||
if head:
|
||||
meta = soup.new_tag("meta", charset="utf-8")
|
||||
head.insert(0, meta)
|
||||
return str(soup)
|
||||
|
||||
@classmethod
|
||||
def _build_readable_html_snapshot(cls, title: str, body: str, source_url: str) -> str:
|
||||
escaped_title = html_lib.escape(title or cls._title_from_url(source_url))
|
||||
escaped_source = html_lib.escape(source_url)
|
||||
escaped_body = html_lib.escape(body or "")
|
||||
return "\n".join(
|
||||
[
|
||||
"<!doctype html>",
|
||||
'<html lang="zh-CN">',
|
||||
"<head>",
|
||||
'<meta charset="utf-8">',
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
||||
f"<title>{escaped_title}</title>",
|
||||
"<style>",
|
||||
"body{margin:0;background:#f5f7fb;color:#1d2129;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;}",
|
||||
"main{box-sizing:border-box;max-width:860px;margin:32px auto;padding:32px;background:#fff;box-shadow:0 2px 12px rgba(0,0,0,.06);}",
|
||||
"h1{margin:0 0 18px;font-size:28px;line-height:1.35;font-weight:700;}",
|
||||
".source{margin:0 0 24px;color:#86909c;font-size:14px;}",
|
||||
".source a{color:#165dff;text-decoration:none;word-break:break-all;}",
|
||||
"article{white-space:pre-wrap;font-size:16px;line-height:1.85;word-break:break-word;}",
|
||||
"</style>",
|
||||
"</head>",
|
||||
"<body>",
|
||||
"<main>",
|
||||
f"<h1>{escaped_title}</h1>",
|
||||
f'<p class="source">来源链接:<a href="{escaped_source}" target="_blank" rel="noreferrer">{escaped_source}</a></p>',
|
||||
f"<article>{escaped_body}</article>",
|
||||
"</main>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def _fetch_rendered_content(cls, source_url: str) -> _RenderedContent | None:
|
||||
try:
|
||||
await cls._validate_url(source_url)
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
try:
|
||||
context = await browser.new_context(
|
||||
user_agent=cls.REQUEST_HEADERS["User-Agent"],
|
||||
extra_http_headers={
|
||||
"Accept-Language": cls.REQUEST_HEADERS["Accept-Language"],
|
||||
},
|
||||
ignore_https_errors=True,
|
||||
)
|
||||
page = await context.new_page()
|
||||
response = await page.goto(
|
||||
source_url,
|
||||
wait_until="domcontentloaded",
|
||||
timeout=cls.RENDER_TIMEOUT_MS,
|
||||
)
|
||||
final_url = page.url or source_url
|
||||
await cls._validate_url(final_url)
|
||||
if response and response.status >= 400:
|
||||
return None
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5_000)
|
||||
except Exception:
|
||||
pass
|
||||
await page.wait_for_timeout(1_000)
|
||||
html = await page.content()
|
||||
await context.close()
|
||||
return _RenderedContent(html=html, final_url=final_url)
|
||||
finally:
|
||||
await browser.close()
|
||||
except Exception as exc:
|
||||
logger.info("Rendered web link fallback failed for %s: %s", source_url, exc)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _extract_markdown(cls, content: str, source_url: str, content_type: str) -> _MarkdownExtraction:
|
||||
if "text/plain" in content_type:
|
||||
title = cls._title_from_url(source_url)
|
||||
body = content.strip()
|
||||
else:
|
||||
soup = cls._make_soup(content)
|
||||
title = cls._extract_title(soup, source_url)
|
||||
fallback_body = cls._extract_fallback_body(cls._make_soup(content))
|
||||
cls._clean_soup(soup, source_url)
|
||||
container = cls._select_content_container(soup)
|
||||
body = cls._html_to_markdown(str(container)) if container else ""
|
||||
if not body:
|
||||
body = fallback_body
|
||||
|
||||
body = cls._normalize_markdown_body(body)
|
||||
body = cls._strip_leading_title_heading(body, title)
|
||||
is_placeholder = cls._is_placeholder_body(body)
|
||||
low_value_reason = cls._get_low_value_reason(title, body)
|
||||
needs_rendered_fallback = is_placeholder or bool(low_value_reason) or cls._is_low_quality_body(body)
|
||||
if is_placeholder:
|
||||
body = ""
|
||||
elif low_value_reason:
|
||||
body = low_value_reason
|
||||
if not body:
|
||||
body = "该网页可能为动态渲染页面,未能提取到静态正文内容。"
|
||||
|
||||
markdown = body.strip() + "\n"
|
||||
return _MarkdownExtraction(
|
||||
title=title,
|
||||
body=body,
|
||||
markdown=markdown,
|
||||
needs_rendered_fallback=needs_rendered_fallback,
|
||||
low_value_reason=low_value_reason,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _make_soup(cls, content: str) -> BeautifulSoup:
|
||||
try:
|
||||
return BeautifulSoup(content, "lxml")
|
||||
except Exception:
|
||||
return BeautifulSoup(content, "html.parser")
|
||||
|
||||
@staticmethod
|
||||
def _strip_leading_title_heading(body: str, title: str) -> str:
|
||||
normalized_title = re.sub(r"\s+", " ", (title or "").strip()).strip("# ")
|
||||
if not normalized_title:
|
||||
return body
|
||||
lines = body.splitlines()
|
||||
index = 0
|
||||
while index < len(lines) and not lines[index].strip():
|
||||
index += 1
|
||||
if index >= len(lines):
|
||||
return body
|
||||
first_line = lines[index].strip()
|
||||
if not first_line.startswith("#"):
|
||||
return body
|
||||
heading_text = re.sub(r"^#+\s*", "", first_line).strip()
|
||||
heading_text = re.sub(r"\s+", " ", heading_text)
|
||||
if heading_text != normalized_title:
|
||||
return body
|
||||
remaining = lines[:index] + lines[index + 1 :]
|
||||
return "\n".join(remaining).strip()
|
||||
|
||||
@classmethod
|
||||
def _clean_soup(cls, soup: BeautifulSoup, source_url: str = "") -> None:
|
||||
for comment in soup.find_all(string=lambda value: isinstance(value, Comment)):
|
||||
comment.extract()
|
||||
|
||||
cls._remove_hidden_elements(soup)
|
||||
for tag_name in cls.SKIP_TAGS:
|
||||
for tag in soup.find_all(tag_name):
|
||||
tag.decompose()
|
||||
|
||||
if source_url:
|
||||
cls._absolutize_links(soup, source_url)
|
||||
cls._remove_decorative_images(soup)
|
||||
|
||||
for tag in soup.find_all(True):
|
||||
if tag.name in {"html", "body"}:
|
||||
continue
|
||||
safe_attrs = {"href", "src", "alt", "title", "colspan", "rowspan", "scope", "headers"}
|
||||
for attr in list(tag.attrs.keys()):
|
||||
if attr not in safe_attrs:
|
||||
del tag[attr]
|
||||
|
||||
@classmethod
|
||||
def _absolutize_links(cls, soup: BeautifulSoup, source_url: str) -> None:
|
||||
for tag in soup.find_all(["a", "img", "source"]):
|
||||
attr = "href" if tag.name == "a" else "src"
|
||||
value = (tag.get(attr) or "").strip()
|
||||
if not value:
|
||||
if tag.name == "img":
|
||||
tag.decompose()
|
||||
continue
|
||||
if value.startswith(("data:", "mailto:", "tel:", "javascript:")):
|
||||
if tag.name == "img" and value.startswith("data:image/svg"):
|
||||
tag.decompose()
|
||||
continue
|
||||
tag[attr] = urljoin(source_url, value)
|
||||
|
||||
@classmethod
|
||||
def _remove_decorative_images(cls, soup: BeautifulSoup) -> None:
|
||||
seen_srcs = set()
|
||||
for image in list(soup.find_all("img")):
|
||||
src = (image.get("src") or "").strip()
|
||||
if not src:
|
||||
image.decompose()
|
||||
continue
|
||||
alt_title = " ".join(
|
||||
str(image.get(attr, ""))
|
||||
for attr in ("alt", "title", "class", "id")
|
||||
).lower()
|
||||
if src in seen_srcs and any(marker in alt_title for marker in cls.DECORATIVE_IMAGE_MARKERS):
|
||||
image.decompose()
|
||||
continue
|
||||
if any(marker in alt_title for marker in cls.DECORATIVE_IMAGE_MARKERS):
|
||||
image.decompose()
|
||||
continue
|
||||
seen_srcs.add(src)
|
||||
|
||||
@classmethod
|
||||
def _remove_hidden_elements(cls, soup: BeautifulSoup) -> None:
|
||||
removable = []
|
||||
for tag in soup.find_all(True):
|
||||
if tag.name in {"html", "body"}:
|
||||
continue
|
||||
attrs = tag.attrs or {}
|
||||
style = str(attrs.get("style", ""))
|
||||
normalized_style = re.sub(r"\s+", "", style).lower()
|
||||
classes = attrs.get("class") or []
|
||||
class_names = classes if isinstance(classes, list) else [classes]
|
||||
is_known_content_root = attrs.get("id") == "js_content" or "rich_media_content" in class_names
|
||||
if is_known_content_root:
|
||||
cleaned_style = re.sub(r"visibility\s*:\s*hidden\s*;?", "", style, flags=re.I)
|
||||
cleaned_style = re.sub(r"opacity\s*:\s*0\s*;?", "", cleaned_style, flags=re.I)
|
||||
cleaned_style = re.sub(r"display\s*:\s*none\s*;?", "", cleaned_style, flags=re.I)
|
||||
if cleaned_style.strip():
|
||||
tag["style"] = cleaned_style.strip()
|
||||
elif "style" in tag.attrs:
|
||||
del tag["style"]
|
||||
continue
|
||||
if (
|
||||
attrs.get("hidden") is not None
|
||||
or str(attrs.get("aria-hidden", "")).lower() == "true"
|
||||
or "display:none" in normalized_style
|
||||
or "visibility:hidden" in normalized_style
|
||||
or "opacity:0" in normalized_style
|
||||
):
|
||||
removable.append(tag)
|
||||
for tag in removable:
|
||||
tag.decompose()
|
||||
|
||||
@classmethod
|
||||
def _select_content_container(cls, soup: BeautifulSoup):
|
||||
for selector in cls.CONTENT_SELECTORS:
|
||||
element = cls._find_element(soup, selector)
|
||||
if element and cls._text_length(element) >= 50:
|
||||
return element
|
||||
|
||||
best = None
|
||||
best_score = 0
|
||||
for element in soup.find_all(["article", "main", "section", "div"]):
|
||||
text = element.get_text(" ", strip=True)
|
||||
if not text:
|
||||
continue
|
||||
direct_children = len(element.find_all(["article", "main", "section", "div"], recursive=False))
|
||||
if direct_children > 8:
|
||||
continue
|
||||
link_text_length = sum(len(link.get_text(" ", strip=True)) for link in element.find_all("a"))
|
||||
score = len(text) - int(link_text_length * 0.6)
|
||||
if score > best_score:
|
||||
best = element
|
||||
best_score = score
|
||||
if best and best_score >= 50:
|
||||
return best
|
||||
return soup.body or soup
|
||||
|
||||
@classmethod
|
||||
def _find_element(cls, soup: BeautifulSoup, selector: dict):
|
||||
if "tag" in selector:
|
||||
return soup.find(selector["tag"])
|
||||
if "class" in selector:
|
||||
return soup.find(class_=selector["class"])
|
||||
if "id" in selector:
|
||||
return soup.find(id=selector["id"])
|
||||
if "role" in selector:
|
||||
return soup.find(attrs={"role": selector["role"]})
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _text_length(element) -> int:
|
||||
return len(element.get_text(" ", strip=True))
|
||||
|
||||
@classmethod
|
||||
def _html_to_markdown(cls, html: str) -> str:
|
||||
markdown = markdownify(
|
||||
html,
|
||||
heading_style="ATX",
|
||||
bullets="-",
|
||||
strip=["style", "script", "link", "meta", "head"],
|
||||
).strip()
|
||||
markdown = re.sub(r"</?[a-zA-Z][^>]*>", "", markdown)
|
||||
markdown = cls._remove_markdown_noise(markdown)
|
||||
return markdown.strip()
|
||||
|
||||
@staticmethod
|
||||
def _remove_markdown_noise(markdown: str) -> str:
|
||||
cleaned_lines = []
|
||||
for line in markdown.splitlines():
|
||||
stripped = line.strip()
|
||||
lower = stripped.lower()
|
||||
if not stripped:
|
||||
cleaned_lines.append("")
|
||||
continue
|
||||
if lower.startswith(("<link ", "<style", "</style", "<script", "</script")):
|
||||
continue
|
||||
if "rel=\"stylesheet\"" in lower or "rel='stylesheet'" in lower:
|
||||
continue
|
||||
if re.search(r"[.#][\w\\-]+\s*\{[^}]{0,240}\}", stripped) and not re.search(r"[\u4e00-\u9fff]", stripped):
|
||||
continue
|
||||
cleaned_lines.append(stripped)
|
||||
return "\n".join(cleaned_lines)
|
||||
|
||||
@staticmethod
|
||||
def _extract_fallback_body(soup: BeautifulSoup) -> str:
|
||||
candidates = []
|
||||
for meta_name in ("description", "keywords"):
|
||||
meta = soup.find("meta", attrs={"name": meta_name})
|
||||
content = meta.get("content", "").strip() if meta else ""
|
||||
if content and not KnowledgeWebLinkImportService._is_placeholder_body(content):
|
||||
candidates.append(content)
|
||||
og_description = soup.find("meta", attrs={"property": "og:description"})
|
||||
og_content = og_description.get("content", "").strip() if og_description else ""
|
||||
if og_content and not KnowledgeWebLinkImportService._is_placeholder_body(og_content):
|
||||
candidates.append(og_content)
|
||||
KnowledgeWebLinkImportService._clean_soup(soup)
|
||||
text = soup.get_text("\n", strip=True)
|
||||
if text and not KnowledgeWebLinkImportService._is_placeholder_body(text):
|
||||
candidates.append(text)
|
||||
return "\n\n".join(dict.fromkeys(candidates))
|
||||
|
||||
@staticmethod
|
||||
def _normalize_markdown_body(body: str, max_line_length: int = 900) -> str:
|
||||
normalized_lines = []
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if len(stripped) <= max_line_length:
|
||||
normalized_lines.append(stripped)
|
||||
continue
|
||||
normalized_lines.extend(
|
||||
stripped[index: index + max_line_length]
|
||||
for index in range(0, len(stripped), max_line_length)
|
||||
)
|
||||
normalized = "\n".join(normalized_lines)
|
||||
return re.sub(r"\n{3,}", "\n\n", normalized).strip()
|
||||
|
||||
@classmethod
|
||||
def _is_placeholder_body(cls, text: str) -> bool:
|
||||
normalized = re.sub(r"\s+", " ", (text or "")).strip().lower()
|
||||
if not normalized:
|
||||
return True
|
||||
if normalized.strip(".… ") == "loading":
|
||||
return True
|
||||
return any(marker in normalized for marker in cls.PLACEHOLDER_BODY_MARKERS)
|
||||
|
||||
@classmethod
|
||||
def _is_low_quality_body(cls, body: str) -> bool:
|
||||
normalized = re.sub(r"\s+", " ", (body or "")).strip()
|
||||
if not normalized:
|
||||
return True
|
||||
|
||||
lower = normalized.lower()
|
||||
artifact_markers = (
|
||||
"<style",
|
||||
"</style",
|
||||
"<script",
|
||||
"</script",
|
||||
"<link",
|
||||
"rel=\"stylesheet\"",
|
||||
"rel='stylesheet'",
|
||||
"display:none",
|
||||
"visibility:hidden",
|
||||
"data-for=\"result\"",
|
||||
)
|
||||
if any(marker in lower for marker in artifact_markers):
|
||||
return True
|
||||
|
||||
css_rule_count = len(re.findall(r"[.#][\w\\-]+\s*\{[^}]{0,300}\}", normalized))
|
||||
if css_rule_count >= 2:
|
||||
return True
|
||||
|
||||
meaningful = re.sub(r"https?://\S+", "", normalized)
|
||||
meaningful = re.sub(r"!\[[^\]]*]\([^)]+\)|\[[^\]]+]\([^)]+\)", "", meaningful)
|
||||
meaningful = re.sub(r"[#*_`>\-\s|::,,.。/\\{}()[\]\"'=;;]+", "", meaningful)
|
||||
if len(meaningful) < cls.MIN_MEANINGFUL_BODY_LENGTH:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _get_low_value_reason(cls, title: str, body: str) -> str:
|
||||
visible_text = cls._visible_markdown_text(body)
|
||||
normalized = re.sub(r"\s+", " ", f"{title} {visible_text}".strip()).lower()
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
matched_markers = sum(1 for marker in cls.LOW_VALUE_PAGE_MARKERS if marker in normalized)
|
||||
if matched_markers >= 2:
|
||||
return "该网页可能是登录、注册或权限入口页面,未能提取到适合入库的正文内容。"
|
||||
|
||||
words_for_navigation = (
|
||||
"首页",
|
||||
"注册",
|
||||
"登录",
|
||||
"管理",
|
||||
"应用",
|
||||
"工具",
|
||||
"工作台",
|
||||
"don't have an account",
|
||||
"register",
|
||||
)
|
||||
nav_hits = sum(1 for word in words_for_navigation if word.lower() in normalized)
|
||||
meaningful = re.sub(r"https?://\S+", "", normalized)
|
||||
meaningful = re.sub(r"!\[[^\]]*]\([^)]+\)|\[[^\]]+]\([^)]+\)", "", meaningful)
|
||||
meaningful = re.sub(r"[#*_`>\-\s|::,,.。/\\{}()[\]\"'=;;]+", "", meaningful)
|
||||
if nav_hits >= 4 and len(meaningful) < 220:
|
||||
return "该网页主要是导航或门户入口页面,正文信息不足,不建议作为知识库网页正文导入。"
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _visible_markdown_text(markdown: str) -> str:
|
||||
text = re.sub(r"!\[([^\]]*)]\([^)]+\)", r"\1", markdown or "")
|
||||
text = re.sub(r"\[([^\]]+)]\([^)]+\)", r"\1", text)
|
||||
return re.sub(r"https?://\S+", "", text)
|
||||
|
||||
@classmethod
|
||||
def _should_use_rendered_extraction(
|
||||
cls,
|
||||
static_extraction: _MarkdownExtraction,
|
||||
rendered_extraction: _MarkdownExtraction,
|
||||
) -> bool:
|
||||
if static_extraction.needs_rendered_fallback and not rendered_extraction.needs_rendered_fallback:
|
||||
return True
|
||||
if (
|
||||
static_extraction.needs_rendered_fallback
|
||||
and rendered_extraction.low_value_reason
|
||||
and rendered_extraction.body != static_extraction.body
|
||||
):
|
||||
return True
|
||||
if cls._is_placeholder_body(static_extraction.body) and rendered_extraction.body:
|
||||
return True
|
||||
return len(rendered_extraction.body) > len(static_extraction.body) * 1.5
|
||||
|
||||
@classmethod
|
||||
def _extract_title(cls, soup: BeautifulSoup, source_url: str) -> str:
|
||||
for selector in cls.TITLE_SELECTORS:
|
||||
candidate = cls._find_element(soup, selector)
|
||||
if not candidate:
|
||||
continue
|
||||
title = candidate.get_text(" ", strip=True)
|
||||
if title and title.strip().lower() not in cls.PLACEHOLDER_TITLES:
|
||||
return title[:200]
|
||||
title_tag = soup.find("title")
|
||||
if title_tag:
|
||||
title = title_tag.get_text(" ", strip=True)
|
||||
if title and title.strip().lower() not in cls.PLACEHOLDER_TITLES:
|
||||
return title[:200]
|
||||
return cls._title_from_url(source_url)
|
||||
|
||||
@staticmethod
|
||||
def _title_from_url(source_url: str) -> str:
|
||||
parsed = urlparse(source_url)
|
||||
title = parsed.path.rstrip("/").rsplit("/", 1)[-1] or parsed.netloc or parsed.hostname or "网页链接"
|
||||
return title[:200]
|
||||
@@ -12,6 +12,7 @@ from bisheng.knowledge.rag.pipeline.loader.etl4lm import Etl4lmLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.excel import ExcelLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.hierarchical import HierarchicalMarkdownLoader, HierarchicalWordLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.html import BishengHtmlLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.media import BishengMediaLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.mineru import MineruLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.ofd import OfdLoader
|
||||
from bisheng.knowledge.rag.pipeline.loader.paddle_ocr import PaddleOcrLoader
|
||||
@@ -43,6 +44,17 @@ FileExtensionMap = {
|
||||
"jpg": {"loader": "_init_image_loader", "transformers": "_init_common_transformers"},
|
||||
"jpeg": {"loader": "_init_image_loader", "transformers": "_init_common_transformers"},
|
||||
"bmp": {"loader": "_init_image_loader", "transformers": "_init_common_transformers"},
|
||||
"mp3": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"wav": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"m4a": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"aac": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"flac": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"ogg": {"loader": "_init_audio_loader", "transformers": "_init_common_transformers"},
|
||||
"mp4": {"loader": "_init_video_loader", "transformers": "_init_common_transformers"},
|
||||
"mov": {"loader": "_init_video_loader", "transformers": "_init_common_transformers"},
|
||||
"avi": {"loader": "_init_video_loader", "transformers": "_init_common_transformers"},
|
||||
"mkv": {"loader": "_init_video_loader", "transformers": "_init_common_transformers"},
|
||||
"webm": {"loader": "_init_video_loader", "transformers": "_init_common_transformers"},
|
||||
}
|
||||
|
||||
|
||||
@@ -247,3 +259,17 @@ class BaseFilePipeline(BasePipeline):
|
||||
if isinstance(pdf_loader, LocalPdfLoader):
|
||||
raise KnowledgeFileNotSupportedError()
|
||||
return pdf_loader
|
||||
|
||||
def _init_audio_loader(self) -> BaseBishengLoader:
|
||||
return BishengMediaLoader(
|
||||
**self._get_loader_common_params(),
|
||||
knowledge_file=getattr(self, "db_file", None),
|
||||
media_kind="audio",
|
||||
)
|
||||
|
||||
def _init_video_loader(self) -> BaseBishengLoader:
|
||||
return BishengMediaLoader(
|
||||
**self._get_loader_common_params(),
|
||||
knowledge_file=getattr(self, "db_file", None),
|
||||
media_kind="video",
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ from functools import cached_property
|
||||
from langchain_core.documents import BaseDocumentTransformer
|
||||
|
||||
from bisheng.api.v1.schemas import FileProcessBase
|
||||
from bisheng.knowledge.domain.models.knowledge_file import KnowledgeFile
|
||||
from bisheng.knowledge.domain.models.knowledge_file import FileSource, KnowledgeFile
|
||||
from bisheng.knowledge.domain.schemas.knowledge_rag_schema import Metadata
|
||||
from bisheng.knowledge.domain.services.knowledge_utils import KnowledgeUtils
|
||||
from bisheng.knowledge.rag.base_file_pipeline import BaseFilePipeline
|
||||
@@ -24,6 +24,9 @@ from bisheng.sensitive_word.domain.services.sensitive_word_policy_service import
|
||||
from bisheng.user.domain.models.user import UserDao
|
||||
from bisheng.utils.file import download_minio_file
|
||||
|
||||
_WEB_LINK_SEPARATORS = ["\n\n", "\n", "。", "\\.", ",", ",", ";", ";", "、", "\\s+", ""]
|
||||
_WEB_LINK_SEPARATOR_RULES = ["after"] * len(_WEB_LINK_SEPARATORS)
|
||||
|
||||
|
||||
class KnowledgeFilePipeline(BaseFilePipeline):
|
||||
|
||||
@@ -33,6 +36,10 @@ class KnowledgeFilePipeline(BaseFilePipeline):
|
||||
if db_file.split_rule and isinstance(db_file.split_rule, str):
|
||||
split_rule = FileProcessBase(**json.loads(db_file.split_rule))
|
||||
split_rule.knowledge_id = db_file.knowledge_id
|
||||
if db_file.file_source == FileSource.WEB_LINK.value:
|
||||
split_rule.separator = _WEB_LINK_SEPARATORS
|
||||
split_rule.separator_rule = _WEB_LINK_SEPARATOR_RULES
|
||||
split_rule.chunk_overlap = 0
|
||||
|
||||
super().__init__(invoke_user_id, db_file.file_name, split_rule, **kwargs)
|
||||
self.db_file = db_file
|
||||
@@ -40,6 +47,14 @@ class KnowledgeFilePipeline(BaseFilePipeline):
|
||||
self.no_summary = no_summary
|
||||
self.need_thumbnail = need_thumbnail
|
||||
|
||||
@cached_property
|
||||
def file_extension(self):
|
||||
if self.db_file.file_source == FileSource.WEB_LINK.value:
|
||||
return "md"
|
||||
if self.file_name:
|
||||
return self.file_name.split(".")[-1].lower()
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def file_metadata(self) -> dict:
|
||||
uploader = UserDao.get_user(self.invoke_user_id)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from bisheng.knowledge.domain.models.knowledge_file import KnowledgeFile
|
||||
from bisheng.knowledge.domain.services.media_transcription_service import (
|
||||
KnowledgeMediaTranscriptionService,
|
||||
)
|
||||
from bisheng.knowledge.rag.pipeline.loader.base import BaseBishengLoader
|
||||
|
||||
|
||||
class BishengMediaLoader(BaseBishengLoader):
|
||||
"""Load audio/video files by transcribing speech to text."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
knowledge_file: KnowledgeFile | None = None,
|
||||
media_kind: str = "audio",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.knowledge_file = knowledge_file
|
||||
self.media_kind = media_kind
|
||||
|
||||
def load(self) -> list[Document]:
|
||||
result = KnowledgeMediaTranscriptionService.transcribe_media(
|
||||
self.file_path,
|
||||
source_file_name=self.file_metadata.get("document_name") or self.file_name,
|
||||
tenant_id=getattr(self.knowledge_file, "tenant_id", None),
|
||||
)
|
||||
preview_path = os.path.join(self.tmp_dir, f"{os.path.splitext(self.file_name)[0]}_transcript.md")
|
||||
with open(preview_path, "w", encoding="utf-8") as f:
|
||||
f.write(result.markdown)
|
||||
self.preview_file_path = preview_path
|
||||
|
||||
metadata = self.file_metadata.copy()
|
||||
metadata["user_metadata"] = {
|
||||
**(metadata.get("user_metadata") or {}),
|
||||
"media_kind": self.media_kind,
|
||||
"transcription_model_id": result.model_id,
|
||||
"transcription_model_name": result.model_name,
|
||||
}
|
||||
|
||||
if self.knowledge_file:
|
||||
self.knowledge_file.user_metadata = {
|
||||
**(self.knowledge_file.user_metadata or {}),
|
||||
"media_kind": self.media_kind,
|
||||
"transcription_model_id": result.model_id,
|
||||
"transcription_model_name": result.model_name,
|
||||
}
|
||||
|
||||
return [Document(page_content=result.text, metadata=metadata)]
|
||||
@@ -0,0 +1,91 @@
|
||||
from bisheng.knowledge.domain.services.media_transcription_service import (
|
||||
KnowledgeMediaTranscriptionService,
|
||||
TranscriptSegment,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_segments_keeps_aliyun_millisecond_timestamps_consistent() -> None:
|
||||
segments = [
|
||||
TranscriptSegment("Again after seeing.", begin_time=3760, end_time=5160),
|
||||
TranscriptSegment(
|
||||
"My entire life's been spent only in one industry.",
|
||||
begin_time=6660,
|
||||
end_time=10000,
|
||||
),
|
||||
TranscriptSegment(
|
||||
"But I've been in it now for about 15 years.",
|
||||
begin_time=10000,
|
||||
end_time=22000,
|
||||
),
|
||||
]
|
||||
|
||||
normalized = KnowledgeMediaTranscriptionService._normalize_segments(
|
||||
segments,
|
||||
media_duration_ms=30_000,
|
||||
)
|
||||
|
||||
assert [(item.begin_time, item.end_time) for item in normalized] == [
|
||||
(3760, 5160),
|
||||
(6660, 10000),
|
||||
(10000, 22000),
|
||||
]
|
||||
markdown = KnowledgeMediaTranscriptionService._build_markdown(
|
||||
source_file_name="jobs.m4a",
|
||||
model_name="paraformer-realtime-v2",
|
||||
segments=normalized,
|
||||
)
|
||||
assert (
|
||||
"[00:00:06 - 00:00:10] My entire life's been spent only in one industry."
|
||||
in markdown
|
||||
)
|
||||
assert "## 入库文本" in markdown
|
||||
assert "## 识别文本" in markdown
|
||||
assert "来源文件" not in markdown
|
||||
assert "ASR 模型" not in markdown
|
||||
assert "01:51:00" not in markdown
|
||||
|
||||
|
||||
def test_normalize_segments_sorts_by_begin_time() -> None:
|
||||
segments = [
|
||||
TranscriptSegment("later", begin_time=22_000, end_time=23_000),
|
||||
TranscriptSegment("first", begin_time=10_000, end_time=20_000),
|
||||
TranscriptSegment("middle", begin_time=20_000, end_time=22_000),
|
||||
]
|
||||
|
||||
normalized = KnowledgeMediaTranscriptionService._normalize_segments(
|
||||
segments,
|
||||
media_duration_ms=30_000,
|
||||
)
|
||||
|
||||
assert [item.text for item in normalized] == ["first", "middle", "later"]
|
||||
|
||||
|
||||
def test_normalize_segments_clamps_invalid_end_time() -> None:
|
||||
segments = [
|
||||
TranscriptSegment("bad range", begin_time=12_000, end_time=10_000),
|
||||
]
|
||||
|
||||
normalized = KnowledgeMediaTranscriptionService._normalize_segments(
|
||||
segments,
|
||||
media_duration_ms=30_000,
|
||||
)
|
||||
|
||||
assert normalized[0].begin_time == 12_000
|
||||
assert normalized[0].end_time == 12_000
|
||||
|
||||
|
||||
def test_normalize_segments_can_scale_second_timestamps_with_duration_hint() -> None:
|
||||
segments = [
|
||||
TranscriptSegment("first", begin_time=10, end_time=20),
|
||||
TranscriptSegment("second", begin_time=20, end_time=30),
|
||||
]
|
||||
|
||||
normalized = KnowledgeMediaTranscriptionService._normalize_segments(
|
||||
segments,
|
||||
media_duration_ms=31_000,
|
||||
)
|
||||
|
||||
assert [(item.begin_time, item.end_time) for item in normalized] == [
|
||||
(10_000, 20_000),
|
||||
(20_000, 30_000),
|
||||
]
|
||||
@@ -0,0 +1,216 @@
|
||||
from bisheng.knowledge.domain.services.web_link_import_service import KnowledgeWebLinkImportService
|
||||
|
||||
|
||||
def test_extract_markdown_removes_stylesheet_and_css_noise() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<head>
|
||||
<title>测试文章</title>
|
||||
<link rel="stylesheet" href="https://example.com/main.css">
|
||||
</head>
|
||||
<body>
|
||||
<style>#app{display:none}.title{color:red}</style>
|
||||
<article>
|
||||
<h1>测试文章</h1>
|
||||
<p>这是第一段正文内容,用来验证网页链接导入时不会把 CSS 样式写进知识库。</p>
|
||||
<p>这是第二段正文内容,包含足够多的文字以通过正文质量检测,并保留真正的页面信息,确保普通文章不会触发浏览器渲染兜底。</p>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
result = KnowledgeWebLinkImportService._extract_markdown(
|
||||
html,
|
||||
"https://example.com/article",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert "rel=\"stylesheet\"" not in result.markdown
|
||||
assert "display:none" not in result.markdown
|
||||
assert "#app" not in result.markdown
|
||||
assert "这是第一段正文内容" in result.markdown
|
||||
assert "来源链接" not in result.markdown
|
||||
assert "导入时间" not in result.markdown
|
||||
assert not result.markdown.lstrip().startswith("# 测试文章")
|
||||
assert not result.needs_rendered_fallback
|
||||
|
||||
|
||||
def test_build_html_snapshot_adds_base_href() -> None:
|
||||
snapshot = KnowledgeWebLinkImportService._build_html_snapshot(
|
||||
"<html><head><title>内网页面</title></head><body><img src='/logo.png'></body></html>",
|
||||
"http://192.168.106.171:3002/apps",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert '<base href="http://192.168.106.171:3002/apps"/>' in snapshot
|
||||
assert '<meta charset="utf-8"/>' in snapshot
|
||||
|
||||
|
||||
def test_build_readable_html_snapshot_uses_extracted_body() -> None:
|
||||
snapshot = KnowledgeWebLinkImportService._build_readable_html_snapshot(
|
||||
"公众号文章",
|
||||
"第一段正文\n\n第二段正文",
|
||||
"https://mp.weixin.qq.com/s/example",
|
||||
)
|
||||
|
||||
assert "<h1>公众号文章</h1>" in snapshot
|
||||
assert "第一段正文" in snapshot
|
||||
assert "第二段正文" in snapshot
|
||||
assert "visibility:hidden" not in snapshot
|
||||
|
||||
|
||||
def test_spa_shell_triggers_rendered_fallback() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<head><title>loading...</title></head>
|
||||
<body>
|
||||
<div id="root">loading... You need to enable JavaScript to run this app.</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
result = KnowledgeWebLinkImportService._extract_markdown(
|
||||
html,
|
||||
"http://192.168.106.171:3002/",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert result.title == "192.168.106.171:3002"
|
||||
assert "动态渲染页面" in result.markdown
|
||||
assert result.needs_rendered_fallback
|
||||
|
||||
|
||||
def test_rendered_extraction_wins_when_static_is_low_quality() -> None:
|
||||
static = KnowledgeWebLinkImportService._extract_markdown(
|
||||
"<html><title>loading...</title><body>loading...</body></html>",
|
||||
"https://example.com/app",
|
||||
"text/html",
|
||||
)
|
||||
rendered = KnowledgeWebLinkImportService._extract_markdown(
|
||||
"""
|
||||
<html>
|
||||
<body>
|
||||
<main>
|
||||
<h1>渲染后的页面</h1>
|
||||
<p>这是浏览器渲染后得到的正文内容,已经不是静态 loading 壳。</p>
|
||||
<p>这里继续补充真实页面信息,确保长度足够并且不会被判定为低质量正文。</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
"https://example.com/app",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert KnowledgeWebLinkImportService._should_use_rendered_extraction(static, rendered)
|
||||
|
||||
|
||||
def test_rendered_low_value_result_wins_over_static_spa_shell() -> None:
|
||||
static = KnowledgeWebLinkImportService._extract_markdown(
|
||||
"<html><title>loading...</title><body>loading...</body></html>",
|
||||
"http://192.168.106.171:3001/build/apps",
|
||||
"text/html",
|
||||
)
|
||||
rendered = KnowledgeWebLinkImportService._extract_markdown(
|
||||
"""
|
||||
<html>
|
||||
<head><title>首钢股份知库工作台</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<p>Convenient, Flexible, Reliable Enterprise-Level Large Model Application Development Platform</p>
|
||||
<a href="/register">Don't have an account? Register</a>
|
||||
<p>v2.6.0-beta2</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
"http://192.168.106.171:3001/build/apps",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert rendered.low_value_reason
|
||||
assert KnowledgeWebLinkImportService._should_use_rendered_extraction(static, rendered)
|
||||
|
||||
|
||||
def test_extract_markdown_absolutizes_links_and_filters_logo_images() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
<main>
|
||||
<img src="/assets/logo.png" alt="logo_picture" />
|
||||
<h1>内网文章</h1>
|
||||
<p>这是一个内网知识页面,正文内容会保留下来,并且页面中的相对链接应当转换成绝对链接。</p>
|
||||
<p>这里继续补充正文信息,避免页面被误判成只有导航入口的低价值页面。</p>
|
||||
<a href="/docs/detail">查看详情</a>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
result = KnowledgeWebLinkImportService._extract_markdown(
|
||||
html,
|
||||
"http://192.168.106.171:3001/build/apps",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert "logo_picture" not in result.markdown
|
||||
assert "http://192.168.106.171:3001/docs/detail" in result.markdown
|
||||
assert "这是一个内网知识页面" in result.markdown
|
||||
|
||||
|
||||
def test_login_page_is_marked_as_low_value() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<head><title>首钢股份知识库工作台</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<img src="/logo.svg" alt="logo_picture" />
|
||||
<p>Convenient, Flexible, Reliable Enterprise-Level Large Model Application Development Platform</p>
|
||||
<a href="/register">Don't have an account? Register</a>
|
||||
<p>v2.6.0-beta2</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
result = KnowledgeWebLinkImportService._extract_markdown(
|
||||
html,
|
||||
"http://192.168.106.171:3001/build/apps",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert result.low_value_reason
|
||||
assert "登录、注册或权限入口页面" in result.markdown
|
||||
assert "logo_picture" not in result.markdown
|
||||
|
||||
|
||||
def test_navigation_page_login_urls_do_not_trigger_login_page_detection() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<head><title>百度一下,你就知道</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<a href="https://www.baidu.com/">百度首页</a>
|
||||
<a href="https://passport.baidu.com/v2/?login&tpl=mn®ister=1">登录</a>
|
||||
<a href="https://news.baidu.com">新闻</a>
|
||||
<a href="https://map.baidu.com">地图</a>
|
||||
<a href="https://wenku.baidu.com">文库</a>
|
||||
<section>
|
||||
<h1>热搜新闻</h1>
|
||||
<p>这里展示搜索门户页面的公开导航内容和热点信息,虽然链接地址里可能带有认证参数,但页面本身不是账号表单。</p>
|
||||
<p>继续补充足够多的可见正文,确保这类门户页面不会因为链接地址中的认证参数而被误判。</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
result = KnowledgeWebLinkImportService._extract_markdown(
|
||||
html,
|
||||
"https://www.baidu.com/",
|
||||
"text/html",
|
||||
)
|
||||
|
||||
assert not result.low_value_reason
|
||||
assert "登录、注册或权限入口页面" not in result.markdown
|
||||
assert "热搜新闻" in result.markdown
|
||||
@@ -21,7 +21,7 @@ server {
|
||||
|
||||
# Increase the client_max_body_size to allow larger file uploads
|
||||
# The default limits for image uploads as of 11/22/23 is 20MB/file, and 25MB/request
|
||||
client_max_body_size 25M;
|
||||
client_max_body_size 1024M;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://api:3080$request_uri;
|
||||
@@ -82,7 +82,7 @@ server {
|
||||
|
||||
# # Increase the client_max_body_size to allow larger file uploads
|
||||
# # The default limits for image uploads as of 11/22/23 is 20MB/file, and 25MB/request
|
||||
# client_max_body_size 25M;
|
||||
# client_max_body_size 1024M;
|
||||
|
||||
# location /api {
|
||||
# proxy_pass http://api:3080/api;
|
||||
|
||||
@@ -39,6 +39,9 @@ export enum FileType {
|
||||
HTML = "html",
|
||||
TXT = "txt",
|
||||
MD = "md",
|
||||
AUDIO = "audio",
|
||||
VIDEO = "video",
|
||||
WEB = "web",
|
||||
WPS = "wps",
|
||||
DPS = "dps",
|
||||
ET = "et",
|
||||
@@ -395,6 +398,9 @@ function extractKnowledgeSpaceList(response: unknown): RawKnowledgeSpace[] {
|
||||
function deriveFileType(raw: any): FileType {
|
||||
// Backend: file_type: 0(dir) | 1(file)
|
||||
if (raw?.type === "folder" || raw?.file_type === 0 || raw?.file_type === "0") return FileType.FOLDER;
|
||||
if (raw?.file_source === "web_link") return FileType.WEB;
|
||||
if (raw?.file_source === "audio_transcript") return FileType.AUDIO;
|
||||
if (raw?.file_source === "video_transcript") return FileType.VIDEO;
|
||||
|
||||
const fileName = raw?.file_name ?? raw?.name ?? raw?.object_name ?? raw?.path ?? "";
|
||||
const ext = String(fileName).split(".").pop()?.toLowerCase() ?? "";
|
||||
@@ -427,6 +433,19 @@ function deriveFileType(raw: any): FileType {
|
||||
return FileType.TXT;
|
||||
case "md":
|
||||
return FileType.MD;
|
||||
case "mp3":
|
||||
case "wav":
|
||||
case "m4a":
|
||||
case "aac":
|
||||
case "flac":
|
||||
case "ogg":
|
||||
return FileType.AUDIO;
|
||||
case "mp4":
|
||||
case "mov":
|
||||
case "avi":
|
||||
case "mkv":
|
||||
case "webm":
|
||||
return FileType.VIDEO;
|
||||
case "wps":
|
||||
return FileType.WPS;
|
||||
case "dps":
|
||||
@@ -534,13 +553,43 @@ export function extractKnowledgeFileSensitiveCheck(raw: any): KnowledgeFileSensi
|
||||
}
|
||||
}
|
||||
|
||||
/** Display title for imported web links. */
|
||||
function ensureWebLinkHtmlName(name: string): string {
|
||||
const trimmed = name.trim().replace(/\.md$/i, "").trim();
|
||||
if (!trimmed) return "";
|
||||
return trimmed.toLowerCase().endsWith(".html") ? trimmed : `${trimmed}.html`;
|
||||
}
|
||||
|
||||
export function resolveWebLinkDisplayName(
|
||||
fileName: string,
|
||||
userMetadata?: Record<string, unknown>,
|
||||
): string {
|
||||
const stem = ensureWebLinkHtmlName(fileName);
|
||||
if (stem) return stem;
|
||||
const webTitle = typeof userMetadata?.web_title === "string" ? userMetadata.web_title.trim() : "";
|
||||
return ensureWebLinkHtmlName(webTitle) || "web-link.html";
|
||||
}
|
||||
|
||||
/** Normalize user-entered web link display name to persisted file_name. */
|
||||
export function toWebLinkFileName(displayName: string): string {
|
||||
return ensureWebLinkHtmlName(displayName);
|
||||
}
|
||||
|
||||
export function isWebLinkKnowledgeFile(file: Pick<KnowledgeFile, "fileSource" | "type">): boolean {
|
||||
return file.fileSource === "web_link" || file.type === FileType.WEB;
|
||||
}
|
||||
|
||||
/** Map a raw space child (file/folder) to the frontend KnowledgeFile model */
|
||||
function mapChild(raw: any, spaceId: string): KnowledgeFile {
|
||||
// Backend keys in children response usually look like:
|
||||
// id, file_name, file_type(0|1), file_level_path, knowledge_id,
|
||||
// status(numeric), update_time/create_time, tags(list of {id,name}) for files
|
||||
const idVal = raw?.id ?? raw?.file_id ?? raw?.knowledge_file_id ?? "";
|
||||
const nameVal = raw?.name ?? raw?.file_name ?? raw?.object_name ?? raw?.file_name ?? raw?.path ?? "";
|
||||
const rawName = raw?.name ?? raw?.file_name ?? raw?.object_name ?? raw?.file_name ?? raw?.path ?? "";
|
||||
const userMetadata = raw?.user_metadata ?? raw?.userMetadata ?? {};
|
||||
const nameVal = raw?.file_source === "web_link"
|
||||
? resolveWebLinkDisplayName(String(rawName), userMetadata)
|
||||
: rawName;
|
||||
|
||||
const tags: FileTag[] = Array.isArray(raw?.tags)
|
||||
? raw.tags
|
||||
@@ -609,6 +658,19 @@ function deriveFileTypeFromName(fileName: string): FileType {
|
||||
case "jpg": return FileType.JPG;
|
||||
case "jpeg": return FileType.JPEG;
|
||||
case "png": return FileType.PNG;
|
||||
case "mp3":
|
||||
case "wav":
|
||||
case "m4a":
|
||||
case "aac":
|
||||
case "flac":
|
||||
case "ogg":
|
||||
return FileType.AUDIO;
|
||||
case "mp4":
|
||||
case "mov":
|
||||
case "avi":
|
||||
case "mkv":
|
||||
case "webm":
|
||||
return FileType.VIDEO;
|
||||
case "wps": return FileType.WPS;
|
||||
case "dps": return FileType.DPS;
|
||||
case "et": return FileType.ET;
|
||||
@@ -1461,6 +1523,27 @@ export async function addFilesApi(
|
||||
});
|
||||
}
|
||||
|
||||
export async function importWebLinkApi(
|
||||
space_id: string,
|
||||
data: { url: string; title?: string; parent_id?: number | null; file_category_code?: string; overwrite?: boolean }
|
||||
): Promise<KnowledgeFile> {
|
||||
const res = await request.post(
|
||||
`/api/v1/knowledge/space/${space_id}/web-links`,
|
||||
data,
|
||||
{ showError: false } as any
|
||||
) as ApiResponse<RawSpaceChild> & { message?: string; msg?: string };
|
||||
if (res?.status_code !== undefined && res.status_code !== 200) {
|
||||
const error = new Error(res.status_message || res.message || res.msg || "import web link failed") as Error & {
|
||||
status_code?: number;
|
||||
status_message?: string;
|
||||
};
|
||||
error.status_code = res.status_code;
|
||||
error.status_message = res.status_message;
|
||||
throw error;
|
||||
}
|
||||
return mapChild(res.data, space_id);
|
||||
}
|
||||
|
||||
/** One file of a folder upload: uploaded body path + its path inside the picked folder. */
|
||||
export interface FolderUploadItemPayload {
|
||||
file_path: string;
|
||||
@@ -1675,19 +1758,35 @@ export async function batchRetryApi(
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Get file preview URL
|
||||
* Returns { original_url, preview_url } — prefer preview_url, fallback to original_url
|
||||
* File preview URLs and source metadata.
|
||||
*/
|
||||
export interface KnowledgeFilePreview {
|
||||
original_url: string;
|
||||
preview_url: string;
|
||||
file_source: string;
|
||||
source_url: string;
|
||||
final_url: string;
|
||||
web_title: string;
|
||||
media_kind: string;
|
||||
html_preview_url: string;
|
||||
}
|
||||
|
||||
export async function getFilePreviewApi(
|
||||
space_id: string,
|
||||
file_id: string
|
||||
): Promise<{ original_url: string; preview_url: string }> {
|
||||
): Promise<KnowledgeFilePreview> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const res = await request.get<any>(`/api/v1/knowledge/space/${space_id}/files/${file_id}/preview`);
|
||||
const data = res?.data ?? res;
|
||||
return {
|
||||
original_url: data?.original_url ?? "",
|
||||
preview_url: data?.preview_url ?? "",
|
||||
file_source: data?.file_source ?? "",
|
||||
source_url: data?.source_url ?? "",
|
||||
final_url: data?.final_url ?? "",
|
||||
web_title: data?.web_title ?? "",
|
||||
media_kind: data?.media_kind ?? "",
|
||||
html_preview_url: data?.html_preview_url ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1525,6 +1525,41 @@
|
||||
},
|
||||
"com_knowledge": {
|
||||
"add_new": "Add",
|
||||
"web_link": "Web link",
|
||||
"web_link_import_title": "Import web link",
|
||||
"web_link_url": "Link URL",
|
||||
"web_link_title": "Display name",
|
||||
"web_link_title_placeholder": "Leave empty to read the page title automatically",
|
||||
"web_link_import_success": "Web link import started",
|
||||
"web_link_url_required": "Please enter a web link",
|
||||
"web_link_http_only": "Only http or https links are supported",
|
||||
"web_link_invalid": "Please enter a valid web link",
|
||||
"web_link_duplicate_content_confirm": "A file with the same content already exists. Overwrite and parse again?",
|
||||
"web_link_duplicate_name_confirm": "A file with the same name already exists. Overwrite and parse again?",
|
||||
"web_link_import_failed": "Failed to import web link",
|
||||
"web_link_host_not_allowed": "This web link host is not allowed",
|
||||
"web_link_address_not_allowed": "This web link address is not allowed",
|
||||
"web_link_host_unresolved": "Web link host cannot be resolved",
|
||||
"web_link_content_too_large": "Web link content is too large",
|
||||
"web_link_content_type_unsupported": "Web link content type is not supported",
|
||||
"web_link_redirects_too_many": "Web link redirects too many times",
|
||||
"web_link_content_empty": "Web link content is empty",
|
||||
"web_link_request_failed": "Web link request failed",
|
||||
"web_page_preview": "Web snapshot",
|
||||
"open_original_page": "Open original page",
|
||||
"web_snapshot_unavailable": "Web snapshot unavailable",
|
||||
"recognized_text": "Recognized text",
|
||||
"knowledge_entry_text": "Knowledge text",
|
||||
"media_file_missing": "Media file does not exist",
|
||||
"media_asr_empty": "ASR returned empty text",
|
||||
"media_asr_api_key_missing": "ASR API key is missing",
|
||||
"media_ffmpeg_missing": "ffmpeg is not installed on the server",
|
||||
"media_audio_extraction_failed": "Media audio extraction failed",
|
||||
"media_asr_provider_unsupported": "Knowledge media transcription only supports Aliyun/Qwen ASR",
|
||||
"media_asr_request_failed": "ASR request failed",
|
||||
"overwrite": "Overwrite",
|
||||
"import": "Import",
|
||||
"importing": "Importing...",
|
||||
"ai_assistant": "AI Assistant",
|
||||
"ai_input_placeholder": "Ask about the current folder, type # to specify tags for the answer…",
|
||||
"ai_input_placeholder_short": "Please enter your question...",
|
||||
@@ -1756,9 +1791,9 @@
|
||||
"subscribe_apply_sent": "Subscription application sent, you can subscribe after approval.",
|
||||
"success": "Success",
|
||||
"supported_formats": "Supported file formats:",
|
||||
"supported_formats_basic": "pdf, ofd, txt, docx, doc, ppt, pptx, md, html, xls, xlsx, csv",
|
||||
"supported_formats_basic": "pdf, ofd, txt, docx, doc, ppt, pptx, md, html, xls, xlsx, csv, mp3, wav, m4a, aac, flac, ogg, mp4, mov, avi, mkv, webm",
|
||||
"supported_formats_tip": "Supported file formats: {{formats}}",
|
||||
"supported_formats_with_etl4lm": "pdf (incl. scanned), ofd, txt, docx, ppt, pptx, md, html, xls, xlsx, csv, doc, png, jpg, jpeg, bmp",
|
||||
"supported_formats_with_etl4lm": "pdf (incl. scanned), ofd, txt, docx, ppt, pptx, md, html, xls, xlsx, csv, doc, png, jpg, jpeg, bmp, mp3, wav, m4a, aac, flac, ogg, mp4, mov, avi, mkv, webm",
|
||||
"tag": "Tag",
|
||||
"tag_save_failed": "Failed to save tag",
|
||||
"tag_save_success": "Tag saved successfully",
|
||||
@@ -2316,4 +2351,4 @@
|
||||
}
|
||||
},
|
||||
"com_linsight_legacy_sop": "Legacy SOP document (pre-migration session)"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1449,6 +1449,41 @@
|
||||
},
|
||||
"com_knowledge": {
|
||||
"add_new": "新規追加",
|
||||
"web_link": "Webリンク",
|
||||
"web_link_import_title": "Webリンクをインポート",
|
||||
"web_link_url": "リンクURL",
|
||||
"web_link_title": "表示名",
|
||||
"web_link_title_placeholder": "空欄の場合はページタイトルを自動取得します",
|
||||
"web_link_import_success": "Webリンクのインポートを開始しました",
|
||||
"web_link_url_required": "Webリンクを入力してください",
|
||||
"web_link_http_only": "http または https のリンクのみサポートしています",
|
||||
"web_link_invalid": "有効なWebリンクを入力してください",
|
||||
"web_link_duplicate_content_confirm": "同じ内容のファイルが既に存在します。上書きして再解析しますか?",
|
||||
"web_link_duplicate_name_confirm": "同名のファイルが既に存在します。上書きして再解析しますか?",
|
||||
"web_link_import_failed": "Webリンクのインポートに失敗しました",
|
||||
"web_link_host_not_allowed": "このWebリンクのホストは許可されていません",
|
||||
"web_link_address_not_allowed": "このWebリンクのアドレスは許可されていません",
|
||||
"web_link_host_unresolved": "Webリンクのホストを解決できません",
|
||||
"web_link_content_too_large": "Webリンクの内容が大きすぎます",
|
||||
"web_link_content_type_unsupported": "Webリンクのコンテンツタイプはサポートされていません",
|
||||
"web_link_redirects_too_many": "Webリンクのリダイレクト回数が多すぎます",
|
||||
"web_link_content_empty": "Webリンクの内容が空です",
|
||||
"web_link_request_failed": "Webリンクのリクエストに失敗しました",
|
||||
"web_page_preview": "Webスナップショット",
|
||||
"open_original_page": "元ページを開く",
|
||||
"web_snapshot_unavailable": "Webスナップショットを利用できません",
|
||||
"recognized_text": "認識テキスト",
|
||||
"knowledge_entry_text": "ナレッジ本文",
|
||||
"media_file_missing": "メディアファイルが存在しません",
|
||||
"media_asr_empty": "ASR の結果が空です",
|
||||
"media_asr_api_key_missing": "ASR API Key が未設定です",
|
||||
"media_ffmpeg_missing": "サーバーに ffmpeg がインストールされていません",
|
||||
"media_audio_extraction_failed": "メディアの音声抽出に失敗しました",
|
||||
"media_asr_provider_unsupported": "ナレッジのメディア文字起こしは Aliyun/Qwen ASR のみ対応しています",
|
||||
"media_asr_request_failed": "ASR リクエストに失敗しました",
|
||||
"overwrite": "上書き",
|
||||
"import": "インポート",
|
||||
"importing": "インポート中...",
|
||||
"ai_assistant": "AI アシスタント",
|
||||
"ai_input_placeholder": "現在のフォルダーについて質問してください。# を入力すると、回答のタグを指定できます…",
|
||||
"ai_input_placeholder_short": "質問を入力してください...",
|
||||
@@ -1679,9 +1714,9 @@
|
||||
"subscribe_apply_sent": "購読申請が送信されました。承認後に購読できます。",
|
||||
"success": "成功",
|
||||
"supported_formats": "サポートされているファイル形式:",
|
||||
"supported_formats_basic": "pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv",
|
||||
"supported_formats_basic": "pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm",
|
||||
"supported_formats_tip": "サポートされているファイル形式:{{formats}}",
|
||||
"supported_formats_with_etl4lm": "pdf(スキャン含む)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp",
|
||||
"supported_formats_with_etl4lm": "pdf(スキャン含む)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm",
|
||||
"tag": "タグ",
|
||||
"tag_save_failed": "タグの保存に失敗しました",
|
||||
"tag_save_success": "タグの保存に成功しました",
|
||||
@@ -2239,4 +2274,4 @@
|
||||
}
|
||||
},
|
||||
"com_linsight_legacy_sop": "旧バージョンの SOP ドキュメント"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1452,6 +1452,41 @@
|
||||
},
|
||||
"com_knowledge": {
|
||||
"add_new": "新增",
|
||||
"web_link": "网页链接",
|
||||
"web_link_import_title": "导入网页链接",
|
||||
"web_link_url": "链接地址",
|
||||
"web_link_title": "显示名称",
|
||||
"web_link_title_placeholder": "留空则自动读取网页标题",
|
||||
"web_link_import_success": "网页链接已开始导入",
|
||||
"web_link_url_required": "请输入网页链接",
|
||||
"web_link_http_only": "仅支持 http 或 https 链接",
|
||||
"web_link_invalid": "请输入有效的网页链接",
|
||||
"web_link_duplicate_content_confirm": "已存在相同内容的文件,是否覆盖并重新解析?",
|
||||
"web_link_duplicate_name_confirm": "已存在同名文件,是否覆盖并重新解析?",
|
||||
"web_link_import_failed": "网页链接导入失败",
|
||||
"web_link_host_not_allowed": "该网页链接主机不允许导入",
|
||||
"web_link_address_not_allowed": "该网页链接地址不允许导入",
|
||||
"web_link_host_unresolved": "网页链接主机无法解析",
|
||||
"web_link_content_too_large": "网页内容超过大小限制",
|
||||
"web_link_content_type_unsupported": "网页链接内容类型不支持",
|
||||
"web_link_redirects_too_many": "网页链接重定向次数过多",
|
||||
"web_link_content_empty": "网页内容为空",
|
||||
"web_link_request_failed": "网页链接请求失败",
|
||||
"web_page_preview": "网页快照",
|
||||
"open_original_page": "打开原网页",
|
||||
"web_snapshot_unavailable": "网页快照不可用",
|
||||
"recognized_text": "识别文本",
|
||||
"knowledge_entry_text": "入库文本",
|
||||
"media_file_missing": "音视频文件不存在",
|
||||
"media_asr_empty": "语音识别结果为空",
|
||||
"media_asr_api_key_missing": "ASR API Key 未配置",
|
||||
"media_ffmpeg_missing": "服务器未安装 ffmpeg",
|
||||
"media_audio_extraction_failed": "音视频音轨提取失败",
|
||||
"media_asr_provider_unsupported": "知识库音视频解析仅支持阿里云/Qwen ASR",
|
||||
"media_asr_request_failed": "ASR 请求失败",
|
||||
"overwrite": "覆盖",
|
||||
"import": "导入",
|
||||
"importing": "导入中...",
|
||||
"ai_assistant": "AI 助手",
|
||||
"ai_input_placeholder": "基于当前文件夹提问,输入#可指定标签回答…",
|
||||
"ai_input_placeholder_short": "请输入你的问题...",
|
||||
@@ -1716,9 +1751,9 @@
|
||||
"users_count": "用户",
|
||||
"window_too_narrow_ai": "窗口宽度不足,无法打开 AI 助手",
|
||||
"yes": "是",
|
||||
"supported_formats_basic": "pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv",
|
||||
"supported_formats_basic": "pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm",
|
||||
"supported_formats_tip": "支持的文件格式为 {{formats}}",
|
||||
"supported_formats_with_etl4lm": "pdf(含扫描件)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp",
|
||||
"supported_formats_with_etl4lm": "pdf(含扫描件)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm",
|
||||
"version": {
|
||||
"menu_version_management": "版本管理",
|
||||
"menu_version_history": "版本历史",
|
||||
@@ -2245,4 +2280,4 @@
|
||||
}
|
||||
},
|
||||
"com_linsight_legacy_sop": "历史 SOP 文档(旧版会话)"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,12 +8,14 @@ import { useCallback, useEffect, useState } from "react";
|
||||
import { useParams, useSearchParams } from "react-router-dom";
|
||||
import { Outlined } from "bisheng-icons";
|
||||
import { getFileDownloadApi, getFilePreviewApi } from "~/api/knowledge";
|
||||
import type { KnowledgeFilePreview } from "~/api/knowledge";
|
||||
import { canOpenPermissionDialog, checkPermission } from "~/api/permission";
|
||||
import { Button, DropdownMenu, DropdownMenuTrigger } from "~/components";
|
||||
import { ActionMenuContent, ActionMenuItem } from "~/components/ActionMenu";
|
||||
import { PermissionDialog } from "~/components/permission";
|
||||
import { FileAiDock } from "~/pages/Subscription/AiChat/FileAiDock";
|
||||
import FilePreview from "./index";
|
||||
import { RichKnowledgePreview } from "./RichKnowledgePreview";
|
||||
import { useLocalize } from "~/hooks";
|
||||
|
||||
/**
|
||||
@@ -35,6 +37,30 @@ function extractExtFromUrl(url: string, fallback: string): string {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function resolvePreviewUrl(url: string): string {
|
||||
if (!url) return "";
|
||||
return /^https?:\/\//.test(url)
|
||||
? url
|
||||
: `${window.location.origin}${__APP_ENV__.BASE_URL}${url}`;
|
||||
}
|
||||
|
||||
const AUDIO_EXTENSIONS = new Set(["mp3", "wav", "m4a", "aac", "flac", "ogg"]);
|
||||
const VIDEO_EXTENSIONS = new Set(["mp4", "mov", "avi", "mkv", "webm"]);
|
||||
|
||||
function isRichPreviewData(data: KnowledgeFilePreview | null): boolean {
|
||||
if (!data) return false;
|
||||
const ext = extractExtFromUrl(data.original_url || data.preview_url || "", "");
|
||||
return (
|
||||
data.file_source === "web_link"
|
||||
|| data.file_source === "audio_transcript"
|
||||
|| data.file_source === "video_transcript"
|
||||
|| data.media_kind === "audio"
|
||||
|| data.media_kind === "video"
|
||||
|| AUDIO_EXTENSIONS.has(ext)
|
||||
|| VIDEO_EXTENSIONS.has(ext)
|
||||
);
|
||||
}
|
||||
|
||||
export default function FilePreviewPage() {
|
||||
const localize = useLocalize();
|
||||
const { fileId } = useParams<{ fileId: string }>();
|
||||
@@ -45,6 +71,7 @@ export default function FilePreviewPage() {
|
||||
// Fetch real preview URL via API
|
||||
const [fileUrl, setFileUrl] = useState<string>("");
|
||||
const [fileType, setFileType] = useState<string>("pdf");
|
||||
const [previewData, setPreviewData] = useState<KnowledgeFilePreview | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [conversionFailed, setConversionFailed] = useState(false);
|
||||
const [canDownload, setCanDownload] = useState(false);
|
||||
@@ -55,8 +82,24 @@ export default function FilePreviewPage() {
|
||||
if (!fileId || !spaceId) { setLoading(false); return; }
|
||||
setLoading(true);
|
||||
setConversionFailed(false);
|
||||
setPreviewData(null);
|
||||
getFilePreviewApi(spaceId, fileId)
|
||||
.then((data) => {
|
||||
const resolvedPreview = {
|
||||
...data,
|
||||
original_url: resolvePreviewUrl(data.original_url),
|
||||
preview_url: resolvePreviewUrl(data.preview_url),
|
||||
html_preview_url: resolvePreviewUrl(data.html_preview_url),
|
||||
};
|
||||
setPreviewData(resolvedPreview);
|
||||
|
||||
if (isRichPreviewData(data)) {
|
||||
const richUrl = resolvedPreview.preview_url || resolvedPreview.html_preview_url || resolvedPreview.original_url;
|
||||
setFileUrl(richUrl);
|
||||
setFileType(data.file_source === "web_link" ? "html" : extractExtFromUrl(data.original_url, "md"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Prefer preview_url, fallback to original_url
|
||||
const chosenUrl = data.preview_url || data.original_url;
|
||||
if (!chosenUrl) {
|
||||
@@ -75,10 +118,7 @@ export default function FilePreviewPage() {
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedUrl = /^https?:\/\//.test(chosenUrl)
|
||||
? chosenUrl
|
||||
: `${window.location.origin}${__APP_ENV__.BASE_URL}${chosenUrl}`;
|
||||
setFileUrl(resolvedUrl);
|
||||
setFileUrl(resolvePreviewUrl(chosenUrl));
|
||||
setFileType(ext);
|
||||
})
|
||||
.catch((err) => console.error("Failed to load preview URL:", err))
|
||||
@@ -207,6 +247,35 @@ export default function FilePreviewPage() {
|
||||
</DropdownMenu>
|
||||
) : null;
|
||||
|
||||
const renderPreview = (compactMode = false) => {
|
||||
if (previewData && isRichPreviewData(previewData)) {
|
||||
return (
|
||||
<RichKnowledgePreview
|
||||
fileName={fileName}
|
||||
preview={previewData}
|
||||
actions={compactMode ? undefined : topBarActions}
|
||||
allowDownload={canDownload}
|
||||
onDownloadFile={handleDownloadFile}
|
||||
compactMode={compactMode}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<FilePreview
|
||||
fileName={fileName}
|
||||
fileType={fileType}
|
||||
fileUrl={fileUrl}
|
||||
actions={compactMode ? undefined : topBarActions}
|
||||
conversionFailed={conversionFailed}
|
||||
allowDownload={canDownload}
|
||||
onDownloadFile={handleDownloadFile}
|
||||
hideHeader={compactMode}
|
||||
hideSidebar={compactMode}
|
||||
hideHeaderDownload={!compactMode}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
// Loading state while fetching preview URL
|
||||
if (loading) {
|
||||
return (
|
||||
@@ -217,7 +286,7 @@ export default function FilePreviewPage() {
|
||||
}
|
||||
|
||||
// No URL available (skip this guard for pptx conversion failure — handled by FilePreview)
|
||||
if (!fileUrl && !conversionFailed) {
|
||||
if (!fileUrl && !conversionFailed && !isRichPreviewData(previewData)) {
|
||||
return (
|
||||
<div className="h-screen flex items-center justify-center bg-white">
|
||||
<div className="text-[#86909c]">{localize("com_knowledge.fetch_preview_link_failed")}</div>
|
||||
@@ -243,16 +312,7 @@ export default function FilePreviewPage() {
|
||||
|
||||
{/* Bare preview — header hidden, viewer fills the container. */}
|
||||
<div className="absolute inset-0">
|
||||
<FilePreview
|
||||
fileName={fileName}
|
||||
fileType={fileType}
|
||||
fileUrl={fileUrl}
|
||||
conversionFailed={conversionFailed}
|
||||
allowDownload={canDownload}
|
||||
onDownloadFile={handleDownloadFile}
|
||||
hideHeader
|
||||
hideSidebar
|
||||
/>
|
||||
{renderPreview(true)}
|
||||
</div>
|
||||
|
||||
{/* Floating top-right download button — styled like ArticlePage's menu button. */}
|
||||
@@ -287,16 +347,7 @@ export default function FilePreviewPage() {
|
||||
)}
|
||||
|
||||
<div className="min-h-0 flex-1">
|
||||
<FilePreview
|
||||
fileName={fileName}
|
||||
fileType={fileType}
|
||||
fileUrl={fileUrl}
|
||||
actions={topBarActions}
|
||||
conversionFailed={conversionFailed}
|
||||
allowDownload={canDownload}
|
||||
hideHeaderDownload
|
||||
onDownloadFile={handleDownloadFile}
|
||||
/>
|
||||
{renderPreview(false)}
|
||||
</div>
|
||||
|
||||
{spaceId && fileId && <FileAiDock spaceId={spaceId} fileId={fileId} />}
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import type { ReactNode } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import rehypeHighlight from "rehype-highlight";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import type { KnowledgeFilePreview } from "~/api/knowledge";
|
||||
import { useLocalize } from "~/hooks";
|
||||
import { TopBar } from "./TopBar";
|
||||
import { HtmlViewer } from "./viewers/HtmlViewer";
|
||||
|
||||
interface RichKnowledgePreviewProps {
|
||||
fileName: string;
|
||||
preview: KnowledgeFilePreview;
|
||||
actions?: ReactNode;
|
||||
allowDownload?: boolean;
|
||||
onDownloadFile?: () => void;
|
||||
compactMode?: boolean;
|
||||
}
|
||||
|
||||
type MediaTab = "recognized" | "entry";
|
||||
|
||||
const MEDIA_SOURCES = new Set(["audio_transcript", "video_transcript"]);
|
||||
const AUDIO_EXTENSIONS = new Set(["mp3", "wav", "m4a", "aac", "flac", "ogg"]);
|
||||
const VIDEO_EXTENSIONS = new Set(["mp4", "mov", "avi", "mkv", "webm"]);
|
||||
|
||||
function getExtensionFromUrl(url?: string): string {
|
||||
if (!url) return "";
|
||||
const path = url.split("?")[0].split("#")[0];
|
||||
const filename = path.split("/").pop() || "";
|
||||
const dotIndex = filename.lastIndexOf(".");
|
||||
return dotIndex >= 0 ? filename.slice(dotIndex + 1).toLowerCase() : "";
|
||||
}
|
||||
|
||||
function isMediaUrl(url?: string): boolean {
|
||||
const ext = getExtensionFromUrl(url);
|
||||
return AUDIO_EXTENSIONS.has(ext) || VIDEO_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
function isMediaPreview(preview: KnowledgeFilePreview): boolean {
|
||||
const ext = getExtensionFromUrl(preview.original_url || preview.preview_url);
|
||||
return (
|
||||
MEDIA_SOURCES.has(preview.file_source)
|
||||
|| preview.media_kind === "audio"
|
||||
|| preview.media_kind === "video"
|
||||
|| AUDIO_EXTENSIONS.has(ext)
|
||||
|| VIDEO_EXTENSIONS.has(ext)
|
||||
);
|
||||
}
|
||||
|
||||
function isVideoPreview(preview: KnowledgeFilePreview): boolean {
|
||||
const ext = getExtensionFromUrl(preview.original_url || preview.preview_url);
|
||||
return preview.file_source === "video_transcript" || preview.media_kind === "video" || VIDEO_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
function extractMarkdownSection(markdown: string, heading: string): string {
|
||||
const pattern = new RegExp(`^##\\s+${heading}\\s*$`, "m");
|
||||
const match = markdown.match(pattern);
|
||||
if (!match || match.index === undefined) return "";
|
||||
const start = match.index + match[0].length;
|
||||
const rest = markdown.slice(start);
|
||||
const nextHeading = rest.search(/^##\s+/m);
|
||||
return (nextHeading >= 0 ? rest.slice(0, nextHeading) : rest).trim();
|
||||
}
|
||||
|
||||
function MarkdownBlock({ content }: { content: string }) {
|
||||
return (
|
||||
<div className="flex-1 overflow-auto bg-[#fbfbfb]">
|
||||
<div className="flex justify-center px-4 py-6">
|
||||
<div className="w-full max-w-[800px] rounded-sm bg-white shadow-md">
|
||||
<div className="prose prose-sm max-w-none p-8 text-[#1d2129]">
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
rehypePlugins={[[rehypeHighlight, { detect: true, ignoreMissing: true }]]}
|
||||
>
|
||||
{content}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MediaTranscriptTabs({ fileUrl }: { fileUrl: string }) {
|
||||
const localize = useLocalize();
|
||||
const [activeTab, setActiveTab] = useState<MediaTab>("recognized");
|
||||
const [content, setContent] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
if (!fileUrl) {
|
||||
setLoading(false);
|
||||
setContent("");
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
setLoading(true);
|
||||
setError("");
|
||||
fetch(fileUrl)
|
||||
.then((response) => {
|
||||
if (!response.ok) throw new Error(localize("com_knowledge.failure_status", { 0: response.status }));
|
||||
return response.text();
|
||||
})
|
||||
.then((text) => {
|
||||
if (!cancelled) setContent(text);
|
||||
})
|
||||
.catch((err: Error) => {
|
||||
if (!cancelled) setError(err.message || localize("com_knowledge.load_file_failed"));
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoading(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [fileUrl]);
|
||||
|
||||
const entryText = extractMarkdownSection(content, "入库文本") || content;
|
||||
const recognizedText = extractMarkdownSection(content, "识别文本") || content;
|
||||
const activeContent = activeTab === "recognized" ? recognizedText : entryText;
|
||||
|
||||
return (
|
||||
<section className="flex min-h-[420px] flex-col overflow-hidden rounded-[8px] border border-[#e5e6eb] bg-white shadow-sm">
|
||||
<div className="flex shrink-0 items-center gap-2 border-b border-[#e5e6eb] bg-white px-4 py-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setActiveTab("recognized")}
|
||||
className={`h-8 rounded-[6px] px-3 text-sm ${activeTab === "recognized" ? "bg-primary text-white" : "bg-[#f2f3f5] text-[#4e5969]"}`}
|
||||
>
|
||||
{localize("com_knowledge.recognized_text")}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setActiveTab("entry")}
|
||||
className={`h-8 rounded-[6px] px-3 text-sm ${activeTab === "entry" ? "bg-primary text-white" : "bg-[#f2f3f5] text-[#4e5969]"}`}
|
||||
>
|
||||
{localize("com_knowledge.knowledge_entry_text")}
|
||||
</button>
|
||||
</div>
|
||||
{loading ? (
|
||||
<div className="flex flex-1 items-center justify-center text-sm text-[#86909c]">
|
||||
{localize("com_knowledge.loading")}
|
||||
</div>
|
||||
) : error ? (
|
||||
<div className="flex flex-1 items-center justify-center text-sm text-[#86909c]">{error}</div>
|
||||
) : (
|
||||
<MarkdownBlock content={activeContent} />
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export function RichKnowledgePreview({
|
||||
fileName,
|
||||
preview,
|
||||
actions,
|
||||
allowDownload = true,
|
||||
onDownloadFile,
|
||||
compactMode = false,
|
||||
}: RichKnowledgePreviewProps) {
|
||||
const localize = useLocalize();
|
||||
const isMedia = isMediaPreview(preview);
|
||||
const isVideo = isVideoPreview(preview);
|
||||
const htmlUrl = preview.html_preview_url;
|
||||
const mediaTextUrl = preview.preview_url && !isMediaUrl(preview.preview_url) ? preview.preview_url : "";
|
||||
|
||||
const title = useMemo(() => {
|
||||
if (preview.file_source === "web_link") {
|
||||
return preview.web_title || fileName;
|
||||
}
|
||||
return fileName;
|
||||
}, [fileName, preview.file_source, preview.web_title]);
|
||||
|
||||
if (isMedia) {
|
||||
return (
|
||||
<div className="flex h-full w-full flex-col overflow-hidden bg-[#f5f7fb]">
|
||||
{!compactMode && (
|
||||
<TopBar
|
||||
fileName={fileName}
|
||||
showZoom={false}
|
||||
onDownload={allowDownload ? onDownloadFile : undefined}
|
||||
actions={actions}
|
||||
/>
|
||||
)}
|
||||
<div className="flex-1 overflow-auto p-5">
|
||||
<div className="mx-auto flex w-full max-w-[980px] flex-col gap-4">
|
||||
<section className="rounded-[8px] border border-[#e5e6eb] bg-white p-4 shadow-sm">
|
||||
<div className="mb-3 text-base font-semibold text-[#1d2129]">{title}</div>
|
||||
{isVideo ? (
|
||||
<video
|
||||
className="max-h-[420px] w-full rounded-[6px] bg-black"
|
||||
src={preview.original_url}
|
||||
controls
|
||||
/>
|
||||
) : (
|
||||
<audio className="w-full" src={preview.original_url} controls />
|
||||
)}
|
||||
</section>
|
||||
{mediaTextUrl ? <MediaTranscriptTabs fileUrl={mediaTextUrl} /> : null}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-full w-full flex-col overflow-hidden bg-[#f5f7fb]">
|
||||
{!compactMode && (
|
||||
<TopBar
|
||||
fileName={fileName}
|
||||
showZoom={false}
|
||||
onDownload={allowDownload ? onDownloadFile : undefined}
|
||||
actions={actions}
|
||||
/>
|
||||
)}
|
||||
<div className="flex min-h-0 flex-1 overflow-hidden">
|
||||
{htmlUrl ? (
|
||||
<HtmlViewer fileUrl={htmlUrl} zoomLevel={100} />
|
||||
) : (
|
||||
<div className="flex h-full items-center justify-center text-sm text-[#86909c]">
|
||||
{localize("com_knowledge.fetch_preview_link_failed")}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import React from 'react';
|
||||
import { Colored } from 'bisheng-icons';
|
||||
import { FileStatus } from '~/api/knowledge';
|
||||
import { FileStatus, FileType } from '~/api/knowledge';
|
||||
import { TxtIcon, FolderIcon } from '~/components/icons';
|
||||
import { cn } from '~/utils';
|
||||
|
||||
@@ -18,6 +18,7 @@ const FILE_TYPE_BG = {
|
||||
csv: 'bg-[linear-gradient(180deg,rgba(226,245,234,0.05)_0%,rgba(0,198,80,0.05)_100%)]',
|
||||
txt: 'bg-[linear-gradient(180deg,rgba(225,227,230,0.05)_0%,rgba(52,64,84,0.05)_100%)]',
|
||||
md: 'bg-[linear-gradient(180deg,rgba(225,227,230,0.05)_0%,rgba(52,64,84,0.05)_100%)]',
|
||||
media: 'bg-[linear-gradient(180deg,rgba(223,238,255,0.05)_0%,rgba(0,114,255,0.05)_100%)]',
|
||||
} as const;
|
||||
|
||||
type FileTypeKey = keyof typeof FILE_TYPE_BG;
|
||||
@@ -33,6 +34,17 @@ const EXTENSION_TO_TYPE: Record<string, FileTypeKey> = {
|
||||
txt: 'txt',
|
||||
md: 'md',
|
||||
markdown: 'md',
|
||||
mp3: 'media',
|
||||
wav: 'media',
|
||||
m4a: 'media',
|
||||
aac: 'media',
|
||||
flac: 'media',
|
||||
ogg: 'media',
|
||||
mp4: 'media',
|
||||
mov: 'media',
|
||||
avi: 'media',
|
||||
mkv: 'media',
|
||||
webm: 'media',
|
||||
};
|
||||
|
||||
const TYPE_TO_ICON: Record<FileTypeKey, React.ReactNode> = {
|
||||
@@ -43,6 +55,7 @@ const TYPE_TO_ICON: Record<FileTypeKey, React.ReactNode> = {
|
||||
csv: <Colored.FileCsv className={iconSlotClass} />,
|
||||
txt: <Colored.FileTxt className={iconSlotClass} />,
|
||||
md: <Colored.FileMd className={iconSlotClass} />,
|
||||
media: <Colored.FileTxt className={iconSlotClass} />,
|
||||
};
|
||||
|
||||
const FileIconRenderer = ({ file, isFolder, iconClassName, thumbBordered, transparentBg }: { file: any; isFolder: boolean; iconClassName?: string; thumbBordered?: boolean; transparentBg?: boolean }) => {
|
||||
@@ -59,7 +72,10 @@ const FileIconRenderer = ({ file, isFolder, iconClassName, thumbBordered, transp
|
||||
}
|
||||
|
||||
const extension: string = file.name?.split('.').pop()?.toLowerCase() ?? '';
|
||||
const typeKey: FileTypeKey | undefined = EXTENSION_TO_TYPE[extension];
|
||||
const typeKey: FileTypeKey | undefined =
|
||||
file.type === FileType.WEB ? 'md'
|
||||
: file.type === FileType.AUDIO || file.type === FileType.VIDEO ? 'media'
|
||||
: EXTENSION_TO_TYPE[extension];
|
||||
|
||||
// Plain-text / structured-text formats (txt / md / csv) never render a
|
||||
// thumbnail. Once parsed they show their colored placeholder icon; before
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { FolderPlus, FolderUp } from "lucide-react";
|
||||
import { FolderPlus, FolderUp, Link2 } from "lucide-react";
|
||||
import { Outlined } from "bisheng-icons";
|
||||
import { KnowledgeSpace, FileStatus, SortType, SortDirection, SpaceRole, VisibilityType } from "~/api/knowledge";
|
||||
import { cn } from "~/utils";
|
||||
@@ -35,6 +35,7 @@ interface KnowledgeSpaceHeaderProps {
|
||||
onCreateFolder: () => void;
|
||||
onTriggerUpload: () => void;
|
||||
onTriggerUploadFolder: () => void;
|
||||
onTriggerWebLink: () => void;
|
||||
canCreateFolder?: boolean;
|
||||
canUploadFile?: boolean;
|
||||
/** Localized comma-joined list of supported upload formats for the upload-button tooltip. */
|
||||
@@ -85,6 +86,7 @@ export function KnowledgeSpaceHeader({
|
||||
onCreateFolder,
|
||||
onTriggerUpload,
|
||||
onTriggerUploadFolder,
|
||||
onTriggerWebLink,
|
||||
canCreateFolder = false,
|
||||
canUploadFile = false,
|
||||
supportedFormatsLabel,
|
||||
@@ -331,6 +333,11 @@ export function KnowledgeSpaceHeader({
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<ActionMenuContent align="end">
|
||||
<ActionMenuItem
|
||||
onClick={onTriggerWebLink}
|
||||
icon={<Link2 />}
|
||||
label={localize("com_knowledge.web_link")}
|
||||
/>
|
||||
<ActionMenuItem
|
||||
onClick={onTriggerUploadFolder}
|
||||
icon={<FolderUp />}
|
||||
|
||||
@@ -2,9 +2,9 @@ import { Fragment, useState, useRef, useEffect, useLayoutEffect, useCallback, ty
|
||||
import { useRecoilValue } from "recoil";
|
||||
import { EmptyStateIllustration } from "~/components/illustrations";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { FolderPlus } from "lucide-react";
|
||||
import { FolderPlus, Link2 } from "lucide-react";
|
||||
import { LoadingIcon } from "~/components/ui/icon/Loading";
|
||||
import { FileStatus, FileType, KnowledgeFile, KnowledgeSpace, SortDirection, SortType, SpaceRole, VisibilityType, batchDownloadApi, batchRetryApi, getFileDownloadApi } from "~/api/knowledge";
|
||||
import { FileStatus, FileType, KnowledgeFile, KnowledgeSpace, SortDirection, SortType, SpaceRole, VisibilityType, batchDownloadApi, batchRetryApi, getFileDownloadApi, importWebLinkApi } from "~/api/knowledge";
|
||||
import { Outlined } from "bisheng-icons";
|
||||
import { NotificationSeverity } from "~/common";
|
||||
import { buildClientShareUrl } from "~/components/CopyShareLinkButton";
|
||||
@@ -15,6 +15,15 @@ import {
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "~/components/ui/DropdownMenu";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
Input,
|
||||
Label,
|
||||
} from "~/components/ui";
|
||||
import { useFileDragDrop } from "../hooks/useFileDragDrop";
|
||||
import { useKnowledgeMove } from "../hooks/useKnowledgeMove";
|
||||
import { useKnowledgeMoveDrag } from "../hooks/useKnowledgeMoveDrag";
|
||||
@@ -23,9 +32,14 @@ import {
|
||||
DEFAULT_MAX_FILE_SIZE_MB,
|
||||
getAllowedExtensions,
|
||||
getFileInputAccept,
|
||||
getMaxFileSizeBytesForFile,
|
||||
getMaxFileSizeMBForFile,
|
||||
isKnowledgeItemUploading,
|
||||
resolveUploadSizeLimits,
|
||||
triggerUrlDownload,
|
||||
type UploadSizeLimits,
|
||||
} from "../knowledgeUtils";
|
||||
import { resolveLocalizedKnowledgeImportError } from "../webLinkI18n";
|
||||
import { bishengConfState } from "~/pages/appChat/store/atoms";
|
||||
import { CompoundSearchInput, SearchParams } from "./CompoundSearchInput";
|
||||
import { EditTagsModal } from "./EditTagsModal";
|
||||
@@ -72,7 +86,7 @@ interface KnowledgeSpaceContentProps {
|
||||
onUploadFile: (files?: FileList | File[]) => void;
|
||||
onUploadFolder: (
|
||||
fileList: FileList | File[],
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number },
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number; limits?: UploadSizeLimits },
|
||||
) => void;
|
||||
onCreateFolder: () => void;
|
||||
onDownloadFile: (fileId: string) => void;
|
||||
@@ -109,6 +123,12 @@ interface KnowledgeSpaceContentProps {
|
||||
onSelectionActiveChange?: (active: boolean) => void;
|
||||
}
|
||||
|
||||
function normalizeWebLinkTitle(title: string): string | undefined {
|
||||
const trimmed = title.trim();
|
||||
if (!trimmed) return undefined;
|
||||
return /\.html$/i.test(trimmed) ? trimmed : `${trimmed}.html`;
|
||||
}
|
||||
|
||||
export function KnowledgeSpaceContent({
|
||||
space,
|
||||
files,
|
||||
@@ -364,7 +384,7 @@ export function KnowledgeSpaceContent({
|
||||
.filter((file) => !file.isCreating && /^\d+$/.test(String(file.id)))
|
||||
.map((file) => `${file.id}:${file.type}`)
|
||||
.join("|");
|
||||
const canUseAddActions = canCreateFolder && !isSearching;
|
||||
const canUseAddActions = (canCreateFolder || canUploadFile) && !isSearching;
|
||||
|
||||
const { showToast } = useToastContext();
|
||||
const confirm = useConfirm();
|
||||
@@ -502,8 +522,8 @@ export function KnowledgeSpaceContent({
|
||||
|
||||
// Read max file size from env config (MB), fallback to default 200MB
|
||||
const bishengConfig = useRecoilValue(bishengConfState);
|
||||
const maxFileSizeMB = bishengConfig?.uploaded_files_maximum_size ?? DEFAULT_MAX_FILE_SIZE_MB;
|
||||
const maxFileSizeBytes = maxFileSizeMB * 1024 * 1024;
|
||||
const uploadSizeLimits = resolveUploadSizeLimits(bishengConfig);
|
||||
const maxFileSizeMB = uploadSizeLimits.defaultMaxMB;
|
||||
const enableEtl4lm = bishengConfig?.enable_etl4lm ?? false;
|
||||
const allowedExtensions = getAllowedExtensions(enableEtl4lm);
|
||||
const fileInputAccept = getFileInputAccept(enableEtl4lm);
|
||||
@@ -522,6 +542,83 @@ export function KnowledgeSpaceContent({
|
||||
folderInputRef.current?.click();
|
||||
};
|
||||
|
||||
const [webLinkDialogOpen, setWebLinkDialogOpen] = useState(false);
|
||||
const [webLinkUrl, setWebLinkUrl] = useState("");
|
||||
const [webLinkTitle, setWebLinkTitle] = useState("");
|
||||
const [webLinkSubmitting, setWebLinkSubmitting] = useState(false);
|
||||
|
||||
const triggerWebLink = () => {
|
||||
if (!canUploadFile) return;
|
||||
setWebLinkDialogOpen(true);
|
||||
};
|
||||
|
||||
const refreshAfterWebLinkImport = () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["file-versions"] });
|
||||
dispatchKnowledgeSpaceFilesRefresh(space.id);
|
||||
onDeleteFile("");
|
||||
};
|
||||
|
||||
const resetWebLinkDialog = () => {
|
||||
setWebLinkUrl("");
|
||||
setWebLinkTitle("");
|
||||
};
|
||||
|
||||
const submitWebLink = async (overwrite = false) => {
|
||||
const trimmedUrl = webLinkUrl.trim();
|
||||
const normalizedTitle = normalizeWebLinkTitle(webLinkTitle);
|
||||
if (!trimmedUrl) {
|
||||
showToast({ message: localize("com_knowledge.web_link_url_required"), status: "error" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(trimmedUrl);
|
||||
if (!["http:", "https:"].includes(parsed.protocol)) {
|
||||
showToast({ message: localize("com_knowledge.web_link_http_only"), status: "error" });
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
showToast({ message: localize("com_knowledge.web_link_invalid"), status: "error" });
|
||||
return;
|
||||
}
|
||||
|
||||
setWebLinkSubmitting(true);
|
||||
try {
|
||||
await importWebLinkApi(space.id, {
|
||||
url: trimmedUrl,
|
||||
title: normalizedTitle,
|
||||
parent_id: currentFolderId ? Number(currentFolderId) : null,
|
||||
overwrite,
|
||||
});
|
||||
showToast({ message: localize("com_knowledge.web_link_import_success"), status: "success" });
|
||||
setWebLinkDialogOpen(false);
|
||||
resetWebLinkDialog();
|
||||
refreshAfterWebLinkImport();
|
||||
} catch (error: any) {
|
||||
const statusCode = error?.status_code ?? error?.statusCode;
|
||||
if (!overwrite && (statusCode === 18021 || statusCode === 18023)) {
|
||||
const confirmed = await confirm({
|
||||
description: localize(
|
||||
statusCode === 18021
|
||||
? "com_knowledge.web_link_duplicate_content_confirm"
|
||||
: "com_knowledge.web_link_duplicate_name_confirm"
|
||||
),
|
||||
confirmText: localize("com_knowledge.overwrite"),
|
||||
cancelText: localize("com_knowledge.cancel"),
|
||||
});
|
||||
if (confirmed) {
|
||||
await submitWebLink(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
showToast({
|
||||
message: resolveLocalizedKnowledgeImportError(error, localize, "com_knowledge.web_link_import_failed"),
|
||||
status: "error",
|
||||
});
|
||||
} finally {
|
||||
setWebLinkSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!canUseAddActions) {
|
||||
setContextMenuOpen(false);
|
||||
@@ -549,8 +646,10 @@ export function KnowledgeSpaceContent({
|
||||
}
|
||||
|
||||
for (let f of filesList) {
|
||||
if (f.size > maxFileSizeBytes) {
|
||||
showToast({ message: localize("com_knowledge.file_exceeds_limit", { name: f.name, size: maxFileSizeMB }), status: "error" });
|
||||
const fileMaxSizeMB = getMaxFileSizeMBForFile(f.name, uploadSizeLimits);
|
||||
const fileMaxSizeBytes = getMaxFileSizeBytesForFile(f.name, uploadSizeLimits);
|
||||
if (f.size > fileMaxSizeBytes) {
|
||||
showToast({ message: localize("com_knowledge.file_exceeds_limit", { name: f.name, size: fileMaxSizeMB }), status: "error" });
|
||||
if (fileInputRef.current) fileInputRef.current.value = "";
|
||||
return;
|
||||
}
|
||||
@@ -575,7 +674,7 @@ export function KnowledgeSpaceContent({
|
||||
const handleFolderChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const filesList = e.target.files;
|
||||
if (filesList && filesList.length > 0 && canUploadFile) {
|
||||
onUploadFolder(filesList, { allowedExtensions, maxSizeMB: maxFileSizeMB });
|
||||
onUploadFolder(filesList, { allowedExtensions, maxSizeMB: maxFileSizeMB, limits: uploadSizeLimits });
|
||||
}
|
||||
if (folderInputRef.current) folderInputRef.current.value = "";
|
||||
};
|
||||
@@ -589,6 +688,7 @@ export function KnowledgeSpaceContent({
|
||||
// nothing. Gate on canUploadFile like onUploadFile.
|
||||
onUploadFolder: canUploadFile ? onUploadFolder : undefined,
|
||||
maxFileSizeMB,
|
||||
uploadSizeLimits,
|
||||
enableEtl4lm,
|
||||
});
|
||||
|
||||
@@ -1116,6 +1216,12 @@ export function KnowledgeSpaceContent({
|
||||
<span className={sidebarListMoreMenuLabelClassName}>{localize("com_knowledge.upload_file")}</span>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{canUploadFile && (
|
||||
<DropdownMenuItem className={sidebarListMoreMenuItemClassName} onClick={triggerWebLink}>
|
||||
<Link2 className={sidebarListMoreMenuIconClassName} />
|
||||
<span className={sidebarListMoreMenuLabelClassName}>{localize("com_knowledge.web_link")}</span>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{canCreateFolder && (
|
||||
<DropdownMenuItem className={sidebarListMoreMenuItemClassName} onClick={() => onCreateFolder()}>
|
||||
<FolderPlus className={sidebarListMoreMenuIconClassName} />
|
||||
@@ -1177,6 +1283,7 @@ export function KnowledgeSpaceContent({
|
||||
onCreateFolder={onCreateFolder}
|
||||
onTriggerUpload={triggerUpload}
|
||||
onTriggerUploadFolder={triggerUploadFolder}
|
||||
onTriggerWebLink={triggerWebLink}
|
||||
canCreateFolder={canCreateFolder}
|
||||
canUploadFile={canUploadFile}
|
||||
supportedFormatsLabel={localize(
|
||||
@@ -1223,10 +1330,24 @@ export function KnowledgeSpaceContent({
|
||||
/>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="start" className={knowledgeSpaceDropdownSurfaceClassName}>
|
||||
<DropdownMenuItem onClick={onCreateFolder} className="cursor-pointer">
|
||||
<FolderPlus className="mr-2 size-4" />
|
||||
{localize("com_knowledge.new_folder")}
|
||||
</DropdownMenuItem>
|
||||
{canUploadFile && (
|
||||
<DropdownMenuItem onClick={triggerUpload} className="cursor-pointer">
|
||||
<Outlined.Upload className="mr-2 size-4" />
|
||||
{localize("com_knowledge.upload_file")}
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{canUploadFile && (
|
||||
<DropdownMenuItem onClick={triggerWebLink} className="cursor-pointer">
|
||||
<Link2 className="mr-2 size-4" />
|
||||
{localize("com_knowledge.web_link")}
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{canCreateFolder && (
|
||||
<DropdownMenuItem onClick={onCreateFolder} className="cursor-pointer">
|
||||
<FolderPlus className="mr-2 size-4" />
|
||||
{localize("com_knowledge.new_folder")}
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
{suppressList ? (
|
||||
@@ -1451,6 +1572,60 @@ export function KnowledgeSpaceContent({
|
||||
}
|
||||
/>
|
||||
|
||||
<Dialog
|
||||
open={webLinkDialogOpen}
|
||||
onOpenChange={(open) => {
|
||||
setWebLinkDialogOpen(open);
|
||||
if (!open) resetWebLinkDialog();
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-w-[520px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{localize("com_knowledge.web_link_import_title")}</DialogTitle>
|
||||
</DialogHeader>
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="knowledge-web-link-url">{localize("com_knowledge.web_link_url")}</Label>
|
||||
<Input
|
||||
id="knowledge-web-link-url"
|
||||
value={webLinkUrl}
|
||||
onChange={(e) => setWebLinkUrl(e.target.value)}
|
||||
placeholder="https://example.com/article"
|
||||
disabled={webLinkSubmitting}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="knowledge-web-link-title">{localize("com_knowledge.web_link_title")}</Label>
|
||||
<Input
|
||||
id="knowledge-web-link-title"
|
||||
value={webLinkTitle}
|
||||
onChange={(e) => setWebLinkTitle(e.target.value)}
|
||||
placeholder={localize("com_knowledge.web_link_title_placeholder")}
|
||||
disabled={webLinkSubmitting}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<button
|
||||
type="button"
|
||||
className="inline-flex h-9 items-center justify-center rounded-md border border-[#e5e6eb] bg-white px-4 text-sm text-[#4e5969] hover:bg-[#f7f8fa]"
|
||||
onClick={() => setWebLinkDialogOpen(false)}
|
||||
disabled={webLinkSubmitting}
|
||||
>
|
||||
{localize("com_knowledge.cancel")}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="inline-flex h-9 items-center justify-center rounded-md bg-primary px-4 text-sm text-white hover:bg-primary/90 disabled:cursor-not-allowed disabled:opacity-60"
|
||||
onClick={() => submitWebLink(false)}
|
||||
disabled={webLinkSubmitting}
|
||||
>
|
||||
{webLinkSubmitting ? localize("com_knowledge.importing") : localize("com_knowledge.import")}
|
||||
</button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{permTarget && (
|
||||
<KnowledgeSpaceShareDialog
|
||||
open={!!permTarget}
|
||||
|
||||
@@ -4,6 +4,9 @@ import {
|
||||
DEFAULT_MAX_FILE_SIZE_MB,
|
||||
getAllowedMimeTypes,
|
||||
getAllowedExtensions,
|
||||
getMaxFileSizeBytesForFile,
|
||||
getMaxFileSizeMBForFile,
|
||||
type UploadSizeLimits,
|
||||
} from "../knowledgeUtils";
|
||||
import { useLocalize } from "~/hooks";
|
||||
|
||||
@@ -22,10 +25,11 @@ interface UseFileDragDropOptions {
|
||||
*/
|
||||
onUploadFolder?: (
|
||||
files: File[],
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number },
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number; limits?: UploadSizeLimits },
|
||||
) => void;
|
||||
/** Maximum single file size in MB (from env config). Falls back to DEFAULT_MAX_FILE_SIZE_MB. */
|
||||
maxFileSizeMB?: number;
|
||||
uploadSizeLimits?: UploadSizeLimits;
|
||||
/** Whether ETL4LM service is deployed; controls which extensions/MIME types are accepted. */
|
||||
enableEtl4lm?: boolean;
|
||||
}
|
||||
@@ -100,12 +104,16 @@ export function useFileDragDrop({
|
||||
onUploadFile,
|
||||
onUploadFolder,
|
||||
maxFileSizeMB,
|
||||
uploadSizeLimits,
|
||||
enableEtl4lm = false,
|
||||
}: UseFileDragDropOptions) {
|
||||
const localize = useLocalize();
|
||||
const dragCounter = useRef(0);
|
||||
const limitMB = maxFileSizeMB ?? DEFAULT_MAX_FILE_SIZE_MB;
|
||||
const limitBytes = limitMB * 1024 * 1024;
|
||||
const limits = useMemo(
|
||||
() => uploadSizeLimits ?? { defaultMaxMB: limitMB, mediaMaxMB: limitMB },
|
||||
[uploadSizeLimits, limitMB],
|
||||
);
|
||||
const allowedMime = useMemo(() => getAllowedMimeTypes(enableEtl4lm), [enableEtl4lm]);
|
||||
const allowedExt = useMemo(() => getAllowedExtensions(enableEtl4lm), [enableEtl4lm]);
|
||||
|
||||
@@ -193,7 +201,11 @@ export function useFileDragDrop({
|
||||
onDragStateChange?.(false);
|
||||
void readFolderFilesRecursive(dirEntry, "").then((files) => {
|
||||
if (files.length > 0) {
|
||||
onUploadFolder(files, { allowedExtensions: allowedExt, maxSizeMB: limitMB });
|
||||
onUploadFolder(files, {
|
||||
allowedExtensions: allowedExt,
|
||||
maxSizeMB: limits.defaultMaxMB,
|
||||
limits,
|
||||
});
|
||||
}
|
||||
});
|
||||
return;
|
||||
@@ -209,8 +221,11 @@ export function useFileDragDrop({
|
||||
}
|
||||
|
||||
for (const f of filesList) {
|
||||
if (f.size > limitBytes) {
|
||||
onDragStateChange?.(true, localize("com_knowledge.file_exceeds_limit", { name: f.name, size: limitMB }));
|
||||
if (f.size > getMaxFileSizeBytesForFile(f.name, limits)) {
|
||||
onDragStateChange?.(true, localize("com_knowledge.file_exceeds_limit", {
|
||||
name: f.name,
|
||||
size: getMaxFileSizeMBForFile(f.name, limits),
|
||||
}));
|
||||
setTimeout(() => onDragStateChange?.(false), 2000);
|
||||
return;
|
||||
}
|
||||
@@ -228,7 +243,7 @@ export function useFileDragDrop({
|
||||
onDragStateChange?.(false);
|
||||
}
|
||||
},
|
||||
[onDragStateChange, onUploadFile, onUploadFolder, limitBytes, limitMB, allowedExt, localize]
|
||||
[onDragStateChange, onUploadFile, onUploadFolder, limits, allowedExt, localize]
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -238,4 +253,3 @@ export function useFileDragDrop({
|
||||
handleDrop,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
isKnowledgeItemPending,
|
||||
MAX_FOLDER_DEPTH,
|
||||
MAX_FOLDER_UPLOAD_COUNT,
|
||||
type UploadSizeLimits,
|
||||
} from "../knowledgeUtils";
|
||||
import { useLocalize } from "~/hooks";
|
||||
import { dispatchKnowledgeSpaceFilesRefresh } from "./useFileManager";
|
||||
@@ -398,7 +399,7 @@ export function useFileUpload({
|
||||
const handleUploadFolder = useCallback(
|
||||
async (
|
||||
fileList: FileList | File[],
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number },
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number; limits?: UploadSizeLimits },
|
||||
) => {
|
||||
if (!activeSpace || !fileList || fileList.length === 0) return;
|
||||
// Re-entry guard. Ignore a second call while the first is still
|
||||
|
||||
@@ -27,6 +27,12 @@ export function isKnowledgeApprovalRejected(file: KnowledgeFile): boolean {
|
||||
return file.approvalStatus === "rejected" || file.approvalStatus === "sensitive_rejected";
|
||||
}
|
||||
|
||||
export {
|
||||
isWebLinkKnowledgeFile,
|
||||
resolveWebLinkDisplayName,
|
||||
toWebLinkFileName,
|
||||
} from "~/api/knowledge";
|
||||
|
||||
/**
|
||||
* True while a file body is still being uploaded (frontend placeholder row)
|
||||
* or a folder row is a not-yet-committed inline-create placeholder. These
|
||||
@@ -66,12 +72,14 @@ export function isKnowledgeItemPending(file: KnowledgeFile): boolean {
|
||||
export const ALLOWED_EXTENSIONS = [
|
||||
"pdf", "ofd", "txt", "docx", "ppt", "pptx", "md", "html",
|
||||
"xls", "xlsx", "csv", "doc", "png", "jpg", "jpeg", "bmp",
|
||||
"wps", "dps", "et",
|
||||
"wps", "dps", "et", "mp3", "wav", "m4a", "aac", "flac", "ogg",
|
||||
"mp4", "mov", "avi", "mkv", "webm",
|
||||
] as const;
|
||||
|
||||
/** Subset used when ETL4LM is NOT deployed — drops images. */
|
||||
const ALLOWED_EXTENSIONS_NO_ETL4LM: readonly string[] = [
|
||||
"pdf", "ofd", "txt", "docx", "doc", "ppt", "pptx", "md", "html", "xls", "xlsx", "csv",
|
||||
"wps", "dps", "et", "mp3", "wav", "m4a", "aac", "flac", "ogg", "mp4", "mov", "avi", "mkv", "webm",
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -89,6 +97,9 @@ export const ALLOWED_MIME_TYPES = [
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation", // pptx
|
||||
"text/markdown", "text/html", "text/csv",
|
||||
"image/png", "image/jpeg", "image/bmp",
|
||||
"audio/mpeg", "audio/mp3", "audio/wav", "audio/x-wav", "audio/mp4", "audio/m4a", "audio/x-m4a",
|
||||
"audio/aac", "audio/flac", "audio/ogg",
|
||||
"video/mp4", "video/quicktime", "video/x-msvideo", "video/avi", "video/x-matroska", "video/webm",
|
||||
"application/vnd.ms-works", "application/kswps", "application/wps-office.wps", // wps
|
||||
"application/vnd.wps-presentation", "application/kswps", // dps
|
||||
"application/vnd.ms-excel", "application/kset", // et
|
||||
@@ -120,6 +131,44 @@ export function getFileInputAccept(enableEtl4lm: boolean): string {
|
||||
/** Default maximum single file size in MB (used when env config is not available) */
|
||||
export const DEFAULT_MAX_FILE_SIZE_MB = 200;
|
||||
|
||||
/** Default maximum media file size in MB when env config is not available */
|
||||
export const DEFAULT_MEDIA_MAX_FILE_SIZE_MB = 1024;
|
||||
|
||||
export const MEDIA_FILE_EXTENSIONS = [
|
||||
"mp3", "wav", "m4a", "aac", "flac", "ogg",
|
||||
"mp4", "mov", "avi", "mkv", "webm",
|
||||
] as const;
|
||||
|
||||
export interface UploadSizeLimits {
|
||||
defaultMaxMB: number;
|
||||
mediaMaxMB: number;
|
||||
}
|
||||
|
||||
export interface UploadSizeEnvConfig {
|
||||
uploaded_files_maximum_size?: number;
|
||||
uploaded_media_maximum_size?: number;
|
||||
}
|
||||
|
||||
export function resolveUploadSizeLimits(config?: UploadSizeEnvConfig | null): UploadSizeLimits {
|
||||
return {
|
||||
defaultMaxMB: config?.uploaded_files_maximum_size ?? DEFAULT_MAX_FILE_SIZE_MB,
|
||||
mediaMaxMB: config?.uploaded_media_maximum_size ?? DEFAULT_MEDIA_MAX_FILE_SIZE_MB,
|
||||
};
|
||||
}
|
||||
|
||||
export function isMediaFileName(name: string): boolean {
|
||||
const ext = name.split(".").pop()?.toLowerCase();
|
||||
return Boolean(ext && (MEDIA_FILE_EXTENSIONS as readonly string[]).includes(ext));
|
||||
}
|
||||
|
||||
export function getMaxFileSizeMBForFile(name: string, limits: UploadSizeLimits): number {
|
||||
return isMediaFileName(name) ? limits.mediaMaxMB : limits.defaultMaxMB;
|
||||
}
|
||||
|
||||
export function getMaxFileSizeBytesForFile(name: string, limits: UploadSizeLimits): number {
|
||||
return getMaxFileSizeMBForFile(name, limits) * 1024 * 1024;
|
||||
}
|
||||
|
||||
/** Maximum number of files per upload batch */
|
||||
export const MAX_UPLOAD_COUNT = 50;
|
||||
|
||||
@@ -148,6 +197,19 @@ export function getFileTypeFromName(name: string): FileType {
|
||||
case "jpg": return FileType.JPG;
|
||||
case "jpeg": return FileType.JPEG;
|
||||
case "png": return FileType.PNG;
|
||||
case "mp3":
|
||||
case "wav":
|
||||
case "m4a":
|
||||
case "aac":
|
||||
case "flac":
|
||||
case "ogg":
|
||||
return FileType.AUDIO;
|
||||
case "mp4":
|
||||
case "mov":
|
||||
case "avi":
|
||||
case "mkv":
|
||||
case "webm":
|
||||
return FileType.VIDEO;
|
||||
case "wps": return FileType.WPS;
|
||||
case "dps": return FileType.DPS;
|
||||
case "et": return FileType.ET;
|
||||
@@ -250,16 +312,16 @@ export interface FolderUploadFilterResult {
|
||||
|
||||
export function filterFolderUploadFiles(
|
||||
files: File[],
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number },
|
||||
options: { allowedExtensions: readonly string[]; maxSizeMB: number; limits?: UploadSizeLimits },
|
||||
): FolderUploadFilterResult {
|
||||
const maxBytes = options.maxSizeMB * 1024 * 1024;
|
||||
const limits = options.limits ?? { defaultMaxMB: options.maxSizeMB, mediaMaxMB: options.maxSizeMB };
|
||||
const valid: File[] = [];
|
||||
let oversizeCount = 0;
|
||||
let unsupportedCount = 0;
|
||||
for (const file of files) {
|
||||
const rel = file.webkitRelativePath || file.name;
|
||||
if (isHiddenPath(rel)) continue; // hidden: silent drop
|
||||
if (file.size > maxBytes) {
|
||||
if (file.size > getMaxFileSizeBytesForFile(file.name, limits)) {
|
||||
oversizeCount++;
|
||||
continue;
|
||||
}
|
||||
@@ -279,10 +341,17 @@ export function filterFolderUploadFiles(
|
||||
* @param maxSizeMB - Maximum file size in MB (from env config or default 200)
|
||||
* Returns an error message string, or null if valid.
|
||||
*/
|
||||
export function validateFileForUpload(file: File, maxSizeMB: number = DEFAULT_MAX_FILE_SIZE_MB): string | null {
|
||||
const maxSizeBytes = maxSizeMB * 1024 * 1024;
|
||||
if (file.size > maxSizeBytes) {
|
||||
return i18next.t("com_knowledge.file_exceeds_limit", { name: file.name, size: maxSizeMB });
|
||||
export function validateFileForUpload(
|
||||
file: File,
|
||||
maxSizeMB: number = DEFAULT_MAX_FILE_SIZE_MB,
|
||||
limits?: UploadSizeLimits,
|
||||
): string | null {
|
||||
const resolvedLimits = limits ?? { defaultMaxMB: maxSizeMB, mediaMaxMB: maxSizeMB };
|
||||
if (file.size > getMaxFileSizeBytesForFile(file.name, resolvedLimits)) {
|
||||
return i18next.t("com_knowledge.file_exceeds_limit", {
|
||||
name: file.name,
|
||||
size: getMaxFileSizeMBForFile(file.name, resolvedLimits),
|
||||
});
|
||||
}
|
||||
const ext = file.name.split(".").pop()?.toLowerCase();
|
||||
if (!ext || !(ALLOWED_EXTENSIONS as readonly string[]).includes(ext)) {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
type LocalizeFn = (key: string, options?: Record<string, unknown>) => string;
|
||||
|
||||
const WEB_LINK_ERROR_KEY_MAP: Record<string, string> = {
|
||||
"web link redirect location is missing": "com_knowledge.web_link_request_failed",
|
||||
"web link content type is not supported": "com_knowledge.web_link_content_type_unsupported",
|
||||
"web link content is too large": "com_knowledge.web_link_content_too_large",
|
||||
"web link redirects too many times": "com_knowledge.web_link_redirects_too_many",
|
||||
"web link content is empty": "com_knowledge.web_link_content_empty",
|
||||
"web link url is empty": "com_knowledge.web_link_url_required",
|
||||
"only http/https web links are supported": "com_knowledge.web_link_http_only",
|
||||
"web link host is missing": "com_knowledge.web_link_invalid",
|
||||
"this web link host is not allowed": "com_knowledge.web_link_host_not_allowed",
|
||||
"this web link address is not allowed": "com_knowledge.web_link_address_not_allowed",
|
||||
"web link host cannot be resolved": "com_knowledge.web_link_host_unresolved",
|
||||
};
|
||||
|
||||
const MEDIA_ERROR_KEY_MAP: Record<string, string> = {
|
||||
"media file does not exist": "com_knowledge.media_file_missing",
|
||||
"asr returned empty text": "com_knowledge.media_asr_empty",
|
||||
"asr api key is missing": "com_knowledge.media_asr_api_key_missing",
|
||||
"ffmpeg is not installed": "com_knowledge.media_ffmpeg_missing",
|
||||
"media audio extraction failed": "com_knowledge.media_audio_extraction_failed",
|
||||
};
|
||||
|
||||
function translateIfExists(localize: LocalizeFn, key: string, options?: Record<string, unknown>) {
|
||||
const value = localize(key, options);
|
||||
return value && value !== key ? value : "";
|
||||
}
|
||||
|
||||
function resolveKnownErrorKey(message: string) {
|
||||
const normalized = message.trim().toLowerCase();
|
||||
if (!normalized) return "";
|
||||
if (normalized.startsWith("web link request failed")) {
|
||||
return "com_knowledge.web_link_request_failed";
|
||||
}
|
||||
if (normalized.startsWith("knowledge media transcription only supports aliyun/qwen asr")) {
|
||||
return "com_knowledge.media_asr_provider_unsupported";
|
||||
}
|
||||
if (normalized.startsWith("asr request failed")) {
|
||||
return "com_knowledge.media_asr_request_failed";
|
||||
}
|
||||
return WEB_LINK_ERROR_KEY_MAP[normalized] || MEDIA_ERROR_KEY_MAP[normalized] || "";
|
||||
}
|
||||
|
||||
export function resolveLocalizedKnowledgeImportError(
|
||||
error: any,
|
||||
localize: LocalizeFn,
|
||||
fallbackKey: string,
|
||||
) {
|
||||
const statusCode = error?.status_code ?? error?.statusCode;
|
||||
const errorData = error?.data ?? error?.errorData ?? {};
|
||||
const rawMessage = (
|
||||
typeof error?.status_message === "string" && error.status_message
|
||||
? error.status_message
|
||||
: typeof error?.message === "string"
|
||||
? error.message
|
||||
: ""
|
||||
).trim();
|
||||
|
||||
if (rawMessage) {
|
||||
const translatedByExactMessage = translateIfExists(localize, `api_errors.${rawMessage}`, errorData);
|
||||
if (translatedByExactMessage) return translatedByExactMessage;
|
||||
|
||||
const knownKey = resolveKnownErrorKey(rawMessage);
|
||||
if (knownKey) {
|
||||
const translatedByKnownKey = translateIfExists(localize, knownKey, errorData);
|
||||
if (translatedByKnownKey) return translatedByKnownKey;
|
||||
}
|
||||
}
|
||||
|
||||
if (statusCode !== undefined && statusCode !== null) {
|
||||
const translated = translateIfExists(localize, `api_errors.${statusCode}`, errorData);
|
||||
if (translated) return translated;
|
||||
}
|
||||
|
||||
return localize(fallbackKey);
|
||||
}
|
||||
@@ -41,7 +41,7 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 200m;
|
||||
client_max_body_size 1024m;
|
||||
add_header Access-Control-Allow-Origin $host;
|
||||
add_header X-Frame-Options SAMEORIGIN;
|
||||
}
|
||||
@@ -50,4 +50,4 @@ server {
|
||||
rewrite ^/workspace(/.*)$ $1 break;
|
||||
proxy_pass http://minio:9000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
client_max_body_size 200m;
|
||||
client_max_body_size 1024m;
|
||||
add_header Access-Control-Allow-Origin "http://192.168.106.120:3002";
|
||||
add_header X-Frame-Options SAMEORIGIN;
|
||||
# proxy_set_header Connection "Upgrade";
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
"linsightFullName": "{{linsightFull}}",
|
||||
"prompt": "Confirmation",
|
||||
"unsupportedFileType": "Unsupported file types: {{extensions}}",
|
||||
"supportedFormatsWithImages": "Supported file formats: pdf (including scanned documents), ofd, txt, docx, ppt, pptx, md, html, xls, xlsx, csv, doc, png, jpg, jpeg, bmp, wps, et, dps; each file supports up to {{maxSize}}MB; pdf supports traceability positioning.",
|
||||
"supportedFormatsWithoutImages": "Supported file formats: pdf, ofd, txt, docx, doc, ppt, pptx, md, html, xls, xlsx, csv, wps, et, dps, each file supports up to {{maxSize}}MB",
|
||||
"supportedFormatsWithImages": "Supported file formats: pdf (including scanned documents), ofd, txt, docx, ppt, pptx, md, html, xls, xlsx, csv, doc, png, jpg, jpeg, bmp, wps, et, dps, mp3, wav, m4a, aac, flac, ogg, mp4, mov, avi, mkv, webm; each file supports up to {{maxSize}}MB; pdf supports traceability positioning.",
|
||||
"supportedFormatsWithoutImages": "Supported file formats: pdf, ofd, txt, docx, doc, ppt, pptx, md, html, xls, xlsx, csv, wps, et, dps, mp3, wav, m4a, aac, flac, ogg, mp4, mov, avi, mkv, webm, each file supports up to {{maxSize}}MB",
|
||||
"pagination": {
|
||||
"totalRecords": "Total {{total}}",
|
||||
"totalWithPageSize": "Total {{total}} items, {{pageSize}} per page",
|
||||
|
||||
@@ -196,6 +196,37 @@
|
||||
"descPlaceholder": "Please enter the knowledge base description",
|
||||
"searchFileName": "Search File Name",
|
||||
"uploadFile": "Upload File",
|
||||
"add": "Add",
|
||||
"webLink": "Web link",
|
||||
"webLinkUrl": "Link URL",
|
||||
"webLinkTitle": "Display name",
|
||||
"webLinkTitlePlaceholder": "Leave empty to read the page title automatically",
|
||||
"webLinkImportStarted": "Web link import started",
|
||||
"webLinkUrlRequired": "Please enter a web link",
|
||||
"webLinkHttpOnly": "Only http or https links are supported",
|
||||
"webLinkInvalid": "Please enter a valid web link",
|
||||
"duplicateWebLinkTitle": "Duplicate web link found",
|
||||
"webLinkOverwriteFailed": "Failed to overwrite web link",
|
||||
"webLinkImportFailed": "Failed to import web link",
|
||||
"webLinkHostNotAllowed": "This web link host is not allowed",
|
||||
"webLinkAddressNotAllowed": "This web link address is not allowed",
|
||||
"webLinkHostUnresolved": "Web link host cannot be resolved",
|
||||
"webLinkContentTooLarge": "Web link content is too large",
|
||||
"webLinkContentTypeUnsupported": "Web link content type is not supported",
|
||||
"webLinkRedirectsTooMany": "Web link redirects too many times",
|
||||
"webLinkContentEmpty": "Web link content is empty",
|
||||
"webLinkRequestFailed": "Web link request failed",
|
||||
"mediaFileMissing": "Media file does not exist",
|
||||
"mediaAsrEmpty": "ASR returned empty text",
|
||||
"mediaAsrApiKeyMissing": "ASR API key is missing",
|
||||
"mediaFfmpegMissing": "ffmpeg is not installed on the server",
|
||||
"mediaAudioExtractionFailed": "Media audio extraction failed",
|
||||
"mediaAsrProviderUnsupported": "Knowledge media transcription only supports Aliyun/Qwen ASR",
|
||||
"mediaAsrRequestFailed": "ASR request failed",
|
||||
"overwrite": "Overwrite",
|
||||
"import": "Import",
|
||||
"importing": "Importing...",
|
||||
"supportedFormatsTip": "Supports document, image, audio and video files",
|
||||
"fileName": "File Name",
|
||||
"previousStep": "Previous Step",
|
||||
"dataProcessing": "Data Processing",
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
"linsightFullName": "{{linsightName}}",
|
||||
"prompt": "ヒント",
|
||||
"unsupportedFileType": "サポートされていないファイル形式: {{extensions}}",
|
||||
"supportedFormatsWithImages": "サポートされているファイル形式は pdf(スキャン含む)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、wps、et、dps です。各ファイルの最大サイズは {{maxSize}}MB です。pdf はトレーサビリティ位置特定をサポートします。",
|
||||
"supportedFormatsWithoutImages": "サポートされているファイル形式は pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、wps、et、dps です。各ファイルの最大サイズは {{maxSize}}MB です。",
|
||||
"supportedFormatsWithImages": "サポートされているファイル形式は pdf(スキャン含む)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、wps、et、dps、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm です。各ファイルの最大サイズは {{maxSize}}MB です。pdf はトレーサビリティ位置特定をサポートします。",
|
||||
"supportedFormatsWithoutImages": "サポートされているファイル形式は pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、wps、et、dps、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm です。各ファイルの最大サイズは {{maxSize}}MB です。",
|
||||
"pagination": {
|
||||
"totalRecords": "合計 {{total}} 件の記録",
|
||||
"totalWithPageSize": "合計 {{total}} 件、1ページ {{pageSize}} 件",
|
||||
|
||||
@@ -181,6 +181,37 @@
|
||||
"descPlaceholder": "説明を入力してください",
|
||||
"searchFileName": "ファイル名を検索",
|
||||
"uploadFile": "アップロード",
|
||||
"add": "新規追加",
|
||||
"webLink": "Webリンク",
|
||||
"webLinkUrl": "リンクURL",
|
||||
"webLinkTitle": "表示名",
|
||||
"webLinkTitlePlaceholder": "空欄の場合はページタイトルを自動取得します",
|
||||
"webLinkImportStarted": "Webリンクのインポートを開始しました",
|
||||
"webLinkUrlRequired": "Webリンクを入力してください",
|
||||
"webLinkHttpOnly": "http または https のリンクのみサポートしています",
|
||||
"webLinkInvalid": "有効なWebリンクを入力してください",
|
||||
"duplicateWebLinkTitle": "重複するWebリンクが見つかりました",
|
||||
"webLinkOverwriteFailed": "Webリンクの上書きに失敗しました",
|
||||
"webLinkImportFailed": "Webリンクのインポートに失敗しました",
|
||||
"webLinkHostNotAllowed": "このWebリンクのホストは許可されていません",
|
||||
"webLinkAddressNotAllowed": "このWebリンクのアドレスは許可されていません",
|
||||
"webLinkHostUnresolved": "Webリンクのホストを解決できません",
|
||||
"webLinkContentTooLarge": "Webリンクの内容が大きすぎます",
|
||||
"webLinkContentTypeUnsupported": "Webリンクのコンテンツタイプはサポートされていません",
|
||||
"webLinkRedirectsTooMany": "Webリンクのリダイレクト回数が多すぎます",
|
||||
"webLinkContentEmpty": "Webリンクの内容が空です",
|
||||
"webLinkRequestFailed": "Webリンクのリクエストに失敗しました",
|
||||
"mediaFileMissing": "メディアファイルが存在しません",
|
||||
"mediaAsrEmpty": "ASR の結果が空です",
|
||||
"mediaAsrApiKeyMissing": "ASR API Key が未設定です",
|
||||
"mediaFfmpegMissing": "サーバーに ffmpeg がインストールされていません",
|
||||
"mediaAudioExtractionFailed": "メディアの音声抽出に失敗しました",
|
||||
"mediaAsrProviderUnsupported": "ナレッジのメディア文字起こしは Aliyun/Qwen ASR のみ対応しています",
|
||||
"mediaAsrRequestFailed": "ASR リクエストに失敗しました",
|
||||
"overwrite": "上書き",
|
||||
"import": "インポート",
|
||||
"importing": "インポート中...",
|
||||
"supportedFormatsTip": "文書、画像、音声、動画ファイルに対応",
|
||||
"segmentStrategy": "セグメントルール",
|
||||
"textComparison": "原文比較",
|
||||
"dataProcessing": "データ処理",
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
"dailyFullName": "{{dailyFullNameZh}}",
|
||||
"prompt": "提示",
|
||||
"unsupportedFileType": "不支持文件类型: {{extensions}}",
|
||||
"supportedFormatsWithImages": "支持的文件格式为 pdf(含扫描件)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、wps、et、dps;每个文件最大支持{{maxSize}}mb;pdf支持溯源定位。",
|
||||
"supportedFormatsWithoutImages": "支持的文件格式为 pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、wps、et、dps,每个文件最大支持{{maxSize}}mb",
|
||||
"supportedFormatsWithImages": "支持的文件格式为 pdf(含扫描件)、ofd、txt、docx、ppt、pptx、md、html、xls、xlsx、csv、doc、png、jpg、jpeg、bmp、wps、et、dps、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm;每个文件最大支持{{maxSize}}mb;pdf支持溯源定位。",
|
||||
"supportedFormatsWithoutImages": "支持的文件格式为 pdf、ofd、txt、docx、doc、ppt、pptx、md、html、xls、xlsx、csv、wps、et、dps、mp3、wav、m4a、aac、flac、ogg、mp4、mov、avi、mkv、webm,每个文件最大支持{{maxSize}}mb",
|
||||
"pagination": {
|
||||
"totalRecords": "共 {{total}} 条记录",
|
||||
"totalWithPageSize": "共 {{total}} 条数据,每页 {{pageSize}} 条",
|
||||
|
||||
@@ -176,6 +176,37 @@
|
||||
"descPlaceholder": "请输入知识库描述",
|
||||
"searchFileName": "搜索文件名称",
|
||||
"uploadFile": "上传文件",
|
||||
"add": "新增",
|
||||
"webLink": "网页链接",
|
||||
"webLinkUrl": "链接地址",
|
||||
"webLinkTitle": "显示名称",
|
||||
"webLinkTitlePlaceholder": "留空则自动读取网页标题",
|
||||
"webLinkImportStarted": "网页链接已开始导入",
|
||||
"webLinkUrlRequired": "请输入网页链接",
|
||||
"webLinkHttpOnly": "仅支持 http 或 https 链接",
|
||||
"webLinkInvalid": "请输入有效的网页链接",
|
||||
"duplicateWebLinkTitle": "发现重复网页链接",
|
||||
"webLinkOverwriteFailed": "网页链接覆盖失败",
|
||||
"webLinkImportFailed": "网页链接导入失败",
|
||||
"webLinkHostNotAllowed": "该网页链接主机不允许导入",
|
||||
"webLinkAddressNotAllowed": "该网页链接地址不允许导入",
|
||||
"webLinkHostUnresolved": "网页链接主机无法解析",
|
||||
"webLinkContentTooLarge": "网页内容超过大小限制",
|
||||
"webLinkContentTypeUnsupported": "网页链接内容类型不支持",
|
||||
"webLinkRedirectsTooMany": "网页链接重定向次数过多",
|
||||
"webLinkContentEmpty": "网页内容为空",
|
||||
"webLinkRequestFailed": "网页链接请求失败",
|
||||
"mediaFileMissing": "音视频文件不存在",
|
||||
"mediaAsrEmpty": "语音识别结果为空",
|
||||
"mediaAsrApiKeyMissing": "ASR API Key 未配置",
|
||||
"mediaFfmpegMissing": "服务器未安装 ffmpeg",
|
||||
"mediaAudioExtractionFailed": "音视频音轨提取失败",
|
||||
"mediaAsrProviderUnsupported": "知识库音视频解析仅支持阿里云/Qwen ASR",
|
||||
"mediaAsrRequestFailed": "ASR 请求失败",
|
||||
"overwrite": "覆盖",
|
||||
"import": "导入",
|
||||
"importing": "导入中...",
|
||||
"supportedFormatsTip": "支持上传文档、图片、音频和视频文件",
|
||||
"segmentStrategy": "分段策略",
|
||||
"docAnalysisStrategy": "文档解析策略",
|
||||
"docSplitStrategy": "文档切分策略",
|
||||
|
||||
@@ -9,17 +9,22 @@ export default function DropZone({ onDrop }) {
|
||||
const { t } = useTranslation()
|
||||
const { appConfig } = useContext(locationContext)
|
||||
const xinChuangFormats = ['.WPS', '.ET', '.DPS'];
|
||||
const mediaFormats = ['.MP3', '.WAV', '.M4A', '.AAC', '.FLAC', '.OGG', '.MP4', '.MOV', '.AVI', '.MKV', '.WEBM'];
|
||||
|
||||
// Define supported file formats (for display purposes only, not for filtering)
|
||||
const supportedFormats = appConfig.enableEtl4lm
|
||||
? ['.PDF', '.OFD', '.TXT', '.DOCX', '.PPT', '.PPTX', '.MD', '.HTML', '.XLS', '.XLSX', '.CSV', '.DOC', '.PNG', '.JPG', '.JPEG', '.BMP', ...xinChuangFormats]
|
||||
: ['.PDF', '.OFD', '.TXT', '.DOCX', '.DOC', '.PPT', '.PPTX', '.MD', '.HTML', '.XLS', '.XLSX', '.CSV', ...xinChuangFormats];
|
||||
? ['.PDF', '.OFD', '.TXT', '.DOCX', '.PPT', '.PPTX', '.MD', '.HTML', '.XLS', '.XLSX', '.CSV', '.DOC', '.PNG', '.JPG', '.JPEG', '.BMP', ...xinChuangFormats, ...mediaFormats]
|
||||
: ['.PDF', '.OFD', '.TXT', '.DOCX', '.DOC', '.PPT', '.PPTX', '.MD', '.HTML', '.XLS', '.XLSX', '.CSV', ...xinChuangFormats, ...mediaFormats];
|
||||
const allowedExts = new Set(
|
||||
supportedFormats.map(ext => ext.toLowerCase().replace('.', ''))
|
||||
);
|
||||
const { getRootProps, getInputProps } = useDropzone({
|
||||
accept: {
|
||||
'application/*': supportedFormats
|
||||
'application/*': supportedFormats,
|
||||
'text/*': supportedFormats,
|
||||
'image/*': supportedFormats,
|
||||
'audio/*': supportedFormats,
|
||||
'video/*': supportedFormats
|
||||
},
|
||||
useFsAccessApi: false,
|
||||
onDrop: (acceptedFiles, disAcceptedFiles) => {
|
||||
|
||||
@@ -6,6 +6,18 @@ import { useTranslation } from "react-i18next";
|
||||
import DropZone from "./DropZone";
|
||||
import ProgressItem from "./ProgressItem";
|
||||
|
||||
const MEDIA_FILE_EXTENSIONS = new Set([
|
||||
'mp3', 'wav', 'm4a', 'aac', 'flac', 'ogg',
|
||||
'mp4', 'mov', 'avi', 'mkv', 'webm',
|
||||
]);
|
||||
|
||||
function getKnowledgeUploadSizeLimitMB(file: File, appConfig: { uploadFileMaxSize?: number; uploadMediaMaxSize?: number }) {
|
||||
const ext = file.name.split('.').pop()?.toLowerCase();
|
||||
return ext && MEDIA_FILE_EXTENSIONS.has(ext)
|
||||
? (appConfig.uploadMediaMaxSize ?? 1024)
|
||||
: (appConfig.uploadFileMaxSize ?? 50);
|
||||
}
|
||||
|
||||
export interface Progress {
|
||||
id: string,
|
||||
file: File,
|
||||
@@ -75,13 +87,11 @@ const KnowledgeUploadComponent = ({
|
||||
|
||||
// beforeupload
|
||||
const handleDrop = (acceptedFiles) => {
|
||||
const sizeLimit = appConfig.uploadFileMaxSize * 1024 * 1024;
|
||||
|
||||
const [bigFiles, files] = acceptedFiles.reduce(
|
||||
([big, small], file) => {
|
||||
if (progressList.some(pros => pros.fileName === file.name)) return [big, small];
|
||||
// 大小校验
|
||||
return file.size < sizeLimit
|
||||
const sizeLimit = getKnowledgeUploadSizeLimitMB(file, appConfig) * 1024 * 1024;
|
||||
return file.size <= sizeLimit
|
||||
? [big, [...small, file]] // 合法文件
|
||||
: [[...big, file.name], small]; // 超大小文件
|
||||
},
|
||||
@@ -92,7 +102,7 @@ const KnowledgeUploadComponent = ({
|
||||
if (bigFiles.length > 0) {
|
||||
message({
|
||||
title: t('prompt'),
|
||||
description: bigFiles.map(str => `${t('code.file')}: ${str} ${t('code.sizeExceedsLimit', { size: appConfig.uploadFileMaxSize + 'MB' })}`).join(';'),
|
||||
description: bigFiles.map(str => `${t('code.file')}: ${str} ${t('code.sizeExceedsLimit', { size: getKnowledgeUploadSizeLimitMB({ name: str } as File, appConfig) + 'MB' })}`).join(';'),
|
||||
variant: 'error'
|
||||
});
|
||||
}
|
||||
@@ -179,4 +189,4 @@ const KnowledgeUploadComponent = ({
|
||||
</div>
|
||||
};
|
||||
|
||||
export default KnowledgeUploadComponent
|
||||
export default KnowledgeUploadComponent
|
||||
|
||||
@@ -489,6 +489,14 @@ export async function subUploadLibFile(data: DefaultUploadFileFc): Promise<any>;
|
||||
export async function subUploadLibFile(data: UploadFileFc | DefaultUploadFileFc) {
|
||||
return await axios.post(`/api/v1/knowledge/process`, data);
|
||||
}
|
||||
|
||||
export async function importKnowledgeWebLinkApi(
|
||||
knowledgeId: string | number,
|
||||
data: { url: string; title?: string; overwrite?: boolean }
|
||||
) {
|
||||
return await axios.post(`/api/v1/knowledge/space/${knowledgeId}/web-links`, data, { silent: true } as any);
|
||||
}
|
||||
|
||||
//调整分段策略
|
||||
export async function rebUploadFile(data) {
|
||||
return await axios.post(`/api/v1/knowledge/process/rebuild`, data);
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Button } from "@/components/bs-ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/bs-ui/dropdownMenu";
|
||||
import Tip from "@/components/bs-ui/tooltip/tip";
|
||||
import { cn } from "@/utils";
|
||||
import { ChevronDown, CircleHelp, Link2, Upload } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
interface AddKnowledgeFileMenuProps {
|
||||
disabled?: boolean;
|
||||
supportedFormatsLabel?: string;
|
||||
buttonClassName?: string;
|
||||
onUploadFile: () => void;
|
||||
onWebLink: () => void;
|
||||
}
|
||||
|
||||
export default function AddKnowledgeFileMenu({
|
||||
disabled = false,
|
||||
supportedFormatsLabel,
|
||||
buttonClassName,
|
||||
onUploadFile,
|
||||
onWebLink,
|
||||
}: AddKnowledgeFileMenuProps) {
|
||||
const { t } = useTranslation("knowledge");
|
||||
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild disabled={disabled}>
|
||||
<Button className={cn("h-9 gap-2 px-4 md:px-6", buttonClassName)}>
|
||||
{t("add", { defaultValue: "新增" })}
|
||||
<ChevronDown className="size-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="min-w-[140px] bg-white p-1">
|
||||
<DropdownMenuItem
|
||||
className="h-9 cursor-pointer gap-2 px-3"
|
||||
onSelect={onUploadFile}
|
||||
>
|
||||
<Upload className="size-4" />
|
||||
<span>{t("uploadFile")}</span>
|
||||
{supportedFormatsLabel ? (
|
||||
<Tip content={supportedFormatsLabel} side="left">
|
||||
<span
|
||||
className="ml-auto inline-flex size-4 items-center justify-center text-[#8a94a6]"
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
>
|
||||
<CircleHelp className="size-3.5" />
|
||||
</span>
|
||||
</Tip>
|
||||
) : null}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
className="h-9 cursor-pointer gap-2 px-3"
|
||||
onSelect={onWebLink}
|
||||
>
|
||||
<Link2 className="size-4" />
|
||||
<span>{t("webLink", { defaultValue: "网页链接" })}</span>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Link, useNavigate, useParams } from "react-router-dom";
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import { Button } from "../../../components/bs-ui/button";
|
||||
import {
|
||||
Table,
|
||||
@@ -28,6 +28,8 @@ import { captureAndAlertRequestErrorHoc } from "../../../controllers/request";
|
||||
import { useTable } from "../../../util/hook";
|
||||
import useKnowledgeStore from "../useKnowledgeStore";
|
||||
import { MetadataManagementDialog } from "./MetadataManagementDialog";
|
||||
import AddKnowledgeFileMenu from "./AddKnowledgeFileMenu";
|
||||
import WebLinkImportDialog from "./WebLinkImportDialog";
|
||||
import FileTagList from "./tags/FileTagList";
|
||||
import KnowledgeTagSelect from "./tags/KnowledgeTagSelect";
|
||||
|
||||
@@ -46,6 +48,17 @@ const STATUS_CONFIG: Record<number, { labelKey: string; colorClass: string; bgCl
|
||||
7: { labelKey: "violation", colorClass: "text-red-500", bgClass: "bg-red-500" },
|
||||
};
|
||||
|
||||
function normalizeWebLinkDisplayName(fileName: string): string {
|
||||
const trimmed = fileName.trim().replace(/\.md$/i, "").trim();
|
||||
if (!trimmed) return fileName;
|
||||
return trimmed.toLowerCase().endsWith(".html") ? trimmed : `${trimmed}.html`;
|
||||
}
|
||||
|
||||
function getKnowledgeFileDisplayName(file: Record<string, any>): string {
|
||||
const fileName = String(file.file_name ?? "");
|
||||
return file.file_source === "web_link" ? normalizeWebLinkDisplayName(fileName) : fileName;
|
||||
}
|
||||
|
||||
function formatSensitiveViolationMessage(hits: any[], t: (key: string, options?: Record<string, any>) => string) {
|
||||
const words = hits
|
||||
.map((item) => String(item?.word ?? "").trim())
|
||||
@@ -151,6 +164,7 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
readFileByLibDatabase({ ...param, id, name: param.keyword })
|
||||
)
|
||||
const [metadataOpen, setMetadataOpen] = useState(false);
|
||||
const [webLinkOpen, setWebLinkOpen] = useState(false);
|
||||
const navigate = useNavigate()
|
||||
|
||||
// Store complete file objects (preserving all original parameters)
|
||||
@@ -334,15 +348,18 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
const dataSouce = useMemo(() => {
|
||||
return datalist.map(el => {
|
||||
const suffix = el.file_name.split('.').pop()?.toLowerCase() || '';
|
||||
const displayFileName = getKnowledgeFileDisplayName(el);
|
||||
if (['xlsx', 'xls', 'csv', 'et'].includes(suffix) && el.parse_type !== "local" && el.parse_type !== "uns") {
|
||||
const excel_rule = JSON.parse(el.split_rule).excel_rule
|
||||
return {
|
||||
...el,
|
||||
display_file_name: displayFileName,
|
||||
strategy: ['', t('everyRowsAsOneSegment', { count: excel_rule?.slice_length })]
|
||||
}
|
||||
}
|
||||
if (!el.split_rule) return {
|
||||
...el,
|
||||
display_file_name: displayFileName,
|
||||
strategy: ['', '']
|
||||
}
|
||||
const rule = JSON.parse(el.split_rule)
|
||||
@@ -350,6 +367,7 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
const data = separator.map((el, i) => `${separator_rule[i] === 'before' ? '✂️' : ''}${el}${separator_rule[i] === 'after' ? '✂️' : ''}`)
|
||||
return {
|
||||
...el,
|
||||
display_file_name: displayFileName,
|
||||
strategy: [data.length > 2 ? data.slice(0, 2).join(',') : '', data.join(',')]
|
||||
}
|
||||
})
|
||||
@@ -501,9 +519,12 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
</Button>
|
||||
)}
|
||||
{canEditKb && (
|
||||
<Link to={`/filelib/upload/${id}`}>
|
||||
<Button className="px-4 md:px-8 h-9">{t('uploadFile')}</Button>
|
||||
</Link>
|
||||
<AddKnowledgeFileMenu
|
||||
buttonClassName="px-4 md:px-8"
|
||||
supportedFormatsLabel={t("supportedFormatsTip", { defaultValue: "" })}
|
||||
onUploadFile={() => navigate(`/filelib/upload/${id}`)}
|
||||
onWebLink={() => setWebLinkOpen(true)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
@@ -677,13 +698,13 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
/>
|
||||
</TableCell>
|
||||
<TableCell className="min-w-[250px]">
|
||||
<Tip content={el.file_name} align="start" >
|
||||
<Tip content={el.display_file_name || el.file_name} align="start" >
|
||||
<div className="flex items-center gap-2">
|
||||
<FileIcon
|
||||
type={el.file_name.split('.').pop().toLowerCase() || 'txt'}
|
||||
type={(el.display_file_name || el.file_name).split('.').pop().toLowerCase() || 'txt'}
|
||||
className="size-[30px] min-w-[30px]"
|
||||
/>
|
||||
{truncateString(el.file_name, 35)}
|
||||
{truncateString(el.display_file_name || el.file_name, 35)}
|
||||
</div>
|
||||
</Tip>
|
||||
</TableCell>
|
||||
@@ -770,6 +791,12 @@ export default function Files({ onPreview, canEditKb = false, canDeleteKb = fals
|
||||
id={id}
|
||||
initialMetadata={metadataFields}
|
||||
/>
|
||||
<WebLinkImportDialog
|
||||
knowledgeId={id}
|
||||
open={webLinkOpen}
|
||||
onOpenChange={setWebLinkOpen}
|
||||
onImported={reload}
|
||||
/>
|
||||
</div>
|
||||
|
||||
)
|
||||
|
||||
@@ -74,6 +74,7 @@ export default function Paragraphs({ fileId, onBack, canEditKb = false, canDelet
|
||||
// Refs
|
||||
const isChangingRef = useRef(false);
|
||||
const [previewUrl, setPreviewUrl] = useState()
|
||||
const [previewData, setPreviewData] = useState<any>(null)
|
||||
const [hasChunkBboxes, setHasChunkBboxes] = useState(false);
|
||||
const latestFileUrlRef = useRef('');
|
||||
const latestPreviewUrlRef = useRef('');
|
||||
@@ -249,6 +250,17 @@ export default function Paragraphs({ fileId, onBack, canEditKb = false, canDelet
|
||||
// Check if there are valid preview_url and original_url
|
||||
const hasPreviewUrl = typeof res.preview_url === 'string' && res.preview_url.trim() !== '';
|
||||
const hasOriginalUrl = typeof res.original_url === 'string' && res.original_url.trim() !== '';
|
||||
setPreviewData({
|
||||
...res,
|
||||
original_url: res.original_url || '',
|
||||
preview_url: res.preview_url || '',
|
||||
file_source: res.file_source || '',
|
||||
source_url: res.source_url || '',
|
||||
final_url: res.final_url || '',
|
||||
web_title: res.web_title || '',
|
||||
media_kind: res.media_kind || '',
|
||||
html_preview_url: res.html_preview_url || '',
|
||||
});
|
||||
|
||||
if (currentFile) {
|
||||
if (hasPreviewUrl) {
|
||||
@@ -279,6 +291,7 @@ export default function Paragraphs({ fileId, onBack, canEditKb = false, canDelet
|
||||
} else {
|
||||
setFileUrl('');
|
||||
setPreviewUrl('');
|
||||
setPreviewData(null);
|
||||
latestOriginalUrlRef.current = '';
|
||||
return '';
|
||||
}
|
||||
@@ -286,6 +299,7 @@ export default function Paragraphs({ fileId, onBack, canEditKb = false, canDelet
|
||||
console.error('Failed to get file URL:', err);
|
||||
setFileUrl('');
|
||||
setPreviewUrl('');
|
||||
setPreviewData(null);
|
||||
setPartitions([]);
|
||||
latestOriginalUrlRef.current = '';
|
||||
return '';
|
||||
@@ -862,6 +876,7 @@ export default function Paragraphs({ fileId, onBack, canEditKb = false, canDelet
|
||||
file={currentFile}
|
||||
chunks={chunks}
|
||||
setChunks={setChunks}
|
||||
previewData={previewData}
|
||||
rules={previewRules}
|
||||
edit={canEditKb}
|
||||
/>
|
||||
|
||||
@@ -10,6 +10,7 @@ import { convertJsonData } from "./ParagraphEdit";
|
||||
import { Partition } from "./PreviewResult";
|
||||
import TxtFileViewer from "./TxtFileViewer";
|
||||
import ExcelPreview from "./ExcelPreview";
|
||||
import RichPreviewFile, { isRichKnowledgePreview } from "./RichPreviewFile";
|
||||
|
||||
export default function PreviewFile({
|
||||
urlState,
|
||||
@@ -22,7 +23,8 @@ export default function PreviewFile({
|
||||
edit = false,
|
||||
resultFiles,
|
||||
etl,
|
||||
previewUrl
|
||||
previewUrl,
|
||||
previewData
|
||||
}: {
|
||||
urlState: { load: boolean; url: string };
|
||||
file: any;
|
||||
@@ -31,6 +33,7 @@ export default function PreviewFile({
|
||||
rawFiles: any[];
|
||||
setChunks: any;
|
||||
edit?: boolean;
|
||||
previewData?: any;
|
||||
}) {
|
||||
const { t } = useTranslation('knowledge')
|
||||
const MemoizedFileView = React.memo(FileView);
|
||||
@@ -225,6 +228,9 @@ export default function PreviewFile({
|
||||
// 加载状态处理
|
||||
if (!load && !url) return <div className="flex justify-center items-center h-full text-gray-400">预览失败</div>;
|
||||
if (!url) return <div className="flex justify-center items-center h-full text-gray-400"><LoadingIcon /></div>;
|
||||
if (isRichKnowledgePreview(previewData)) {
|
||||
return <RichPreviewFile file={file} previewData={previewData} />;
|
||||
}
|
||||
|
||||
// 新版文件预览
|
||||
switch (suffix) {
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { LoadingIcon } from "@/components/bs-icons/loading";
|
||||
import { ExternalLink } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { MarkdownView } from "./PreviewParagraph";
|
||||
import TxtFileViewer from "./TxtFileViewer";
|
||||
|
||||
declare const __APP_ENV__: any;
|
||||
|
||||
type PreviewData = {
|
||||
original_url?: string;
|
||||
preview_url?: string;
|
||||
file_source?: string;
|
||||
source_url?: string;
|
||||
final_url?: string;
|
||||
web_title?: string;
|
||||
media_kind?: string;
|
||||
html_preview_url?: string;
|
||||
};
|
||||
|
||||
type MediaTab = "recognized" | "entry";
|
||||
type WebTab = "html" | "text";
|
||||
|
||||
const AUDIO_EXTENSIONS = new Set(["mp3", "wav", "m4a", "aac", "flac", "ogg"]);
|
||||
const VIDEO_EXTENSIONS = new Set(["mp4", "mov", "avi", "mkv", "webm"]);
|
||||
|
||||
function normalizeUrl(url?: string) {
|
||||
if (!url) return "";
|
||||
return url.replace(/https?:\/\/[^/]+/, __APP_ENV__.BASE_URL);
|
||||
}
|
||||
|
||||
function getExtensionFromUrl(url?: string) {
|
||||
if (!url) return "";
|
||||
const path = url.split("?")[0].split("#")[0];
|
||||
const filename = path.split("/").pop() || "";
|
||||
const dotIndex = filename.lastIndexOf(".");
|
||||
return dotIndex >= 0 ? filename.slice(dotIndex + 1).toLowerCase() : "";
|
||||
}
|
||||
|
||||
function isMediaUrl(url?: string) {
|
||||
const ext = getExtensionFromUrl(url);
|
||||
return AUDIO_EXTENSIONS.has(ext) || VIDEO_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
export function isRichKnowledgePreview(previewData?: PreviewData) {
|
||||
if (!previewData) return false;
|
||||
const ext = getExtensionFromUrl(previewData.original_url || previewData.preview_url);
|
||||
return (
|
||||
previewData.file_source === "web_link"
|
||||
|| previewData.file_source === "audio_transcript"
|
||||
|| previewData.file_source === "video_transcript"
|
||||
|| previewData.media_kind === "audio"
|
||||
|| previewData.media_kind === "video"
|
||||
|| AUDIO_EXTENSIONS.has(ext)
|
||||
|| VIDEO_EXTENSIONS.has(ext)
|
||||
);
|
||||
}
|
||||
|
||||
function isMediaPreview(previewData?: PreviewData) {
|
||||
if (!previewData) return false;
|
||||
const ext = getExtensionFromUrl(previewData.original_url || previewData.preview_url);
|
||||
return (
|
||||
previewData.file_source === "audio_transcript"
|
||||
|| previewData.file_source === "video_transcript"
|
||||
|| previewData.media_kind === "audio"
|
||||
|| previewData.media_kind === "video"
|
||||
|| AUDIO_EXTENSIONS.has(ext)
|
||||
|| VIDEO_EXTENSIONS.has(ext)
|
||||
);
|
||||
}
|
||||
|
||||
function isVideoPreview(previewData?: PreviewData) {
|
||||
if (!previewData) return false;
|
||||
const ext = getExtensionFromUrl(previewData.original_url || previewData.preview_url);
|
||||
return previewData.file_source === "video_transcript" || previewData.media_kind === "video" || VIDEO_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
function extractMarkdownSection(markdown: string, heading: string) {
|
||||
const pattern = new RegExp(`^##\\s+${heading}\\s*$`, "m");
|
||||
const match = markdown.match(pattern);
|
||||
if (!match || match.index === undefined) return "";
|
||||
const start = match.index + match[0].length;
|
||||
const rest = markdown.slice(start);
|
||||
const nextHeading = rest.search(/^##\s+/m);
|
||||
return (nextHeading >= 0 ? rest.slice(0, nextHeading) : rest).trim();
|
||||
}
|
||||
|
||||
function TextFromUrl({ fileUrl, section }: { fileUrl: string; section?: string }) {
|
||||
const [content, setContent] = useState("");
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setLoading(true);
|
||||
setError("");
|
||||
fetch(normalizeUrl(fileUrl))
|
||||
.then((response) => {
|
||||
if (!response.ok) throw new Error(`Failed to fetch file: ${response.status}`);
|
||||
return response.text();
|
||||
})
|
||||
.then((text) => {
|
||||
if (!cancelled) setContent(text);
|
||||
})
|
||||
.catch((err: Error) => {
|
||||
if (!cancelled) setError(err.message);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoading(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [fileUrl]);
|
||||
|
||||
if (loading) {
|
||||
return <div className="flex h-full items-center justify-center text-gray-400"><LoadingIcon /></div>;
|
||||
}
|
||||
if (error) {
|
||||
return <div className="flex h-full items-center justify-center text-sm text-red-500">{error}</div>;
|
||||
}
|
||||
|
||||
const sectionText = section ? extractMarkdownSection(content, section) : "";
|
||||
const text = sectionText || content;
|
||||
return (
|
||||
<div className="h-full overflow-y-auto bg-gray-50 p-4">
|
||||
<MarkdownView noHead data={{ text }} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function RichPreviewFile({ file, previewData }: { file: any; previewData: PreviewData }) {
|
||||
const [mediaTab, setMediaTab] = useState<MediaTab>("recognized");
|
||||
const [webTab, setWebTab] = useState<WebTab>("html");
|
||||
const isMedia = isMediaPreview(previewData);
|
||||
const isVideo = isVideoPreview(previewData);
|
||||
const sourceUrl = previewData?.final_url || previewData?.source_url || "";
|
||||
const mediaTextUrl = previewData?.preview_url && !isMediaUrl(previewData.preview_url) ? previewData.preview_url : "";
|
||||
const textUrl = isMedia ? mediaTextUrl : previewData?.preview_url || previewData?.original_url || "";
|
||||
const htmlUrl = previewData?.html_preview_url || "";
|
||||
const originalUrl = previewData?.original_url || "";
|
||||
const mediaUrl = normalizeUrl(originalUrl);
|
||||
const title = previewData?.web_title || file?.file_name || file?.fileName || file?.name || "";
|
||||
|
||||
if (isMedia) {
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col gap-3 overflow-hidden bg-gray-50 p-3">
|
||||
<div className="rounded-md border bg-white p-3 shadow-sm">
|
||||
<div className="mb-2 text-sm font-medium text-gray-800">{title}</div>
|
||||
{isVideo ? (
|
||||
<video className="max-h-[360px] w-full rounded bg-black" src={mediaUrl} controls />
|
||||
) : (
|
||||
<audio className="w-full" src={mediaUrl} controls />
|
||||
)}
|
||||
</div>
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden rounded-md border bg-white shadow-sm">
|
||||
<div className="flex shrink-0 gap-2 border-b px-3 py-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMediaTab("recognized")}
|
||||
className={`h-8 rounded-md px-3 text-sm ${mediaTab === "recognized" ? "bg-primary text-white" : "bg-gray-100 text-gray-600"}`}
|
||||
>
|
||||
识别文本
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMediaTab("entry")}
|
||||
className={`h-8 rounded-md px-3 text-sm ${mediaTab === "entry" ? "bg-primary text-white" : "bg-gray-100 text-gray-600"}`}
|
||||
>
|
||||
入库文本
|
||||
</button>
|
||||
</div>
|
||||
{mediaTextUrl ? (
|
||||
<TextFromUrl fileUrl={mediaTextUrl} section={mediaTab === "recognized" ? "识别文本" : "入库文本"} />
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col overflow-hidden bg-gray-50">
|
||||
<div className="flex shrink-0 items-center justify-between border-b bg-white px-3 py-2">
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setWebTab("html")}
|
||||
className={`h-8 rounded-md px-3 text-sm ${webTab === "html" ? "bg-primary text-white" : "bg-gray-100 text-gray-600"}`}
|
||||
>
|
||||
网页预览
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setWebTab("text")}
|
||||
className={`h-8 rounded-md px-3 text-sm ${webTab === "text" ? "bg-primary text-white" : "bg-gray-100 text-gray-600"}`}
|
||||
>
|
||||
入库文本
|
||||
</button>
|
||||
</div>
|
||||
{sourceUrl ? (
|
||||
<a className="flex items-center gap-1 text-sm text-primary" href={sourceUrl} target="_blank" rel="noreferrer">
|
||||
<ExternalLink className="size-4" />
|
||||
打开原网页
|
||||
</a>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-hidden">
|
||||
{webTab === "html" ? (
|
||||
htmlUrl ? <TxtFileViewer html filePath={htmlUrl} /> : (
|
||||
<div className="flex h-full items-center justify-center text-sm text-gray-500">
|
||||
暂无网页快照,请查看入库文本或打开原网页。
|
||||
</div>
|
||||
)
|
||||
) : textUrl ? (
|
||||
<TextFromUrl fileUrl={textUrl} />
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
import { Button } from "@/components/bs-ui/button";
|
||||
import { bsConfirm } from "@/components/bs-ui/alertDialog/useConfirm";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/bs-ui/dialog";
|
||||
import { Input } from "@/components/bs-ui/input";
|
||||
import { useToast } from "@/components/bs-ui/toast/use-toast";
|
||||
import { importKnowledgeWebLinkApi } from "@/controllers/API";
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
const WEB_LINK_DUPLICATE_ERROR_CODES = new Set([18021, 18023]);
|
||||
|
||||
const WEB_LINK_ERROR_KEY_MAP: Record<string, string> = {
|
||||
"web link redirect location is missing": "webLinkRequestFailed",
|
||||
"web link content type is not supported": "webLinkContentTypeUnsupported",
|
||||
"web link content is too large": "webLinkContentTooLarge",
|
||||
"web link redirects too many times": "webLinkRedirectsTooMany",
|
||||
"web link content is empty": "webLinkContentEmpty",
|
||||
"web link url is empty": "webLinkUrlRequired",
|
||||
"only http/https web links are supported": "webLinkHttpOnly",
|
||||
"web link host is missing": "webLinkInvalid",
|
||||
"this web link host is not allowed": "webLinkHostNotAllowed",
|
||||
"this web link address is not allowed": "webLinkAddressNotAllowed",
|
||||
"web link host cannot be resolved": "webLinkHostUnresolved",
|
||||
"media file does not exist": "mediaFileMissing",
|
||||
"asr returned empty text": "mediaAsrEmpty",
|
||||
"asr api key is missing": "mediaAsrApiKeyMissing",
|
||||
"ffmpeg is not installed": "mediaFfmpegMissing",
|
||||
"media audio extraction failed": "mediaAudioExtractionFailed",
|
||||
};
|
||||
|
||||
function normalizeWebLinkTitle(title: string): string | undefined {
|
||||
const trimmed = title.trim();
|
||||
if (!trimmed) return undefined;
|
||||
return /\.html$/i.test(trimmed) ? trimmed : `${trimmed}.html`;
|
||||
}
|
||||
|
||||
interface WebLinkImportDialogProps {
|
||||
knowledgeId?: string;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
onImported?: () => void;
|
||||
}
|
||||
|
||||
export default function WebLinkImportDialog({
|
||||
knowledgeId,
|
||||
open,
|
||||
onOpenChange,
|
||||
onImported,
|
||||
}: WebLinkImportDialogProps) {
|
||||
const { t } = useTranslation("knowledge");
|
||||
const { message } = useToast();
|
||||
const [webLinkUrl, setWebLinkUrl] = useState("");
|
||||
const [webLinkTitle, setWebLinkTitle] = useState("");
|
||||
const [webLinkLoading, setWebLinkLoading] = useState(false);
|
||||
|
||||
const isDuplicateError = (error: any) => WEB_LINK_DUPLICATE_ERROR_CODES.has(Number(error?.status_code));
|
||||
const resolveErrorMessage = (error: any, fallbackKey: string, fallbackValue: string) => {
|
||||
const statusCode = error?.status_code ?? error?.statusCode;
|
||||
const statusMessage = (
|
||||
typeof error?.status_message === "string" && error.status_message
|
||||
? error.status_message
|
||||
: typeof error?.message === "string"
|
||||
? error.message
|
||||
: ""
|
||||
).trim();
|
||||
|
||||
const normalizedStatusMessage = statusMessage.toLowerCase();
|
||||
const mappedKey = normalizedStatusMessage.startsWith("web link request failed")
|
||||
? "webLinkRequestFailed"
|
||||
: normalizedStatusMessage.startsWith("knowledge media transcription only supports aliyun/qwen asr")
|
||||
? "mediaAsrProviderUnsupported"
|
||||
: normalizedStatusMessage.startsWith("asr request failed")
|
||||
? "mediaAsrRequestFailed"
|
||||
: WEB_LINK_ERROR_KEY_MAP[normalizedStatusMessage];
|
||||
if (mappedKey) {
|
||||
const translated = t(mappedKey, { defaultValue: "" });
|
||||
if (translated) return translated;
|
||||
}
|
||||
|
||||
if (statusCode !== undefined && statusCode !== null) {
|
||||
const translated = t(`bs:errors.${statusCode}`, { ...(error?.data || {}), defaultValue: "" });
|
||||
if (translated) return translated;
|
||||
}
|
||||
|
||||
return t(fallbackKey, { defaultValue: fallbackValue });
|
||||
};
|
||||
|
||||
const importWebLink = async (normalizedUrl: string, overwrite = false) => {
|
||||
const normalizedTitle = normalizeWebLinkTitle(webLinkTitle);
|
||||
await importKnowledgeWebLinkApi(knowledgeId!, {
|
||||
url: normalizedUrl,
|
||||
title: normalizedTitle,
|
||||
...(overwrite ? { overwrite: true } : {}),
|
||||
});
|
||||
message({
|
||||
variant: "success",
|
||||
description: t("webLinkImportStarted", { defaultValue: "网页链接已开始导入" }),
|
||||
});
|
||||
setWebLinkUrl("");
|
||||
setWebLinkTitle("");
|
||||
onOpenChange(false);
|
||||
onImported?.();
|
||||
};
|
||||
|
||||
const handleImportWebLink = async () => {
|
||||
const normalizedUrl = webLinkUrl.trim();
|
||||
if (!knowledgeId) {
|
||||
message({
|
||||
variant: "warning",
|
||||
description: t("knowledgeIdMissing", { defaultValue: "知识库不存在" }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!normalizedUrl) {
|
||||
message({
|
||||
variant: "warning",
|
||||
description: t("webLinkUrlRequired", { defaultValue: "请输入网页链接" }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(normalizedUrl);
|
||||
if (!["http:", "https:"].includes(parsed.protocol)) {
|
||||
message({
|
||||
variant: "warning",
|
||||
description: t("webLinkHttpOnly", { defaultValue: "仅支持 http 或 https 链接" }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
message({
|
||||
variant: "warning",
|
||||
description: t("webLinkInvalid", { defaultValue: "请输入有效的网页链接" }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setWebLinkLoading(true);
|
||||
try {
|
||||
await importWebLink(normalizedUrl);
|
||||
} catch (error: any) {
|
||||
if (isDuplicateError(error)) {
|
||||
bsConfirm({
|
||||
title: t("duplicateWebLinkTitle", { defaultValue: "发现重复网页链接" }),
|
||||
desc: normalizeWebLinkTitle(webLinkTitle) || normalizedUrl,
|
||||
canelTxt: t("cancel", { defaultValue: "取消" }),
|
||||
okTxt: t("overwrite", { defaultValue: "覆盖" }),
|
||||
onCancel: () => {
|
||||
setWebLinkUrl("");
|
||||
setWebLinkTitle("");
|
||||
onOpenChange(false);
|
||||
},
|
||||
onOk: async (close) => {
|
||||
close();
|
||||
onOpenChange(false);
|
||||
try {
|
||||
await importWebLink(normalizedUrl, true);
|
||||
} catch (overwriteError: any) {
|
||||
message({
|
||||
variant: "error",
|
||||
description: resolveErrorMessage(
|
||||
overwriteError,
|
||||
"webLinkOverwriteFailed",
|
||||
"网页链接覆盖失败",
|
||||
),
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
message({
|
||||
variant: "error",
|
||||
description: resolveErrorMessage(error, "webLinkImportFailed", "网页链接导入失败"),
|
||||
});
|
||||
} finally {
|
||||
setWebLinkLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(nextOpen) => !webLinkLoading && onOpenChange(nextOpen)}>
|
||||
<DialogContent className="max-w-[520px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t("webLink", { defaultValue: "网页链接" })}</DialogTitle>
|
||||
</DialogHeader>
|
||||
<form
|
||||
className="space-y-4"
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (webLinkLoading) return;
|
||||
void handleImportWebLink();
|
||||
}}
|
||||
>
|
||||
<label className="block space-y-2 text-sm text-[#1d2129]">
|
||||
<span className="font-medium">{t("webLinkUrl", { defaultValue: "链接地址" })}</span>
|
||||
<Input
|
||||
value={webLinkUrl}
|
||||
onChange={(event) => setWebLinkUrl(event.currentTarget.value)}
|
||||
placeholder="https://example.com/page"
|
||||
disabled={webLinkLoading}
|
||||
/>
|
||||
</label>
|
||||
<label className="block space-y-2 text-sm text-[#1d2129]">
|
||||
<span className="font-medium">{t("webLinkTitle", { defaultValue: "显示名称" })}</span>
|
||||
<Input
|
||||
value={webLinkTitle}
|
||||
onChange={(event) => setWebLinkTitle(event.currentTarget.value)}
|
||||
placeholder={t("webLinkTitlePlaceholder", {
|
||||
defaultValue: "留空则自动读取网页标题",
|
||||
})}
|
||||
disabled={webLinkLoading}
|
||||
/>
|
||||
</label>
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={() => onOpenChange(false)} disabled={webLinkLoading}>
|
||||
{t("cancel", { defaultValue: "取消" })}
|
||||
</Button>
|
||||
<Button type="submit" disabled={webLinkLoading}>
|
||||
{webLinkLoading
|
||||
? t("importing", { defaultValue: "导入中..." })
|
||||
: t("import", { defaultValue: "导入" })}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user