mirror of
https://github.com/AstrBotDevs/AstrBot.git
synced 2026-10-28 02:27:46 +08:00
feat: add epub support for knowledge base document upload (#7594)
* feat: add EPUB parsing support for knowledge base and file reader * feat: update supported file formats for document upload in knowledge base * feat: enhance EPUB parser to support spine order and generic containers * makeitdown parse epub * update parser * fix
This commit is contained in:
@@ -73,7 +73,7 @@ class FileProbe:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedDocument:
|
||||
kind: Literal["docx", "pdf"]
|
||||
kind: Literal["docx", "epub", "pdf"]
|
||||
file_bytes: bytes
|
||||
text: str
|
||||
|
||||
@@ -371,6 +371,18 @@ def _is_docx_bytes(file_bytes: bytes) -> bool:
|
||||
return any(name.startswith("word/") for name in names)
|
||||
|
||||
|
||||
def _is_epub_bytes(file_bytes: bytes) -> bool:
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(file_bytes)) as archive:
|
||||
names = set(archive.namelist())
|
||||
with archive.open("mimetype") as mimetype_file:
|
||||
mimetype = mimetype_file.read(64).decode("utf-8").strip()
|
||||
except (KeyError, OSError, UnicodeDecodeError, zipfile.BadZipFile):
|
||||
return False
|
||||
|
||||
return mimetype == "application/epub+zip" and "META-INF/container.xml" in names
|
||||
|
||||
|
||||
async def _parse_local_docx_text(file_bytes: bytes, file_name: str) -> str:
|
||||
from astrbot.core.knowledge_base.parsers.markitdown_parser import (
|
||||
MarkitdownParser,
|
||||
@@ -387,23 +399,48 @@ async def _parse_local_pdf_text(file_bytes: bytes, file_name: str) -> str:
|
||||
return result.text
|
||||
|
||||
|
||||
async def _parse_local_epub_text(file_bytes: bytes, file_name: str) -> str:
|
||||
from astrbot.core.knowledge_base.parsers.epub_parser import EpubParser
|
||||
|
||||
result = await EpubParser().parse(file_bytes, file_name)
|
||||
return result.text
|
||||
|
||||
|
||||
async def _parse_local_supported_document(
|
||||
path: str,
|
||||
sample: bytes,
|
||||
) -> ParsedDocument | None:
|
||||
file_name = Path(path).name
|
||||
suffix = Path(path).suffix.lower()
|
||||
if _looks_like_pdf(path, sample):
|
||||
file_bytes = await _read_local_file_bytes(path)
|
||||
text = await _parse_local_pdf_text(file_bytes, file_name)
|
||||
return ParsedDocument(kind="pdf", file_bytes=file_bytes, text=text)
|
||||
|
||||
if Path(path).suffix.lower() == ".docx" or _looks_like_zip_container(sample):
|
||||
if suffix == ".epub":
|
||||
file_bytes = await _read_local_file_bytes(path)
|
||||
if not _is_epub_bytes(file_bytes):
|
||||
return None
|
||||
text = await _parse_local_epub_text(file_bytes, file_name)
|
||||
return ParsedDocument(kind="epub", file_bytes=file_bytes, text=text)
|
||||
|
||||
if suffix == ".docx":
|
||||
file_bytes = await _read_local_file_bytes(path)
|
||||
if not _is_docx_bytes(file_bytes):
|
||||
return None
|
||||
text = await _parse_local_docx_text(file_bytes, file_name)
|
||||
return ParsedDocument(kind="docx", file_bytes=file_bytes, text=text)
|
||||
|
||||
if _looks_like_zip_container(sample):
|
||||
file_bytes = await _read_local_file_bytes(path)
|
||||
if _is_epub_bytes(file_bytes):
|
||||
text = await _parse_local_epub_text(file_bytes, file_name)
|
||||
return ParsedDocument(kind="epub", file_bytes=file_bytes, text=text)
|
||||
if _is_docx_bytes(file_bytes):
|
||||
text = await _parse_local_docx_text(file_bytes, file_name)
|
||||
return ParsedDocument(kind="docx", file_bytes=file_bytes, text=text)
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -109,6 +109,10 @@ Text chunk to process:
|
||||
return [chunk]
|
||||
|
||||
|
||||
def _compact_chunks(chunks: list[str]) -> list[str]:
|
||||
return [chunk.strip() for chunk in chunks if chunk and chunk.strip()]
|
||||
|
||||
|
||||
class KBHelper:
|
||||
vec_db: BaseVecDB
|
||||
kb: KnowledgeBase
|
||||
@@ -249,7 +253,7 @@ class KBHelper:
|
||||
|
||||
if pre_chunked_text is not None:
|
||||
# 如果提供了预分块文本,直接使用
|
||||
chunks_text = pre_chunked_text
|
||||
chunks_text = _compact_chunks(pre_chunked_text)
|
||||
file_size = sum(len(chunk) for chunk in chunks_text)
|
||||
logger.info(f"使用预分块文本进行上传,共 {len(chunks_text)} 个块。")
|
||||
else:
|
||||
@@ -316,6 +320,7 @@ class KBHelper:
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
)
|
||||
chunks_text = _compact_chunks(chunks_text)
|
||||
except KnowledgeBaseUploadError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
@@ -728,6 +733,8 @@ class KBHelper:
|
||||
elif isinstance(result, list):
|
||||
final_chunks.extend(result)
|
||||
|
||||
final_chunks = _compact_chunks(final_chunks)
|
||||
|
||||
logger.info(
|
||||
f"文本修复完成: {len(initial_chunks)} 个原始块 -> {len(final_chunks)} 个最终块。"
|
||||
)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
"""文档解析器模块"""
|
||||
|
||||
from .base import BaseParser, MediaItem, ParseResult
|
||||
from .epub_parser import EpubParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .text_parser import TextParser
|
||||
|
||||
__all__ = [
|
||||
"BaseParser",
|
||||
"EpubParser",
|
||||
"MediaItem",
|
||||
"PDFParser",
|
||||
"ParseResult",
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""EPUB document parser."""
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
from astrbot.core.knowledge_base.parsers.base import BaseParser, ParseResult
|
||||
|
||||
_KEYS = (
|
||||
"Title|Author|Creator|Language|Publisher|Date|Modified|Identifier|ISBN|Description|"
|
||||
"Subject|Rights|Source|Series|标题|书名|作者|语言|出版社|日期|出版日期|标识符|简介|描述|"
|
||||
"主题|版权|来源|系列|タイトル|書名|著者|言語|出版社|日付|識別子|説明|件名|権利|ソース|シリーズ"
|
||||
)
|
||||
_META_RE = re.compile(rf"^\s*(?:[-*]\s*)?\*\*(?:{_KEYS})\s*[::]\*\*\s+\S")
|
||||
_TOC_HEAD_RE = re.compile(
|
||||
r"^\s{0,3}(?:#{1,6}\s*)?(?:table of contents|contents|toc|目录|目次|もくじ)\s*$",
|
||||
re.I,
|
||||
)
|
||||
_LINK_RE = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)]+)\)")
|
||||
_IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
_EMPTY_IMG_LINK_RE = re.compile(
|
||||
r"\[\s*\]\([^)]+\.(?:png|jpe?g|gif|webp|svg)(?:#[^)]+)?\)", re.I
|
||||
)
|
||||
_FOOTNOTE_LABEL_RE = re.compile(
|
||||
r"^(?:\d{1,3}|[ivxlcdm]{1,8}|[*†‡§¶]|↩|↑|back|return|返回|回到正文)$", re.I
|
||||
)
|
||||
_FOOTNOTE_HREF_RE = re.compile(
|
||||
r"(?:^#|[#/_-](?:fn|footnote|note|noteref|backlink|return|filepos)\b)", re.I
|
||||
)
|
||||
_DOTTED_TOC_RE = re.compile(r"^\s*.+?\.{2,}\s*(?:\d+|[ivxlcdm]+)\s*$", re.I)
|
||||
_SEP_RE = re.compile(r"^\s*(?:[-=*_]){3,}\s*$")
|
||||
_NOISE_RE = re.compile(
|
||||
r"^\s*(?:\[\s*)?(?:\d{1,3}|[ivxlcdm]{1,8}|[*†‡§¶]|↩|↑)(?:\s*\])?\s*$", re.I
|
||||
)
|
||||
_GENERIC_ALT_RE = re.compile(
|
||||
r"^(?:image|img|picture|photo|illustration|figure|fig|cover|插图|图片|图像|封面)\s*[\d._-]*$",
|
||||
re.I,
|
||||
)
|
||||
_FILENAME_ALT_RE = re.compile(r"^[\w.\- ]+\.(?:png|jpe?g|gif|webp|svg)$", re.I)
|
||||
|
||||
|
||||
def _n(s: str) -> str:
|
||||
return (
|
||||
html.unescape(s)
|
||||
.replace("\r\n", "\n")
|
||||
.replace("\r", "\n")
|
||||
.replace("\ufeff", "")
|
||||
.replace("\u00a0", " ")
|
||||
.replace("\u200b", "")
|
||||
)
|
||||
|
||||
|
||||
def _is_internal(href: str) -> bool:
|
||||
href = html.unescape(href).strip().lower()
|
||||
return (
|
||||
href.startswith("#")
|
||||
or href.endswith(".html")
|
||||
or href.endswith(".xhtml")
|
||||
or ".html#" in href
|
||||
or ".xhtml#" in href
|
||||
)
|
||||
|
||||
|
||||
def _is_toc_line(s: str) -> bool:
|
||||
s = s.strip()
|
||||
if not s:
|
||||
return False
|
||||
s = re.sub(r"^\s*(?:[-*+]|\d+\.)\s+", "", s)
|
||||
m = re.fullmatch(r"\[([^\]]+)\]\(([^)]+)\)", s)
|
||||
return bool((m and _is_internal(m.group(2))) or _DOTTED_TOC_RE.match(s))
|
||||
|
||||
|
||||
def _strip_head(text: str) -> str:
|
||||
lines = _n(text).split("\n")
|
||||
i = 0
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
start = i
|
||||
while i < len(lines) and _META_RE.match(lines[i].strip()):
|
||||
i += 1
|
||||
if i - start >= 2:
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
else:
|
||||
i = start
|
||||
toc0, had_head = i, False
|
||||
if i < len(lines) and _TOC_HEAD_RE.match(lines[i].strip()):
|
||||
had_head = True
|
||||
i += 1
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
toc = 0
|
||||
while i < len(lines) and i - toc0 < 120:
|
||||
s = lines[i].strip()
|
||||
if not s:
|
||||
if toc and i + 1 < len(lines) and _is_toc_line(lines[i + 1]):
|
||||
i += 1
|
||||
continue
|
||||
break
|
||||
if not _is_toc_line(s):
|
||||
break
|
||||
toc += 1
|
||||
i += 1
|
||||
if toc >= 2 and (had_head or toc >= 3):
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
return "\n".join(lines[i:]).strip()
|
||||
return "\n".join(lines[toc0:]).strip()
|
||||
|
||||
|
||||
def _strip_links(text: str) -> str:
|
||||
def repl(m: re.Match[str]) -> str:
|
||||
label = html.unescape(m.group(1)).strip()
|
||||
href = html.unescape(m.group(2)).strip().lower()
|
||||
if not _is_internal(href):
|
||||
return m.group(0)
|
||||
if _FOOTNOTE_HREF_RE.search(href) or (
|
||||
href.startswith("#") and _FOOTNOTE_LABEL_RE.fullmatch(label)
|
||||
):
|
||||
return ""
|
||||
return label
|
||||
|
||||
return _LINK_RE.sub(repl, _n(text))
|
||||
|
||||
|
||||
def _img_alt(m: re.Match[str]) -> str:
|
||||
alt = re.sub(r"\s+", " ", html.unescape(m.group(1)).strip())
|
||||
if not alt or _GENERIC_ALT_RE.fullmatch(alt) or _FILENAME_ALT_RE.fullmatch(alt):
|
||||
return ""
|
||||
return alt
|
||||
|
||||
|
||||
def _sanitize(text: str) -> str:
|
||||
out, prev_blank, prev = [], True, ""
|
||||
for raw in _n(text).split("\n"):
|
||||
line = _IMG_RE.sub(_img_alt, raw)
|
||||
line = _EMPTY_IMG_LINK_RE.sub("", line).rstrip()
|
||||
s = line.strip()
|
||||
if not s:
|
||||
if not prev_blank:
|
||||
out.append("")
|
||||
prev_blank = True
|
||||
continue
|
||||
if _SEP_RE.match(s) or _NOISE_RE.match(s):
|
||||
continue
|
||||
norm = re.sub(r"^\s{0,3}#{1,6}\s*", "", s).strip("*_ ").casefold()
|
||||
if norm and norm == prev and len(norm) <= 120:
|
||||
continue
|
||||
out.append(line)
|
||||
prev_blank = False
|
||||
prev = norm
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
class EpubParser(BaseParser):
|
||||
"""Parse EPUB files via MarkItDown."""
|
||||
|
||||
async def parse(self, file_content: bytes, file_name: str) -> ParseResult:
|
||||
from .markitdown_parser import MarkitdownParser
|
||||
|
||||
result = await MarkitdownParser().parse(file_content, file_name)
|
||||
text = _sanitize(_strip_links(_strip_head(result.text)))
|
||||
return ParseResult(text=text, media=result.media)
|
||||
@@ -6,6 +6,10 @@ async def select_parser(ext: str) -> BaseParser:
|
||||
from .markitdown_parser import MarkitdownParser
|
||||
|
||||
return MarkitdownParser()
|
||||
if ext == ".epub":
|
||||
from .epub_parser import EpubParser
|
||||
|
||||
return EpubParser()
|
||||
if ext == ".pdf":
|
||||
from .pdf_parser import PDFParser
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"upload": {
|
||||
"title": "Upload Files to Knowledge Base",
|
||||
"subtitle": "Supports txt, pdf, word, excel and other formats",
|
||||
"subtitle": "Supports txt, pdf, epub, word, excel and other formats",
|
||||
"dropzone": "Drag and drop files here or click to upload",
|
||||
"chunkSettings": {
|
||||
"title": "Chunk Settings",
|
||||
@@ -152,4 +152,4 @@
|
||||
"preRequisite": "Hint: Please go to the plugin market to install astrbot_plugin_url_2_knowledge_base and follow the instructions in the plugin documentation to complete the playwright installation before using this feature.",
|
||||
"allChunksUploaded": "All chunks uploaded successfully"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
"title": "Upload Document",
|
||||
"selectFile": "Select File",
|
||||
"dropzone": "Drop files here or click to select",
|
||||
"supportedFormats": "Supported formats: ",
|
||||
"supportedFormats": "Supported formats: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
|
||||
"maxSize": "Max file size: 128MB",
|
||||
"chunkSettings": "Chunk Settings",
|
||||
"batchSettings": "Batch Settings",
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
"title": "Добавление контента",
|
||||
"selectFile": "Файл",
|
||||
"dropzone": "Нажмите или перетащите файл сюда",
|
||||
"supportedFormats": "Форматы: ",
|
||||
"supportedFormats": "Форматы: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
|
||||
"maxSize": "Максимум: 128MB",
|
||||
"chunkSettings": "Фрагментация",
|
||||
"batchSettings": "Пакетная обработка",
|
||||
@@ -115,4 +115,4 @@
|
||||
"saveFailed": "Ошибка сохранения",
|
||||
"tips": "Внимание! Изменение этих параметров повлияет на будущую выдачу базы знаний."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"upload": {
|
||||
"title": "上传文件到知识库",
|
||||
"subtitle": "支持 txt、pdf、word、excel 等多种格式",
|
||||
"subtitle": "支持 txt、pdf、epub、word、excel 等多种格式",
|
||||
"dropzone": "拖放文件到这里或点击上传",
|
||||
"chunkSettings": {
|
||||
"title": "分片设置",
|
||||
@@ -152,4 +152,4 @@
|
||||
"preRequisite": "提示:请先前往插件市场安装 astrbot_plugin_url_2_knowledge_base 并根据插件文档内的指示完成 playwright 安装后才可使用本功能",
|
||||
"allChunksUploaded": "所有分片上传成功"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
"title": "上传文档",
|
||||
"selectFile": "选择文件",
|
||||
"dropzone": "拖放文件到这里或点击选择",
|
||||
"supportedFormats": "支持的格式: ",
|
||||
"supportedFormats": "支持的格式: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
|
||||
"maxSize": "最大文件大小: 128MB",
|
||||
"chunkSettings": "分块设置",
|
||||
"batchSettings": "批处理设置",
|
||||
|
||||
@@ -830,6 +830,7 @@ export default {
|
||||
if (files.length > 0) {
|
||||
this.selectedFile = files[0];
|
||||
}
|
||||
event.target.value = '';
|
||||
},
|
||||
|
||||
onFileDrop(event) {
|
||||
@@ -845,6 +846,8 @@ export default {
|
||||
switch (extension) {
|
||||
case 'pdf':
|
||||
return 'mdi-file-pdf-box';
|
||||
case 'epub':
|
||||
return 'mdi-book-open-page-variant';
|
||||
case 'doc':
|
||||
case 'docx':
|
||||
return 'mdi-file-word-box';
|
||||
@@ -882,11 +885,7 @@ export default {
|
||||
formData.append('chunk_overlap', this.overlap);
|
||||
}
|
||||
|
||||
axios.post('/api/plug/alkaid/kb/collection/add_file', formData, {
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data'
|
||||
}
|
||||
})
|
||||
axios.post('/api/plug/alkaid/kb/collection/add_file', formData)
|
||||
.then(response => {
|
||||
if (response.data.status === 'ok') {
|
||||
this.showSnackbar(this.tm('messages.operationSuccess', { message: response.data.message }));
|
||||
|
||||
@@ -382,6 +382,7 @@ const deleteChunk = async (chunk: any) => {
|
||||
const getFileIcon = (fileType: string) => {
|
||||
const type = fileType?.toLowerCase() || ''
|
||||
if (type.includes('pdf')) return 'mdi-file-pdf-box'
|
||||
if (type.includes('epub')) return 'mdi-book-open-page-variant'
|
||||
if (type.includes('md')) return 'mdi-language-markdown'
|
||||
if (type.includes('txt')) return 'mdi-file-document-outline'
|
||||
return 'mdi-file'
|
||||
@@ -390,6 +391,7 @@ const getFileIcon = (fileType: string) => {
|
||||
const getFileColor = (fileType: string) => {
|
||||
const type = fileType?.toLowerCase() || ''
|
||||
if (type.includes('pdf')) return 'error'
|
||||
if (type.includes('epub')) return 'warning'
|
||||
if (type.includes('md')) return 'info'
|
||||
if (type.includes('txt')) return 'success'
|
||||
return 'grey'
|
||||
|
||||
@@ -84,12 +84,10 @@
|
||||
@dragover.prevent="isDragging = true" @dragleave="isDragging = false" @click="fileInput?.click()">
|
||||
<v-icon size="64" color="primary">mdi-cloud-upload</v-icon>
|
||||
<p class="mt-4 text-h6">{{ t('upload.dropzone') }}</p>
|
||||
<p class="text-caption text-medium-emphasis mt-2">{{ t('upload.supportedFormats') }}.txt, .md, .pdf,
|
||||
.docx,
|
||||
.xls, .xlsx</p>
|
||||
<p class="text-caption text-medium-emphasis mt-2">{{ t('upload.supportedFormats') }}</p>
|
||||
<p class="text-caption text-medium-emphasis">{{ t('upload.maxSize') }}</p>
|
||||
<p class="text-caption text-medium-emphasis">最多可上传 10 个文件</p>
|
||||
<input ref="fileInput" type="file" multiple hidden accept=".txt,.md,.pdf,.docx,.xls,.xlsx"
|
||||
<input ref="fileInput" type="file" multiple hidden accept=".txt,.md,.pdf,.docx,.epub,.xls,.xlsx"
|
||||
@change="handleFileSelect" />
|
||||
</div>
|
||||
|
||||
@@ -369,6 +367,7 @@ const handleFileSelect = (event: Event) => {
|
||||
const newFiles = Array.from(target.files)
|
||||
addFiles(newFiles)
|
||||
}
|
||||
target.value = ''
|
||||
}
|
||||
|
||||
// 添加文件(检查数量限制)
|
||||
@@ -432,9 +431,7 @@ const uploadFiles = async () => {
|
||||
formData.append('tasks_limit', uploadSettings.value.tasks_limit.toString())
|
||||
formData.append('max_retries', uploadSettings.value.max_retries.toString())
|
||||
|
||||
const response = await axios.post('/api/kb/document/upload', formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' }
|
||||
})
|
||||
const response = await axios.post('/api/kb/document/upload', formData)
|
||||
|
||||
if (response.data.status === 'ok') {
|
||||
const result = response.data.data
|
||||
@@ -717,6 +714,7 @@ const deleteDocument = async () => {
|
||||
const getFileIcon = (fileType: string) => {
|
||||
const type = fileType?.toLowerCase() || ''
|
||||
if (type.includes('pdf')) return 'mdi-file-pdf-box'
|
||||
if (type.includes('epub')) return 'mdi-book-open-page-variant'
|
||||
if (type.includes('md') || type.includes('markdown')) return 'mdi-language-markdown'
|
||||
if (type.includes('txt')) return 'mdi-file-document-outline'
|
||||
if (type.includes('url')) return 'mdi-link-variant'
|
||||
@@ -726,6 +724,7 @@ const getFileIcon = (fileType: string) => {
|
||||
const getFileColor = (fileType: string) => {
|
||||
const type = fileType?.toLowerCase() || ''
|
||||
if (type.includes('pdf')) return 'error'
|
||||
if (type.includes('epub')) return 'warning'
|
||||
if (type.includes('md')) return 'info'
|
||||
if (type.includes('txt')) return 'success'
|
||||
if (type.includes('url')) return 'primary'
|
||||
|
||||
+1
-1
@@ -53,4 +53,4 @@ shipyard-python-sdk>=0.2.4
|
||||
shipyard-neo-sdk>=0.2.0
|
||||
packaging>=24.2
|
||||
qrcode>=8.2
|
||||
python-ripgrep==0.0.8
|
||||
python-ripgrep==0.0.8
|
||||
|
||||
@@ -92,6 +92,95 @@ def _make_large_text() -> str:
|
||||
return "".join(f"line-{index:05d}-{'x' * 48}\n" for index in range(6000))
|
||||
|
||||
|
||||
def _make_epub_bytes(*, chapter_count: int = 1) -> bytes:
|
||||
manifest_items = [
|
||||
'<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
|
||||
]
|
||||
spine_items = ['<itemref idref="nav"/>']
|
||||
nav_links = []
|
||||
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, mode="w") as archive:
|
||||
archive.writestr(
|
||||
"mimetype",
|
||||
"application/epub+zip",
|
||||
compress_type=zipfile.ZIP_STORED,
|
||||
)
|
||||
archive.writestr(
|
||||
"META-INF/container.xml",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
""",
|
||||
)
|
||||
|
||||
for index in range(1, chapter_count + 1):
|
||||
manifest_items.append(
|
||||
f'<item id="chapter{index}" href="chapter{index}.xhtml" '
|
||||
'media-type="application/xhtml+xml"/>'
|
||||
)
|
||||
spine_items.append(f'<itemref idref="chapter{index}"/>')
|
||||
nav_links.append(f'<li><a href="chapter{index}.xhtml">Chapter {index}</a></li>')
|
||||
archive.writestr(
|
||||
f"OEBPS/chapter{index}.xhtml",
|
||||
f"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Chapter {index}</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Chapter {index}</h1>
|
||||
<p>Paragraph {index}</p>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
)
|
||||
|
||||
archive.writestr(
|
||||
"OEBPS/nav.xhtml",
|
||||
"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Navigation</title>
|
||||
</head>
|
||||
<body>
|
||||
<nav epub:type="toc" xmlns:epub="http://www.idpf.org/2007/ops">
|
||||
<ol>
|
||||
{links}
|
||||
</ol>
|
||||
</nav>
|
||||
</body>
|
||||
</html>
|
||||
""".format(links="".join(nav_links)),
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/content.opf",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">test-book</dc:identifier>
|
||||
<dc:title>Test Book</dc:title>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
{manifest}
|
||||
</manifest>
|
||||
<spine>
|
||||
{spine}
|
||||
</spine>
|
||||
</package>
|
||||
""".format(
|
||||
manifest="".join(manifest_items),
|
||||
spine="".join(spine_items),
|
||||
),
|
||||
)
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def test_detect_text_encoding_allows_utf8_probe_cut_mid_character():
|
||||
sample = '{"results": ["中文内容"]}'.encode()[:-1]
|
||||
|
||||
@@ -230,6 +319,36 @@ async def test_file_read_tool_reads_docx_via_parser_and_magic(
|
||||
assert result == "doc-line-1\ndoc-line-2\n"
|
||||
|
||||
|
||||
def test_is_epub_bytes_rejects_plain_zip_archive():
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, mode="w") as archive:
|
||||
archive.writestr("README.txt", "hello")
|
||||
|
||||
assert file_read_utils._is_epub_bytes(buffer.getvalue()) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_read_tool_reads_epub_via_parser_and_magic(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
):
|
||||
workspace = _setup_local_fs_tools(monkeypatch, tmp_path)
|
||||
epub_path = workspace / "novel.bin"
|
||||
epub_path.write_bytes(_make_epub_bytes(chapter_count=2))
|
||||
|
||||
async def _fake_parse_epub(_file_bytes: bytes, _file_name: str) -> str:
|
||||
return "# Chapter 1\n\nParagraph 1\n"
|
||||
|
||||
monkeypatch.setattr(file_read_utils, "_parse_local_epub_text", _fake_parse_epub)
|
||||
|
||||
result = await fs_tools.FileReadTool().call(
|
||||
_make_context(),
|
||||
path="novel.bin",
|
||||
)
|
||||
|
||||
assert result == "# Chapter 1\n\nParagraph 1\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_read_tool_stores_long_converted_document_in_workspace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from astrbot.core.knowledge_base.parsers.epub_parser import EpubParser
|
||||
from astrbot.core.knowledge_base.parsers.util import select_parser
|
||||
|
||||
|
||||
def _make_epub_bytes() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, mode="w") as archive:
|
||||
archive.writestr(
|
||||
"mimetype",
|
||||
"application/epub+zip",
|
||||
compress_type=zipfile.ZIP_STORED,
|
||||
)
|
||||
archive.writestr(
|
||||
"META-INF/container.xml",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/nav.xhtml",
|
||||
"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Navigation</title>
|
||||
</head>
|
||||
<body>
|
||||
<nav epub:type="toc" xmlns:epub="http://www.idpf.org/2007/ops">
|
||||
<ol>
|
||||
<li><a href="chapter2.xhtml">Chapter 2</a></li>
|
||||
<li><a href="chapter1.xhtml">Chapter 1</a></li>
|
||||
</ol>
|
||||
</nav>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/chapter2.xhtml",
|
||||
"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Chapter 2</title>
|
||||
</head>
|
||||
<body>
|
||||
<h2>Second</h2>
|
||||
<p>Beta paragraph.</p>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/chapter1.xhtml",
|
||||
"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Chapter 1</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>First</h1>
|
||||
<p>Alpha paragraph.</p>
|
||||
<ul>
|
||||
<li>Point A</li>
|
||||
<li>Point B</li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/content.opf",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">test-book</dc:identifier>
|
||||
<dc:title>Test Book</dc:title>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>
|
||||
<item id="chapter2" href="chapter2.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="chapter1" href="chapter1.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="nav"/>
|
||||
<itemref idref="chapter2"/>
|
||||
<itemref idref="chapter1"/>
|
||||
</spine>
|
||||
</package>
|
||||
""",
|
||||
)
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _make_epub_bytes_with_generic_content() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, mode="w") as archive:
|
||||
archive.writestr(
|
||||
"mimetype",
|
||||
"application/epub+zip",
|
||||
compress_type=zipfile.ZIP_STORED,
|
||||
)
|
||||
archive.writestr(
|
||||
"META-INF/container.xml",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/chapter1.xhtml",
|
||||
"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>Chapter 1</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>First</h1>
|
||||
Lead text
|
||||
<p>Piura<a href="text00000.html#filepos0000045863">*5</a>, continued.</p>
|
||||
<img src="Image00000.jpg" alt="" />
|
||||
<div>Inside div</div>
|
||||
<section>Inside section</section>
|
||||
<table>
|
||||
<tr>
|
||||
<td>Cell A</td>
|
||||
<td>Cell B</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/content.opf",
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">test-book</dc:identifier>
|
||||
<dc:title>Test Book</dc:title>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="chapter1" href="chapter1.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="chapter1"/>
|
||||
</spine>
|
||||
</package>
|
||||
""",
|
||||
)
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_parser_supports_epub():
|
||||
parser = await select_parser(".epub")
|
||||
|
||||
assert isinstance(parser, EpubParser)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_epub_parser_reads_spine_order_as_text():
|
||||
result = await EpubParser().parse(_make_epub_bytes(), "book.epub")
|
||||
|
||||
assert result.media == []
|
||||
assert "**Title:**" not in result.text
|
||||
assert "[Chapter 2](chapter2.xhtml)" not in result.text
|
||||
assert result.text.startswith("1. Chapter 2")
|
||||
assert "2. Chapter 1" in result.text
|
||||
assert "Beta paragraph." in result.text
|
||||
assert "# First" in result.text
|
||||
assert "* Point A" in result.text
|
||||
assert result.text.index("1. Chapter 2") < result.text.index("## Second")
|
||||
assert result.text.index("## Second") < result.text.index("# First")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_epub_parser_preserves_generic_container_text():
|
||||
result = await EpubParser().parse(
|
||||
_make_epub_bytes_with_generic_content(),
|
||||
"book.epub",
|
||||
)
|
||||
|
||||
assert "**Title:**" not in result.text
|
||||
assert "# First" in result.text
|
||||
assert "Lead text" in result.text
|
||||
assert r"Piura\*5, continued." in result.text
|
||||
assert "filepos" not in result.text
|
||||
assert r"[\*5]" not in result.text
|
||||
assert "Image00000.jpg" not in result.text
|
||||
assert "![](" not in result.text
|
||||
assert "\n\n\n" not in result.text
|
||||
assert "Inside div" in result.text
|
||||
assert "Inside section" in result.text
|
||||
assert "| Cell A | Cell B |" in result.text
|
||||
Reference in New Issue
Block a user