feat: add epub support for knowledge base document upload (#7594)

* feat: add EPUB parsing support for knowledge base and file reader

* feat: update supported file formats for document upload in knowledge base

* feat: enhance EPUB parser to support spine order and generic containers

* makeitdown parse epub

* update parser

* fix
This commit is contained in:
Aster
2026-04-20 15:24:07 +08:00
committed by GitHub
parent 43989471e1
commit 76ee4f27dd
16 changed files with 566 additions and 24 deletions
+39 -2
View File
@@ -73,7 +73,7 @@ class FileProbe:
@dataclass(frozen=True)
class ParsedDocument:
kind: Literal["docx", "pdf"]
kind: Literal["docx", "epub", "pdf"]
file_bytes: bytes
text: str
@@ -371,6 +371,18 @@ def _is_docx_bytes(file_bytes: bytes) -> bool:
return any(name.startswith("word/") for name in names)
def _is_epub_bytes(file_bytes: bytes) -> bool:
try:
with zipfile.ZipFile(io.BytesIO(file_bytes)) as archive:
names = set(archive.namelist())
with archive.open("mimetype") as mimetype_file:
mimetype = mimetype_file.read(64).decode("utf-8").strip()
except (KeyError, OSError, UnicodeDecodeError, zipfile.BadZipFile):
return False
return mimetype == "application/epub+zip" and "META-INF/container.xml" in names
async def _parse_local_docx_text(file_bytes: bytes, file_name: str) -> str:
from astrbot.core.knowledge_base.parsers.markitdown_parser import (
MarkitdownParser,
@@ -387,23 +399,48 @@ async def _parse_local_pdf_text(file_bytes: bytes, file_name: str) -> str:
return result.text
async def _parse_local_epub_text(file_bytes: bytes, file_name: str) -> str:
from astrbot.core.knowledge_base.parsers.epub_parser import EpubParser
result = await EpubParser().parse(file_bytes, file_name)
return result.text
async def _parse_local_supported_document(
path: str,
sample: bytes,
) -> ParsedDocument | None:
file_name = Path(path).name
suffix = Path(path).suffix.lower()
if _looks_like_pdf(path, sample):
file_bytes = await _read_local_file_bytes(path)
text = await _parse_local_pdf_text(file_bytes, file_name)
return ParsedDocument(kind="pdf", file_bytes=file_bytes, text=text)
if Path(path).suffix.lower() == ".docx" or _looks_like_zip_container(sample):
if suffix == ".epub":
file_bytes = await _read_local_file_bytes(path)
if not _is_epub_bytes(file_bytes):
return None
text = await _parse_local_epub_text(file_bytes, file_name)
return ParsedDocument(kind="epub", file_bytes=file_bytes, text=text)
if suffix == ".docx":
file_bytes = await _read_local_file_bytes(path)
if not _is_docx_bytes(file_bytes):
return None
text = await _parse_local_docx_text(file_bytes, file_name)
return ParsedDocument(kind="docx", file_bytes=file_bytes, text=text)
if _looks_like_zip_container(sample):
file_bytes = await _read_local_file_bytes(path)
if _is_epub_bytes(file_bytes):
text = await _parse_local_epub_text(file_bytes, file_name)
return ParsedDocument(kind="epub", file_bytes=file_bytes, text=text)
if _is_docx_bytes(file_bytes):
text = await _parse_local_docx_text(file_bytes, file_name)
return ParsedDocument(kind="docx", file_bytes=file_bytes, text=text)
return None
return None
+8 -1
View File
@@ -109,6 +109,10 @@ Text chunk to process:
return [chunk]
def _compact_chunks(chunks: list[str]) -> list[str]:
return [chunk.strip() for chunk in chunks if chunk and chunk.strip()]
class KBHelper:
vec_db: BaseVecDB
kb: KnowledgeBase
@@ -249,7 +253,7 @@ class KBHelper:
if pre_chunked_text is not None:
# 如果提供了预分块文本,直接使用
chunks_text = pre_chunked_text
chunks_text = _compact_chunks(pre_chunked_text)
file_size = sum(len(chunk) for chunk in chunks_text)
logger.info(f"使用预分块文本进行上传,共 {len(chunks_text)} 个块。")
else:
@@ -316,6 +320,7 @@ class KBHelper:
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
chunks_text = _compact_chunks(chunks_text)
except KnowledgeBaseUploadError:
raise
except Exception as exc:
@@ -728,6 +733,8 @@ class KBHelper:
elif isinstance(result, list):
final_chunks.extend(result)
final_chunks = _compact_chunks(final_chunks)
logger.info(
f"文本修复完成: {len(initial_chunks)} 个原始块 -> {len(final_chunks)} 个最终块。"
)
@@ -1,11 +1,13 @@
"""文档解析器模块"""
from .base import BaseParser, MediaItem, ParseResult
from .epub_parser import EpubParser
from .pdf_parser import PDFParser
from .text_parser import TextParser
__all__ = [
"BaseParser",
"EpubParser",
"MediaItem",
"PDFParser",
"ParseResult",
@@ -0,0 +1,162 @@
"""EPUB document parser."""
import html
import re
from astrbot.core.knowledge_base.parsers.base import BaseParser, ParseResult
_KEYS = (
"Title|Author|Creator|Language|Publisher|Date|Modified|Identifier|ISBN|Description|"
"Subject|Rights|Source|Series|标题|书名|作者|语言|出版社|日期|出版日期|标识符|简介|描述|"
"主题|版权|来源|系列|タイトル|書名|著者|言語|出版社|日付|識別子|説明|件名|権利|ソース|シリーズ"
)
_META_RE = re.compile(rf"^\s*(?:[-*]\s*)?\*\*(?:{_KEYS})\s*[::]\*\*\s+\S")
_TOC_HEAD_RE = re.compile(
r"^\s{0,3}(?:#{1,6}\s*)?(?:table of contents|contents|toc|目录|目次|もくじ)\s*$",
re.I,
)
_LINK_RE = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)]+)\)")
_IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
_EMPTY_IMG_LINK_RE = re.compile(
r"\[\s*\]\([^)]+\.(?:png|jpe?g|gif|webp|svg)(?:#[^)]+)?\)", re.I
)
_FOOTNOTE_LABEL_RE = re.compile(
r"^(?:\d{1,3}|[ivxlcdm]{1,8}|[*†‡§¶]|↩|↑|back|return|返回|回到正文)$", re.I
)
_FOOTNOTE_HREF_RE = re.compile(
r"(?:^#|[#/_-](?:fn|footnote|note|noteref|backlink|return|filepos)\b)", re.I
)
_DOTTED_TOC_RE = re.compile(r"^\s*.+?\.{2,}\s*(?:\d+|[ivxlcdm]+)\s*$", re.I)
_SEP_RE = re.compile(r"^\s*(?:[-=*_]){3,}\s*$")
_NOISE_RE = re.compile(
r"^\s*(?:\[\s*)?(?:\d{1,3}|[ivxlcdm]{1,8}|[*†‡§¶]|↩|↑)(?:\s*\])?\s*$", re.I
)
_GENERIC_ALT_RE = re.compile(
r"^(?:image|img|picture|photo|illustration|figure|fig|cover|插图|图片|图像|封面)\s*[\d._-]*$",
re.I,
)
_FILENAME_ALT_RE = re.compile(r"^[\w.\- ]+\.(?:png|jpe?g|gif|webp|svg)$", re.I)
def _n(s: str) -> str:
return (
html.unescape(s)
.replace("\r\n", "\n")
.replace("\r", "\n")
.replace("\ufeff", "")
.replace("\u00a0", " ")
.replace("\u200b", "")
)
def _is_internal(href: str) -> bool:
href = html.unescape(href).strip().lower()
return (
href.startswith("#")
or href.endswith(".html")
or href.endswith(".xhtml")
or ".html#" in href
or ".xhtml#" in href
)
def _is_toc_line(s: str) -> bool:
s = s.strip()
if not s:
return False
s = re.sub(r"^\s*(?:[-*+]|\d+\.)\s+", "", s)
m = re.fullmatch(r"\[([^\]]+)\]\(([^)]+)\)", s)
return bool((m and _is_internal(m.group(2))) or _DOTTED_TOC_RE.match(s))
def _strip_head(text: str) -> str:
lines = _n(text).split("\n")
i = 0
while i < len(lines) and not lines[i].strip():
i += 1
start = i
while i < len(lines) and _META_RE.match(lines[i].strip()):
i += 1
if i - start >= 2:
while i < len(lines) and not lines[i].strip():
i += 1
else:
i = start
toc0, had_head = i, False
if i < len(lines) and _TOC_HEAD_RE.match(lines[i].strip()):
had_head = True
i += 1
while i < len(lines) and not lines[i].strip():
i += 1
toc = 0
while i < len(lines) and i - toc0 < 120:
s = lines[i].strip()
if not s:
if toc and i + 1 < len(lines) and _is_toc_line(lines[i + 1]):
i += 1
continue
break
if not _is_toc_line(s):
break
toc += 1
i += 1
if toc >= 2 and (had_head or toc >= 3):
while i < len(lines) and not lines[i].strip():
i += 1
return "\n".join(lines[i:]).strip()
return "\n".join(lines[toc0:]).strip()
def _strip_links(text: str) -> str:
def repl(m: re.Match[str]) -> str:
label = html.unescape(m.group(1)).strip()
href = html.unescape(m.group(2)).strip().lower()
if not _is_internal(href):
return m.group(0)
if _FOOTNOTE_HREF_RE.search(href) or (
href.startswith("#") and _FOOTNOTE_LABEL_RE.fullmatch(label)
):
return ""
return label
return _LINK_RE.sub(repl, _n(text))
def _img_alt(m: re.Match[str]) -> str:
alt = re.sub(r"\s+", " ", html.unescape(m.group(1)).strip())
if not alt or _GENERIC_ALT_RE.fullmatch(alt) or _FILENAME_ALT_RE.fullmatch(alt):
return ""
return alt
def _sanitize(text: str) -> str:
out, prev_blank, prev = [], True, ""
for raw in _n(text).split("\n"):
line = _IMG_RE.sub(_img_alt, raw)
line = _EMPTY_IMG_LINK_RE.sub("", line).rstrip()
s = line.strip()
if not s:
if not prev_blank:
out.append("")
prev_blank = True
continue
if _SEP_RE.match(s) or _NOISE_RE.match(s):
continue
norm = re.sub(r"^\s{0,3}#{1,6}\s*", "", s).strip("*_ ").casefold()
if norm and norm == prev and len(norm) <= 120:
continue
out.append(line)
prev_blank = False
prev = norm
return "\n".join(out).strip()
class EpubParser(BaseParser):
"""Parse EPUB files via MarkItDown."""
async def parse(self, file_content: bytes, file_name: str) -> ParseResult:
from .markitdown_parser import MarkitdownParser
result = await MarkitdownParser().parse(file_content, file_name)
text = _sanitize(_strip_links(_strip_head(result.text)))
return ParseResult(text=text, media=result.media)
@@ -6,6 +6,10 @@ async def select_parser(ext: str) -> BaseParser:
from .markitdown_parser import MarkitdownParser
return MarkitdownParser()
if ext == ".epub":
from .epub_parser import EpubParser
return EpubParser()
if ext == ".pdf":
from .pdf_parser import PDFParser
@@ -70,7 +70,7 @@
},
"upload": {
"title": "Upload Files to Knowledge Base",
"subtitle": "Supports txt, pdf, word, excel and other formats",
"subtitle": "Supports txt, pdf, epub, word, excel and other formats",
"dropzone": "Drag and drop files here or click to upload",
"chunkSettings": {
"title": "Chunk Settings",
@@ -152,4 +152,4 @@
"preRequisite": "Hint: Please go to the plugin market to install astrbot_plugin_url_2_knowledge_base and follow the instructions in the plugin documentation to complete the playwright installation before using this feature.",
"allChunksUploaded": "All chunks uploaded successfully"
}
}
}
@@ -46,7 +46,7 @@
"title": "Upload Document",
"selectFile": "Select File",
"dropzone": "Drop files here or click to select",
"supportedFormats": "Supported formats: ",
"supportedFormats": "Supported formats: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
"maxSize": "Max file size: 128MB",
"chunkSettings": "Chunk Settings",
"batchSettings": "Batch Settings",
@@ -46,7 +46,7 @@
"title": "Добавление контента",
"selectFile": "Файл",
"dropzone": "Нажмите или перетащите файл сюда",
"supportedFormats": "Форматы: ",
"supportedFormats": "Форматы: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
"maxSize": "Максимум: 128MB",
"chunkSettings": "Фрагментация",
"batchSettings": "Пакетная обработка",
@@ -115,4 +115,4 @@
"saveFailed": "Ошибка сохранения",
"tips": "Внимание! Изменение этих параметров повлияет на будущую выдачу базы знаний."
}
}
}
@@ -70,7 +70,7 @@
},
"upload": {
"title": "上传文件到知识库",
"subtitle": "支持 txt、pdf、word、excel 等多种格式",
"subtitle": "支持 txt、pdf、epub、word、excel 等多种格式",
"dropzone": "拖放文件到这里或点击上传",
"chunkSettings": {
"title": "分片设置",
@@ -152,4 +152,4 @@
"preRequisite": "提示:请先前往插件市场安装 astrbot_plugin_url_2_knowledge_base 并根据插件文档内的指示完成 playwright 安装后才可使用本功能",
"allChunksUploaded": "所有分片上传成功"
}
}
}
@@ -46,7 +46,7 @@
"title": "上传文档",
"selectFile": "选择文件",
"dropzone": "拖放文件到这里或点击选择",
"supportedFormats": "支持的格式: ",
"supportedFormats": "支持的格式: .txt, .md, .pdf, .docx, .epub, .xls, .xlsx",
"maxSize": "最大文件大小: 128MB",
"chunkSettings": "分块设置",
"batchSettings": "批处理设置",
+4 -5
View File
@@ -830,6 +830,7 @@ export default {
if (files.length > 0) {
this.selectedFile = files[0];
}
event.target.value = '';
},
onFileDrop(event) {
@@ -845,6 +846,8 @@ export default {
switch (extension) {
case 'pdf':
return 'mdi-file-pdf-box';
case 'epub':
return 'mdi-book-open-page-variant';
case 'doc':
case 'docx':
return 'mdi-file-word-box';
@@ -882,11 +885,7 @@ export default {
formData.append('chunk_overlap', this.overlap);
}
axios.post('/api/plug/alkaid/kb/collection/add_file', formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
})
axios.post('/api/plug/alkaid/kb/collection/add_file', formData)
.then(response => {
if (response.data.status === 'ok') {
this.showSnackbar(this.tm('messages.operationSuccess', { message: response.data.message }));
@@ -382,6 +382,7 @@ const deleteChunk = async (chunk: any) => {
const getFileIcon = (fileType: string) => {
const type = fileType?.toLowerCase() || ''
if (type.includes('pdf')) return 'mdi-file-pdf-box'
if (type.includes('epub')) return 'mdi-book-open-page-variant'
if (type.includes('md')) return 'mdi-language-markdown'
if (type.includes('txt')) return 'mdi-file-document-outline'
return 'mdi-file'
@@ -390,6 +391,7 @@ const getFileIcon = (fileType: string) => {
const getFileColor = (fileType: string) => {
const type = fileType?.toLowerCase() || ''
if (type.includes('pdf')) return 'error'
if (type.includes('epub')) return 'warning'
if (type.includes('md')) return 'info'
if (type.includes('txt')) return 'success'
return 'grey'
@@ -84,12 +84,10 @@
@dragover.prevent="isDragging = true" @dragleave="isDragging = false" @click="fileInput?.click()">
<v-icon size="64" color="primary">mdi-cloud-upload</v-icon>
<p class="mt-4 text-h6">{{ t('upload.dropzone') }}</p>
<p class="text-caption text-medium-emphasis mt-2">{{ t('upload.supportedFormats') }}.txt, .md, .pdf,
.docx,
.xls, .xlsx</p>
<p class="text-caption text-medium-emphasis mt-2">{{ t('upload.supportedFormats') }}</p>
<p class="text-caption text-medium-emphasis">{{ t('upload.maxSize') }}</p>
<p class="text-caption text-medium-emphasis">最多可上传 10 个文件</p>
<input ref="fileInput" type="file" multiple hidden accept=".txt,.md,.pdf,.docx,.xls,.xlsx"
<input ref="fileInput" type="file" multiple hidden accept=".txt,.md,.pdf,.docx,.epub,.xls,.xlsx"
@change="handleFileSelect" />
</div>
@@ -369,6 +367,7 @@ const handleFileSelect = (event: Event) => {
const newFiles = Array.from(target.files)
addFiles(newFiles)
}
target.value = ''
}
// 添加文件(检查数量限制)
@@ -432,9 +431,7 @@ const uploadFiles = async () => {
formData.append('tasks_limit', uploadSettings.value.tasks_limit.toString())
formData.append('max_retries', uploadSettings.value.max_retries.toString())
const response = await axios.post('/api/kb/document/upload', formData, {
headers: { 'Content-Type': 'multipart/form-data' }
})
const response = await axios.post('/api/kb/document/upload', formData)
if (response.data.status === 'ok') {
const result = response.data.data
@@ -717,6 +714,7 @@ const deleteDocument = async () => {
const getFileIcon = (fileType: string) => {
const type = fileType?.toLowerCase() || ''
if (type.includes('pdf')) return 'mdi-file-pdf-box'
if (type.includes('epub')) return 'mdi-book-open-page-variant'
if (type.includes('md') || type.includes('markdown')) return 'mdi-language-markdown'
if (type.includes('txt')) return 'mdi-file-document-outline'
if (type.includes('url')) return 'mdi-link-variant'
@@ -726,6 +724,7 @@ const getFileIcon = (fileType: string) => {
const getFileColor = (fileType: string) => {
const type = fileType?.toLowerCase() || ''
if (type.includes('pdf')) return 'error'
if (type.includes('epub')) return 'warning'
if (type.includes('md')) return 'info'
if (type.includes('txt')) return 'success'
if (type.includes('url')) return 'primary'
+1 -1
View File
@@ -53,4 +53,4 @@ shipyard-python-sdk>=0.2.4
shipyard-neo-sdk>=0.2.0
packaging>=24.2
qrcode>=8.2
python-ripgrep==0.0.8
python-ripgrep==0.0.8
+119
View File
@@ -92,6 +92,95 @@ def _make_large_text() -> str:
return "".join(f"line-{index:05d}-{'x' * 48}\n" for index in range(6000))
def _make_epub_bytes(*, chapter_count: int = 1) -> bytes:
manifest_items = [
'<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
]
spine_items = ['<itemref idref="nav"/>']
nav_links = []
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, mode="w") as archive:
archive.writestr(
"mimetype",
"application/epub+zip",
compress_type=zipfile.ZIP_STORED,
)
archive.writestr(
"META-INF/container.xml",
"""<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
""",
)
for index in range(1, chapter_count + 1):
manifest_items.append(
f'<item id="chapter{index}" href="chapter{index}.xhtml" '
'media-type="application/xhtml+xml"/>'
)
spine_items.append(f'<itemref idref="chapter{index}"/>')
nav_links.append(f'<li><a href="chapter{index}.xhtml">Chapter {index}</a></li>')
archive.writestr(
f"OEBPS/chapter{index}.xhtml",
f"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Chapter {index}</title>
</head>
<body>
<h1>Chapter {index}</h1>
<p>Paragraph {index}</p>
</body>
</html>
""",
)
archive.writestr(
"OEBPS/nav.xhtml",
"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Navigation</title>
</head>
<body>
<nav epub:type="toc" xmlns:epub="http://www.idpf.org/2007/ops">
<ol>
{links}
</ol>
</nav>
</body>
</html>
""".format(links="".join(nav_links)),
)
archive.writestr(
"OEBPS/content.opf",
"""<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:identifier id="bookid">test-book</dc:identifier>
<dc:title>Test Book</dc:title>
<dc:language>en</dc:language>
</metadata>
<manifest>
{manifest}
</manifest>
<spine>
{spine}
</spine>
</package>
""".format(
manifest="".join(manifest_items),
spine="".join(spine_items),
),
)
return buffer.getvalue()
def test_detect_text_encoding_allows_utf8_probe_cut_mid_character():
sample = '{"results": ["中文内容"]}'.encode()[:-1]
@@ -230,6 +319,36 @@ async def test_file_read_tool_reads_docx_via_parser_and_magic(
assert result == "doc-line-1\ndoc-line-2\n"
def test_is_epub_bytes_rejects_plain_zip_archive():
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, mode="w") as archive:
archive.writestr("README.txt", "hello")
assert file_read_utils._is_epub_bytes(buffer.getvalue()) is False
@pytest.mark.asyncio
async def test_file_read_tool_reads_epub_via_parser_and_magic(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
):
workspace = _setup_local_fs_tools(monkeypatch, tmp_path)
epub_path = workspace / "novel.bin"
epub_path.write_bytes(_make_epub_bytes(chapter_count=2))
async def _fake_parse_epub(_file_bytes: bytes, _file_name: str) -> str:
return "# Chapter 1\n\nParagraph 1\n"
monkeypatch.setattr(file_read_utils, "_parse_local_epub_text", _fake_parse_epub)
result = await fs_tools.FileReadTool().call(
_make_context(),
path="novel.bin",
)
assert result == "# Chapter 1\n\nParagraph 1\n"
@pytest.mark.asyncio
async def test_file_read_tool_stores_long_converted_document_in_workspace(
monkeypatch: pytest.MonkeyPatch,
+211
View File
@@ -0,0 +1,211 @@
from __future__ import annotations
import io
import zipfile
import pytest
from astrbot.core.knowledge_base.parsers.epub_parser import EpubParser
from astrbot.core.knowledge_base.parsers.util import select_parser
def _make_epub_bytes() -> bytes:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, mode="w") as archive:
archive.writestr(
"mimetype",
"application/epub+zip",
compress_type=zipfile.ZIP_STORED,
)
archive.writestr(
"META-INF/container.xml",
"""<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
""",
)
archive.writestr(
"OEBPS/nav.xhtml",
"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Navigation</title>
</head>
<body>
<nav epub:type="toc" xmlns:epub="http://www.idpf.org/2007/ops">
<ol>
<li><a href="chapter2.xhtml">Chapter 2</a></li>
<li><a href="chapter1.xhtml">Chapter 1</a></li>
</ol>
</nav>
</body>
</html>
""",
)
archive.writestr(
"OEBPS/chapter2.xhtml",
"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Chapter 2</title>
</head>
<body>
<h2>Second</h2>
<p>Beta paragraph.</p>
</body>
</html>
""",
)
archive.writestr(
"OEBPS/chapter1.xhtml",
"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Chapter 1</title>
</head>
<body>
<h1>First</h1>
<p>Alpha paragraph.</p>
<ul>
<li>Point A</li>
<li>Point B</li>
</ul>
</body>
</html>
""",
)
archive.writestr(
"OEBPS/content.opf",
"""<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:identifier id="bookid">test-book</dc:identifier>
<dc:title>Test Book</dc:title>
<dc:language>en</dc:language>
</metadata>
<manifest>
<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>
<item id="chapter2" href="chapter2.xhtml" media-type="application/xhtml+xml"/>
<item id="chapter1" href="chapter1.xhtml" media-type="application/xhtml+xml"/>
</manifest>
<spine>
<itemref idref="nav"/>
<itemref idref="chapter2"/>
<itemref idref="chapter1"/>
</spine>
</package>
""",
)
return buffer.getvalue()
def _make_epub_bytes_with_generic_content() -> bytes:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, mode="w") as archive:
archive.writestr(
"mimetype",
"application/epub+zip",
compress_type=zipfile.ZIP_STORED,
)
archive.writestr(
"META-INF/container.xml",
"""<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>
""",
)
archive.writestr(
"OEBPS/chapter1.xhtml",
"""<?xml version="1.0" encoding="utf-8"?>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Chapter 1</title>
</head>
<body>
<h1>First</h1>
Lead text
<p>Piura<a href="text00000.html#filepos0000045863">*5</a>, continued.</p>
<img src="Image00000.jpg" alt="" />
<div>Inside div</div>
<section>Inside section</section>
<table>
<tr>
<td>Cell A</td>
<td>Cell B</td>
</tr>
</table>
</body>
</html>
""",
)
archive.writestr(
"OEBPS/content.opf",
"""<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:identifier id="bookid">test-book</dc:identifier>
<dc:title>Test Book</dc:title>
<dc:language>en</dc:language>
</metadata>
<manifest>
<item id="chapter1" href="chapter1.xhtml" media-type="application/xhtml+xml"/>
</manifest>
<spine>
<itemref idref="chapter1"/>
</spine>
</package>
""",
)
return buffer.getvalue()
@pytest.mark.asyncio
async def test_select_parser_supports_epub():
parser = await select_parser(".epub")
assert isinstance(parser, EpubParser)
@pytest.mark.asyncio
async def test_epub_parser_reads_spine_order_as_text():
result = await EpubParser().parse(_make_epub_bytes(), "book.epub")
assert result.media == []
assert "**Title:**" not in result.text
assert "[Chapter 2](chapter2.xhtml)" not in result.text
assert result.text.startswith("1. Chapter 2")
assert "2. Chapter 1" in result.text
assert "Beta paragraph." in result.text
assert "# First" in result.text
assert "* Point A" in result.text
assert result.text.index("1. Chapter 2") < result.text.index("## Second")
assert result.text.index("## Second") < result.text.index("# First")
@pytest.mark.asyncio
async def test_epub_parser_preserves_generic_container_text():
result = await EpubParser().parse(
_make_epub_bytes_with_generic_content(),
"book.epub",
)
assert "**Title:**" not in result.text
assert "# First" in result.text
assert "Lead text" in result.text
assert r"Piura\*5, continued." in result.text
assert "filepos" not in result.text
assert r"[\*5]" not in result.text
assert "Image00000.jpg" not in result.text
assert "![](" not in result.text
assert "\n\n\n" not in result.text
assert "Inside div" in result.text
assert "Inside section" in result.text
assert "| Cell A | Cell B |" in result.text