mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-10-15 10:58:21 +08:00
xiaowang
f3cd3d9724
English: Accept .html and .htm knowledge uploads across frontend, Go, and DocReader. Parse static HTML with charset-aware decoding, preserve document structure and local links, and keep scripts/network execution disabled. Add regression coverage for validation, engine registration, encoding, and parser behavior. 中文：打通知识库本地 .html 与 .htm 文件从前端、Go 服务到 DocReader 的完整上传链路。使用可识别字符编码的静态解析，保留文档结构和本地链接，不执行脚本或发起网络请求，并补充文件校验、引擎注册、编码及解析行为的回归测试。
38 lines
1.2 KiB
Python
38 lines
1.2 KiB
Python
"""Static HTML file parser."""
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from docreader.models.document import Document
|
|
from docreader.parser.base_parser import BaseParser
|
|
from docreader.parser.chain_parser import PipelineParser
|
|
from docreader.parser.markdown_parser import MarkdownParser
|
|
from docreader.parser.mhtml_parser import MHTMLParser
|
|
|
|
|
|
class HTMLToMarkdownParser(BaseParser):
|
|
"""Convert uploaded HTML bytes to Markdown without browser or network access."""
|
|
|
|
def parse_into_text(self, content: bytes) -> Document:
|
|
if not content.strip():
|
|
return Document()
|
|
|
|
# Inspect the original bytes so BOMs and HTML charset declarations are
|
|
# honored before the shared Markdown conversion runs.
|
|
html = BeautifulSoup(content, "lxml").decode()
|
|
markdown = MHTMLParser(
|
|
file_name=self.file_name,
|
|
file_type=self.file_type,
|
|
extract_images=False,
|
|
).html_to_markdown(
|
|
html,
|
|
strip_internal_links=False,
|
|
fallback_to_raw_html=False,
|
|
)
|
|
return Document(content=markdown or "")
|
|
|
|
|
|
class HTMLParser(PipelineParser):
|
|
"""Extract static HTML content and normalize the resulting Markdown."""
|
|
|
|
_parser_cls = (HTMLToMarkdownParser, MarkdownParser)
|