mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-24 16:29:01 +08:00
begoniezhao
2d66abedf0
移除日志设置与冗余代码，优化导入、类型提示及OCR后端管理 统一调整各文件模块导入路径为绝对导入 调整导入路径，移除部分导入，优化日志及注释 升级文档解析器为 Docx2Parser，优化超时与图片处理逻辑
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
import re
|
|
from typing import Callable, List
|
|
|
|
|
|
def split_text_keep_separator(text: str, separator: str) -> List[str]:
|
|
"""Split text with separator and keep the separator at the end of each split."""
|
|
parts = text.split(separator)
|
|
result = [separator + s if i > 0 else s for i, s in enumerate(parts)]
|
|
return [s for s in result if s]
|
|
|
|
|
|
def split_by_sep(sep: str, keep_sep: bool = True) -> Callable[[str], List[str]]:
|
|
"""Split text by separator."""
|
|
if keep_sep:
|
|
return lambda text: split_text_keep_separator(text, sep)
|
|
else:
|
|
return lambda text: text.split(sep)
|
|
|
|
|
|
def split_by_char() -> Callable[[str], List[str]]:
|
|
"""Split text by character."""
|
|
return lambda text: list(text)
|
|
|
|
|
|
def split_by_regex(regex: str) -> Callable[[str], List[str]]:
|
|
"""Split text by regex."""
|
|
pattern = re.compile(f"({regex})")
|
|
return lambda text: list(filter(None, pattern.split(text)))
|
|
|
|
|
|
def match_by_regex(regex: str) -> Callable[[str], bool]:
|
|
"""Split text by regex."""
|
|
pattern = re.compile(regex)
|
|
return lambda text: bool(pattern.match(text))
|