WeKnora/docreader/parser/text_parser.py

import logging
from .base_parser import BaseParser
from typing import Dict, Any, Tuple, Union

logger = logging.getLogger(__name__)


class TextParser(BaseParser):
    """
    Text document parser for processing plain text files.
    This parser handles text extraction and chunking from plain text documents.
    """

    def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
        """
        Parse text document content by decoding bytes to string.

        This is a straightforward parser that simply converts the binary content
        to text using appropriate character encoding.

        Args:
            content: Raw document content as bytes

        Returns:
            Parsed text content as string
        """
        logger.info(f"Parsing text document, content size: {len(content)} bytes")
        text = self.decode_bytes(content)
        logger.info(
            f"Successfully parsed text document, extracted {len(text)} characters"
        )
        return text


if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )
    logger.info("Running TextParser in standalone mode")

    # Sample text for testing
    text = """## 标题1
    ![alt text](image.png)
    ## 标题2
    ![alt text](image2.png)
    ## 标题3
    ![alt text](image3.png)"""
    logger.info(f"Test text content: {text}")

    # Define separators for text splitting
    seperators = ["\n\n", "\n", "。"]
    parser = TextParser(separators=seperators)
    logger.info("Splitting text into units")
    units = parser._split_into_units(text)
    logger.info(f"Split text into {len(units)} units")
    logger.info(f"Units: {units}")