fix(knowledge): hard-split separator-proof fragments instead of failing the file

ETL4LM returns markdown tables whose rows can run past 10000 chars with no
\n / 。 / . inside them (one observed row: 7820 chars across 601 cells, the
result of a merged cell being expanded into every sub-cell). Such a fragment
survived ElemCharacterTextSplitter untouched — _split_text appends it verbatim
once new_separators is empty — and then tripped SplitterTransformer's
max_chunk_limit, failing the whole file with 10912. PaddleOCR and MinerU never
hit this because both rewrite tables to per-row markdown before splitting; ETL
is the only PDF loader without that step.

Add hard_split_limit to ElemCharacterTextSplitter, injected by
SplitterTransformer from its max chunk cap. _force_split kicks in only past
that limit, so documents that split fine today keep byte-identical chunk
boundaries and only files that would have been rejected change. This mirrors
what HierarchicalSplitterTransformer._split_long_text already does for the
hierarchical mode.

Also guard the separator_rule lookup against separators containing "".

Backport of 1ae4ca919. This branch predates KNOWLEDGE_MAX_CHUNK_CHARS, so the
cap stays the local 10000 literal and the test defines it inline; ruff
reformatted splitter.py while applying the change.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/backend/bisheng/knowledge/rag/pipeline/transformer/splitter.py

@@ -1,5 +1,6 @@
 import json
-from typing import Any, List, Optional, Sequence
+from collections.abc import Sequence
+from typing import Any
 
 from langchain_core.documents import BaseDocumentTransformer, Document
 
@@ -12,29 +13,37 @@ class SplitterTransformer(BaseDocumentTransformer):
     Splits text documents using ElemCharacterTextSplitter.
     """
 
-    def __init__(self,
-                 separator: Optional[List[str]] = None,
-                 separator_rule: Optional[List[str]] = None,
-                 chunk_size: int = 1000,
-                 chunk_overlap: int = 100, **kwargs) -> None:
+    def __init__(
+        self,
+        separator: list[str] | None = None,
+        separator_rule: list[str] | None = None,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 100,
+        **kwargs,
+    ) -> None:
+        self.max_chunk_limit = 10000
         self.text_splitter = ElemCharacterTextSplitter(
             separators=separator,
             separator_rule=separator_rule,
             chunk_size=chunk_size,
             chunk_overlap=chunk_overlap,
             is_separator_regex=True,
-            **kwargs)
-        self.max_chunk_limit = 10000
+            # Degrade separator-proof fragments to a character-level split instead
+            # of letting them reach the max_chunk_limit check below and fail the file.
+            hard_split_limit=self.max_chunk_limit,
+            **kwargs,
+        )
 
-    def transform_documents(
-            self, documents: Sequence[Document], **kwargs: Any
-    ) -> Sequence[Document]:
+    def transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> Sequence[Document]:
         documents = self.text_splitter.split_documents(documents)
         for index, one in enumerate(documents):
             one.metadata["chunk_index"] = index
             one.metadata["bbox"] = json.dumps({"chunk_bboxes": one.metadata.get("chunk_bboxes", "")})
-            one.metadata['page'] = one.metadata["chunk_bboxes"][0].get("page") if one.metadata.get("chunk_bboxes", None) \
-                else one.metadata.get('page', 0)
+            one.metadata["page"] = (
+                one.metadata["chunk_bboxes"][0].get("page")
+                if one.metadata.get("chunk_bboxes", None)
+                else one.metadata.get("page", 0)
+            )
             if len(one.page_content) > 10000:
                 raise KnowledgeFileChunkMaxError()
         return documents

src/backend/bisheng_langchain/text_splitter.py

@@ -7,7 +7,7 @@ import copy
 import logging
 import re
 from collections import Counter
-from typing import (Any, Iterable, List, Optional)
+from typing import Any, Iterable, List, Optional
 
 from langchain_classic.docstore.document import Document
 from langchain_classic.text_splitter import RecursiveCharacterTextSplitter
@@ -15,14 +15,12 @@ from langchain_classic.text_splitter import RecursiveCharacterTextSplitter
 logger = logging.getLogger(__name__)
 
 
-def _split_text_with_regex(
-        text: str, separator: str, keep_separator: bool, separator_rule: str
-) -> List[str]:
+def _split_text_with_regex(text: str, separator: str, keep_separator: bool, separator_rule: str) -> List[str]:
     # Now that we have the separator, split the text
     if separator:
         if keep_separator:
             # The parentheses in the pattern keep the delimiters in the result.
-            _splits = re.split(f'({separator})', text)
+            _splits = re.split(f"({separator})", text)
 
             if separator_rule == "before":
                 splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
@@ -36,7 +34,7 @@ def _split_text_with_regex(
             splits = re.split(separator, text)
     else:
         splits = list(text)
-    return [s for s in splits if s != '']
+    return [s for s in splits if s != ""]
 
 
 class IntervalSearch(object):
@@ -84,24 +82,22 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
     """
 
     def __init__(
-            self,
-            separators: Optional[List[str]] = None,
-            separator_rule: Optional[List[str]] = None,
-            is_separator_regex: bool = False,
-            keep_separator: bool = True,
-            **kwargs: Any,
+        self,
+        separators: Optional[List[str]] = None,
+        separator_rule: Optional[List[str]] = None,
+        is_separator_regex: bool = False,
+        keep_separator: bool = True,
+        hard_split_limit: Optional[int] = None,
+        **kwargs: Any,
     ) -> None:
         """Create a new TextSplitter."""
-        super().__init__(
-            separators=separators,
-            keep_separator=keep_separator,
-            **kwargs
-        )
-        self._separators = separators or ['\n\n', '\n', ' ', '']
-        self._separator_rule = separator_rule or ['after' for _ in range(4)]
+        super().__init__(separators=separators, keep_separator=keep_separator, **kwargs)
+        self._separators = separators or ["\n\n", "\n", " ", ""]
+        self._separator_rule = separator_rule or ["after" for _ in range(4)]
         self.separator_rule = {one: self._separator_rule[index] for index, one in enumerate(separators)}
         self._is_separator_regex = is_separator_regex
-        self._chunk_overlap = kwargs.get('chunk_overlap', 0)
+        self._chunk_overlap = kwargs.get("chunk_overlap", 0)
+        self.hard_split_limit = hard_split_limit
 
     def split_documents(self, documents: Iterable[Document]) -> List[Document]:
         texts, metadatas = [], []
@@ -116,17 +112,17 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
         final_chunks = []
         # Get appropriate separator to use
         separator = separators[-1]
-        separator_rule = 'after'
+        separator_rule = "after"
         new_separators = []
         for i, _s in enumerate(separators):
             _separator = _s if self._is_separator_regex else re.escape(_s)
-            separator_rule = self.separator_rule[_s]
-            if _s == '':
+            separator_rule = self.separator_rule.get(_s, "after")
+            if _s == "":
                 separator = _s
                 break
             if re.search(_separator, text):
                 separator = _s
-                new_separators = separators[i + 1:]
+                new_separators = separators[i + 1 :]
                 break
 
         _separator = separator if self._is_separator_regex else re.escape(separator)
@@ -134,7 +130,7 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
 
         # Now go merging things, recursively splitting longer texts.
         _good_splits = []
-        _separator = '' if self._keep_separator else separator
+        _separator = "" if self._keep_separator else separator
         for s in splits:
             if self._length_function(s) < self._chunk_size:
                 _good_splits.append(s)
@@ -144,7 +140,7 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
                     final_chunks.extend(merged_text)
                     _good_splits = []
                 if not new_separators:
-                    final_chunks.append(s)
+                    final_chunks.extend(self._force_split(s))
                 else:
                     other_info = self._split_text(s, new_separators)
                     final_chunks.extend(other_info)
@@ -153,20 +149,33 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
             final_chunks.extend(merged_text)
         return final_chunks
 
+    def _force_split(self, text: str) -> List[str]:
+        """Last-resort character-level split for a fragment no separator can break.
+
+        Only kicks in past ``hard_split_limit``, so fragments that are over
+        chunk_size but still ingestible keep the boundaries they have today —
+        the caller's configured separators stay the only thing shaping ordinary
+        documents. Without this, a fragment carrying none of the configured
+        separators (an OCR'd 600-column markdown table row, a long digit run,
+        a base64 blob) survives as one oversized chunk and fails the whole file
+        downstream.
+        """
+        if not self.hard_split_limit or self._length_function(text) <= self.hard_split_limit:
+            return [text]
+        return self._merge_splits(list(text), "")
+
     def split_text(self, text: str) -> List[str]:
         return self._split_text(text, self._separators)
 
-    def create_documents(
-            self, texts: List[str], metadatas: Optional[List[dict]] = None
-    ) -> List[Document]:
+    def create_documents(self, texts: List[str], metadatas: Optional[List[dict]] = None) -> List[Document]:
         """Create documents from a list of texts."""
         documents = []
         for i, text in enumerate(texts):
             index = -1
-            indexes = metadatas[i].pop('indexes', [])
-            pages = metadatas[i].pop('pages', [])
-            types = metadatas[i].pop('types', [])
-            bboxes = metadatas[i].pop('bboxes', [])
+            indexes = metadatas[i].pop("indexes", [])
+            pages = metadatas[i].pop("pages", [])
+            types = metadatas[i].pop("types", [])
+            bboxes = metadatas[i].pop("bboxes", [])
             searcher = IntervalSearch(indexes)
             split_texts = self.split_text(text)
             for chunk in split_texts:
@@ -175,15 +184,14 @@ class ElemCharacterTextSplitter(RecursiveCharacterTextSplitter):
                     index = text.find(chunk, index + 1)
                     inter0 = [index, index + len(chunk) - 1]
                     norm_inter = searcher.find(inter0)
-                    new_metadata['chunk_bboxes'] = []
+                    new_metadata["chunk_bboxes"] = []
                     for j in range(norm_inter[0], norm_inter[1] + 1):
-                        new_metadata['chunk_bboxes'].append(
-                            {'page': pages[j], 'bbox': bboxes[j]})
+                        new_metadata["chunk_bboxes"].append({"page": pages[j], "bbox": bboxes[j]})
 
                     c = Counter([types[j] for j in norm_inter])
                     chunk_type = c.most_common(1)[0][0]
-                    new_metadata['chunk_type'] = chunk_type
-                    new_metadata['source'] = metadatas[i].get('source', '')
+                    new_metadata["chunk_type"] = chunk_type
+                    new_metadata["source"] = metadatas[i].get("source", "")
 
                 # for chunk in split_texts:
                 #     new_metadata = {}

src/backend/test/knowledge/rag/test_splitter_hard_limit.py

@@ -0,0 +1,117 @@
+"""Hard-split fallback for fragments no configured separator can break.
+
+Regression cover for the ETL4LM path: its Table elements are markdown tables
+whose rows can run thousands of characters with no ``\\n`` / ``。`` / ``.`` inside
+(one observed row was 7820 chars across 601 cells). Such a fragment used to
+survive ``ElemCharacterTextSplitter`` untouched and then fail the whole file at
+``SplitterTransformer``'s max_chunk_limit check. PaddleOCR/MinerU never hit this
+because they rewrite tables to per-row markdown before splitting.
+"""
+
+from langchain_core.documents import Document
+
+from bisheng.knowledge.rag.pipeline.transformer.splitter import SplitterTransformer
+
+# On this branch SplitterTransformer hardcodes the cap rather than reading a shared constant.
+KNOWLEDGE_MAX_CHUNK_CHARS = 10000
+DEFAULT_SEPARATORS = ["\n\n", "\n", "。", "\\."]
+CHUNK_SIZE = 1000
+
+
+def _transformer(**kwargs) -> SplitterTransformer:
+    params = {
+        "separator": DEFAULT_SEPARATORS,
+        "separator_rule": ["after"] * len(DEFAULT_SEPARATORS),
+        "chunk_size": CHUNK_SIZE,
+        "chunk_overlap": 0,
+    }
+    params.update(kwargs)
+    return SplitterTransformer(**params)
+
+
+def _wide_table_row(char_len: int) -> str:
+    """A single markdown table row: no newline, no period, only ' | ' between cells."""
+    cell = "呼气吸气当肺活化时乙醇消毒液与碘酒的混合物喷洒完毕后用酒精棉球擦拭"
+    row = "|"
+    while len(row) < char_len:
+        row += f" {cell} |"
+    return row
+
+
+def test_oversized_separator_proof_fragment_is_split_instead_of_rejected():
+    """The real failure: a >10000 char fragment with none of the default separators."""
+    text = _wide_table_row(KNOWLEDGE_MAX_CHUNK_CHARS + 3000)
+    assert "\n" not in text and "。" not in text and "." not in text
+
+    chunks = _transformer().transform_documents([Document(page_content=text, metadata={})])
+
+    assert len(chunks) > 1
+    assert max(len(one.page_content) for one in chunks) <= CHUNK_SIZE
+    # No text is dropped on the way through.
+    assert "".join(one.page_content for one in chunks).replace(" ", "") == text.replace(" ", "")
+
+
+def test_fragment_under_hard_limit_keeps_current_boundaries():
+    """Zero-regression guard: below the hard limit nothing changes.
+
+    A 3596-char unbreakable fragment is exactly what the reported document
+    produces today; it must still come through as one chunk, or every stored
+    document would re-chunk differently after this change.
+    """
+    text = _wide_table_row(3596)
+
+    chunks = _transformer().transform_documents([Document(page_content=text, metadata={})])
+
+    assert len(chunks) == 1
+    assert chunks[0].page_content == text
+
+
+def test_hard_split_respects_configured_chunk_size():
+    text = _wide_table_row(KNOWLEDGE_MAX_CHUNK_CHARS + 500)
+
+    chunks = _transformer(chunk_size=300).transform_documents([Document(page_content=text, metadata={})])
+
+    assert max(len(one.page_content) for one in chunks) <= 300
+
+
+def test_normal_text_splitting_is_untouched():
+    """Ordinary prose must split exactly where the separators say, as before."""
+    paragraph = "这是一段普通的中文正文内容用于验证切分行为没有发生变化。" * 60
+    text = "\n\n".join([paragraph, paragraph])
+
+    chunks = _transformer().transform_documents([Document(page_content=text, metadata={})])
+
+    assert all(len(one.page_content) <= CHUNK_SIZE for one in chunks)
+    # Every boundary still lands after a full sentence, i.e. the separator did the work.
+    assert all(one.page_content.rstrip().endswith("。") for one in chunks)
+
+
+def test_hard_split_chunks_still_resolve_bboxes():
+    """chunk_bboxes come from text.find() in create_documents, which hard-split chunks keep."""
+    text = _wide_table_row(KNOWLEDGE_MAX_CHUNK_CHARS + 2000)
+    metadata = {
+        "indexes": [[0, len(text)]],
+        "bboxes": [[95, 390, 489, 430]],
+        "pages": [95],
+        "types": ["table"],
+        "source": "wide_table.pdf",
+    }
+
+    chunks = _transformer().transform_documents([Document(page_content=text, metadata=metadata)])
+
+    assert len(chunks) > 1
+    for one in chunks:
+        assert one.metadata["chunk_bboxes"], "hard-split chunk lost its bbox attribution"
+        assert one.metadata["chunk_type"] == "table"
+        assert one.metadata["page"] == 95
+
+
+def test_empty_string_separator_does_not_raise_keyerror():
+    """separator_rule is keyed by the caller's separators; '' must not blow up."""
+    text = _wide_table_row(2000)
+
+    chunks = _transformer(separator=[*DEFAULT_SEPARATORS, ""], separator_rule=["after"] * 5).transform_documents(
+        [Document(page_content=text, metadata={})]
+    )
+
+    assert max(len(one.page_content) for one in chunks) <= CHUNK_SIZE