From ad69240ac1740e8c0539ece833f3c8daeee3fe65 Mon Sep 17 00:00:00 2001
From: wizardchen <wizardchen@tencent.com>
Date: Wed, 6 May 2026 20:59:32 +0800
Subject: [PATCH] fix(chunker): heuristic splitter drops boundaries inside
 protected spans
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Heuristic boundary detection (numbered sections, all-caps headings,
\\n{3,} blank blocks, etc.) ran on the raw text and could land inside
atomic regions handled by protectedPatterns — most notably LaTeX
$$...$$ blocks, Markdown tables, fenced code, and image/link refs.
A boundary inside such a region would cause the bin-packer to slice
through protected content, defeating the protection.

Convert protectedSpans output to rune offsets once and filter the
boundary list before bin-packing. Boundaries on a span edge are kept
(they align with the span) — only strictly-interior ones are dropped.
---
 .../chunker/heuristic_splitter.go             | 31 +++++++++++++++++++
 .../chunker/heuristic_splitter_test.go        | 29 +++++++++++++++++
 internal/infrastructure/chunker/splitter.go   | 29 +++++++++++++++++
 3 files changed, 89 insertions(+)

diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go
index 5e642ccde..3bac30987 100644
--- a/internal/infrastructure/chunker/heuristic_splitter.go
+++ b/internal/infrastructure/chunker/heuristic_splitter.go
@@ -42,6 +42,13 @@ func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chu
 	}
 
 	bounds := findHeuristicBoundaries(text, cfg.Languages)
+	// Drop any boundary that falls strictly inside a protected region (table,
+	// fenced code block, LaTeX block, etc.) — splitting there would cut
+	// through atomic content. Boundaries on a span edge are kept since they
+	// align with the protected region start/end.
+	if prot := protectedSpansRune(text, protectedSpans(text)); len(prot) > 0 {
+		bounds = dropBoundsInsideSpans(bounds, prot)
+	}
 	if len(bounds) == 0 {
 		return SplitText(text, cfg)
 	}
@@ -180,6 +187,30 @@ func findHeuristicBoundaries(text string, langs []string) []boundary {
 	return deduped
 }
 
+// dropBoundsInsideSpans returns bounds with entries that fall strictly
+// inside any of the (rune-offset) protected spans removed. Bounds at a
+// span's start or end are kept — they align with the span edge and don't
+// split protected content. spans must be sorted by start.
+func dropBoundsInsideSpans(bounds []boundary, spans []span) []boundary {
+	if len(spans) == 0 {
+		return bounds
+	}
+	out := bounds[:0]
+boundLoop:
+	for _, b := range bounds {
+		for _, s := range spans {
+			if s.start >= b.runeStart {
+				break // remaining spans start at or after b — can't contain b
+			}
+			if b.runeStart < s.end {
+				continue boundLoop
+			}
+		}
+		out = append(out, b)
+	}
+	return out
+}
+
 // allRuneIndices returns every rune offset where needle starts in text.
 // Only used for single-rune needles like form-feed.
 func allRuneIndices(text, needle string) []int {
diff --git a/internal/infrastructure/chunker/heuristic_splitter_test.go b/internal/infrastructure/chunker/heuristic_splitter_test.go
index c26854d5f..2b6926d7f 100644
--- a/internal/infrastructure/chunker/heuristic_splitter_test.go
+++ b/internal/infrastructure/chunker/heuristic_splitter_test.go
@@ -140,6 +140,35 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
 	}
 }
 
+// Heuristic boundaries that fall inside protected regions (LaTeX block,
+// table, link, etc.) must be dropped so the bin-packer doesn't break
+// atomic content. Without the protected-span filter, a numbered-section
+// looking line inside a $$...$$ math block would be picked as a boundary.
+func TestSplitByHeuristics_DropsBoundariesInsideProtectedSpans(t *testing.T) {
+	body := strings.Repeat("filler. ", 30)
+	// LaTeX block whose middle line matches NumberedSectionPattern. The
+	// filter should drop that boundary so the math block stays intact.
+	doc := body + "\n\n$$\nx = 1\n1. equation step one\ny = 2\n$$\n\n" + body
+
+	bounds := findHeuristicBoundaries(doc, nil)
+	prot := protectedSpansRune(doc, protectedSpans(doc))
+	if len(prot) == 0 {
+		t.Fatalf("expected protected spans for doc, got none")
+	}
+	filtered := dropBoundsInsideSpans(bounds, prot)
+	for _, b := range filtered {
+		for _, s := range prot {
+			if b.runeStart > s.start && b.runeStart < s.end {
+				t.Errorf("boundary %d still inside protected span [%d,%d)", b.runeStart, s.start, s.end)
+			}
+		}
+	}
+	// And it should actually have removed at least one boundary.
+	if len(filtered) >= len(bounds) {
+		t.Errorf("filter removed nothing: before=%d after=%d", len(bounds), len(filtered))
+	}
+}
+
 func chunkLengths(chunks []Chunk) []int {
 	out := make([]int, len(chunks))
 	for i, c := range chunks {
diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go
index a3baf0ee8..514e2c56d 100644
--- a/internal/infrastructure/chunker/splitter.go
+++ b/internal/infrastructure/chunker/splitter.go
@@ -121,6 +121,35 @@ type span struct {
 	start, end int
 }
 
+// protectedSpansRune converts byte-offset protected spans to rune offsets
+// in a single forward pass over text. Used by callers that work in rune
+// space (e.g. the heuristic splitter) to avoid choosing chunk boundaries
+// that cut through protected content. byteSpans must be sorted by start
+// (protectedSpans guarantees this).
+func protectedSpansRune(text string, byteSpans []span) []span {
+	if len(byteSpans) == 0 {
+		return nil
+	}
+	out := make([]span, 0, len(byteSpans))
+	runeIdx := 0
+	byteIdx := 0
+	for _, s := range byteSpans {
+		for byteIdx < s.start && byteIdx < len(text) {
+			_, size := utf8.DecodeRuneInString(text[byteIdx:])
+			byteIdx += size
+			runeIdx++
+		}
+		startRune := runeIdx
+		for byteIdx < s.end && byteIdx < len(text) {
+			_, size := utf8.DecodeRuneInString(text[byteIdx:])
+			byteIdx += size
+			runeIdx++
+		}
+		out = append(out, span{start: startRune, end: runeIdx})
+	}
+	return out
+}
+
 // protectedSpans finds all non-overlapping protected regions in text.
 func protectedSpans(text string) []span {
 	type match struct {