fix(chunker): heuristic splitter drops boundaries inside protected spans

Heuristic boundary detection (numbered sections, all-caps headings,
\\n{3,} blank blocks, etc.) ran on the raw text and could land inside
atomic regions handled by protectedPatterns — most notably LaTeX
$$...$$ blocks, Markdown tables, fenced code, and image/link refs.
A boundary inside such a region would cause the bin-packer to slice
through protected content, defeating the protection.

Convert protectedSpans output to rune offsets once and filter the
boundary list before bin-packing. Boundaries on a span edge are kept
(they align with the span) — only strictly-interior ones are dropped.
This commit is contained in:
wizardchen
2026-05-06 21:16:48 +08:00
committed by lyingbug
parent 151e999db1
commit ad69240ac1
3 changed files with 89 additions and 0 deletions
@@ -42,6 +42,13 @@ func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chu
}
bounds := findHeuristicBoundaries(text, cfg.Languages)
// Drop any boundary that falls strictly inside a protected region (table,
// fenced code block, LaTeX block, etc.) — splitting there would cut
// through atomic content. Boundaries on a span edge are kept since they
// align with the protected region start/end.
if prot := protectedSpansRune(text, protectedSpans(text)); len(prot) > 0 {
bounds = dropBoundsInsideSpans(bounds, prot)
}
if len(bounds) == 0 {
return SplitText(text, cfg)
}
@@ -180,6 +187,30 @@ func findHeuristicBoundaries(text string, langs []string) []boundary {
return deduped
}
// dropBoundsInsideSpans returns bounds with entries that fall strictly
// inside any of the (rune-offset) protected spans removed. Bounds at a
// span's start or end are kept — they align with the span edge and don't
// split protected content. spans must be sorted by start.
func dropBoundsInsideSpans(bounds []boundary, spans []span) []boundary {
if len(spans) == 0 {
return bounds
}
out := bounds[:0]
boundLoop:
for _, b := range bounds {
for _, s := range spans {
if s.start >= b.runeStart {
break // remaining spans start at or after b — can't contain b
}
if b.runeStart < s.end {
continue boundLoop
}
}
out = append(out, b)
}
return out
}
// allRuneIndices returns every rune offset where needle starts in text.
// Only used for single-rune needles like form-feed.
func allRuneIndices(text, needle string) []int {
@@ -140,6 +140,35 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
}
}
// Heuristic boundaries that fall inside protected regions (LaTeX block,
// table, link, etc.) must be dropped so the bin-packer doesn't break
// atomic content. Without the protected-span filter, a numbered-section
// looking line inside a $$...$$ math block would be picked as a boundary.
func TestSplitByHeuristics_DropsBoundariesInsideProtectedSpans(t *testing.T) {
body := strings.Repeat("filler. ", 30)
// LaTeX block whose middle line matches NumberedSectionPattern. The
// filter should drop that boundary so the math block stays intact.
doc := body + "\n\n$$\nx = 1\n1. equation step one\ny = 2\n$$\n\n" + body
bounds := findHeuristicBoundaries(doc, nil)
prot := protectedSpansRune(doc, protectedSpans(doc))
if len(prot) == 0 {
t.Fatalf("expected protected spans for doc, got none")
}
filtered := dropBoundsInsideSpans(bounds, prot)
for _, b := range filtered {
for _, s := range prot {
if b.runeStart > s.start && b.runeStart < s.end {
t.Errorf("boundary %d still inside protected span [%d,%d)", b.runeStart, s.start, s.end)
}
}
}
// And it should actually have removed at least one boundary.
if len(filtered) >= len(bounds) {
t.Errorf("filter removed nothing: before=%d after=%d", len(bounds), len(filtered))
}
}
func chunkLengths(chunks []Chunk) []int {
out := make([]int, len(chunks))
for i, c := range chunks {
@@ -121,6 +121,35 @@ type span struct {
start, end int
}
// protectedSpansRune converts byte-offset protected spans to rune offsets
// in a single forward pass over text. Used by callers that work in rune
// space (e.g. the heuristic splitter) to avoid choosing chunk boundaries
// that cut through protected content. byteSpans must be sorted by start
// (protectedSpans guarantees this).
func protectedSpansRune(text string, byteSpans []span) []span {
if len(byteSpans) == 0 {
return nil
}
out := make([]span, 0, len(byteSpans))
runeIdx := 0
byteIdx := 0
for _, s := range byteSpans {
for byteIdx < s.start && byteIdx < len(text) {
_, size := utf8.DecodeRuneInString(text[byteIdx:])
byteIdx += size
runeIdx++
}
startRune := runeIdx
for byteIdx < s.end && byteIdx < len(text) {
_, size := utf8.DecodeRuneInString(text[byteIdx:])
byteIdx += size
runeIdx++
}
out = append(out, span{start: startRune, end: runeIdx})
}
return out
}
// protectedSpans finds all non-overlapping protected regions in text.
func protectedSpans(text string) []span {
type match struct {