mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-19 02:18:25 +08:00
fix(chunker): heuristic splitter drops boundaries inside protected spans
Heuristic boundary detection (numbered sections, all-caps headings,
\\n{3,} blank blocks, etc.) ran on the raw text and could land inside
atomic regions handled by protectedPatterns — most notably LaTeX
$$...$$ blocks, Markdown tables, fenced code, and image/link refs.
A boundary inside such a region would cause the bin-packer to slice
through protected content, defeating the protection.
Convert protectedSpans output to rune offsets once and filter the
boundary list before bin-packing. Boundaries on a span edge are kept
(they align with the span) — only strictly-interior ones are dropped.
This commit is contained in:
@@ -42,6 +42,13 @@ func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chu
|
||||
}
|
||||
|
||||
bounds := findHeuristicBoundaries(text, cfg.Languages)
|
||||
// Drop any boundary that falls strictly inside a protected region (table,
|
||||
// fenced code block, LaTeX block, etc.) — splitting there would cut
|
||||
// through atomic content. Boundaries on a span edge are kept since they
|
||||
// align with the protected region start/end.
|
||||
if prot := protectedSpansRune(text, protectedSpans(text)); len(prot) > 0 {
|
||||
bounds = dropBoundsInsideSpans(bounds, prot)
|
||||
}
|
||||
if len(bounds) == 0 {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
@@ -180,6 +187,30 @@ func findHeuristicBoundaries(text string, langs []string) []boundary {
|
||||
return deduped
|
||||
}
|
||||
|
||||
// dropBoundsInsideSpans returns bounds with entries that fall strictly
|
||||
// inside any of the (rune-offset) protected spans removed. Bounds at a
|
||||
// span's start or end are kept — they align with the span edge and don't
|
||||
// split protected content. spans must be sorted by start.
|
||||
func dropBoundsInsideSpans(bounds []boundary, spans []span) []boundary {
|
||||
if len(spans) == 0 {
|
||||
return bounds
|
||||
}
|
||||
out := bounds[:0]
|
||||
boundLoop:
|
||||
for _, b := range bounds {
|
||||
for _, s := range spans {
|
||||
if s.start >= b.runeStart {
|
||||
break // remaining spans start at or after b — can't contain b
|
||||
}
|
||||
if b.runeStart < s.end {
|
||||
continue boundLoop
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// allRuneIndices returns every rune offset where needle starts in text.
|
||||
// Only used for single-rune needles like form-feed.
|
||||
func allRuneIndices(text, needle string) []int {
|
||||
|
||||
@@ -140,6 +140,35 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic boundaries that fall inside protected regions (LaTeX block,
|
||||
// table, link, etc.) must be dropped so the bin-packer doesn't break
|
||||
// atomic content. Without the protected-span filter, a numbered-section
|
||||
// looking line inside a $$...$$ math block would be picked as a boundary.
|
||||
func TestSplitByHeuristics_DropsBoundariesInsideProtectedSpans(t *testing.T) {
|
||||
body := strings.Repeat("filler. ", 30)
|
||||
// LaTeX block whose middle line matches NumberedSectionPattern. The
|
||||
// filter should drop that boundary so the math block stays intact.
|
||||
doc := body + "\n\n$$\nx = 1\n1. equation step one\ny = 2\n$$\n\n" + body
|
||||
|
||||
bounds := findHeuristicBoundaries(doc, nil)
|
||||
prot := protectedSpansRune(doc, protectedSpans(doc))
|
||||
if len(prot) == 0 {
|
||||
t.Fatalf("expected protected spans for doc, got none")
|
||||
}
|
||||
filtered := dropBoundsInsideSpans(bounds, prot)
|
||||
for _, b := range filtered {
|
||||
for _, s := range prot {
|
||||
if b.runeStart > s.start && b.runeStart < s.end {
|
||||
t.Errorf("boundary %d still inside protected span [%d,%d)", b.runeStart, s.start, s.end)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And it should actually have removed at least one boundary.
|
||||
if len(filtered) >= len(bounds) {
|
||||
t.Errorf("filter removed nothing: before=%d after=%d", len(bounds), len(filtered))
|
||||
}
|
||||
}
|
||||
|
||||
func chunkLengths(chunks []Chunk) []int {
|
||||
out := make([]int, len(chunks))
|
||||
for i, c := range chunks {
|
||||
|
||||
@@ -121,6 +121,35 @@ type span struct {
|
||||
start, end int
|
||||
}
|
||||
|
||||
// protectedSpansRune converts byte-offset protected spans to rune offsets
|
||||
// in a single forward pass over text. Used by callers that work in rune
|
||||
// space (e.g. the heuristic splitter) to avoid choosing chunk boundaries
|
||||
// that cut through protected content. byteSpans must be sorted by start
|
||||
// (protectedSpans guarantees this).
|
||||
func protectedSpansRune(text string, byteSpans []span) []span {
|
||||
if len(byteSpans) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]span, 0, len(byteSpans))
|
||||
runeIdx := 0
|
||||
byteIdx := 0
|
||||
for _, s := range byteSpans {
|
||||
for byteIdx < s.start && byteIdx < len(text) {
|
||||
_, size := utf8.DecodeRuneInString(text[byteIdx:])
|
||||
byteIdx += size
|
||||
runeIdx++
|
||||
}
|
||||
startRune := runeIdx
|
||||
for byteIdx < s.end && byteIdx < len(text) {
|
||||
_, size := utf8.DecodeRuneInString(text[byteIdx:])
|
||||
byteIdx += size
|
||||
runeIdx++
|
||||
}
|
||||
out = append(out, span{start: startRune, end: runeIdx})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// protectedSpans finds all non-overlapping protected regions in text.
|
||||
func protectedSpans(text string) []span {
|
||||
type match struct {
|
||||
|
||||
Reference in New Issue
Block a user