mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-08-29 02:04:30 +08:00
feat(chunker): foundation for adaptive 3-tier chunking
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
@@ -7484,7 +7484,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg)
|
||||
pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg)
|
||||
parsed = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
@@ -7501,7 +7501,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
}
|
||||
opts.ParentChunks = parentChunks
|
||||
} else {
|
||||
splitChunks := chunker.SplitText(clean, chunkCfg)
|
||||
splitChunks := chunker.Split(clean, chunkCfg)
|
||||
parsed = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
@@ -8340,7 +8340,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
|
||||
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
|
||||
chunks = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
@@ -8359,7 +8359,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
|
||||
len(pcResult.Parents), len(pcResult.Children), knowledge.ID)
|
||||
} else {
|
||||
splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg)
|
||||
splitChunks := chunker.Split(convertResult.MarkdownContent, chunkCfg)
|
||||
chunks = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
// Package chunker - patterns.go is the source of truth for multilingual
|
||||
// regex patterns used by the heading-aware and heuristic splitters.
|
||||
//
|
||||
// Patterns are grouped by purpose (chapter markers, numbering, separators)
|
||||
// and tagged with a priority that the heuristic splitter uses to rank
|
||||
// candidate chunk boundaries.
|
||||
package chunker
|
||||
|
||||
import "regexp"
|
||||
|
||||
// BoundaryPriority levels for heuristic chunk boundaries. Higher = stronger.
|
||||
const (
|
||||
PrioFormFeed = 100
|
||||
PrioNumberedHead = 90
|
||||
PrioChapterMarker = 85
|
||||
PrioAllCapsHeading = 70
|
||||
PrioVisualSep = 60
|
||||
PrioPageFooter = 50
|
||||
PrioBlankBlock = 40
|
||||
)
|
||||
|
||||
// MarkdownHeadingPattern matches an ATX-style Markdown heading at line start.
|
||||
// Capture groups: (1) hashes, (2) heading text.
|
||||
var MarkdownHeadingPattern = regexp.MustCompile(`(?m)^(#{1,6})\s+(.+?)\s*#*\s*$`)
|
||||
|
||||
// FormFeedPattern matches the form-feed control character used by some PDF
|
||||
// converters as a page break marker.
|
||||
var FormFeedPattern = regexp.MustCompile(`\f`)
|
||||
|
||||
// NumberedSectionPattern matches lines starting with numeric or roman numbering
|
||||
// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results".
|
||||
var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){0,3}|[IVX]{1,5})\.[ \t]+\S.{0,200}$`)
|
||||
|
||||
// AllCapsHeadingPattern matches short all-caps lines (likely section titles
|
||||
// rendered without Markdown headings). It requires at least 4 letters and
|
||||
// up to ~10 words. Trailing colons are tolerated.
|
||||
var AllCapsHeadingPattern = regexp.MustCompile(`(?m)^[ \t]*([A-ZÄÖÜ][A-ZÄÖÜ \-]{3,80}):?\s*$`)
|
||||
|
||||
// VisualSeparatorPattern matches horizontal rules / divider lines used as
|
||||
// section separators in plain text or pre-Markdown documents.
|
||||
var VisualSeparatorPattern = regexp.MustCompile(`(?m)^[ \t]*(?:-{3,}|={3,}|\*{3,}|_{3,})[ \t]*$`)
|
||||
|
||||
// ExcessiveBlanksPattern matches three or more consecutive newlines, which
|
||||
// usually denote a hard section break.
|
||||
var ExcessiveBlanksPattern = regexp.MustCompile(`\n{3,}`)
|
||||
|
||||
// PageFooterPattern matches typical "Seite X von Y" / "Page X of Y" lines.
|
||||
var PageFooterPattern = regexp.MustCompile(`(?mi)^[ \t]*(?:Seite|Page|页码?)\s+\d+(?:\s*(?:von|of|/)\s*\d+)?[ \t]*$`)
|
||||
|
||||
// GermanChapterPattern matches German chapter / section markers.
|
||||
var GermanChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Kapitel|Abschnitt|Teil)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`)
|
||||
|
||||
// EnglishChapterPattern matches English chapter / section markers.
|
||||
var EnglishChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Chapter|Section|Part)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`)
|
||||
|
||||
// ChineseChapterPattern matches CJK chapter / section markers like 第一章, 第3节.
|
||||
var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[一二三四五六七八九十百千零〇0-9]+(?:章|节|節|部分|篇)[ \t]?.{0,200}$`)
|
||||
|
||||
// SentenceSeparators returns sentence-level separators tuned for the language.
|
||||
// Used for fine-grained sub-splitting when a section is still too large.
|
||||
func SentenceSeparators(lang string) []string {
|
||||
switch lang {
|
||||
case LangChinese:
|
||||
return []string{"。", "！", "？", "；", "\n"}
|
||||
case LangGerman, LangEnglish:
|
||||
return []string{". ", "! ", "? ", "; ", "\n"}
|
||||
default:
|
||||
return []string{"。", "！", "？", "；", ". ", "! ", "? ", "; ", "\n"}
|
||||
}
|
||||
}
|
||||
|
||||
// ChapterPatternsForLangs returns the chapter-marker regexes that apply for
|
||||
// the given language hints. An empty / unknown list returns all of them so
|
||||
// that auto-detected documents still match.
|
||||
func ChapterPatternsForLangs(langs []string) []*regexp.Regexp {
|
||||
if len(langs) == 0 {
|
||||
return []*regexp.Regexp{GermanChapterPattern, EnglishChapterPattern, ChineseChapterPattern}
|
||||
}
|
||||
var out []*regexp.Regexp
|
||||
for _, l := range langs {
|
||||
switch l {
|
||||
case LangGerman:
|
||||
out = append(out, GermanChapterPattern)
|
||||
case LangEnglish:
|
||||
out = append(out, EnglishChapterPattern)
|
||||
case LangChinese:
|
||||
out = append(out, ChineseChapterPattern)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = []*regexp.Regexp{GermanChapterPattern, EnglishChapterPattern, ChineseChapterPattern}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package chunker
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMarkdownHeadingPattern_BasicLevels(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"# Heading 1", true},
|
||||
{"## Heading 2", true},
|
||||
{"###### Heading 6", true},
|
||||
{"####### Too many", false},
|
||||
{"#NoSpace", false},
|
||||
{" # Indented", false},
|
||||
{"plain text", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := MarkdownHeadingPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("MarkdownHeadingPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumberedSectionPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"1. Introduction", true},
|
||||
{"2.3 Methodology", false}, // requires dot after number
|
||||
{"2.3. Methodology", true},
|
||||
{"IV. Results", true},
|
||||
{"1.Introduction", false}, // requires whitespace
|
||||
{"1.", false}, // requires title
|
||||
{"plain text", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := NumberedSectionPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("NumberedSectionPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGermanChapterPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"Kapitel 1: Einführung", true},
|
||||
{"Abschnitt 2.3 Methodik", true},
|
||||
{"Abschnitt 3 Methodik", true},
|
||||
{"Teil II Ergebnisse", true},
|
||||
{"chapter 1", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := GermanChapterPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("GermanChapterPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnglishChapterPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"Chapter 1: Intro", true},
|
||||
{"Section 5 Methods", true},
|
||||
{"Part IV Results", true},
|
||||
{"Kapitel 1", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := EnglishChapterPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("EnglishChapterPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChineseChapterPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"第一章 引言", true},
|
||||
{"第3节 方法论", true},
|
||||
{"第二部分 结果", true},
|
||||
{"Chapter 1", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := ChineseChapterPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("ChineseChapterPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisualSeparatorPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"---", true},
|
||||
{"========", true},
|
||||
{"***", true},
|
||||
{"____", true},
|
||||
{"--", false}, // needs at least 3
|
||||
{"-- text", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := VisualSeparatorPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("VisualSeparatorPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageFooterPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"Seite 3 von 24", true},
|
||||
{"Page 5 of 12", true},
|
||||
{"page 7", true},
|
||||
{"Seite 9", true},
|
||||
{"页 3", true},
|
||||
{"页码 3 / 12", true},
|
||||
{"Some text", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := PageFooterPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("PageFooterPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllCapsHeadingPattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
match bool
|
||||
}{
|
||||
{"INTRODUCTION", true},
|
||||
{"METHODS AND MATERIALS", true},
|
||||
{"Mixed Case Heading", false},
|
||||
{"ABC", false}, // <4 chars
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := AllCapsHeadingPattern.MatchString(c.in)
|
||||
if got != c.match {
|
||||
t.Errorf("AllCapsHeadingPattern(%q): got %v want %v", c.in, got, c.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSentenceSeparators(t *testing.T) {
|
||||
if got := SentenceSeparators(LangChinese); got[0] != "。" {
|
||||
t.Errorf("Chinese should start with 。, got %v", got)
|
||||
}
|
||||
if got := SentenceSeparators(LangEnglish); got[0] != ". " {
|
||||
t.Errorf("English should start with '. ', got %v", got)
|
||||
}
|
||||
if got := SentenceSeparators("xx"); len(got) < 5 {
|
||||
t.Errorf("Unknown lang should return mixed (>=5 separators), got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChapterPatternsForLangs(t *testing.T) {
|
||||
if got := ChapterPatternsForLangs(nil); len(got) != 3 {
|
||||
t.Errorf("nil langs should return all 3, got %d", len(got))
|
||||
}
|
||||
if got := ChapterPatternsForLangs([]string{LangGerman}); len(got) != 1 {
|
||||
t.Errorf("only DE requested should return 1, got %d", len(got))
|
||||
}
|
||||
if got := ChapterPatternsForLangs([]string{LangGerman, LangChinese}); len(got) != 2 {
|
||||
t.Errorf("DE+ZH requested should return 2, got %d", len(got))
|
||||
}
|
||||
if got := ChapterPatternsForLangs([]string{"xx"}); len(got) != 3 {
|
||||
t.Errorf("unknown lang should fall back to all 3, got %d", len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// Package chunker - profiler.go scans a document once to gather structure
|
||||
// indicators that drive strategy selection (heading-aware vs. heuristic vs.
|
||||
// recursive). Profiling is cheap (a few regex passes plus rune counting)
|
||||
// and runs before any chunking decision is made.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DocProfile holds the document-level signals used to choose a chunking tier.
|
||||
type DocProfile struct {
|
||||
TotalChars int
|
||||
TotalLines int
|
||||
AvgLineLen float64
|
||||
StdLineLen float64
|
||||
|
||||
// Markdown structure
|
||||
MdHeadingCounts map[int]int // level (1..6) → count
|
||||
MdHeadingTotal int
|
||||
|
||||
// Heuristic indicators
|
||||
NumberedSectionCount int
|
||||
AllCapsShortLineCount int
|
||||
BlankParagraphBreaks int
|
||||
FormFeedCount int
|
||||
VisualSepCount int
|
||||
GermanChapterCount int
|
||||
EnglishChapterCount int
|
||||
ChineseChapterCount int
|
||||
RepeatedFooterCount int
|
||||
|
||||
// Content characteristics
|
||||
HasTables bool
|
||||
HasCode bool
|
||||
CodeRatio float64
|
||||
|
||||
// Detected language hints (best-effort)
|
||||
DetectedLangs []string
|
||||
}
|
||||
|
||||
// HeadingDensity returns the share of lines that are Markdown headings.
|
||||
func (p *DocProfile) HeadingDensity() float64 {
|
||||
if p.TotalLines == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(p.MdHeadingTotal) / float64(p.TotalLines)
|
||||
}
|
||||
|
||||
// DominantHeadingLevel returns the heading level (1..6) that should drive
|
||||
// section splitting. It picks the lowest level (closest to root) that has
|
||||
// at least 3 occurrences, falling back to the most frequent level. Returns
|
||||
// 0 when no usable Markdown structure exists.
|
||||
func (p *DocProfile) DominantHeadingLevel() int {
|
||||
if p.MdHeadingTotal == 0 {
|
||||
return 0
|
||||
}
|
||||
for level := 1; level <= 6; level++ {
|
||||
if p.MdHeadingCounts[level] >= 3 {
|
||||
return level
|
||||
}
|
||||
}
|
||||
// fallback: most frequent level
|
||||
bestLevel, bestCount := 0, 0
|
||||
for level := 1; level <= 6; level++ {
|
||||
if p.MdHeadingCounts[level] > bestCount {
|
||||
bestLevel, bestCount = level, p.MdHeadingCounts[level]
|
||||
}
|
||||
}
|
||||
return bestLevel
|
||||
}
|
||||
|
||||
// HeuristicMarkerTotal sums the non-Markdown structural markers.
|
||||
func (p *DocProfile) HeuristicMarkerTotal() int {
|
||||
return p.NumberedSectionCount +
|
||||
p.GermanChapterCount + p.EnglishChapterCount + p.ChineseChapterCount +
|
||||
p.AllCapsShortLineCount + p.VisualSepCount + p.FormFeedCount
|
||||
}
|
||||
|
||||
// ProfileDocument runs a single pass over text and returns its profile.
|
||||
func ProfileDocument(text string) *DocProfile {
|
||||
p := &DocProfile{
|
||||
MdHeadingCounts: make(map[int]int),
|
||||
}
|
||||
if text == "" {
|
||||
return p
|
||||
}
|
||||
|
||||
p.TotalChars = len([]rune(text))
|
||||
p.FormFeedCount = strings.Count(text, "\f")
|
||||
|
||||
lines := strings.Split(text, "\n")
|
||||
p.TotalLines = len(lines)
|
||||
|
||||
// First pass: per-line markers and length stats
|
||||
var lengths []float64
|
||||
inFence := false
|
||||
codeChars := 0
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Toggle fenced-code state. We use a 3-backtick prefix detector here
|
||||
// rather than a full regex so we don't have to fight with the
|
||||
// protected-pattern logic later.
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
inFence = !inFence
|
||||
p.HasCode = true
|
||||
continue
|
||||
}
|
||||
if inFence {
|
||||
codeChars += len([]rune(line))
|
||||
continue
|
||||
}
|
||||
|
||||
runeLen := len([]rune(line))
|
||||
lengths = append(lengths, float64(runeLen))
|
||||
|
||||
if matchHeading(line, &p.MdHeadingCounts) {
|
||||
p.MdHeadingTotal++
|
||||
continue
|
||||
}
|
||||
if NumberedSectionPattern.MatchString(line) {
|
||||
p.NumberedSectionCount++
|
||||
}
|
||||
if GermanChapterPattern.MatchString(line) {
|
||||
p.GermanChapterCount++
|
||||
}
|
||||
if EnglishChapterPattern.MatchString(line) {
|
||||
p.EnglishChapterCount++
|
||||
}
|
||||
if ChineseChapterPattern.MatchString(line) {
|
||||
p.ChineseChapterCount++
|
||||
}
|
||||
if AllCapsHeadingPattern.MatchString(line) {
|
||||
p.AllCapsShortLineCount++
|
||||
}
|
||||
if VisualSeparatorPattern.MatchString(line) {
|
||||
p.VisualSepCount++
|
||||
}
|
||||
if PageFooterPattern.MatchString(line) {
|
||||
p.RepeatedFooterCount++
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "|") && strings.HasSuffix(trimmed, "|") {
|
||||
p.HasTables = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(lengths) > 0 {
|
||||
var sum float64
|
||||
for _, l := range lengths {
|
||||
sum += l
|
||||
}
|
||||
p.AvgLineLen = sum / float64(len(lengths))
|
||||
var variance float64
|
||||
for _, l := range lengths {
|
||||
d := l - p.AvgLineLen
|
||||
variance += d * d
|
||||
}
|
||||
variance /= float64(len(lengths))
|
||||
p.StdLineLen = math.Sqrt(variance)
|
||||
}
|
||||
|
||||
if p.TotalChars > 0 {
|
||||
p.CodeRatio = float64(codeChars) / float64(p.TotalChars)
|
||||
}
|
||||
|
||||
p.BlankParagraphBreaks = strings.Count(text, "\n\n\n")
|
||||
|
||||
// Sample a slice of the document for language detection — avoids paying
|
||||
// O(N) scan cost on huge inputs while still giving a stable signal.
|
||||
sample := text
|
||||
if len(sample) > 4096 {
|
||||
sample = sample[:4096]
|
||||
}
|
||||
lang := DetectLanguage(sample)
|
||||
p.DetectedLangs = []string{lang}
|
||||
if lang == LangMixed {
|
||||
// Provide all three for downstream pattern selection.
|
||||
p.DetectedLangs = []string{LangEnglish, LangGerman, LangChinese}
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// matchHeading checks whether line is an ATX heading and increments the
|
||||
// appropriate level counter when so. Returns true on match.
|
||||
func matchHeading(line string, counts *map[int]int) bool {
|
||||
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return false
|
||||
}
|
||||
level := len(m[1])
|
||||
if level < 1 || level > 6 {
|
||||
return false
|
||||
}
|
||||
(*counts)[level]++
|
||||
return true
|
||||
}
|
||||
|
||||
// StrategyTier identifies which chunking implementation should run.
|
||||
type StrategyTier string
|
||||
|
||||
const (
|
||||
TierHeading StrategyTier = "heading"
|
||||
TierHeuristic StrategyTier = "heuristic"
|
||||
TierRecursive StrategyTier = "recursive"
|
||||
TierLegacy StrategyTier = "legacy"
|
||||
)
|
||||
|
||||
// SelectStrategy returns the ordered tier chain to attempt for this document.
|
||||
// The first tier is the primary choice; subsequent tiers are fallbacks if
|
||||
// validation rejects the previous output. The "legacy" tier is appended as
|
||||
// a final safety net so callers always receive at least one chunk-set.
|
||||
func SelectStrategy(p *DocProfile) []StrategyTier {
|
||||
if p == nil {
|
||||
return []StrategyTier{TierRecursive, TierLegacy}
|
||||
}
|
||||
var chain []StrategyTier
|
||||
|
||||
// Tier 1 candidate: Markdown heading-aware
|
||||
if p.MdHeadingTotal >= 3 && p.HeadingDensity() > 0.005 && p.DominantHeadingLevel() > 0 {
|
||||
chain = append(chain, TierHeading)
|
||||
}
|
||||
|
||||
// Tier 2 candidate: heuristic boundary detection
|
||||
if p.HeuristicMarkerTotal() >= 5 || p.FormFeedCount > 0 ||
|
||||
p.GermanChapterCount+p.EnglishChapterCount+p.ChineseChapterCount > 0 {
|
||||
chain = append(chain, TierHeuristic)
|
||||
}
|
||||
|
||||
// Always end with recursive (Tier 3) and then legacy as ultimate fallback.
|
||||
chain = append(chain, TierRecursive, TierLegacy)
|
||||
return chain
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProfileDocument_Empty(t *testing.T) {
|
||||
p := ProfileDocument("")
|
||||
if p.TotalChars != 0 || p.TotalLines != 0 {
|
||||
t.Errorf("empty doc should have zero stats, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_MarkdownHeadings(t *testing.T) {
|
||||
doc := `# Title
|
||||
Some intro text here.
|
||||
|
||||
## Section 1
|
||||
Body of section 1.
|
||||
|
||||
## Section 2
|
||||
Body of section 2.
|
||||
|
||||
### Subsection 2.1
|
||||
Detail.
|
||||
|
||||
## Section 3
|
||||
More body.`
|
||||
p := ProfileDocument(doc)
|
||||
if p.MdHeadingCounts[1] != 1 {
|
||||
t.Errorf("expected 1 H1, got %d", p.MdHeadingCounts[1])
|
||||
}
|
||||
if p.MdHeadingCounts[2] != 3 {
|
||||
t.Errorf("expected 3 H2, got %d", p.MdHeadingCounts[2])
|
||||
}
|
||||
if p.MdHeadingCounts[3] != 1 {
|
||||
t.Errorf("expected 1 H3, got %d", p.MdHeadingCounts[3])
|
||||
}
|
||||
if p.MdHeadingTotal != 5 {
|
||||
t.Errorf("expected 5 headings total, got %d", p.MdHeadingTotal)
|
||||
}
|
||||
if p.DominantHeadingLevel() != 2 {
|
||||
t.Errorf("dominant level should be 2 (≥3 occurrences), got %d", p.DominantHeadingLevel())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_DominantLevelFallback(t *testing.T) {
|
||||
// No level reaches 3 occurrences — should fall back to most frequent.
|
||||
doc := "# Single H1\n## H2 a\n## H2 b\n"
|
||||
p := ProfileDocument(doc)
|
||||
if p.DominantHeadingLevel() != 2 {
|
||||
t.Errorf("expected fallback to level 2 (most frequent), got %d", p.DominantHeadingLevel())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_NumberedSections(t *testing.T) {
|
||||
doc := `1. Introduction
|
||||
text
|
||||
|
||||
2. Methodology
|
||||
text
|
||||
|
||||
3. Results
|
||||
text`
|
||||
p := ProfileDocument(doc)
|
||||
if p.NumberedSectionCount < 3 {
|
||||
t.Errorf("expected ≥3 numbered sections, got %d", p.NumberedSectionCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_GermanChapters(t *testing.T) {
|
||||
doc := "Kapitel 1: Einführung\n\nText\n\nKapitel 2: Hauptteil\n\nText"
|
||||
p := ProfileDocument(doc)
|
||||
if p.GermanChapterCount != 2 {
|
||||
t.Errorf("expected 2 German chapters, got %d", p.GermanChapterCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_ChineseChapters(t *testing.T) {
|
||||
doc := "第一章 引言\n\n内容\n\n第二章 方法\n\n内容"
|
||||
p := ProfileDocument(doc)
|
||||
if p.ChineseChapterCount != 2 {
|
||||
t.Errorf("expected 2 Chinese chapters, got %d", p.ChineseChapterCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_FormFeed(t *testing.T) {
|
||||
doc := "page 1 content\f\npage 2 content\f\npage 3 content"
|
||||
p := ProfileDocument(doc)
|
||||
if p.FormFeedCount != 2 {
|
||||
t.Errorf("expected 2 form feeds, got %d", p.FormFeedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_DetectsCodeBlock(t *testing.T) {
|
||||
doc := "Some prose.\n\n```go\nfunc main() {}\n```\n\nMore prose."
|
||||
p := ProfileDocument(doc)
|
||||
if !p.HasCode {
|
||||
t.Error("expected HasCode=true for fenced block")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_DetectsTable(t *testing.T) {
|
||||
doc := "Intro.\n\n| col a | col b |\n| --- | --- |\n| 1 | 2 |\n"
|
||||
p := ProfileDocument(doc)
|
||||
if !p.HasTables {
|
||||
t.Error("expected HasTables=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileDocument_LineStatistics(t *testing.T) {
|
||||
doc := "short\nthis is a longer line of text\nanother line here"
|
||||
p := ProfileDocument(doc)
|
||||
if p.TotalLines != 3 {
|
||||
t.Errorf("expected 3 lines, got %d", p.TotalLines)
|
||||
}
|
||||
if p.AvgLineLen <= 0 {
|
||||
t.Error("expected positive avg line len")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectStrategy_HeadingDoc(t *testing.T) {
|
||||
doc := "# A\nbody\n## B\nbody\n## C\nbody\n## D\nbody"
|
||||
p := ProfileDocument(doc)
|
||||
chain := SelectStrategy(p)
|
||||
if chain[0] != TierHeading {
|
||||
t.Errorf("expected heading tier first, got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectStrategy_HeuristicDoc(t *testing.T) {
|
||||
doc := strings.Repeat("Kapitel 1: Foo\nbody body body\n\n", 1) +
|
||||
strings.Repeat("Kapitel 2: Bar\nbody body body\n\n", 1)
|
||||
p := ProfileDocument(doc)
|
||||
chain := SelectStrategy(p)
|
||||
// no markdown headings → heuristic must come first (heading tier skipped)
|
||||
if chain[0] != TierHeuristic {
|
||||
t.Errorf("expected heuristic tier first, got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectStrategy_PlainDoc(t *testing.T) {
|
||||
doc := "just a paragraph of plain text without any structure indicators at all here"
|
||||
p := ProfileDocument(doc)
|
||||
chain := SelectStrategy(p)
|
||||
if chain[0] != TierRecursive {
|
||||
t.Errorf("expected recursive tier first for unstructured doc, got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectStrategy_AlwaysFallsBackToLegacy(t *testing.T) {
|
||||
for _, doc := range []string{"", "simple", "# H1\nbody"} {
|
||||
p := ProfileDocument(doc)
|
||||
chain := SelectStrategy(p)
|
||||
if chain[len(chain)-1] != TierLegacy {
|
||||
t.Errorf("chain must end with legacy, got %v for doc=%q", chain, doc)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,18 +27,35 @@ type ImageRef struct {
|
||||
End int
|
||||
}
|
||||
|
||||
// SplitterConfig configures the text splitter.
|
||||
// SplitterConfig configures the text splitter. Strategy and TokenLimit are
|
||||
// honored by the strategy entry point in strategy.go; the legacy SplitText
|
||||
// path uses only ChunkSize/Overlap/Separators.
|
||||
type SplitterConfig struct {
|
||||
ChunkSize int
|
||||
ChunkOverlap int
|
||||
Separators []string
|
||||
|
||||
// Strategy selects an adaptive tier. Empty = legacy (backwards-compatible).
|
||||
// See strategy.go for valid values.
|
||||
Strategy string
|
||||
// TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize chars.
|
||||
TokenLimit int
|
||||
// Languages hints multilingual heuristic patterns. Empty = auto-detect.
|
||||
Languages []string
|
||||
}
|
||||
|
||||
// Default sizes used by all entry points (DefaultConfig, ensureDefaults,
|
||||
// and buildSplitterConfig in the knowledge service).
|
||||
const (
|
||||
DefaultChunkSize = 512
|
||||
DefaultChunkOverlap = 80 // ≈ 15% of DefaultChunkSize
|
||||
)
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
func DefaultConfig() SplitterConfig {
|
||||
return SplitterConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 64,
|
||||
ChunkSize: DefaultChunkSize,
|
||||
ChunkOverlap: DefaultChunkOverlap,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package chunker - strategy.go is the public entry point for adaptive
|
||||
// chunking. Callers invoke Split / SplitParentChild instead of the legacy
|
||||
// SplitText / SplitTextParentChild functions; the strategy resolver picks
|
||||
// a tier based on document profile and the SplitterConfig.Strategy hint.
|
||||
//
|
||||
// The legacy entry points still exist in splitter.go for backwards
|
||||
// compatibility — strategy.go simply layers a tier-selector on top.
|
||||
package chunker
|
||||
|
||||
import "github.com/Tencent/WeKnora/internal/logger"
|
||||
|
||||
// Strategy values for SplitterConfig.Strategy.
|
||||
const (
|
||||
StrategyAuto = "auto"
|
||||
StrategyHeading = "heading"
|
||||
StrategyHeuristic = "heuristic"
|
||||
StrategyRecursive = "recursive"
|
||||
StrategyLegacy = "legacy"
|
||||
)
|
||||
|
||||
// Split chunks text using the strategy configured in cfg. When cfg.Strategy
|
||||
// is empty or "auto" the document profiler picks the tier. The function
|
||||
// always returns a non-nil result: on tier failure the chain falls through
|
||||
// to the legacy splitter, which is the original Tier 3 implementation.
|
||||
func Split(text string, cfg SplitterConfig) []Chunk {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
cfg = ensureDefaults(cfg)
|
||||
|
||||
chain := resolveChain(text, cfg)
|
||||
totalChars := len([]rune(text))
|
||||
|
||||
for _, tier := range chain {
|
||||
out := runTier(tier, text, cfg)
|
||||
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
||||
return out
|
||||
} else {
|
||||
logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
}
|
||||
}
|
||||
// Last-ditch fallback: always return *something*.
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
|
||||
// It runs the tier selector for parent splitting, then re-splits each
|
||||
// parent into children with the small-chunk config.
|
||||
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
||||
if text == "" {
|
||||
return ParentChildResult{}
|
||||
}
|
||||
parentCfg = ensureDefaults(parentCfg)
|
||||
childCfg = ensureDefaults(childCfg)
|
||||
|
||||
parents := Split(text, parentCfg)
|
||||
if len(parents) == 0 {
|
||||
return ParentChildResult{}
|
||||
}
|
||||
|
||||
var newParents []Chunk
|
||||
var children []ChildChunk
|
||||
childSeq := 0
|
||||
for _, parent := range parents {
|
||||
subs := Split(parent.Content, childCfg)
|
||||
|
||||
parentIndex := -1
|
||||
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
|
||||
parentIndex = len(newParents)
|
||||
newParents = append(newParents, parent)
|
||||
}
|
||||
for _, sub := range subs {
|
||||
sub.Seq = childSeq
|
||||
sub.Start += parent.Start
|
||||
sub.End += parent.Start
|
||||
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
|
||||
childSeq++
|
||||
}
|
||||
}
|
||||
return ParentChildResult{Parents: newParents, Children: children}
|
||||
}
|
||||
|
||||
// resolveChain returns the strategy chain to attempt. An explicit non-auto
|
||||
// strategy bypasses the profiler entirely and pins to the requested tier.
|
||||
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
|
||||
switch cfg.Strategy {
|
||||
case StrategyHeading:
|
||||
return []StrategyTier{TierHeading, TierLegacy}
|
||||
case StrategyHeuristic:
|
||||
return []StrategyTier{TierHeuristic, TierLegacy}
|
||||
case StrategyRecursive:
|
||||
return []StrategyTier{TierRecursive, TierLegacy}
|
||||
case StrategyLegacy, "":
|
||||
// Empty == legacy preserves backwards compatibility with stored
|
||||
// ChunkingConfig rows that pre-date the Strategy field.
|
||||
return []StrategyTier{TierLegacy}
|
||||
case StrategyAuto:
|
||||
fallthrough
|
||||
default:
|
||||
profile := ProfileDocument(text)
|
||||
return SelectStrategy(profile)
|
||||
}
|
||||
}
|
||||
|
||||
// runTier dispatches the splitter implementation for the given tier.
|
||||
// Heading and heuristic tiers are stubbed in this scaffold and currently
|
||||
// fall through to the legacy splitter — they are filled in in later phases.
|
||||
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
||||
switch tier {
|
||||
case TierHeading:
|
||||
return splitByHeadings(text, cfg)
|
||||
case TierHeuristic:
|
||||
return splitByHeuristics(text, cfg)
|
||||
case TierRecursive, TierLegacy:
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
// ensureDefaults fills in zero-value config fields with sane defaults.
|
||||
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
|
||||
// so direct callers of this package get the same numbers.
|
||||
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
||||
if cfg.ChunkSize <= 0 {
|
||||
cfg.ChunkSize = DefaultChunkSize
|
||||
}
|
||||
if cfg.ChunkOverlap <= 0 {
|
||||
cfg.ChunkOverlap = DefaultChunkOverlap
|
||||
}
|
||||
if len(cfg.Separators) == 0 {
|
||||
cfg.Separators = []string{"\n\n", "\n", "。"}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// splitByHeadings is overridden by heading_splitter.go.
|
||||
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
// splitByHeuristics is overridden by heuristic_splitter.go.
|
||||
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplit_EmptyText(t *testing.T) {
|
||||
if got := Split("", DefaultConfig()); got != nil {
|
||||
t.Errorf("empty text should return nil, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplit_LegacyStrategy_MatchesSplitText(t *testing.T) {
|
||||
text := strings.Repeat("Hello world.\n\n", 30)
|
||||
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n"}, Strategy: StrategyLegacy}
|
||||
a := Split(text, cfg)
|
||||
b := SplitText(text, cfg)
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("legacy strategy should match SplitText: got %d vs %d chunks", len(a), len(b))
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Content != b[i].Content {
|
||||
t.Errorf("chunk %d differs", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplit_EmptyStrategyEqualsLegacy(t *testing.T) {
|
||||
text := strings.Repeat("Sentence one. Sentence two.\n", 20)
|
||||
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 10}
|
||||
a := Split(text, cfg)
|
||||
cfg.Strategy = StrategyLegacy
|
||||
b := Split(text, cfg)
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("empty Strategy should equal legacy: %d vs %d", len(a), len(b))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplit_AutoStrategy_PicksHeadingForMarkdownDoc(t *testing.T) {
|
||||
doc := strings.Repeat("# A\nbody\n## B\nbody\n## C\nbody\n## D\nbody\n", 1)
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto}
|
||||
// Until heading_splitter is wired, this falls through to SplitText —
|
||||
// just assert we get a valid result.
|
||||
chunks := Split(doc, cfg)
|
||||
if len(chunks) == 0 {
|
||||
t.Error("auto strategy should produce chunks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitParentChild_LegacyStrategy(t *testing.T) {
|
||||
text := strings.Repeat("This is a sentence. Another one.\n\n", 50)
|
||||
parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy}
|
||||
childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Strategy: StrategyLegacy}
|
||||
res := SplitParentChild(text, parentCfg, childCfg)
|
||||
if len(res.Children) == 0 {
|
||||
t.Fatal("expected children chunks")
|
||||
}
|
||||
for i, c := range res.Children {
|
||||
if c.ParentIndex >= 0 && c.ParentIndex >= len(res.Parents) {
|
||||
t.Errorf("child[%d] has invalid ParentIndex %d (parents=%d)", i, c.ParentIndex, len(res.Parents))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDefaults(t *testing.T) {
|
||||
cfg := ensureDefaults(SplitterConfig{})
|
||||
if cfg.ChunkSize != DefaultChunkSize {
|
||||
t.Errorf("expected default ChunkSize %d, got %d", DefaultChunkSize, cfg.ChunkSize)
|
||||
}
|
||||
if cfg.ChunkOverlap != DefaultChunkOverlap {
|
||||
t.Errorf("expected default ChunkOverlap %d, got %d", DefaultChunkOverlap, cfg.ChunkOverlap)
|
||||
}
|
||||
if len(cfg.Separators) == 0 {
|
||||
t.Error("expected default separators")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChunks_Empty(t *testing.T) {
|
||||
if v := ValidateChunks(nil, 1000, 500); v.OK {
|
||||
t.Error("nil chunks should be invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChunks_SingleChunkLargeDoc(t *testing.T) {
|
||||
c := []Chunk{{Content: strings.Repeat("a", 5000)}}
|
||||
if v := ValidateChunks(c, 5000, 500); v.OK {
|
||||
t.Error("single 10x-too-large chunk should be invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChunks_AcceptsReasonableOutput(t *testing.T) {
|
||||
chunks := []Chunk{
|
||||
{Content: strings.Repeat("a", 480)},
|
||||
{Content: strings.Repeat("b", 510)},
|
||||
{Content: strings.Repeat("c", 460)},
|
||||
}
|
||||
if v := ValidateChunks(chunks, 1500, 512); !v.OK {
|
||||
t.Errorf("reasonable chunks should validate, got: %s", v.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChunks_RejectsOversized(t *testing.T) {
|
||||
chunks := []Chunk{
|
||||
{Content: strings.Repeat("a", 100)},
|
||||
{Content: strings.Repeat("b", 5000)}, // > 2x chunkSize
|
||||
}
|
||||
if v := ValidateChunks(chunks, 5100, 1000); v.OK {
|
||||
t.Error("chunk >2x size should be invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChunks_TolerantTinyTail(t *testing.T) {
|
||||
chunks := []Chunk{
|
||||
{Content: strings.Repeat("a", 480)},
|
||||
{Content: strings.Repeat("b", 510)},
|
||||
{Content: "tail"},
|
||||
}
|
||||
if v := ValidateChunks(chunks, 994, 512); !v.OK {
|
||||
t.Errorf("tiny last chunk should be tolerated, got: %s", v.Reason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Package chunker - tokens.go provides language-aware token count approximation.
|
||||
//
|
||||
// We avoid pulling in a tokenizer dependency (e.g. tiktoken) and instead use
|
||||
// per-language chars-per-token ratios derived from common embedding model
|
||||
// vocabularies. The numbers are conservative — they tend to slightly
|
||||
// over-estimate token counts so that chunks stay safely under model limits.
|
||||
package chunker
|
||||
|
||||
import "unicode"
|
||||
|
||||
// Language identifiers used by the token estimator and the heuristic splitter.
|
||||
const (
|
||||
LangEnglish = "en"
|
||||
LangGerman = "de"
|
||||
LangChinese = "zh"
|
||||
LangMixed = "mixed"
|
||||
)
|
||||
|
||||
// charsPerToken holds approximate chars/token ratios per language.
|
||||
// Numbers err on the conservative side so estimates over-shoot a little.
|
||||
var charsPerToken = map[string]float64{
|
||||
LangEnglish: 4.0,
|
||||
LangGerman: 4.5,
|
||||
LangChinese: 1.7,
|
||||
LangMixed: 3.0,
|
||||
}
|
||||
|
||||
// ApproxTokenCount returns a conservative token estimate for s in the given
|
||||
// language. An empty or unknown lang falls back to "mixed".
|
||||
func ApproxTokenCount(s string, lang string) int {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
ratio, ok := charsPerToken[lang]
|
||||
if !ok {
|
||||
ratio = charsPerToken[LangMixed]
|
||||
}
|
||||
runes := []rune(s)
|
||||
approx := float64(len(runes)) / ratio
|
||||
if approx < 1 {
|
||||
return 1
|
||||
}
|
||||
return int(approx + 0.5)
|
||||
}
|
||||
|
||||
// DetectLanguage returns a coarse language label by counting CJK runes vs.
|
||||
// Latin runes. The result is one of LangChinese, LangGerman, LangEnglish or
|
||||
// LangMixed. Detection is cheap and meant only for heuristic dispatch — it
|
||||
// is NOT a replacement for proper language identification.
|
||||
func DetectLanguage(s string) string {
|
||||
if s == "" {
|
||||
return LangMixed
|
||||
}
|
||||
var cjk, latin, umlaut int
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hangul, r) || unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r):
|
||||
cjk++
|
||||
case isGermanUmlaut(r):
|
||||
umlaut++
|
||||
latin++
|
||||
case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z'):
|
||||
latin++
|
||||
}
|
||||
}
|
||||
total := cjk + latin
|
||||
if total == 0 {
|
||||
return LangMixed
|
||||
}
|
||||
cjkRatio := float64(cjk) / float64(total)
|
||||
latinRatio := float64(latin) / float64(total)
|
||||
// Mixed: meaningful presence of both scripts (>=15% each).
|
||||
if cjkRatio >= 0.15 && latinRatio >= 0.15 {
|
||||
return LangMixed
|
||||
}
|
||||
if cjkRatio > 0.3 {
|
||||
return LangChinese
|
||||
}
|
||||
if umlaut > 0 || hasGermanWords(s) {
|
||||
return LangGerman
|
||||
}
|
||||
return LangEnglish
|
||||
}
|
||||
|
||||
func isGermanUmlaut(r rune) bool {
|
||||
switch r {
|
||||
case 'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasGermanWords does a tiny stop-word check to bias towards "de" when the
|
||||
// text uses common German function words. Cheap heuristic — false positives
|
||||
// on borrowed terms are acceptable.
|
||||
func hasGermanWords(s string) bool {
|
||||
const sample = 512
|
||||
if len(s) > sample {
|
||||
s = s[:sample]
|
||||
}
|
||||
for _, w := range []string{" der ", " die ", " das ", " und ", " ist ", " nicht ", " mit ", " auf "} {
|
||||
if containsLower(s, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsLower(haystack, needle string) bool {
|
||||
if len(haystack) < len(needle) {
|
||||
return false
|
||||
}
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
match := true
|
||||
for j := 0; j < len(needle); j++ {
|
||||
h := haystack[i+j]
|
||||
if h >= 'A' && h <= 'Z' {
|
||||
h += 'a' - 'A'
|
||||
}
|
||||
if h != needle[j] {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CharsForTokenLimit converts a token limit into an approximate character
|
||||
// budget for a given language. Used to size chunks so they fit within an
|
||||
// embedding model's max-token window with a small safety margin.
|
||||
func CharsForTokenLimit(tokens int, lang string) int {
|
||||
if tokens <= 0 {
|
||||
return 0
|
||||
}
|
||||
ratio, ok := charsPerToken[lang]
|
||||
if !ok {
|
||||
ratio = charsPerToken[LangMixed]
|
||||
}
|
||||
// 0.9 safety factor so we under-shoot the model limit instead of overshooting.
|
||||
return int(float64(tokens) * ratio * 0.9)
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package chunker
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestApproxTokenCount_English(t *testing.T) {
|
||||
got := ApproxTokenCount("The quick brown fox jumps over the lazy dog.", LangEnglish)
|
||||
// 44 chars / 4 ≈ 11 tokens
|
||||
if got < 9 || got > 13 {
|
||||
t.Errorf("English token estimate out of range: got %d, want 9..13", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxTokenCount_Chinese(t *testing.T) {
|
||||
got := ApproxTokenCount("这是一段中文测试内容用于检验分词估算", LangChinese)
|
||||
// 18 runes / 1.7 ≈ 10
|
||||
if got < 9 || got > 12 {
|
||||
t.Errorf("Chinese token estimate out of range: got %d, want 9..12", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxTokenCount_Empty(t *testing.T) {
|
||||
if got := ApproxTokenCount("", LangEnglish); got != 0 {
|
||||
t.Errorf("empty string should return 0 tokens, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxTokenCount_UnknownLang(t *testing.T) {
|
||||
got := ApproxTokenCount("Hello world hello world", "xx")
|
||||
if got <= 0 {
|
||||
t.Errorf("unknown lang should fall back to mixed, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectLanguage_English(t *testing.T) {
|
||||
if got := DetectLanguage("The quick brown fox jumps over the lazy dog."); got != LangEnglish {
|
||||
t.Errorf("expected English, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectLanguage_German(t *testing.T) {
|
||||
if got := DetectLanguage("Der schnelle braune Fuchs springt über den faulen Hund."); got != LangGerman {
|
||||
t.Errorf("expected German, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectLanguage_GermanByStopwords(t *testing.T) {
|
||||
// No umlauts but plenty of German function words.
|
||||
if got := DetectLanguage("Das ist ein Test und nicht mit Umlauten."); got != LangGerman {
|
||||
t.Errorf("expected German via stopwords, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectLanguage_Chinese(t *testing.T) {
|
||||
if got := DetectLanguage("这是一段中文测试内容"); got != LangChinese {
|
||||
t.Errorf("expected Chinese, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectLanguage_Mixed(t *testing.T) {
|
||||
got := DetectLanguage("This 这是 mixed 测试 content with 多语言 inside")
|
||||
if got != LangMixed {
|
||||
t.Errorf("expected Mixed, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCharsForTokenLimit_AppliesSafetyMargin(t *testing.T) {
|
||||
got := CharsForTokenLimit(1000, LangEnglish)
|
||||
// 1000 * 4 * 0.9 = 3600
|
||||
if got < 3500 || got > 3700 {
|
||||
t.Errorf("char budget for 1000 EN tokens out of range: got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCharsForTokenLimit_ZeroTokens(t *testing.T) {
|
||||
if got := CharsForTokenLimit(0, LangEnglish); got != 0 {
|
||||
t.Errorf("zero tokens should give zero chars, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// Package chunker - validator.go inspects a tier's output and decides whether
|
||||
// it is good enough to ship or whether the strategy chain should fall through
|
||||
// to the next tier. The validator is intentionally permissive: a single
|
||||
// "obviously broken" output is rejected, but plausible-looking variation is
|
||||
// accepted so we don't oscillate between tiers.
|
||||
package chunker
|
||||
|
||||
import "math"
|
||||
|
||||
// ValidationResult captures the verdict and reason for a chunk-set.
|
||||
type ValidationResult struct {
|
||||
OK bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
// ValidateChunks checks whether the given chunks form a usable result for a
|
||||
// document of totalChars characters with a target chunkSize. Returns OK=true
|
||||
// when no broken-output indicator triggers.
|
||||
func ValidateChunks(chunks []Chunk, totalChars, chunkSize int) ValidationResult {
|
||||
if len(chunks) == 0 {
|
||||
return ValidationResult{Reason: "no chunks produced"}
|
||||
}
|
||||
|
||||
// A single chunk for a document much larger than chunkSize means the
|
||||
// strategy did not actually split — fail so the next tier runs.
|
||||
if len(chunks) == 1 && totalChars > 2*chunkSize {
|
||||
return ValidationResult{Reason: "single chunk for large document"}
|
||||
}
|
||||
|
||||
// Compute size statistics.
|
||||
var sum, sumSq float64
|
||||
maxLen, minLen := 0, math.MaxInt32
|
||||
for _, c := range chunks {
|
||||
l := len([]rune(c.Content))
|
||||
sum += float64(l)
|
||||
sumSq += float64(l * l)
|
||||
if l > maxLen {
|
||||
maxLen = l
|
||||
}
|
||||
if l < minLen {
|
||||
minLen = l
|
||||
}
|
||||
}
|
||||
avg := sum / float64(len(chunks))
|
||||
|
||||
// All but the last chunk should carry meaningful content. We allow the
|
||||
// last chunk to be tiny because tail residue is normal.
|
||||
tinyCount := 0
|
||||
for i, c := range chunks {
|
||||
if i == len(chunks)-1 {
|
||||
continue
|
||||
}
|
||||
if len([]rune(c.Content)) < 50 {
|
||||
tinyCount++
|
||||
}
|
||||
}
|
||||
if tinyCount > len(chunks)/4 && tinyCount > 2 {
|
||||
return ValidationResult{Reason: "too many tiny chunks"}
|
||||
}
|
||||
|
||||
// Reject when no chunk reached at least 25% of the target — the splitter
|
||||
// is fragmenting too aggressively to be useful.
|
||||
if maxLen < chunkSize/4 && totalChars > chunkSize {
|
||||
return ValidationResult{Reason: "all chunks far below target size"}
|
||||
}
|
||||
|
||||
// Sanity check on absolute upper bound. Anything past 2x chunkSize is a
|
||||
// red flag — the splitter ignored its size budget.
|
||||
if maxLen > 2*chunkSize && chunkSize > 0 {
|
||||
return ValidationResult{Reason: "chunk exceeds 2x target size"}
|
||||
}
|
||||
|
||||
_ = avg
|
||||
return ValidationResult{OK: true}
|
||||
}
|
||||
Reference in New Issue
Block a user