mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-19 02:18:25 +08:00
refactor(chunker): drop redundant TierRecursive
TierRecursive and TierLegacy both invoked SplitText with identical output, but only TierLegacy got the "always-return-on-validation-failure" safety-net behavior. Auto chains thus ran SplitText twice on the fallback path and produced two-line debug rejection traces with the same reason. Inline TierRecursive into TierLegacy: SelectStrategy and the explicit StrategyRecursive entry point now emit single-legacy chains. The StrategyRecursive public constant stays so existing ChunkingConfig rows keep parsing — it's just an alias for legacy now. No user-visible behavior change; one fewer SplitText call per failed auto fallback.
This commit is contained in:
@@ -210,7 +210,6 @@ type StrategyTier string
|
||||
const (
|
||||
TierHeading StrategyTier = "heading"
|
||||
TierHeuristic StrategyTier = "heuristic"
|
||||
TierRecursive StrategyTier = "recursive"
|
||||
TierLegacy StrategyTier = "legacy"
|
||||
)
|
||||
|
||||
@@ -220,7 +219,7 @@ const (
|
||||
// a final safety net so callers always receive at least one chunk-set.
|
||||
func SelectStrategy(p *DocProfile) []StrategyTier {
|
||||
if p == nil {
|
||||
return []StrategyTier{TierRecursive, TierLegacy}
|
||||
return []StrategyTier{TierLegacy}
|
||||
}
|
||||
var chain []StrategyTier
|
||||
|
||||
@@ -235,7 +234,8 @@ func SelectStrategy(p *DocProfile) []StrategyTier {
|
||||
chain = append(chain, TierHeuristic)
|
||||
}
|
||||
|
||||
// Always end with recursive (Tier 3) and then legacy as ultimate fallback.
|
||||
chain = append(chain, TierRecursive, TierLegacy)
|
||||
// Legacy is the ultimate fallback: always returns chunks even when
|
||||
// validation fails, so callers never get an empty result.
|
||||
chain = append(chain, TierLegacy)
|
||||
return chain
|
||||
}
|
||||
|
||||
@@ -144,8 +144,8 @@ func TestSelectStrategy_PlainDoc(t *testing.T) {
|
||||
doc := "just a paragraph of plain text without any structure indicators at all here"
|
||||
p := ProfileDocument(doc)
|
||||
chain := SelectStrategy(p)
|
||||
if chain[0] != TierRecursive {
|
||||
t.Errorf("expected recursive tier first for unstructured doc, got %v", chain)
|
||||
if chain[0] != TierLegacy {
|
||||
t.Errorf("expected legacy tier first for unstructured doc, got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -185,7 +185,9 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *
|
||||
case StrategyHeuristic:
|
||||
return []StrategyTier{TierHeuristic, TierLegacy}, nil
|
||||
case StrategyRecursive:
|
||||
return []StrategyTier{TierRecursive, TierLegacy}, nil
|
||||
// "recursive" is a public-API alias for "legacy": both invoke
|
||||
// SplitText. Kept for backwards compatibility with stored configs.
|
||||
return []StrategyTier{TierLegacy}, nil
|
||||
case StrategyLegacy, "":
|
||||
// Empty == legacy preserves backwards compatibility with stored
|
||||
// ChunkingConfig rows that pre-date the Strategy field.
|
||||
@@ -200,16 +202,16 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *
|
||||
|
||||
// runTier dispatches the splitter implementation for the given tier.
|
||||
// splitByHeadings / splitByHeuristics are package-level vars overridden
|
||||
// from heading_splitter.go / heuristic_splitter.go via init(); recursive
|
||||
// and legacy share the same SplitText path. The default branch is kept
|
||||
// as defensive belt-and-suspenders for future StrategyTier additions.
|
||||
// from heading_splitter.go / heuristic_splitter.go via init(); legacy
|
||||
// runs SplitText. The default branch is defensive for future
|
||||
// StrategyTier additions.
|
||||
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
||||
switch tier {
|
||||
case TierHeading:
|
||||
return splitByHeadings(text, cfg)
|
||||
case TierHeuristic:
|
||||
return splitByHeuristics(text, cfg)
|
||||
case TierRecursive, TierLegacy:
|
||||
case TierLegacy:
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
return SplitText(text, cfg)
|
||||
|
||||
Reference in New Issue
Block a user