From 262b172c28f7d13ba53ed24d08b32d628b580735 Mon Sep 17 00:00:00 2001 From: sandaogouchen <143976836+sandaogouchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:20:03 +0800 Subject: [PATCH] fix(chunking): preserve parent-child text child embeddings (#2680) * fix(chunking): preserve parent-child text child embeddings * refactor(chunking): simplify text chunk index selection --- .../application/service/knowledge_process.go | 9 +- .../knowledge_process_parent_child_test.go | 200 ++++++++++++++++++ internal/infrastructure/chunker/strategy.go | 5 + .../infrastructure/chunker/strategy_test.go | 45 ++++ 4 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 internal/application/service/knowledge_process_parent_child_test.go diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index fb5396330..97c3a4b18 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -442,14 +442,11 @@ func (s *knowledgeService) processChunks(ctx context.Context, return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex }) - // 仅为文本类型的Chunk设置前后关系（child chunks only, parents already linked above） + // Collect retrievable text chunks only. ParentChunkID only controls parent expansion after retrieval. + // When ParentChunkID is empty, retrieval keeps the standalone child content without loading a parent. textChunks := make([]*types.Chunk, 0, len(chunks)) for _, chunk := range insertChunks { - if chunk.ChunkType == types.ChunkTypeText && chunk.ParentChunkID != "" { - // This is a child chunk in parent-child mode - textChunks = append(textChunks, chunk) - } else if chunk.ChunkType == types.ChunkTypeText && !hasParentChild { - // Normal flat chunk (no parent-child mode) + if chunk.ChunkType == types.ChunkTypeText { textChunks = append(textChunks, chunk) } } diff --git a/internal/application/service/knowledge_process_parent_child_test.go b/internal/application/service/knowledge_process_parent_child_test.go new file mode 100644 index 000000000..a217a86a4 --- /dev/null +++ b/internal/application/service/knowledge_process_parent_child_test.go @@ -0,0 +1,200 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +type parentChildKnowledgeRepo struct { + interfaces.KnowledgeRepository + knowledge *types.Knowledge +} + +func (r *parentChildKnowledgeRepo) GetKnowledgeByID( + context.Context, uint64, string, +) (*types.Knowledge, error) { + return r.knowledge, nil +} + +func (r *parentChildKnowledgeRepo) UpdateKnowledge( + context.Context, *types.Knowledge, +) error { + return nil +} + +type parentChildChunkService struct { + interfaces.ChunkService + created []*types.Chunk +} + +func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, string) error { + return nil +} + +func (s *parentChildChunkService) CreateChunks(_ context.Context, chunks []*types.Chunk) error { + s.created = append([]*types.Chunk(nil), chunks...) + return nil +} + +type parentChildModelService struct { + interfaces.ModelService + embedder embedding.Embedder +} + +func (s parentChildModelService) GetEmbeddingModel(context.Context, string) (embedding.Embedder, error) { + return s.embedder, nil +} + +type parentChildEmbedder struct{} + +func (parentChildEmbedder) Embed(context.Context, string) ([]float32, error) { + return []float32{1}, nil +} + +func (parentChildEmbedder) BatchEmbed(context.Context, []string) ([][]float32, error) { + return [][]float32{{1}}, nil +} + +func (parentChildEmbedder) BatchEmbedWithPool( + context.Context, embedding.Embedder, []string, +) ([][]float32, error) { + return [][]float32{{1}}, nil +} + +func (parentChildEmbedder) GetModelName() string { return "parent-child-test" } +func (parentChildEmbedder) GetDimensions() int { return 1 } +func (parentChildEmbedder) GetModelID() string { return "parent-child-test" } + +type parentChildRetrieveEngine struct { + interfaces.RetrieveEngineService + indexed []*types.IndexInfo +} + +func (e *parentChildRetrieveEngine) EngineType() types.RetrieverEngineType { + return types.PostgresRetrieverEngineType +} + +func (e *parentChildRetrieveEngine) Support() []types.RetrieverType { + return []types.RetrieverType{types.VectorRetrieverType} +} + +func (e *parentChildRetrieveEngine) DeleteByKnowledgeIDList( + context.Context, []string, int, string, +) error { + return nil +} + +func (e *parentChildRetrieveEngine) EstimateStorageSize( + context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType, +) int64 { + return 0 +} + +func (e *parentChildRetrieveEngine) BatchIndex( + _ context.Context, + _ embedding.Embedder, + infos []*types.IndexInfo, + _ []types.RetrieverType, +) error { + e.indexed = append([]*types.IndexInfo(nil), infos...) + return nil +} + +type parentChildRetrieveRegistry struct { + interfaces.RetrieveEngineRegistry + engine interfaces.RetrieveEngineService +} + +func (r parentChildRetrieveRegistry) GetRetrieveEngineService( + types.RetrieverEngineType, +) (interfaces.RetrieveEngineService, error) { + return r.engine, nil +} + +type parentChildGraphRepo struct { + interfaces.RetrieveGraphRepository +} + +func (parentChildGraphRepo) DelGraph(context.Context, []types.NameSpace) error { + return nil +} + +type parentChildTenantRepo struct { + interfaces.TenantRepository +} + +func (parentChildTenantRepo) AdjustStorageUsed(context.Context, uint64, int64) error { + return nil +} + +type parentChildTaskEnqueuer struct{} + +func (parentChildTaskEnqueuer) Enqueue(*asynq.Task, ...asynq.Option) (*asynq.TaskInfo, error) { + return nil, nil +} + +func TestProcessChunksIndexesEveryTextChild(t *testing.T) { + knowledge := &types.Knowledge{ + ID: "knowledge-1", + TenantID: 1, + KnowledgeBaseID: "kb-1", + ParseStatus: types.ParseStatusProcessing, + } + chunkService := &parentChildChunkService{} + retrieveEngine := &parentChildRetrieveEngine{} + tenant := &types.Tenant{ + ID: 1, + RetrieverEngines: types.RetrieverEngines{Engines: []types.RetrieverEngineParams{ + { + RetrieverType: types.VectorRetrieverType, + RetrieverEngineType: types.PostgresRetrieverEngineType, + }, + }}, + } + ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant) + svc := &knowledgeService{ + repo: &parentChildKnowledgeRepo{knowledge: knowledge}, + chunkService: chunkService, + modelService: parentChildModelService{embedder: parentChildEmbedder{}}, + retrieveEngine: parentChildRetrieveRegistry{engine: retrieveEngine}, + graphEngine: parentChildGraphRepo{}, + tenantRepo: parentChildTenantRepo{}, + task: parentChildTaskEnqueuer{}, + } + kb := &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 1, + EmbeddingModelID: "embedding-1", + IndexingStrategy: types.IndexingStrategy{VectorEnabled: true}, + } + chunks := []types.ParsedChunk{ + {Content: "linked child", Seq: 0, Start: 0, End: 12, ParentIndex: 0}, + {Content: "standalone child", Seq: 1, Start: 12, End: 28, ParentIndex: -1}, + } + + svc.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{ + ParentChunks: []types.ParsedParentChunk{ + {Content: "parent context", Seq: 0, Start: 0, End: 28}, + }, + }) + + var textChunkIDs []string + for _, chunk := range chunkService.created { + if chunk.ChunkType == types.ChunkTypeText { + textChunkIDs = append(textChunkIDs, chunk.ID) + } + } + require.Len(t, textChunkIDs, 2) + + indexedSourceIDs := make([]string, 0, len(retrieveEngine.indexed)) + for _, info := range retrieveEngine.indexed { + indexedSourceIDs = append(indexedSourceIDs, info.SourceID) + } + require.ElementsMatch(t, textChunkIDs, indexedSourceIDs) +} diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 9fc8d319c..053bf3957 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -215,6 +215,8 @@ func NormalizeLineEndings(text string) string { // DeriveParentChildConfigs produces the exact parent and child splitter // configurations used by knowledge ingestion. Keeping this here lets preview // and ingestion remain in lockstep as the parent-child defaults evolve. +// Languages are copied to both levels so parent and child splitters use the same boundary rules. +// TokenLimit is copied only to children because parents keep the configured context window. func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (parent, child SplitterConfig) { if parentSize <= 0 { parentSize = 4096 @@ -227,12 +229,15 @@ func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (p ChunkOverlap: base.ChunkOverlap, Separators: base.Separators, Strategy: base.Strategy, + Languages: base.Languages, } child = SplitterConfig{ ChunkSize: childSize, ChunkOverlap: childSize / 5, Separators: base.Separators, Strategy: base.Strategy, + TokenLimit: base.TokenLimit, + Languages: base.Languages, } return } diff --git a/internal/infrastructure/chunker/strategy_test.go b/internal/infrastructure/chunker/strategy_test.go index 91677c75f..4c565ec00 100644 --- a/internal/infrastructure/chunker/strategy_test.go +++ b/internal/infrastructure/chunker/strategy_test.go @@ -263,6 +263,51 @@ func TestDeriveParentChildConfigs_DefaultSizes(t *testing.T) { } } +func TestDeriveParentChildConfigs_AppliesEmbeddingTokenLimitOnlyToChildren(t *testing.T) { + const ( + parentSize = 4096 + childSize = 1024 + tokenLimit = 200 + ) + base := SplitterConfig{ + ChunkSize: DefaultChunkSize, + ChunkOverlap: DefaultChunkOverlap, + Separators: []string{"。"}, + Strategy: StrategyLegacy, + TokenLimit: tokenLimit, + Languages: []string{LangChinese}, + } + + parent, child := DeriveParentChildConfigs(base, parentSize, childSize) + if parent.TokenLimit != 0 { + t.Fatalf("parent token limit: got %d want 0", parent.TokenLimit) + } + if len(parent.Languages) != 1 || parent.Languages[0] != LangChinese { + t.Fatalf("parent languages: got %v want [%s]", parent.Languages, LangChinese) + } + if child.TokenLimit != tokenLimit { + t.Fatalf("child token limit: got %d want %d", child.TokenLimit, tokenLimit) + } + if len(child.Languages) != 1 || child.Languages[0] != LangChinese { + t.Fatalf("child languages: got %v want [%s]", child.Languages, LangChinese) + } + + text := strings.Repeat("这是用于验证中文分块预算的句子。", 100) + result := SplitParentChild(text, parent, child) + if len(result.Parents) != 1 { + t.Fatalf("parents: got %d want 1", len(result.Parents)) + } + if got := len([]rune(result.Parents[0].Content)); got != len([]rune(text)) { + t.Fatalf("parent length: got %d want %d", got, len([]rune(text))) + } + budget := CharsForTokenLimit(tokenLimit, LangChinese) + for i, c := range result.Children { + if got := len([]rune(c.Content)); got > budget { + t.Fatalf("child[%d] length: got %d want <= %d", i, got, budget) + } + } +} + func TestValidateChunks_Empty(t *testing.T) { if v := ValidateChunks(nil, 1000, 500); v.OK { t.Error("nil chunks should be invalid")