fix(chunking): preserve parent-child text child embeddings (#2680)

* fix(chunking): preserve parent-child text child embeddings

* refactor(chunking): simplify text chunk index selection

internal/application/service/knowledge_process.go

@@ -442,14 +442,11 @@ func (s *knowledgeService) processChunks(ctx context.Context,
 		return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex
 	})
 
-	// 仅为文本类型的Chunk设置前后关系（child chunks only, parents already linked above）
+	// Collect retrievable text chunks only. ParentChunkID only controls parent expansion after retrieval.
+	// When ParentChunkID is empty, retrieval keeps the standalone child content without loading a parent.
 	textChunks := make([]*types.Chunk, 0, len(chunks))
 	for _, chunk := range insertChunks {
-		if chunk.ChunkType == types.ChunkTypeText && chunk.ParentChunkID != "" {
-			// This is a child chunk in parent-child mode
-			textChunks = append(textChunks, chunk)
-		} else if chunk.ChunkType == types.ChunkTypeText && !hasParentChild {
-			// Normal flat chunk (no parent-child mode)
+		if chunk.ChunkType == types.ChunkTypeText {
 			textChunks = append(textChunks, chunk)
 		}
 	}

internal/application/service/knowledge_process_parent_child_test.go

@@ -0,0 +1,200 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/models/embedding"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+	"github.com/hibiken/asynq"
+	"github.com/stretchr/testify/require"
+)
+
+type parentChildKnowledgeRepo struct {
+	interfaces.KnowledgeRepository
+	knowledge *types.Knowledge
+}
+
+func (r *parentChildKnowledgeRepo) GetKnowledgeByID(
+	context.Context, uint64, string,
+) (*types.Knowledge, error) {
+	return r.knowledge, nil
+}
+
+func (r *parentChildKnowledgeRepo) UpdateKnowledge(
+	context.Context, *types.Knowledge,
+) error {
+	return nil
+}
+
+type parentChildChunkService struct {
+	interfaces.ChunkService
+	created []*types.Chunk
+}
+
+func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, string) error {
+	return nil
+}
+
+func (s *parentChildChunkService) CreateChunks(_ context.Context, chunks []*types.Chunk) error {
+	s.created = append([]*types.Chunk(nil), chunks...)
+	return nil
+}
+
+type parentChildModelService struct {
+	interfaces.ModelService
+	embedder embedding.Embedder
+}
+
+func (s parentChildModelService) GetEmbeddingModel(context.Context, string) (embedding.Embedder, error) {
+	return s.embedder, nil
+}
+
+type parentChildEmbedder struct{}
+
+func (parentChildEmbedder) Embed(context.Context, string) ([]float32, error) {
+	return []float32{1}, nil
+}
+
+func (parentChildEmbedder) BatchEmbed(context.Context, []string) ([][]float32, error) {
+	return [][]float32{{1}}, nil
+}
+
+func (parentChildEmbedder) BatchEmbedWithPool(
+	context.Context, embedding.Embedder, []string,
+) ([][]float32, error) {
+	return [][]float32{{1}}, nil
+}
+
+func (parentChildEmbedder) GetModelName() string { return "parent-child-test" }
+func (parentChildEmbedder) GetDimensions() int   { return 1 }
+func (parentChildEmbedder) GetModelID() string   { return "parent-child-test" }
+
+type parentChildRetrieveEngine struct {
+	interfaces.RetrieveEngineService
+	indexed []*types.IndexInfo
+}
+
+func (e *parentChildRetrieveEngine) EngineType() types.RetrieverEngineType {
+	return types.PostgresRetrieverEngineType
+}
+
+func (e *parentChildRetrieveEngine) Support() []types.RetrieverType {
+	return []types.RetrieverType{types.VectorRetrieverType}
+}
+
+func (e *parentChildRetrieveEngine) DeleteByKnowledgeIDList(
+	context.Context, []string, int, string,
+) error {
+	return nil
+}
+
+func (e *parentChildRetrieveEngine) EstimateStorageSize(
+	context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType,
+) int64 {
+	return 0
+}
+
+func (e *parentChildRetrieveEngine) BatchIndex(
+	_ context.Context,
+	_ embedding.Embedder,
+	infos []*types.IndexInfo,
+	_ []types.RetrieverType,
+) error {
+	e.indexed = append([]*types.IndexInfo(nil), infos...)
+	return nil
+}
+
+type parentChildRetrieveRegistry struct {
+	interfaces.RetrieveEngineRegistry
+	engine interfaces.RetrieveEngineService
+}
+
+func (r parentChildRetrieveRegistry) GetRetrieveEngineService(
+	types.RetrieverEngineType,
+) (interfaces.RetrieveEngineService, error) {
+	return r.engine, nil
+}
+
+type parentChildGraphRepo struct {
+	interfaces.RetrieveGraphRepository
+}
+
+func (parentChildGraphRepo) DelGraph(context.Context, []types.NameSpace) error {
+	return nil
+}
+
+type parentChildTenantRepo struct {
+	interfaces.TenantRepository
+}
+
+func (parentChildTenantRepo) AdjustStorageUsed(context.Context, uint64, int64) error {
+	return nil
+}
+
+type parentChildTaskEnqueuer struct{}
+
+func (parentChildTaskEnqueuer) Enqueue(*asynq.Task, ...asynq.Option) (*asynq.TaskInfo, error) {
+	return nil, nil
+}
+
+func TestProcessChunksIndexesEveryTextChild(t *testing.T) {
+	knowledge := &types.Knowledge{
+		ID:              "knowledge-1",
+		TenantID:        1,
+		KnowledgeBaseID: "kb-1",
+		ParseStatus:     types.ParseStatusProcessing,
+	}
+	chunkService := &parentChildChunkService{}
+	retrieveEngine := &parentChildRetrieveEngine{}
+	tenant := &types.Tenant{
+		ID: 1,
+		RetrieverEngines: types.RetrieverEngines{Engines: []types.RetrieverEngineParams{
+			{
+				RetrieverType:       types.VectorRetrieverType,
+				RetrieverEngineType: types.PostgresRetrieverEngineType,
+			},
+		}},
+	}
+	ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)
+	svc := &knowledgeService{
+		repo:           &parentChildKnowledgeRepo{knowledge: knowledge},
+		chunkService:   chunkService,
+		modelService:   parentChildModelService{embedder: parentChildEmbedder{}},
+		retrieveEngine: parentChildRetrieveRegistry{engine: retrieveEngine},
+		graphEngine:    parentChildGraphRepo{},
+		tenantRepo:     parentChildTenantRepo{},
+		task:           parentChildTaskEnqueuer{},
+	}
+	kb := &types.KnowledgeBase{
+		ID:               "kb-1",
+		TenantID:         1,
+		EmbeddingModelID: "embedding-1",
+		IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
+	}
+	chunks := []types.ParsedChunk{
+		{Content: "linked child", Seq: 0, Start: 0, End: 12, ParentIndex: 0},
+		{Content: "standalone child", Seq: 1, Start: 12, End: 28, ParentIndex: -1},
+	}
+
+	svc.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{
+		ParentChunks: []types.ParsedParentChunk{
+			{Content: "parent context", Seq: 0, Start: 0, End: 28},
+		},
+	})
+
+	var textChunkIDs []string
+	for _, chunk := range chunkService.created {
+		if chunk.ChunkType == types.ChunkTypeText {
+			textChunkIDs = append(textChunkIDs, chunk.ID)
+		}
+	}
+	require.Len(t, textChunkIDs, 2)
+
+	indexedSourceIDs := make([]string, 0, len(retrieveEngine.indexed))
+	for _, info := range retrieveEngine.indexed {
+		indexedSourceIDs = append(indexedSourceIDs, info.SourceID)
+	}
+	require.ElementsMatch(t, textChunkIDs, indexedSourceIDs)
+}

internal/infrastructure/chunker/strategy.go

@@ -215,6 +215,8 @@ func NormalizeLineEndings(text string) string {
 // DeriveParentChildConfigs produces the exact parent and child splitter
 // configurations used by knowledge ingestion. Keeping this here lets preview
 // and ingestion remain in lockstep as the parent-child defaults evolve.
+// Languages are copied to both levels so parent and child splitters use the same boundary rules.
+// TokenLimit is copied only to children because parents keep the configured context window.
 func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (parent, child SplitterConfig) {
 	if parentSize <= 0 {
 		parentSize = 4096
@@ -227,12 +229,15 @@ func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (p
 		ChunkOverlap: base.ChunkOverlap,
 		Separators:   base.Separators,
 		Strategy:     base.Strategy,
+		Languages:    base.Languages,
 	}
 	child = SplitterConfig{
 		ChunkSize:    childSize,
 		ChunkOverlap: childSize / 5,
 		Separators:   base.Separators,
 		Strategy:     base.Strategy,
+		TokenLimit:   base.TokenLimit,
+		Languages:    base.Languages,
 	}
 	return
 }

internal/infrastructure/chunker/strategy_test.go

@@ -263,6 +263,51 @@ func TestDeriveParentChildConfigs_DefaultSizes(t *testing.T) {
 	}
 }
 
+func TestDeriveParentChildConfigs_AppliesEmbeddingTokenLimitOnlyToChildren(t *testing.T) {
+	const (
+		parentSize = 4096
+		childSize  = 1024
+		tokenLimit = 200
+	)
+	base := SplitterConfig{
+		ChunkSize:    DefaultChunkSize,
+		ChunkOverlap: DefaultChunkOverlap,
+		Separators:   []string{"。"},
+		Strategy:     StrategyLegacy,
+		TokenLimit:   tokenLimit,
+		Languages:    []string{LangChinese},
+	}
+
+	parent, child := DeriveParentChildConfigs(base, parentSize, childSize)
+	if parent.TokenLimit != 0 {
+		t.Fatalf("parent token limit: got %d want 0", parent.TokenLimit)
+	}
+	if len(parent.Languages) != 1 || parent.Languages[0] != LangChinese {
+		t.Fatalf("parent languages: got %v want [%s]", parent.Languages, LangChinese)
+	}
+	if child.TokenLimit != tokenLimit {
+		t.Fatalf("child token limit: got %d want %d", child.TokenLimit, tokenLimit)
+	}
+	if len(child.Languages) != 1 || child.Languages[0] != LangChinese {
+		t.Fatalf("child languages: got %v want [%s]", child.Languages, LangChinese)
+	}
+
+	text := strings.Repeat("这是用于验证中文分块预算的句子。", 100)
+	result := SplitParentChild(text, parent, child)
+	if len(result.Parents) != 1 {
+		t.Fatalf("parents: got %d want 1", len(result.Parents))
+	}
+	if got := len([]rune(result.Parents[0].Content)); got != len([]rune(text)) {
+		t.Fatalf("parent length: got %d want %d", got, len([]rune(text)))
+	}
+	budget := CharsForTokenLimit(tokenLimit, LangChinese)
+	for i, c := range result.Children {
+		if got := len([]rune(c.Content)); got > budget {
+			t.Fatalf("child[%d] length: got %d want <= %d", i, got, budget)
+		}
+	}
+}
+
 func TestValidateChunks_Empty(t *testing.T) {
 	if v := ValidateChunks(nil, 1000, 500); v.OK {
 		t.Error("nil chunks should be invalid")