diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 79b9a36ae..831d4d063 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -241,10 +241,16 @@ func (r *knowledgeRepository) CheckKnowledgeExists( switch params.Type { case "file": - // If file hash exists, prioritize exact match using hash + // File content is only a duplicate within the same file type. This keeps + // same-content documents with distinct formats (for example, .md and + // .txt) available as separate knowledge items. if params.FileHash != "" { var knowledge types.Knowledge - err := query.Where("file_hash = ?", params.FileHash).First(&knowledge).Error + duplicateQuery := query.Where("type = ? AND file_hash = ?", "file", params.FileHash) + if params.FileType != "" { + duplicateQuery = duplicateQuery.Where("file_type = ?", params.FileType) + } + err := duplicateQuery.First(&knowledge).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return false, nil, nil diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.go new file mode 100644 index 000000000..55ae0a230 --- /dev/null +++ b/internal/application/repository/knowledge_duplicate_test.go @@ -0,0 +1,50 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckKnowledgeExists_FileHashIsScopedByFileType(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db) + ctx := context.Background() + tenantID := uint64(1) + kbID := uuid.NewString() + const fileHash = "same-content-hash" + + require.NoError(t, db.Exec(` + INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, file_name, file_type, file_hash, parse_status) + VALUES (?, ?, ?, 'file', 'document.md', 'document.md', 'md', ?, 'completed') + `, uuid.NewString(), tenantID, kbID, fileHash).Error) + + t.Run("same content with another file type is allowed", func(t *testing.T) { + exists, knowledge, err := repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ + Type: "file", + FileHash: fileHash, + FileType: "txt", + }) + + require.NoError(t, err) + assert.False(t, exists) + assert.Nil(t, knowledge) + }) + + t.Run("same content and file type remains a duplicate", func(t *testing.T) { + exists, knowledge, err := repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ + Type: "file", + FileHash: fileHash, + FileType: "md", + }) + + require.NoError(t, err) + assert.True(t, exists) + require.NotNil(t, knowledge) + assert.Equal(t, "md", knowledge.FileType) + }) +} diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index 84f5393c9..fc5523405 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -82,6 +82,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "file", FileName: fileName, + FileType: getFileType(fileName), FileSize: file.Size, FileHash: hash, }) diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 461a618dd..8b68aafd8 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -445,6 +445,7 @@ func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error { type KnowledgeCheckParams struct { // File parameters FileName string + FileType string FileSize int64 FileHash string // URL parameters