fix(knowledge): harden file-type scoped duplicate detection

Use case-insensitive file_type matching, scope the filename fallback by type, and document the dedup contract for datasource sweep handling.
This commit is contained in:
wizardchen
2026-07-29 10:36:17 +08:00
committed by lyingbug
parent 18b95ebb3a
commit 5db186a0fc
5 changed files with 27 additions and 9 deletions
+10 -6
View File
@@ -248,7 +248,7 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
var knowledge types.Knowledge
duplicateQuery := query.Where("type = ? AND file_hash = ?", "file", params.FileHash)
if params.FileType != "" {
duplicateQuery = duplicateQuery.Where("file_type = ?", params.FileType)
duplicateQuery = duplicateQuery.Where("LOWER(file_type) = ?", strings.ToLower(params.FileType))
}
err := duplicateQuery.First(&knowledge).Error
if err != nil {
@@ -260,13 +260,17 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
return true, &knowledge, nil
}
// If no hash or hash doesn't match, use filename and size
// If no hash or hash doesn't match, use filename, size, and file type.
if params.FileName != "" && params.FileSize > 0 {
var knowledge types.Knowledge
err := query.Where(
"file_name = ? AND file_size = ?",
params.FileName, params.FileSize,
).First(&knowledge).Error
duplicateQuery := query.Where(
"type = ? AND file_name = ? AND file_size = ?",
"file", params.FileName, params.FileSize,
)
if params.FileType != "" {
duplicateQuery = duplicateQuery.Where("LOWER(file_type) = ?", strings.ToLower(params.FileType))
}
err := duplicateQuery.First(&knowledge).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return false, nil, nil
@@ -47,4 +47,17 @@ func TestCheckKnowledgeExists_FileHashIsScopedByFileType(t *testing.T) {
require.NotNil(t, knowledge)
assert.Equal(t, "md", knowledge.FileType)
})
t.Run("file type matching is case-insensitive", func(t *testing.T) {
exists, knowledge, err := repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{
Type: "file",
FileHash: fileHash,
FileType: "MD",
})
require.NoError(t, err)
assert.True(t, exists)
require.NotNil(t, knowledge)
assert.Equal(t, "md", knowledge.FileType)
})
}
@@ -1253,8 +1253,8 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
// dupIsSameNode reports whether a duplicate-content error means the parent still
// exists in the KB *under this item's own external_id* — i.e. a content-dedup hit
// against this same node, so reconciling its subtree is safe. Deduplication keys
// on file_hash alone (CheckKnowledgeExists), so an updated node whose rebuilt body
// against this same node, so reconciling its subtree is safe. File deduplication
// keys on file_hash plus file_type (CheckKnowledgeExists), so an updated node whose rebuilt body
// happens to hash-collide with a DIFFERENT knowledge item (another node, or a
// manually-uploaded file with no external_id) would otherwise sweep this node's
// children even though its own parent row was just deleted for the update and
@@ -160,7 +160,7 @@ func TestIngestItem_ReplacesSubtreeSweepsOnDuplicateParent(t *testing.T) {
// TestIngestItem_NoSweepWhenDuplicateIsDifferentNode guards the data-loss path:
// when an updated node's rebuilt body content-hash-collides with a DIFFERENT
// knowledge item (dedup keys on file_hash alone), the parent row under this
// knowledge item (dedup keys on file_hash plus file_type), the parent row under this
// node's external_id no longer exists (it was deleted for the update and never
// recreated), so the subtree must NOT be swept — deleting the children would
// destroy them with no parent to replace them.
+1
View File
@@ -445,6 +445,7 @@ func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {
type KnowledgeCheckParams struct {
// File parameters
FileName string
// FileType scopes file-hash deduplication; callers checking file uploads should set it.
FileType string
FileSize int64
FileHash string