mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-08-30 16:53:21 +08:00
feat(wiki): prune empty folders after document retract
Automatically remove wiki folders that become empty when knowledge is retracted, using the durable finalize lane so pruning waits until ingest drains. Repository DeleteFolder now atomically checks emptiness to avoid races with concurrent page or child-folder writes.
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -455,6 +456,10 @@ var ErrWikiFolderNotFound = errors.New("wiki folder not found")
|
||||
// already exists under the same parent.
|
||||
var ErrWikiFolderConflict = errors.New("wiki folder name conflict")
|
||||
|
||||
// ErrWikiFolderNotEmpty is returned when a folder still has a live page or
|
||||
// child folder at the instant an atomic delete is attempted.
|
||||
var ErrWikiFolderNotEmpty = errors.New("wiki folder is not empty")
|
||||
|
||||
func (r *wikiPageRepository) CreateFolder(ctx context.Context, folder *types.WikiFolder) error {
|
||||
return r.db.WithContext(ctx).Create(folder).Error
|
||||
}
|
||||
@@ -535,14 +540,34 @@ func (r *wikiPageRepository) UpdateFolder(ctx context.Context, folder *types.Wik
|
||||
}
|
||||
|
||||
func (r *wikiPageRepository) DeleteFolder(ctx context.Context, kbID string, id string) error {
|
||||
result := r.db.WithContext(ctx).
|
||||
Where("knowledge_base_id = ? AND id = ?", kbID, id).
|
||||
Delete(&types.WikiFolder{})
|
||||
// Keep the emptiness test in the same SQL statement as the soft delete.
|
||||
// A page move or child-folder create can race the service's earlier checks;
|
||||
// a check-then-delete sequence would otherwise leave a dangling folder_id.
|
||||
result := r.db.WithContext(ctx).Exec(`
|
||||
UPDATE wiki_folders
|
||||
SET deleted_at = ?
|
||||
WHERE knowledge_base_id = ? AND id = ? AND deleted_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM wiki_pages
|
||||
WHERE knowledge_base_id = ? AND folder_id = ? AND deleted_at IS NULL
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM wiki_folders AS child
|
||||
WHERE child.knowledge_base_id = ? AND child.parent_id = ? AND child.deleted_at IS NULL
|
||||
)`, time.Now(), kbID, id, kbID, id, kbID, id)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return ErrWikiFolderNotFound
|
||||
var count int64
|
||||
if err := r.db.WithContext(ctx).Model(&types.WikiFolder{}).
|
||||
Where("knowledge_base_id = ? AND id = ?", kbID, id).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return ErrWikiFolderNotFound
|
||||
}
|
||||
return ErrWikiFolderNotEmpty
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -205,6 +205,14 @@ func TestFolderTree_CRUDAndChildListing(t *testing.T) {
|
||||
pages, err := repo.ListPagesByFolderIDs(ctx, "kb-f", []string{"f-ai", "f-llm"})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, pages, 3)
|
||||
|
||||
// Repository deletion re-checks emptiness atomically, so a concurrent page
|
||||
// move / child create cannot slip between the service check and soft delete.
|
||||
err = repo.DeleteFolder(ctx, "kb-f", "f-ai")
|
||||
assert.ErrorIs(t, err, ErrWikiFolderNotEmpty)
|
||||
require.NoError(t, repo.DeleteFolder(ctx, "kb-f", "f-people"))
|
||||
_, err = repo.GetFolderByID(ctx, "kb-f", "f-people")
|
||||
assert.ErrorIs(t, err, ErrWikiFolderNotFound)
|
||||
}
|
||||
|
||||
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
|
||||
|
||||
@@ -272,10 +272,14 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno
|
||||
|
||||
var deletedSlugs []string
|
||||
var retractSlugs []string
|
||||
var affectedFolderIDs []string
|
||||
for _, page := range pages {
|
||||
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
if page.FolderID != "" {
|
||||
affectedFolderIDs = append(affectedFolderIDs, page.FolderID)
|
||||
}
|
||||
|
||||
remaining := removeSourceRef(page.SourceRefs, knowledgeID)
|
||||
|
||||
@@ -318,6 +322,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno
|
||||
DocSummary: docSummary,
|
||||
Language: lang,
|
||||
PageSlugs: allAffectedSlugs,
|
||||
FolderIDs: uniqueWikiFolderIDs(affectedFolderIDs),
|
||||
})
|
||||
logger.Infof(ctx, "wiki cleanup: enqueued retract task for knowledge %s (%d known slugs: %v)",
|
||||
knowledgeID, len(allAffectedSlugs), allAffectedSlugs)
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type folderPruneWikiServiceStub struct {
|
||||
interfaces.WikiPageService
|
||||
calls int
|
||||
folderIDs []string
|
||||
}
|
||||
|
||||
func (s *folderPruneWikiServiceStub) PruneEmptyFolderChains(
|
||||
_ context.Context, _ string, folderIDs []string,
|
||||
) ([]string, error) {
|
||||
s.calls++
|
||||
s.folderIDs = append([]string(nil), folderIDs...)
|
||||
return folderIDs, nil
|
||||
}
|
||||
|
||||
type folderPruneKBServiceStub struct {
|
||||
interfaces.KnowledgeBaseService
|
||||
}
|
||||
|
||||
func (s *folderPruneKBServiceStub) GetKnowledgeBaseByIDOnly(
|
||||
_ context.Context, id string,
|
||||
) (*types.KnowledgeBase, error) {
|
||||
return &types.KnowledgeBase{
|
||||
ID: id,
|
||||
IndexingStrategy: types.IndexingStrategy{
|
||||
WikiEnabled: true,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type folderPrunePendingRepoStub struct {
|
||||
interfaces.TaskPendingOpsRepository
|
||||
rows []*types.TaskPendingOp
|
||||
ingestPending int64
|
||||
deletedRowIDs []int64
|
||||
finalizeCounts int
|
||||
}
|
||||
|
||||
func (s *folderPrunePendingRepoStub) PeekBatch(
|
||||
_ context.Context, _, _, _ string, _ int,
|
||||
) ([]*types.TaskPendingOp, error) {
|
||||
return s.rows, nil
|
||||
}
|
||||
|
||||
func (s *folderPrunePendingRepoStub) PendingCount(
|
||||
_ context.Context, taskType, _, _ string,
|
||||
) (int64, error) {
|
||||
if taskType == wikiTaskType {
|
||||
return s.ingestPending, nil
|
||||
}
|
||||
s.finalizeCounts++
|
||||
return int64(len(s.rows)), nil
|
||||
}
|
||||
|
||||
func (s *folderPrunePendingRepoStub) DeleteByIDs(_ context.Context, ids []int64) error {
|
||||
s.deletedRowIDs = append(s.deletedRowIDs, ids...)
|
||||
return nil
|
||||
}
|
||||
|
||||
type folderPruneTaskStub struct {
|
||||
interfaces.TaskEnqueuer
|
||||
enqueuedTypes []string
|
||||
}
|
||||
|
||||
func (s *folderPruneTaskStub) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
s.enqueuedTypes = append(s.enqueuedTypes, task.Type())
|
||||
return &asynq.TaskInfo{}, nil
|
||||
}
|
||||
|
||||
func makeFolderPruneFinalizeTask(t *testing.T) (*asynq.Task, *types.TaskPendingOp) {
|
||||
t.Helper()
|
||||
rowPayload, err := json.Marshal(wikiFinalizeRow{FolderIDs: []string{"folder-a"}})
|
||||
require.NoError(t, err)
|
||||
taskPayload, err := json.Marshal(WikiIngestPayload{TenantID: 1, KnowledgeBaseID: "kb-1"})
|
||||
require.NoError(t, err)
|
||||
return asynq.NewTask(types.TypeWikiFinalize, taskPayload), &types.TaskPendingOp{
|
||||
ID: 11, Op: wikiFinalizeOpFolderPrune, Payload: rowPayload,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessWikiFinalizeDefersFolderPruneWhileIngestIsPending(t *testing.T) {
|
||||
task, row := makeFolderPruneFinalizeTask(t)
|
||||
wikiSvc := &folderPruneWikiServiceStub{}
|
||||
pendingRepo := &folderPrunePendingRepoStub{rows: []*types.TaskPendingOp{row}, ingestPending: 1}
|
||||
taskQueue := &folderPruneTaskStub{}
|
||||
svc := &wikiIngestService{
|
||||
wikiService: wikiSvc, kbService: &folderPruneKBServiceStub{},
|
||||
pendingRepo: pendingRepo, task: taskQueue,
|
||||
}
|
||||
|
||||
require.NoError(t, svc.ProcessWikiFinalize(context.Background(), task))
|
||||
require.Zero(t, wikiSvc.calls, "must not prune a folder reserved by an in-flight ingest")
|
||||
require.Empty(t, pendingRepo.deletedRowIDs, "durable prune row must remain for retry")
|
||||
require.Equal(t, []string{types.TypeWikiFinalize}, taskQueue.enqueuedTypes)
|
||||
}
|
||||
|
||||
func TestProcessWikiFinalizePrunesFolderAfterIngestDrains(t *testing.T) {
|
||||
task, row := makeFolderPruneFinalizeTask(t)
|
||||
wikiSvc := &folderPruneWikiServiceStub{}
|
||||
pendingRepo := &folderPrunePendingRepoStub{rows: []*types.TaskPendingOp{row}}
|
||||
svc := &wikiIngestService{
|
||||
wikiService: wikiSvc, kbService: &folderPruneKBServiceStub{},
|
||||
pendingRepo: pendingRepo, task: &folderPruneTaskStub{},
|
||||
}
|
||||
|
||||
require.NoError(t, svc.ProcessWikiFinalize(context.Background(), task))
|
||||
require.Equal(t, 1, wikiSvc.calls)
|
||||
require.Equal(t, []string{"folder-a"}, wikiSvc.folderIDs)
|
||||
require.Equal(t, []int64{11}, pendingRepo.deletedRowIDs)
|
||||
}
|
||||
@@ -183,6 +183,13 @@ const (
|
||||
// wikiFinalizeOpChange rows carry a doc-level add/remove change entry for
|
||||
// the index-intro change description.
|
||||
wikiFinalizeOpChange = "change"
|
||||
// wikiFinalizeOpFolderPrune rows carry folders that may have become empty
|
||||
// after a document retract. Keeping this in the durable finalize lane lets
|
||||
// us wait until every ingest op for the KB has settled before deleting the
|
||||
// directories; taxonomy planning creates folders before reduce writes the
|
||||
// corresponding pages, so pruning any earlier can invalidate in-flight
|
||||
// folder assignments.
|
||||
wikiFinalizeOpFolderPrune = "folder_prune"
|
||||
|
||||
wikiFinalizeAdded = "added"
|
||||
wikiFinalizeRemoved = "removed"
|
||||
@@ -201,6 +208,11 @@ const (
|
||||
wikiFinalizeLockTTL = 60 * time.Second
|
||||
wikiFinalizeLockRenew = 20 * time.Second
|
||||
|
||||
// Folder cleanup is maintenance, not user-blocking work. When an ingest is
|
||||
// still active, retry slowly so pruning never competes with the primary wiki
|
||||
// pipeline for worker capacity.
|
||||
wikiFolderPruneRetryDelay = 1 * time.Minute
|
||||
|
||||
// wikiIngestCleanupTimeout bounds detached tail cleanup after the asynq
|
||||
// task context has been cancelled or hit its timeout.
|
||||
wikiIngestCleanupTimeout = 10 * time.Second
|
||||
@@ -215,12 +227,13 @@ type wikiFinalizeChange struct {
|
||||
}
|
||||
|
||||
// wikiFinalizeRow is the JSON payload of a task_pending_ops row in the
|
||||
// finalize lane. Exactly one of {Slug, Change} is set, distinguished by the
|
||||
// row's Op column (wikiFinalizeOpSlug / wikiFinalizeOpChange).
|
||||
// finalize lane. Exactly one of {Slug, Change, FolderIDs} is set,
|
||||
// distinguished by the row's Op column.
|
||||
type wikiFinalizeRow struct {
|
||||
Slug string `json:"slug,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Change *wikiFinalizeChange `json:"change,omitempty"`
|
||||
Slug string `json:"slug,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Change *wikiFinalizeChange `json:"change,omitempty"`
|
||||
FolderIDs []string `json:"folder_ids,omitempty"`
|
||||
}
|
||||
|
||||
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
|
||||
@@ -253,6 +266,7 @@ type WikiRetractPayload struct {
|
||||
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
|
||||
Language string `json:"language,omitempty"`
|
||||
PageSlugs []string `json:"page_slugs"`
|
||||
FolderIDs []string `json:"folder_ids,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -281,6 +295,7 @@ type WikiPendingOp struct {
|
||||
DocTitle string `json:"doc_title,omitempty"`
|
||||
DocSummary string `json:"doc_summary,omitempty"`
|
||||
PageSlugs []string `json:"page_slugs,omitempty"`
|
||||
FolderIDs []string `json:"folder_ids,omitempty"`
|
||||
|
||||
// dbID is set by peekPendingList from task_pending_ops.id. Zero in
|
||||
// constructions made outside the queue (e.g. legacy tests).
|
||||
@@ -489,6 +504,7 @@ func EnqueueWikiRetract(
|
||||
DocTitle: payload.DocTitle,
|
||||
DocSummary: payload.DocSummary,
|
||||
PageSlugs: payload.PageSlugs,
|
||||
FolderIDs: payload.FolderIDs,
|
||||
Language: payload.Language,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(op)
|
||||
@@ -559,6 +575,7 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
affectedSlugs []string,
|
||||
freshTitleBySlug map[string]string,
|
||||
changes []wikiFinalizeChange,
|
||||
folderIDs []string,
|
||||
) {
|
||||
if s.pendingRepo == nil {
|
||||
return
|
||||
@@ -599,9 +616,42 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err)
|
||||
}
|
||||
}
|
||||
if len(folderIDs) > 0 {
|
||||
row := wikiFinalizeRow{FolderIDs: uniqueWikiFolderIDs(folderIDs)}
|
||||
if b, err := json.Marshal(row); err == nil {
|
||||
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiFinalizeTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
ScopeID: payload.KnowledgeBaseID,
|
||||
Op: wikiFinalizeOpFolderPrune,
|
||||
DedupKey: "",
|
||||
Payload: b,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue folder prune row failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.scheduleFinalize(ctx, payload)
|
||||
}
|
||||
|
||||
func uniqueWikiFolderIDs(values []string) []string {
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scheduleFinalize enqueues a debounced, coalesced KB-global finalize trigger.
|
||||
// asynq.TaskID ("wiki-finalize-<kbID>") makes concurrent schedules within the
|
||||
// debounce window collapse into one pending task; the conflict error is the
|
||||
@@ -626,6 +676,25 @@ func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIn
|
||||
}
|
||||
}
|
||||
|
||||
// scheduleFinalizeRetry is used when folder pruning is waiting for ingest
|
||||
// rows to drain. It deliberately has no stable TaskID: the currently-running
|
||||
// finalize task still owns that ID until it returns, so reusing it here would
|
||||
// coalesce the only retry away. Duplicate retries are harmless because the
|
||||
// durable prune row is deleted exactly once and an empty lane is a no-op.
|
||||
func (s *wikiIngestService) scheduleFinalizeRetry(ctx context.Context, payload WikiIngestPayload) {
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
b, _ := json.Marshal(payload)
|
||||
t := asynq.NewTask(types.TypeWikiFinalize, b,
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(wikiIngestMaxRetry),
|
||||
asynq.Timeout(30*time.Minute),
|
||||
asynq.ProcessIn(wikiFolderPruneRetryDelay),
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: schedule deferred folder prune failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// peekPendingList loads up to `limit` ops from task_pending_ops for
|
||||
// this KB, ordered FIFO. Rows are NOT removed; callers must
|
||||
// DeleteByIDs once they have been consumed (or IncrFailCount + leave
|
||||
|
||||
@@ -413,6 +413,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
var failedOps []WikiPendingOp
|
||||
slugUpdates := make(map[string][]SlugUpdate)
|
||||
var docResults []*docIngestResult
|
||||
var retractFolderIDs []string
|
||||
// rateLimited flips true when any map/reduce LLM failure looks like an
|
||||
// upstream 429/quota trip. It steers the follow-up scheduler onto the
|
||||
// longer wikiRateLimitBackoff so retries don't keep hammering an already
|
||||
@@ -442,12 +443,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// PageSlugs as "figure it out yourself" — see
|
||||
// cleanupWikiOnKnowledgeDelete's comment (3).
|
||||
slugSet := make(map[string]struct{}, len(op.PageSlugs))
|
||||
folderSet := make(map[string]struct{}, len(op.FolderIDs))
|
||||
for _, slug := range op.PageSlugs {
|
||||
if slug == "" {
|
||||
continue
|
||||
}
|
||||
slugSet[slug] = struct{}{}
|
||||
}
|
||||
for _, folderID := range op.FolderIDs {
|
||||
if folderID != "" {
|
||||
folderSet[folderID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if op.KnowledgeID != "" {
|
||||
livePages, err := s.wikiService.ListPagesBySourceRef(mapCtx, payload.KnowledgeBaseID, op.KnowledgeID)
|
||||
if err != nil {
|
||||
@@ -464,6 +471,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
continue
|
||||
}
|
||||
slugSet[p.Slug] = struct{}{}
|
||||
if p.FolderID != "" {
|
||||
folderSet[p.FolderID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -483,6 +493,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
Language: types.LanguageLocaleName(op.Language),
|
||||
})
|
||||
}
|
||||
for folderID := range folderSet {
|
||||
retractFolderIDs = append(retractFolderIDs, folderID)
|
||||
}
|
||||
mapMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -763,7 +776,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
freshTitleBySlug[p.Slug] = p.Title
|
||||
}
|
||||
}
|
||||
if len(allPagesAffected) > 0 || len(docResults) > 0 {
|
||||
if len(allPagesAffected) > 0 || len(docResults) > 0 || retractHandled > 0 || len(retractFolderIDs) > 0 {
|
||||
changes := make([]wikiFinalizeChange, 0, len(docResults)+len(pendingOps))
|
||||
for _, r := range docResults {
|
||||
changes = append(changes, wikiFinalizeChange{
|
||||
@@ -777,7 +790,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
})
|
||||
}
|
||||
}
|
||||
s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes)
|
||||
s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes, retractFolderIDs)
|
||||
}
|
||||
|
||||
// Close postprocess.wiki spans for every successfully-mapped doc.
|
||||
@@ -1001,12 +1014,17 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
// refs, and the index-intro change description. We collect ids up front so
|
||||
// we can drain the lane even on the KB-disabled short-circuit below.
|
||||
ids := make([]int64, 0, len(rows))
|
||||
pruneRowIDs := make([]int64, 0)
|
||||
affectedSet := make(map[string]struct{}, len(rows))
|
||||
var affectedSlugs []string
|
||||
var freshRefs []linkRef
|
||||
var folderPruneIDs []string
|
||||
var changeDesc strings.Builder
|
||||
for _, r := range rows {
|
||||
ids = append(ids, r.ID)
|
||||
if r.Op == wikiFinalizeOpFolderPrune {
|
||||
pruneRowIDs = append(pruneRowIDs, r.ID)
|
||||
}
|
||||
if len(r.Payload) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -1015,6 +1033,10 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
logger.Warnf(ctx, "wiki finalize: unmarshal row id=%d failed: %v", r.ID, err)
|
||||
continue
|
||||
}
|
||||
if r.Op == wikiFinalizeOpFolderPrune {
|
||||
folderPruneIDs = append(folderPruneIDs, row.FolderIDs...)
|
||||
continue
|
||||
}
|
||||
if row.Change != nil {
|
||||
if row.Change.Action == wikiFinalizeRemoved {
|
||||
fmt.Fprintf(&changeDesc, "<document_removed>\n<title>%s</title>\n<summary>%s</summary>\n</document_removed>\n\n", row.Change.DocTitle, row.Change.DocSummary)
|
||||
@@ -1080,11 +1102,50 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
s.injectCrossLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, freshRefs, batchCtx)
|
||||
}
|
||||
|
||||
// A retract may leave one or more generated folders empty. Do not prune
|
||||
// while any ingest row for this KB is queued or claimed: taxonomy planning
|
||||
// creates folders before reduce writes pages, so an apparently-empty folder
|
||||
// can still be owned by an in-flight batch. The durable prune rows stay in
|
||||
// the finalize lane and are retried after the ingest lane drains.
|
||||
pruneDeferred := false
|
||||
deletedFolders := 0
|
||||
if len(folderPruneIDs) > 0 {
|
||||
pending, pErr := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID)
|
||||
if pErr != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: cannot verify ingest drain before folder prune: %v", pErr)
|
||||
pruneDeferred = true
|
||||
} else if pending > 0 {
|
||||
pruneDeferred = true
|
||||
} else {
|
||||
deleted, pruneErr := s.wikiService.PruneEmptyFolderChains(
|
||||
ctx, payload.KnowledgeBaseID, uniqueWikiFolderIDs(folderPruneIDs))
|
||||
if pruneErr != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: prune empty folders failed: %v", pruneErr)
|
||||
pruneDeferred = true
|
||||
} else {
|
||||
deletedFolders = len(deleted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drain the processed rows. Best-effort convergence mirrors the legacy
|
||||
// in-batch behaviour: a failed index rebuild is logged (not retried),
|
||||
// so we delete regardless to avoid re-doing the whole pass forever.
|
||||
idsToTrim := ids
|
||||
if pruneDeferred && len(pruneRowIDs) > 0 {
|
||||
deferred := make(map[int64]struct{}, len(pruneRowIDs))
|
||||
for _, id := range pruneRowIDs {
|
||||
deferred[id] = struct{}{}
|
||||
}
|
||||
idsToTrim = idsToTrim[:0:0]
|
||||
for _, id := range ids {
|
||||
if _, keep := deferred[id]; !keep {
|
||||
idsToTrim = append(idsToTrim, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
drainCtx, drainCancel := wikiIngestCleanupContext(ctx)
|
||||
err = s.trimPendingList(drainCtx, ids)
|
||||
err = s.trimPendingList(drainCtx, idsToTrim)
|
||||
drainCancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki finalize: trim pending rows: %w", err)
|
||||
@@ -1093,14 +1154,20 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
// If more finalize rows landed while we were working, reschedule so they
|
||||
// get their own convergence pass.
|
||||
rescheduled := false
|
||||
if pruneDeferred {
|
||||
s.scheduleFinalizeRetry(ctx, payload)
|
||||
rescheduled = true
|
||||
}
|
||||
if n, cErr := s.pendingRepo.PendingCount(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID); cErr == nil && n > 0 {
|
||||
s.scheduleFinalize(ctx, payload)
|
||||
if !pruneDeferred {
|
||||
s.scheduleFinalize(ctx, payload)
|
||||
}
|
||||
rescheduled = true
|
||||
}
|
||||
|
||||
logger.Infof(ctx,
|
||||
"wiki finalize: kb=%s rows=%d affected_slugs=%d index_rebuilt=%v rescheduled=%v elapsed=%s",
|
||||
payload.KnowledgeBaseID, len(rows), len(affectedSlugs), indexRebuilt, rescheduled,
|
||||
"wiki finalize: kb=%s rows=%d affected_slugs=%d deleted_folders=%d folder_prune_deferred=%v index_rebuilt=%v rescheduled=%v elapsed=%s",
|
||||
payload.KnowledgeBaseID, len(rows), len(affectedSlugs), deletedFolders, pruneDeferred, indexRebuilt, rescheduled,
|
||||
time.Since(startedAt).Round(time.Millisecond),
|
||||
)
|
||||
return nil
|
||||
|
||||
@@ -1460,18 +1460,97 @@ func (s *wikiPageService) DeleteFolder(ctx context.Context, kbID string, id stri
|
||||
return err
|
||||
}
|
||||
if len(children) > 0 {
|
||||
return errors.New("folder is not empty: it still has sub-folders")
|
||||
return repository.ErrWikiFolderNotEmpty
|
||||
}
|
||||
pages, err := s.repo.ListPagesByFolderIDs(ctx, kbID, []string{id})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(pages) > 0 {
|
||||
return errors.New("folder is not empty: it still contains pages")
|
||||
return repository.ErrWikiFolderNotEmpty
|
||||
}
|
||||
return s.repo.DeleteFolder(ctx, kbID, id)
|
||||
}
|
||||
|
||||
// PruneEmptyFolderChains removes folders that became empty after retracting a
|
||||
// document, followed by any ancestors made empty by those removals. It only
|
||||
// considers the supplied folder chains, so intentionally-empty folders
|
||||
// elsewhere in the wiki are preserved. Callers must wait for the KB's ingest
|
||||
// queue to drain before invoking this method: taxonomy planning creates a
|
||||
// folder before reduce writes the page that will reference it.
|
||||
func (s *wikiPageService) PruneEmptyFolderChains(
|
||||
ctx context.Context, kbID string, folderIDs []string,
|
||||
) ([]string, error) {
|
||||
if len(folderIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
all, err := s.repo.ListAllFolders(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]*types.WikiFolder, len(all))
|
||||
for _, folder := range all {
|
||||
if folder != nil {
|
||||
byID[folder.ID] = folder
|
||||
}
|
||||
}
|
||||
|
||||
candidates := make(map[string]*types.WikiFolder)
|
||||
for _, id := range folderIDs {
|
||||
seen := make(map[string]struct{})
|
||||
for id != types.WikiFolderRootID {
|
||||
if _, cycle := seen[id]; cycle {
|
||||
break
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
folder := byID[id]
|
||||
if folder == nil {
|
||||
break
|
||||
}
|
||||
candidates[id] = folder
|
||||
id = folder.ParentID
|
||||
}
|
||||
}
|
||||
|
||||
ordered := make([]*types.WikiFolder, 0, len(candidates))
|
||||
for _, folder := range candidates {
|
||||
ordered = append(ordered, folder)
|
||||
}
|
||||
sort.Slice(ordered, func(i, j int) bool {
|
||||
if ordered[i].Depth == ordered[j].Depth {
|
||||
return ordered[i].Path > ordered[j].Path
|
||||
}
|
||||
return ordered[i].Depth > ordered[j].Depth
|
||||
})
|
||||
|
||||
deleted := make([]string, 0, len(ordered))
|
||||
for _, folder := range ordered {
|
||||
children, err := s.repo.ListChildFolders(ctx, kbID, folder.ID)
|
||||
if err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
if len(children) > 0 {
|
||||
continue
|
||||
}
|
||||
pages, err := s.repo.ListPagesByFolderIDs(ctx, kbID, []string{folder.ID})
|
||||
if err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
if len(pages) > 0 {
|
||||
continue
|
||||
}
|
||||
if err := s.repo.DeleteFolder(ctx, kbID, folder.ID); err != nil {
|
||||
if errors.Is(err, repository.ErrWikiFolderNotFound) ||
|
||||
errors.Is(err, repository.ErrWikiFolderNotEmpty) {
|
||||
continue
|
||||
}
|
||||
return deleted, err
|
||||
}
|
||||
deleted = append(deleted, folder.ID)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// InjectCrossLinks scans affected pages and injects [[wiki-links]] for mentions
|
||||
// of other wiki page titles in the content. Pure text replacement, no LLM call.
|
||||
// Shares the linkifyContent helper with the ingest pipeline so both paths honor
|
||||
|
||||
@@ -1,11 +1,58 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestPruneEmptyFolderChainsDeletesOnlyEmptyCandidateAncestors(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{}))
|
||||
|
||||
ctx := context.Background()
|
||||
repo := repository.NewWikiPageRepository(db)
|
||||
svc := NewWikiPageService(repo, nil, nil, nil, nil)
|
||||
now := time.Now()
|
||||
createFolder := func(id, parentID, name, path string, depth int) {
|
||||
require.NoError(t, repo.CreateFolder(ctx, &types.WikiFolder{
|
||||
ID: id, TenantID: 1, KnowledgeBaseID: "kb-prune", ParentID: parentID,
|
||||
Name: name, Path: path, Depth: depth, CreatedAt: now, UpdatedAt: now,
|
||||
}))
|
||||
}
|
||||
createFolder("topic", "", "Topic", "Topic", 1)
|
||||
createFolder("empty-leaf", "topic", "Empty", "Topic/Empty", 2)
|
||||
createFolder("occupied-leaf", "topic", "Occupied", "Topic/Occupied", 2)
|
||||
createFolder("empty-chain", "", "Empty chain", "Empty chain", 1)
|
||||
createFolder("empty-chain-leaf", "empty-chain", "Leaf", "Empty chain/Leaf", 2)
|
||||
createFolder("unrelated-empty", "", "Keep me", "Keep me", 1)
|
||||
|
||||
require.NoError(t, repo.Create(ctx, &types.WikiPage{
|
||||
ID: "page-1", TenantID: 1, KnowledgeBaseID: "kb-prune", Slug: "entity/occupied",
|
||||
Title: "Occupied", PageType: types.WikiPageTypeEntity, Status: types.WikiPageStatusPublished,
|
||||
FolderID: "occupied-leaf", Version: 1, CreatedAt: now, UpdatedAt: now,
|
||||
}))
|
||||
|
||||
deleted, err := svc.PruneEmptyFolderChains(ctx, "kb-prune", []string{"empty-leaf", "empty-chain-leaf"})
|
||||
require.NoError(t, err)
|
||||
require.ElementsMatch(t, []string{"empty-leaf", "empty-chain-leaf", "empty-chain"}, deleted)
|
||||
|
||||
_, err = repo.GetFolderByID(ctx, "kb-prune", "topic")
|
||||
require.NoError(t, err, "ancestor with another occupied child must remain")
|
||||
_, err = repo.GetFolderByID(ctx, "kb-prune", "occupied-leaf")
|
||||
require.NoError(t, err)
|
||||
_, err = repo.GetFolderByID(ctx, "kb-prune", "unrelated-empty")
|
||||
require.NoError(t, err, "empty folders outside the affected chains must remain")
|
||||
}
|
||||
|
||||
func TestStripWikiInlineChunkCitations(t *testing.T) {
|
||||
input := "[**橡皮障夹**](#)**钳**\n\n夹钳是用于夹持橡皮障夹的专用器械[c003]。手柄便于操作 [c003]。多个来源[c003, c1000]。"
|
||||
want := "[**橡皮障夹**](#)**钳**\n\n夹钳是用于夹持橡皮障夹的专用器械。手柄便于操作。多个来源。"
|
||||
|
||||
@@ -314,7 +314,7 @@ func writeWikiFolderError(c *gin.Context, err error) {
|
||||
switch {
|
||||
case stderrors.Is(err, repository.ErrWikiFolderNotFound), stderrors.Is(err, repository.ErrWikiPageNotFound):
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
|
||||
case stderrors.Is(err, repository.ErrWikiFolderConflict):
|
||||
case stderrors.Is(err, repository.ErrWikiFolderConflict), stderrors.Is(err, repository.ErrWikiFolderNotEmpty):
|
||||
c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
|
||||
default:
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
|
||||
@@ -185,6 +185,10 @@ type WikiPageService interface {
|
||||
// DeleteFolder removes an empty folder. Fails if it still contains pages
|
||||
// or child folders (the UI must move or delete contents first).
|
||||
DeleteFolder(ctx context.Context, kbID string, id string) error
|
||||
// PruneEmptyFolderChains deletes candidate folders that are still empty,
|
||||
// walking upward through newly-empty ancestors. Candidates are supplied by
|
||||
// document retract cleanup so unrelated user-created empty folders remain.
|
||||
PruneEmptyFolderChains(ctx context.Context, kbID string, folderIDs []string) ([]string, error)
|
||||
// FindOrCreateFolderPath resolves a category path (e.g. ["AI","RAG"]) to a
|
||||
// folder id, creating any missing intermediate folders. Returns the leaf
|
||||
// folder id and the canonical (cleaned) path. An empty/blank path resolves
|
||||
|
||||
Reference in New Issue
Block a user