diff --git a/docs/docs.go b/docs/docs.go index 5058b0c60..bd8edfc7d 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -6350,6 +6350,71 @@ const docTemplate = `{ } } }, + "/knowledgebase/{kb_id}/wiki/categories": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Retrieve direct child directories for a wiki page type and optional parent_path", + "produces": [ + "application/json" + ], + "tags": [ + "Wiki" + ], + "summary": "List wiki category paths", + "parameters": [ + { + "type": "string", + "description": "Knowledge base ID", + "name": "kb_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Wiki page type; comma-separated for multiple (e.g. entity,concept)", + "name": "page_type", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Parent category path, slash-separated", + "name": "parent_path", + "in": "query" + }, + { + "type": "integer", + "description": "Page number (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "page_size", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledgebase/{kb_id}/wiki/graph": { "get": { "security": [ @@ -6682,7 +6747,7 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by page type", + "description": "Filter by page type; comma-separated for multiple (e.g. entity,concept)", "name": "page_type", "in": "query" }, @@ -13730,6 +13795,9 @@ const docTemplate = `{ "dimension": { "type": "integer" }, + "supports_dimension_override": { + "type": "boolean" + }, "truncate_prompt_tokens": { "type": "integer" } @@ -17095,6 +17163,43 @@ const docTemplate = `{ "WebSearchProviderTypeSearxng" ] }, + "github_com_Tencent_WeKnora_internal_types.WikiCategoryPath": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse": { + "type": "object", + "properties": { + "page": { + "type": "integer" + }, + "page_size": { + "type": "integer" + }, + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath" + } + }, + "total": { + "type": "integer" + }, + "total_pages": { + "type": "integer" + } + } + }, "github_com_Tencent_WeKnora_internal_types.WikiConfig": { "type": "object", "properties": { @@ -17223,14 +17328,32 @@ const docTemplate = `{ "github_com_Tencent_WeKnora_internal_types.WikiIndexEntry": { "type": "object", "properties": { + "category_path": { + "type": "array", + "items": { + "type": "string" + } + }, + "depth": { + "type": "integer" + }, + "parent_slug": { + "type": "string" + }, "slug": { "type": "string" }, + "sort_order": { + "type": "integer" + }, "summary": { "type": "string" }, "title": { "type": "string" + }, + "wiki_path": { + "type": "string" } } }, @@ -17350,6 +17473,13 @@ const docTemplate = `{ "type": "string" } }, + "category_path": { + "description": "CategoryPath is the directory breadcrumb that groups this page in the\nwiki browser, e.g. [\"AI\", \"LLM 应用\", \"RAG\"]. It is intentionally\nlabel-based so intermediate directory nodes do not need to be real pages.", + "type": "array", + "items": { + "type": "string" + } + }, "chunk_refs": { "description": "ChunkRefs records the specific source-document chunks this page was\nbuilt from — one UUID per cited chunk. Populated during ingest from\nthe chunk-citation pass; refreshed wholesale whenever the page is\nre-materialized. Empty for summary pages (they are document-level\nsynopses and don't carry chunk-level citations). Use this when you\nneed to surface the underlying evidence for a wiki page, or to\nretract citations when a source document is deleted.", "type": "array", @@ -17373,6 +17503,10 @@ const docTemplate = `{ } ] }, + "depth": { + "description": "Depth is len(CategoryPath), cached for filtering / display.", + "type": "integer" + }, "id": { "description": "Unique identifier (UUID)", "type": "string" @@ -17406,10 +17540,18 @@ const docTemplate = `{ "description": "Page type: summary, entity, concept, index, log, synthesis, comparison", "type": "string" }, + "parent_slug": { + "description": "ParentSlug optionally points at the wiki page that should act as this\npage's semantic parent in the directory tree. The parent may be empty\nwhen the page is grouped only by CategoryPath.", + "type": "string" + }, "slug": { "description": "URL-friendly slug for addressing, e.g. \"entity/acme-corp\", \"concept/rag\"\nUnique within a knowledge base", "type": "string" }, + "sort_order": { + "description": "SortOrder allows generated or manually edited pages to control sibling\nordering before falling back to title.", + "type": "integer" + }, "source_refs": { "description": "References to source knowledge IDs that contributed to this page.\nFormat matches the legacy \"\u003cknowledge_id\u003e|\u003cdoc_title\u003e\" convention used\nacross the ingest pipeline, so retract / display code can split on ` + "`" + `|` + "`" + `\nto recover the title. Document-level granularity.", "type": "array", @@ -17440,6 +17582,10 @@ const docTemplate = `{ "version": { "description": "Version number. Incremented only when a user-visible content field\n(title, content, summary, page_type, status) actually changes; pure\nbookkeeping writes (link maintenance, same-content re-ingest, status\nsync from background jobs) leave it untouched so it can be used as a\nreal \"the page was edited\" signal.", "type": "integer" + }, + "wiki_path": { + "description": "WikiPath is a normalized, sortable path derived from page_type,\ncategory_path, and title. It keeps large directory listings cheap to sort.", + "type": "string" } } }, @@ -18203,6 +18349,9 @@ const docTemplate = `{ "source": { "description": "为空时按需默认为 \"remote\"", "type": "string" + }, + "supportsDimensionOverride": { + "type": "boolean" } } }, @@ -18437,6 +18586,9 @@ const docTemplate = `{ "source": { "description": "为空时按需默认为 \"remote\"", "type": "string" + }, + "supportsDimensionOverride": { + "type": "boolean" } } }, diff --git a/docs/swagger.json b/docs/swagger.json index a2f311860..39cdafc74 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -6343,6 +6343,71 @@ } } }, + "/knowledgebase/{kb_id}/wiki/categories": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Retrieve direct child directories for a wiki page type and optional parent_path", + "produces": [ + "application/json" + ], + "tags": [ + "Wiki" + ], + "summary": "List wiki category paths", + "parameters": [ + { + "type": "string", + "description": "Knowledge base ID", + "name": "kb_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Wiki page type; comma-separated for multiple (e.g. entity,concept)", + "name": "page_type", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Parent category path, slash-separated", + "name": "parent_path", + "in": "query" + }, + { + "type": "integer", + "description": "Page number (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "page_size", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledgebase/{kb_id}/wiki/graph": { "get": { "security": [ @@ -6675,7 +6740,7 @@ }, { "type": "string", - "description": "Filter by page type", + "description": "Filter by page type; comma-separated for multiple (e.g. entity,concept)", "name": "page_type", "in": "query" }, @@ -13723,6 +13788,9 @@ "dimension": { "type": "integer" }, + "supports_dimension_override": { + "type": "boolean" + }, "truncate_prompt_tokens": { "type": "integer" } @@ -17088,6 +17156,43 @@ "WebSearchProviderTypeSearxng" ] }, + "github_com_Tencent_WeKnora_internal_types.WikiCategoryPath": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse": { + "type": "object", + "properties": { + "page": { + "type": "integer" + }, + "page_size": { + "type": "integer" + }, + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath" + } + }, + "total": { + "type": "integer" + }, + "total_pages": { + "type": "integer" + } + } + }, "github_com_Tencent_WeKnora_internal_types.WikiConfig": { "type": "object", "properties": { @@ -17216,14 +17321,32 @@ "github_com_Tencent_WeKnora_internal_types.WikiIndexEntry": { "type": "object", "properties": { + "category_path": { + "type": "array", + "items": { + "type": "string" + } + }, + "depth": { + "type": "integer" + }, + "parent_slug": { + "type": "string" + }, "slug": { "type": "string" }, + "sort_order": { + "type": "integer" + }, "summary": { "type": "string" }, "title": { "type": "string" + }, + "wiki_path": { + "type": "string" } } }, @@ -17343,6 +17466,13 @@ "type": "string" } }, + "category_path": { + "description": "CategoryPath is the directory breadcrumb that groups this page in the\nwiki browser, e.g. [\"AI\", \"LLM 应用\", \"RAG\"]. It is intentionally\nlabel-based so intermediate directory nodes do not need to be real pages.", + "type": "array", + "items": { + "type": "string" + } + }, "chunk_refs": { "description": "ChunkRefs records the specific source-document chunks this page was\nbuilt from — one UUID per cited chunk. Populated during ingest from\nthe chunk-citation pass; refreshed wholesale whenever the page is\nre-materialized. Empty for summary pages (they are document-level\nsynopses and don't carry chunk-level citations). Use this when you\nneed to surface the underlying evidence for a wiki page, or to\nretract citations when a source document is deleted.", "type": "array", @@ -17366,6 +17496,10 @@ } ] }, + "depth": { + "description": "Depth is len(CategoryPath), cached for filtering / display.", + "type": "integer" + }, "id": { "description": "Unique identifier (UUID)", "type": "string" @@ -17399,10 +17533,18 @@ "description": "Page type: summary, entity, concept, index, log, synthesis, comparison", "type": "string" }, + "parent_slug": { + "description": "ParentSlug optionally points at the wiki page that should act as this\npage's semantic parent in the directory tree. The parent may be empty\nwhen the page is grouped only by CategoryPath.", + "type": "string" + }, "slug": { "description": "URL-friendly slug for addressing, e.g. \"entity/acme-corp\", \"concept/rag\"\nUnique within a knowledge base", "type": "string" }, + "sort_order": { + "description": "SortOrder allows generated or manually edited pages to control sibling\nordering before falling back to title.", + "type": "integer" + }, "source_refs": { "description": "References to source knowledge IDs that contributed to this page.\nFormat matches the legacy \"\u003cknowledge_id\u003e|\u003cdoc_title\u003e\" convention used\nacross the ingest pipeline, so retract / display code can split on `|`\nto recover the title. Document-level granularity.", "type": "array", @@ -17433,6 +17575,10 @@ "version": { "description": "Version number. Incremented only when a user-visible content field\n(title, content, summary, page_type, status) actually changes; pure\nbookkeeping writes (link maintenance, same-content re-ingest, status\nsync from background jobs) leave it untouched so it can be used as a\nreal \"the page was edited\" signal.", "type": "integer" + }, + "wiki_path": { + "description": "WikiPath is a normalized, sortable path derived from page_type,\ncategory_path, and title. It keeps large directory listings cheap to sort.", + "type": "string" } } }, @@ -18196,6 +18342,9 @@ "source": { "description": "为空时按需默认为 \"remote\"", "type": "string" + }, + "supportsDimensionOverride": { + "type": "boolean" } } }, @@ -18430,6 +18579,9 @@ "source": { "description": "为空时按需默认为 \"remote\"", "type": "string" + }, + "supportsDimensionOverride": { + "type": "boolean" } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 1f0ee667a..ec4d4f8ae 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -850,6 +850,8 @@ definitions: properties: dimension: type: integer + supports_dimension_override: + type: boolean truncate_prompt_tokens: type: integer type: object @@ -3345,6 +3347,30 @@ definitions: - WebSearchProviderTypeOllama - WebSearchProviderTypeBaidu - WebSearchProviderTypeSearxng + github_com_Tencent_WeKnora_internal_types.WikiCategoryPath: + properties: + count: + type: integer + path: + items: + type: string + type: array + type: object + github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse: + properties: + page: + type: integer + page_size: + type: integer + paths: + items: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath' + type: array + total: + type: integer + total_pages: + type: integer + type: object github_com_Tencent_WeKnora_internal_types.WikiConfig: properties: extraction_granularity: @@ -3450,12 +3476,24 @@ definitions: type: object github_com_Tencent_WeKnora_internal_types.WikiIndexEntry: properties: + category_path: + items: + type: string + type: array + depth: + type: integer + parent_slug: + type: string slug: type: string + sort_order: + type: integer summary: type: string title: type: string + wiki_path: + type: string type: object github_com_Tencent_WeKnora_internal_types.WikiIndexGroup: properties: @@ -3549,6 +3587,14 @@ definitions: items: type: string type: array + category_path: + description: |- + CategoryPath is the directory breadcrumb that groups this page in the + wiki browser, e.g. ["AI", "LLM 应用", "RAG"]. It is intentionally + label-based so intermediate directory nodes do not need to be real pages. + items: + type: string + type: array chunk_refs: description: |- ChunkRefs records the specific source-document chunks this page was @@ -3571,6 +3617,9 @@ definitions: allOf: - $ref: '#/definitions/gorm.DeletedAt' description: Soft delete + depth: + description: Depth is len(CategoryPath), cached for filtering / display. + type: integer id: description: Unique identifier (UUID) type: string @@ -3596,11 +3645,22 @@ definitions: description: 'Page type: summary, entity, concept, index, log, synthesis, comparison' type: string + parent_slug: + description: |- + ParentSlug optionally points at the wiki page that should act as this + page's semantic parent in the directory tree. The parent may be empty + when the page is grouped only by CategoryPath. + type: string slug: description: |- URL-friendly slug for addressing, e.g. "entity/acme-corp", "concept/rag" Unique within a knowledge base type: string + sort_order: + description: |- + SortOrder allows generated or manually edited pages to control sibling + ordering before falling back to title. + type: integer source_refs: description: |- References to source knowledge IDs that contributed to this page. @@ -3633,6 +3693,11 @@ definitions: sync from background jobs) leave it untouched so it can be used as a real "the page was edited" signal. type: integer + wiki_path: + description: |- + WikiPath is a normalized, sortable path derived from page_type, + category_path, and title. It keeps large directory listings cheap to sort. + type: string type: object github_com_Tencent_WeKnora_internal_types.WikiPageIssue: properties: @@ -4159,6 +4224,8 @@ definitions: source: description: 为空时按需默认为 "remote" type: string + supportsDimensionOverride: + type: boolean required: - modelName type: object @@ -4325,6 +4392,8 @@ definitions: source: description: 为空时按需默认为 "remote" type: string + supportsDimensionOverride: + type: boolean required: - modelName type: object @@ -8834,6 +8903,49 @@ paths: summary: Auto-fix wiki issues tags: - Wiki + /knowledgebase/{kb_id}/wiki/categories: + get: + description: Retrieve direct child directories for a wiki page type and optional + parent_path + parameters: + - description: Knowledge base ID + in: path + name: kb_id + required: true + type: string + - description: Wiki page type; comma-separated for multiple (e.g. entity,concept) + in: query + name: page_type + required: true + type: string + - description: Parent category path, slash-separated + in: query + name: parent_path + type: string + - description: Page number (1-based) + in: query + name: page + type: integer + - description: Page size + in: query + name: page_size + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + security: + - Bearer: [] + summary: List wiki category paths + tags: + - Wiki /knowledgebase/{kb_id}/wiki/graph: get: description: |- @@ -9050,7 +9162,7 @@ paths: name: kb_id required: true type: string - - description: Filter by page type + - description: Filter by page type; comma-separated for multiple (e.g. entity,concept) in: query name: page_type type: string diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index f200a8c75..330af0228 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -1,5 +1,25 @@ package agent +// wikiExistingTaxonomyXML is injected into the page-modify prompt so each page's +// CATEGORY line can reuse folder labels already present in the knowledge base +// instead of inventing synonyms. Category assignment happens at page-write +// (reduce) time, where the LLM sees the whole entity — not per source document. +const wikiExistingTaxonomyXML = ` + +{{.ExistingTaxonomy}} + +` + +// wikiExistingTaxonomyContinuityRules extends Directory Taxonomy Rules with +// cross-document folder reuse requirements. +const wikiExistingTaxonomyContinuityRules = ` +- If above lists folders already used in this knowledge base, you MUST reuse those **exact folder labels** (character-for-character) when the new item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节" already has a "传统习俗" subfolder and that path fits semantically). +- Prefer nesting under an existing parent folder over creating a new top-level folder with a similar name. +- When multiple existing paths could fit, pick the most specific existing path whose labels match the document's section hierarchy. +- Only introduce a new folder label when no existing folder is a reasonable match. +- You may extend an existing path with one new trailing folder only when the document clearly introduces a genuinely new sub-section. +` + // Wiki ingest prompt templates for LLM-powered wiki page generation. // These prompts are used by the wiki ingest pipeline to extract structured // knowledge from raw documents and build/update wiki pages. @@ -313,8 +333,22 @@ The block above is assembled from VERBATIM source chunks that {{.AvailableSlugs}} + +{{.CurrentCategory}} + +` + wikiExistingTaxonomyXML + ` 1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing} +1b. The SECOND line of your output MUST be: CATEGORY: {a navigation taxonomy path for this page, folders from broad to narrow separated by " / ", e.g. "CATEGORY: 人物" or "CATEGORY: 组织 / 企业"}. Follow the Directory Taxonomy Rules below. Almost every page can be filed under at least one broad top-level folder, so leaving it empty should be rare. + +### Directory Taxonomy Rules (for the CATEGORY line) +The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label that stays true across the whole knowledge base — never the role it happens to play in one document. +- Litmus test: keep a label only if it would still fit when this same {{.PageType}} shows up in a completely unrelated document. If a label only describes how it relates to THIS document's topic (who organized, sponsored, acknowledged, authored, or took part in something), that is a ROLE, not a category — leave it out and let the page body and [[wiki-links]] carry the relationship. +- Always assign at least the single broadest top-level folder that fits (think "which shelf would this always sit on in a library"). A coarse one-level category is expected for most pages; reserve a second level for a genuinely durable sub-domain. +- Do NOT use the item type itself as a folder (e.g. "实体", "概念", "摘要"). +- Do NOT put slashes inside a single label. Use at most 2 levels. +- Treat as the default and keep it unless new evidence clearly points to a better stable classification. +- Only output an empty "CATEGORY:" in the rare case where the page genuinely cannot be classified under any broad, durable folder.` + wikiExistingTaxonomyContinuityRules + ` {{if .HasRetractions}} 2. REMOVE facts/claims that were ONLY sourced from the and are NOT present in any or . {{end}} @@ -332,12 +366,51 @@ The block above is assembled from VERBATIM source chunks that 6. Maintain the existing page structure and formatting style. Use "# {{.PageTitle}}" as the top-level heading if the page does not already have one. Do NOT introduce new heading levels beyond what the source or existing page justifies. 7. **Image rule**: Include relevant images using Markdown syntax: ![caption](url) from new information if applicable. The URL inside ![caption](url) is an opaque token; reproduce it EXACTLY and VERBATIM, do not alter, shorten, or normalize it. {{if .HasRetractions}} -8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*" +8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\nCATEGORY:\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*" {{end}} 9. Write in {{.Language}}. -Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.` +Output the SUMMARY line first, then the CATEGORY line, then the updated Markdown content. Do not include any other preamble.` + +// WikiTaxonomyReconcilePrompt is a per-knowledge-base maintenance prompt. It is +// given the full set of distinct category_path folders currently in use and asks +// the LLM to collapse synonymous folders onto a single canonical path. The +// result is applied in Go by re-pointing matching pages' category_path — slugs +// never change. This runs occasionally (after a batch introduces new folders, or +// on manual trigger), NOT per page, so its cost is O(folder count). +const WikiTaxonomyReconcilePrompt = `You are a taxonomy curator for a wiki knowledge base. Below is the complete list of directory paths (category_path) currently used to file wiki pages. Some of them are synonyms or near-duplicates that should share ONE canonical path. + + +{{.CurrentTaxonomy}} + + + +Identify groups of paths that mean the SAME thing and pick the single best canonical path for each group, then output a remap from every non-canonical path to its canonical path. + +Rules: +- Only merge paths that are genuinely synonymous or where one is clearly a less-canonical spelling/wording of another (e.g. "春节习俗" → "春节 / 传统习俗"; "AI厂商" and "AI行业 / 厂商" → one of them). +- Prefer the more standard, concise, and reusable label as the canonical target. Prefer an existing path over inventing a new one. +- Do NOT merge paths that refer to DIFFERENT things, even if they look similar. +- Do NOT merge across clearly different domains. +- Do NOT output an entry whose "from" equals its "to". +- A canonical "to" path may itself be one of the listed paths; never map a path onto a synonym chain (every "to" must be a final canonical path, not another "from"). +- Keep canonical paths to at most 2 levels. +- Each path is an array of folder labels from broad to narrow. Write ALL labels in {{.Language}}. +- If nothing should be merged, return {"remap": []}. + +### JSON Formatting Rules +- Output ONLY valid JSON, no preamble. +- Do NOT use literal newlines inside JSON string values. + + +Output format: +{ + "remap": [ + {"from": ["春节习俗"], "to": ["春节", "传统习俗"]}, + {"from": ["AI厂商"], "to": ["AI行业", "厂商"]} + ] +}` // WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only). const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page. diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index e3bbb2cd9..ec536518c 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -28,6 +28,13 @@ func NewWikiPageRepository(db *gorm.DB) interfaces.WikiPageRepository { return &wikiPageRepository{db: db} } +func (r *wikiPageRepository) wikiCategoryRankOrder() string { + if r.db != nil && r.db.Dialector != nil && r.db.Dialector.Name() == "sqlite" { + return "CASE WHEN COALESCE(json_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC" + } + return "CASE WHEN COALESCE(jsonb_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC" +} + // Create inserts a new wiki page record func (r *wikiPageRepository) Create(ctx context.Context, page *types.WikiPage) error { return r.db.WithContext(ctx).Create(page).Error @@ -101,6 +108,11 @@ func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPag "source_refs": page.SourceRefs, "chunk_refs": page.ChunkRefs, "page_metadata": page.PageMetadata, + "parent_slug": page.ParentSlug, + "category_path": page.CategoryPath, + "wiki_path": page.WikiPath, + "depth": page.Depth, + "sort_order": page.SortOrder, "updated_at": page.UpdatedAt, }) if result.Error != nil { @@ -138,13 +150,38 @@ func (r *wikiPageRepository) GetBySlug(ctx context.Context, kbID string, slug st return &page, nil } +// splitWikiPageTypes parses a page_type filter that may carry several +// comma-separated types (e.g. "entity,concept") into a deduplicated slice. +// An empty/whitespace-only input yields nil, meaning "no type filter". +func splitWikiPageTypes(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + seen := make(map[string]struct{}) + out := make([]string, 0, 4) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if _, ok := seen[part]; ok { + continue + } + seen[part] = struct{}{} + out = append(out, part) + } + return out +} + // List retrieves wiki pages with filtering and pagination func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRequest) ([]*types.WikiPage, int64, error) { query := r.db.WithContext(ctx).Model(&types.WikiPage{}). Where("knowledge_base_id = ?", req.KnowledgeBaseID) - if req.PageType != "" { - query = query.Where("page_type = ?", req.PageType) + if pageTypes := splitWikiPageTypes(req.PageType); len(pageTypes) == 1 { + query = query.Where("page_type = ?", pageTypes[0]) + } else if len(pageTypes) > 1 { + query = query.Where("page_type IN ?", pageTypes) } if req.Status != "" { query = query.Where("status = ?", req.Status) @@ -157,6 +194,24 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe "%"+req.Query+"%", ) } + // Directory filters are pushed to SQL so the DB does the counting and + // pagination instead of loading every page of the type into memory. `depth` + // is a cached column (= len(category_path)); `category_path` is a JSON column + // whose stored text is json.Marshal of the cleaned path, so we compare + // against the same encoding. Postgres needs an explicit jsonb cast for array + // equality; SQLite stores JSON as TEXT and compares directly. + if req.CategoryDepth != nil { + query = query.Where("depth = ?", *req.CategoryDepth) + } + if wantPath := cleanWikiRepositoryCategoryPath(req.CategoryPath); len(wantPath) > 0 { + if encoded, err := json.Marshal([]string(wantPath)); err == nil { + if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" { + query = query.Where("category_path::jsonb = ?::jsonb", string(encoded)) + } else { + query = query.Where("category_path = ?", string(encoded)) + } + } + } var total int64 if err := query.Count(&total).Error; err != nil { @@ -167,7 +222,7 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe sortBy := "updated_at" if req.SortBy != "" { switch req.SortBy { - case "title", "created_at", "updated_at", "page_type": + case "title", "created_at", "updated_at", "page_type", "wiki_path", "sort_order", "depth": sortBy = req.SortBy } } @@ -175,9 +230,15 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe if req.SortOrder == "asc" { sortOrder = "ASC" } - query = query.Order(fmt.Sprintf("%s %s", sortBy, sortOrder)) + if sortBy == "wiki_path" { + query = query.Order(r.wikiCategoryRankOrder()). + Order(fmt.Sprintf("wiki_path %s", sortOrder)). + Order("sort_order ASC"). + Order("title ASC") + } else { + query = query.Order(fmt.Sprintf("%s %s", sortBy, sortOrder)) + } - // Pagination page := req.Page if page < 1 { page = 1 @@ -187,6 +248,8 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe pageSize = 20 } offset := (page - 1) * pageSize + + // Pagination query = query.Offset(offset).Limit(pageSize) var pages []*types.WikiPage @@ -249,7 +312,10 @@ func (r *wikiPageRepository) ListByTypeLight( var entries []types.WikiIndexEntry if err := base. - Select("slug", "title", "summary"). + Select("slug", "title", "summary", "parent_slug", "category_path", "wiki_path", "depth", "sort_order"). + Order(r.wikiCategoryRankOrder()). + Order("wiki_path ASC"). + Order("sort_order ASC"). Order("title ASC"). Limit(limit). Offset(offset). @@ -368,6 +434,199 @@ func (r *wikiPageRepository) ListBySlugs( return out, nil } +// ListDistinctCategoryPaths scans entity/concept pages and returns unique +// non-empty category_path values. Rows are read in wiki_path order so the +// prompt tree is stable; deduplication happens in Go with a maxPaths cap. +func (r *wikiPageRepository) ListDistinctCategoryPaths( + ctx context.Context, + kbID string, + maxPaths int, +) ([][]string, error) { + if maxPaths <= 0 { + maxPaths = 150 + } + + type row struct { + CategoryPath types.StringArray `gorm:"column:category_path"` + } + var rows []row + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Select("category_path"). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?", + kbID, + []string{types.WikiPageTypeEntity, types.WikiPageTypeConcept}, + types.WikiPageStatusArchived, + ). + Order(r.wikiCategoryRankOrder()). + Order("wiki_path ASC"). + Scan(&rows).Error; err != nil { + return nil, err + } + + seen := make(map[string]struct{}, len(rows)) + out := make([][]string, 0, len(rows)) + for _, row := range rows { + if len(row.CategoryPath) == 0 { + continue + } + clean := cleanWikiRepositoryCategoryPath(row.CategoryPath) + if len(clean) == 0 { + continue + } + key := strings.Join(clean, "\x00") + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, clean) + if len(out) >= maxPaths { + break + } + } + return out, nil +} + +// ListDistinctCategoryPathsByType scans one page type and returns unique +// non-empty category_path values. Unlike ListDistinctCategoryPaths, this is +// used by the UI and must work for every visible page_type bucket. +func (r *wikiPageRepository) ListDistinctCategoryPathsByType( + ctx context.Context, + kbID string, + pageTypes []string, + parentPath []string, + maxPaths int, +) ([]types.WikiCategoryPath, error) { + if maxPaths <= 0 { + maxPaths = 2000 + } + if len(pageTypes) == 0 { + return nil, nil + } + + type row struct { + CategoryPath types.StringArray `gorm:"column:category_path"` + } + var rows []row + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Select("category_path"). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?", + kbID, + pageTypes, + types.WikiPageStatusArchived, + ). + Order(r.wikiCategoryRankOrder()). + Order("wiki_path ASC"). + Scan(&rows).Error; err != nil { + return nil, err + } + + parent := cleanWikiRepositoryCategoryPath(parentPath) + + counts := make(map[string]int64, len(rows)) + pathsByKey := make(map[string][]string, len(rows)) + order := make([]string, 0, len(rows)) + for _, row := range rows { + if len(row.CategoryPath) == 0 { + continue + } + clean := cleanWikiRepositoryCategoryPath(row.CategoryPath) + if len(clean) == 0 { + continue + } + if len(clean) <= len(parent) || !wikiPathHasPrefix(clean, parent) { + continue + } + path := clean[:len(parent)+1] + key := strings.Join(path, "\x00") + if _, ok := pathsByKey[key]; !ok { + cp := append([]string(nil), path...) + pathsByKey[key] = cp + order = append(order, key) + } + counts[key]++ + } + + out := make([]types.WikiCategoryPath, 0, len(order)) + for _, key := range order { + out = append(out, types.WikiCategoryPath{ + Path: pathsByKey[key], + Count: counts[key], + }) + } + return out, nil +} + +func cleanWikiRepositoryCategoryPath(parts []string) []string { + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + for _, label := range cleanWikiRepositoryCategoryPart(part) { + if containsWikiRepositoryString(cleaned, label) { + continue + } + cleaned = append(cleaned, label) + } + } + return cleaned +} + +func cleanWikiRepositoryCategoryPart(part string) []string { + part = strings.TrimSpace(part) + if part == "" { + return nil + } + + part = strings.NewReplacer("／", "/", "｜", "/", "|", "/").Replace(part) + rawParts := strings.Split(part, "/") + cleaned := make([]string, 0, len(rawParts)) + for _, raw := range rawParts { + label := strings.TrimSpace(raw) + label = strings.Trim(label, `"'“”‘’[]（）()`) + label = strings.TrimSpace(label) + if label == "" || isWikiRepositoryTypeCategoryLabel(label) { + continue + } + cleaned = append(cleaned, label) + } + return cleaned +} + +func isWikiRepositoryTypeCategoryLabel(label string) bool { + normalized := strings.ToLower(strings.TrimSpace(label)) + normalized = strings.TrimSuffix(normalized, "s") + switch normalized { + case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面": + return true + default: + return false + } +} + +func containsWikiRepositoryString(slice []string, s string) bool { + for _, v := range slice { + if v == s { + return true + } + } + return false +} + +func wikiPathHasPrefix(path []string, prefix []string) bool { + if len(prefix) == 0 { + return true + } + if len(path) < len(prefix) { + return false + } + for i, part := range prefix { + if path[i] != part { + return false + } + } + return true +} + // ListSummariesByKnowledgeIDs returns summary-page content keyed by the // knowledge id that authored it. The page_type filter is applied first // (only summary pages have content suitable for retract framing); within diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go index 4f3d5430f..1efc76157 100644 --- a/internal/application/repository/wiki_page_test.go +++ b/internal/application/repository/wiki_page_test.go @@ -29,6 +29,11 @@ CREATE TABLE IF NOT EXISTS wiki_pages ( status VARCHAR(32) NOT NULL DEFAULT 'published', content TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', + parent_slug VARCHAR(255) NOT NULL DEFAULT '', + category_path TEXT DEFAULT '[]', + wiki_path VARCHAR(1024) NOT NULL DEFAULT '', + depth INTEGER NOT NULL DEFAULT 0, + sort_order INTEGER NOT NULL DEFAULT 0, source_refs TEXT DEFAULT '[]', chunk_refs TEXT DEFAULT '[]', in_links TEXT DEFAULT '[]', @@ -68,12 +73,57 @@ func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage { Status: status, Content: "body of " + slug, Summary: "summary of " + slug, + WikiPath: pageType + "/" + title, Version: 1, CreatedAt: time.Now(), UpdatedAt: time.Now(), } } +func makeCategorizedWikiPage(kbID, slug, pageType, status string, categoryPath ...string) *types.WikiPage { + page := makeWikiPage(kbID, slug, pageType, status) + page.CategoryPath = types.StringArray(categoryPath) + if len(categoryPath) > 0 { + page.WikiPath = pageType + "/" + strings.Join(categoryPath, "/") + "/" + page.Title + page.Depth = len(categoryPath) + } + return page +} + +// TestList_WikiPathSortReturnsCategorizedPagesFirst protects the sidebar's +// IDE-like tree contract. Pagination happens in the repository, so the DB +// must return pages with category_path before loose root pages; otherwise the +// frontend cannot know about directories hiding on later pages. +func TestList_WikiPathSortReturnsCategorizedPagesFirst(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + pages := []*types.WikiPage{ + makeWikiPage("kb-a", "entity/000-root", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + makeCategorizedWikiPage("kb-a", "entity/999-child", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "zzz-folder"), + makeWikiPage("kb-a", "entity/001-root", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + } + for _, p := range pages { + require.NoError(t, repo.Create(ctx, p)) + } + + got, total, err := repo.List(ctx, &types.WikiPageListRequest{ + KnowledgeBaseID: "kb-a", + PageType: types.WikiPageTypeEntity, + Page: 1, + PageSize: 10, + SortBy: "wiki_path", + SortOrder: "asc", + }) + require.NoError(t, err) + assert.Equal(t, int64(3), total) + require.Len(t, got, 3) + assert.Equal(t, "entity/999-child", got[0].Slug) + assert.Equal(t, "entity/000-root", got[1].Slug) + assert.Equal(t, "entity/001-root", got[2].Slug) +} + // TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies // that the index-view projection only emits the slug/title/summary // triples and respects the archived-status filter. This is the whole diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 9dbde1df9..447bb0256 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -658,6 +658,12 @@ type WikiBatchContext struct { // Already Normalize()'d — consumers can assume it is one of the // three valid values. ExtractionGranularity types.WikiExtractionGranularity + + // ExistingTaxonomy returns a formatted directory tree of category_path + // folders already used in this KB. Loaded once per batch and injected + // into entity/concept extraction prompts so new pages reuse established + // folder labels instead of inventing synonyms. + ExistingTaxonomy func(ctx context.Context) string } // SlugUpdate represents a single update operation for a specific slug @@ -1197,6 +1203,126 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef { return refs } +// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when +// building the existing-taxonomy prompt block. Distinct paths are deduped +// in Go; this bound keeps the query cheap on large KBs. +const wikiTaxonomyScanRowLimit = 2000 + +// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes +// are rendered into the LLM prompt to control token usage. +const wikiTaxonomyPromptMaxPaths = 150 + +func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string { + if batchCtx == nil || batchCtx.ExistingTaxonomy == nil { + return "(none — this knowledge base has no established folders yet)" + } + text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx)) + if text == "" { + return "(none — this knowledge base has no established folders yet)" + } + return text +} + +type wikiTaxonomyNode struct { + children map[string]*wikiTaxonomyNode +} + +func insertWikiTaxonomyPath(root *wikiTaxonomyNode, path []string) { + if root == nil || len(path) == 0 { + return + } + cur := root + for _, part := range path { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if cur.children == nil { + cur.children = make(map[string]*wikiTaxonomyNode) + } + child := cur.children[part] + if child == nil { + child = &wikiTaxonomyNode{} + cur.children[part] = child + } + cur = child + } +} + +func appendWikiTaxonomyNode(buf *strings.Builder, label string, node *wikiTaxonomyNode, depth int) { + if label != "" { + fmt.Fprintf(buf, "%s%s\n", strings.Repeat(" ", depth), label) + } + if node == nil || len(node.children) == 0 { + return + } + keys := make([]string, 0, len(node.children)) + for k := range node.children { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + appendWikiTaxonomyNode(buf, k, node.children[k], depth+1) + } +} + +// formatExistingTaxonomyForPrompt renders distinct category_path values as an +// indented folder tree for LLM extraction prompts. +func formatExistingTaxonomyForPrompt(paths [][]string) string { + if len(paths) == 0 { + return "" + } + root := &wikiTaxonomyNode{} + for _, path := range paths { + insertWikiTaxonomyPath(root, path) + } + if len(root.children) == 0 { + return "" + } + var buf strings.Builder + keys := make([]string, 0, len(root.children)) + for k := range root.children { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + appendWikiTaxonomyNode(&buf, k, root.children[k], 0) + } + return strings.TrimSpace(buf.String()) +} + +// dedupeWikiCategoryPaths returns unique non-empty category_path slices, +// preserving first-seen order up to maxPaths. +func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string { + if maxPaths <= 0 { + maxPaths = wikiTaxonomyPromptMaxPaths + } + seen := make(map[string]struct{}, len(paths)) + out := make([][]string, 0, len(paths)) + for _, raw := range paths { + clean := make([]string, 0, len(raw)) + for _, part := range raw { + part = strings.TrimSpace(part) + if part != "" { + clean = append(clean, part) + } + } + if len(clean) == 0 { + continue + } + key := strings.Join(clean, "\x00") + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, clean) + if len(out) >= maxPaths { + break + } + } + return out +} + // getExistingPageSlugsForKnowledge returns all page slugs that currently // reference a given knowledge ID in their source_refs. Used to snapshot // state before re-ingest so the reduce phase can reconcile additions vs @@ -1435,6 +1561,50 @@ func splitSummaryLine(raw string) (summary string, content string) { return "", raw } +// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content +// (the page body AFTER the SUMMARY line has been stripped) and returns the +// parsed folder labels plus the remaining content with that line removed. +// +// Category assignment lives in the page-modify output (reduce time) rather +// than per-document extraction so it reflects WHAT the whole entity IS, not +// the role it played in one source document. An empty/absent CATEGORY line +// yields nil so callers keep the page's existing category instead of wiping it. +func splitCategoryLine(content string) (category []string, rest string) { + trimmed := strings.TrimLeft(content, " \t\r\n") + if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY：") { + return nil, content + } + var line, remainder string + if idx := strings.IndexByte(trimmed, '\n'); idx < 0 { + line = trimmed + } else { + line = trimmed[:idx] + remainder = trimmed[idx+1:] + } + line = strings.TrimPrefix(line, "CATEGORY:") + line = strings.TrimPrefix(line, "CATEGORY：") + return parseCategoryPath(line), strings.TrimSpace(remainder) +} + +// parseCategoryPath splits a "a / b" breadcrumb string into clean folder +// labels. Separator variants and wrapping quotes/brackets are normalized; the +// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is +// left to normalizeWikiHierarchy on the page. +func parseCategoryPath(line string) []string { + line = strings.NewReplacer("／", "/", "｜", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line) + parts := strings.Split(line, "/") + out := make([]string, 0, len(parts)) + for _, p := range parts { + label := strings.TrimSpace(p) + label = strings.Trim(label, `"'“”‘’[]（）()`) + label = strings.TrimSpace(label) + if label != "" { + out = append(out, label) + } + } + return out +} + // buildLogEntry builds a WikiLogEntry struct for the current batch. It is // pure (no DB access) so callers can accumulate entries cheaply under their // lock and flush them in a single AppendBatch call at the end of the batch. diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 4f1530f86..7023c8577 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -357,6 +357,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task ExtractionGranularity: granularity, } + var taxonomyOnce sync.Once + var taxonomyText string + batchCtx.ExistingTaxonomy = func(ctx context.Context) string { + taxonomyOnce.Do(func() { + if s.wikiService == nil { + return + } + paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths) + if err != nil { + logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err) + return + } + taxonomyText = formatExistingTaxonomyForPrompt(paths) + }) + return taxonomyText + } + + // Snapshot the KB's category folders before this batch runs so we can tell + // afterwards whether new (possibly synonymous) folders were introduced and + // a taxonomy reconcile pass is worth running. + startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID) + // 1. MAP PHASE (Parallel extraction and generation of updates) var mapMu sync.Mutex var failedOps []WikiPendingOp @@ -628,6 +650,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } } + // Collapse synonymous category folders introduced by this batch (cold + // start, intra-batch races, long-term drift). Runs only when a genuinely + // new folder appeared vs the pre-batch snapshot; re-points pages' category + // without touching slugs, then refreshes the index. + if len(ingestPagesAffected) > 0 { + s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys) + } + // Clean dead [[slug]] references whenever ANY page was touched this // batch (not just retracts). Reduce-phase failures can leave stale // references in pages we just rewrote (e.g. summary pages cite @@ -960,7 +990,7 @@ func (s *wikiIngestService) mapOneDocument( return } candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts) - citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang) + citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang, batchCtx) s.tracker().EndSpan(ctx, classifySpan, types.JSONMap{ "cited_slugs": len(citations), "new_slugs": len(newSlugs), @@ -1585,6 +1615,11 @@ func (s *wikiIngestService) reduceSlugUpdates( } pageAliases := strings.Join(page.Aliases, ", ") + currentCategory := strings.Join(page.CategoryPath, " / ") + if strings.TrimSpace(currentCategory) == "" { + currentCategory = "(none yet)" + } + var updatedContent string updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{ "HasAdditions": hasAdditionsStr, @@ -1598,11 +1633,22 @@ func (s *wikiIngestService) reduceSlugUpdates( "DeletedContent": deletedContent.String(), "RemainingSourcesContent": remainingSourcesContent.String(), "AvailableSlugs": relatedSlugs.String(), + "CurrentCategory": currentCategory, + "ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx), "Language": language, }) if err == nil && updatedContent != "" { updatedSummary, updatedBody := splitSummaryLine(updatedContent) + // CATEGORY is the page-level intrinsic classification produced by + // the editor model. Only overwrite when the model actually emitted + // a non-empty path so a lazy/empty line never wipes a good category. + category, bodyAfterCategory := splitCategoryLine(updatedBody) + updatedBody = bodyAfterCategory + if len(category) > 0 { + page.CategoryPath = types.StringArray(category) + page.WikiPath = "" + } if updatedBody != "" { page.Content = updatedBody } else { diff --git a/internal/application/service/wiki_ingest_cite.go b/internal/application/service/wiki_ingest_cite.go index 7f6ec99e1..48c684406 100644 --- a/internal/application/service/wiki_ingest_cite.go +++ b/internal/application/service/wiki_ingest_cite.go @@ -279,6 +279,7 @@ func (s *wikiIngestService) classifyChunkCitations( candidatesXML string, chunks []*types.Chunk, lang string, + batchCtx *WikiBatchContext, ) (map[string][]string, []newSlugFromCitation, int) { batches := splitChunksIntoCitationBatches(chunks) if len(batches) == 0 || strings.TrimSpace(candidatesXML) == "" { diff --git a/internal/application/service/wiki_ingest_taxonomy.go b/internal/application/service/wiki_ingest_taxonomy.go new file mode 100644 index 000000000..b53cf6346 --- /dev/null +++ b/internal/application/service/wiki_ingest_taxonomy.go @@ -0,0 +1,259 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/agent" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" +) + +// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass +// will rewrite, so an over-eager remap cannot fan out into an unbounded write +// storm on a very large KB. Pages beyond the cap are picked up by the next run. +const wikiTaxonomyReconcileMaxPages = 5000 + +// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the +// KB to apply a category remap. +const wikiTaxonomyReconcilePageSize = 500 + +type wikiTaxonomyRemapEntry struct { + From []string `json:"from"` + To []string `json:"to"` +} + +type wikiTaxonomyRemapResult struct { + Remap []wikiTaxonomyRemapEntry `json:"remap"` +} + +// ReconcileTaxonomy is the manual entry point for collapsing synonymous +// category folders across a wiki knowledge base. It resolves the KB's synthesis +// chat model itself so it can be invoked outside an ingest batch (e.g. from a +// future maintenance handler). +func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error { + kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) + if err != nil { + return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err) + } + if !kb.IsWikiEnabled() { + return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID) + } + + synthesisModelID := "" + if kb.WikiConfig != nil { + synthesisModelID = kb.WikiConfig.SynthesisModelID + } + if synthesisModelID == "" { + synthesisModelID = kb.SummaryModelID + } + if synthesisModelID == "" { + return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID) + } + chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID) + if err != nil { + return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err) + } + + return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx)) +} + +// reconcileTaxonomy collapses synonymous category_path folders across the KB +// onto a single canonical path. It is per-KB (cost O(folder count), not page +// count): one LLM call clusters the distinct folders, then matching pages are +// re-pointed in Go. Slugs are NEVER touched — only the navigation category. +func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error { + if s.wikiService == nil || chatModel == nil { + return nil + } + paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) + if err != nil { + return fmt.Errorf("list distinct category paths: %w", err) + } + if len(paths) < 2 { + return nil // nothing to merge + } + + remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang) + if len(remap) == 0 { + return nil + } + + updated := s.applyTaxonomyRemap(ctx, kbID, remap) + if updated > 0 { + logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID) + if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err) + } + } + return nil +} + +// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced +// at least one category folder that was not present in startKeys (the snapshot +// taken before the batch ran). This keeps the extra LLM call off the hot path +// for batches that only reuse established folders. +func (s *wikiIngestService) maybeReconcileTaxonomy( + ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool, +) { + if s.wikiService == nil { + return + } + paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) + if err != nil || len(paths) < 2 { + return + } + introducedNewFolder := false + for _, p := range paths { + if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] { + introducedNewFolder = true + break + } + } + if !introducedNewFolder { + return + } + if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err) + } +} + +// snapshotCategoryKeys returns the set of canonical category keys currently in +// use, used as the "before" baseline for maybeReconcileTaxonomy. +func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool { + out := map[string]bool{} + if s.wikiService == nil { + return out + } + paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) + if err != nil { + return out + } + for _, p := range paths { + out[categoryPathKey(cleanCategoryPathParts(p))] = true + } + return out +} + +// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a +// map keyed by the canonical key of each non-canonical path to its replacement. +func (s *wikiIngestService) buildTaxonomyRemap( + ctx context.Context, chatModel chat.Chat, paths [][]string, lang string, +) map[string][]string { + taxonomyText := formatExistingTaxonomyForPrompt(paths) + if strings.TrimSpace(taxonomyText) == "" { + return nil + } + raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{ + "CurrentTaxonomy": taxonomyText, + "Language": lang, + }) + if err != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err) + return nil + } + raw = cleanLLMJSON(raw) + var parsed wikiTaxonomyRemapResult + if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw) + return nil + } + + out := make(map[string][]string, len(parsed.Remap)) + for _, e := range parsed.Remap { + from := cleanCategoryPathParts(e.From) + to := cleanCategoryPathParts(e.To) + if len(from) == 0 || len(to) == 0 { + continue + } + fromKey := categoryPathKey(from) + if fromKey == categoryPathKey(to) { + continue // no-op rule + } + out[fromKey] = to + } + return out +} + +// applyTaxonomyRemap scans entity/concept pages and re-points any whose +// category matches a remap rule. Returns the number of pages updated. +func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int { + var cursor string + var updated int + for updated < wikiTaxonomyReconcileMaxPages { + pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize) + if err != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err) + break + } + if len(pages) == 0 { + break + } + for _, p := range pages { + if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept { + continue + } + newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap) + if !ok { + continue + } + p.CategoryPath = types.StringArray(newPath) + p.WikiPath = "" // force recompute in normalizeWikiHierarchy + if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil { + logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err) + continue + } + updated++ + if updated >= wikiTaxonomyReconcileMaxPages { + break + } + } + if next == "" { + break + } + cursor = next + } + return updated +} + +// remapCategoryPath returns the canonical category for current when it matches +// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can +// be unit-tested in isolation. +func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) { + cleaned := cleanCategoryPathParts(current) + if len(cleaned) == 0 { + return nil, false + } + to, ok := remap[categoryPathKey(cleaned)] + if !ok { + return nil, false + } + if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) { + return nil, false + } + return append([]string(nil), to...), true +} + +// cleanCategoryPathParts normalizes a category path the same way the page +// hierarchy normalizer does (separator/quote cleanup, type-label stripping, +// depth cap) so remap keys match what is stored on pages. +func cleanCategoryPathParts(parts []string) []string { + out := make([]string, 0, len(parts)) + for _, part := range parts { + for _, c := range cleanWikiCategoryPart(part) { + out = append(out, c) + if len(out) >= 3 { + return out + } + } + } + return out +} + +// categoryPathKey builds a stable comparison key for a cleaned category path. +func categoryPathKey(parts []string) string { + return strings.Join(parts, "\u0000") +} diff --git a/internal/application/service/wiki_ingest_taxonomy_test.go b/internal/application/service/wiki_ingest_taxonomy_test.go new file mode 100644 index 000000000..d8fab8f9d --- /dev/null +++ b/internal/application/service/wiki_ingest_taxonomy_test.go @@ -0,0 +1,185 @@ +package service + +import ( + "context" + "strings" + "testing" +) + +func TestSplitCategoryLine(t *testing.T) { + tests := []struct { + name string + content string + wantCat []string + wantRest string + }{ + { + name: "two level category", + content: "CATEGORY: 组织 / 企业\n# 标题\n正文", + wantCat: []string{"组织", "企业"}, + wantRest: "# 标题\n正文", + }, + { + name: "single level category", + content: "CATEGORY: 人物\n# 陈洋", + wantCat: []string{"人物"}, + wantRest: "# 陈洋", + }, + { + name: "empty category keeps existing (nil) and strips line", + content: "CATEGORY:\n# 标题", + wantCat: nil, + wantRest: "# 标题", + }, + { + name: "no category line left untouched", + content: "# 标题\n正文", + wantCat: nil, + wantRest: "# 标题\n正文", + }, + { + name: "fullwidth colon and separators", + content: "CATEGORY：组织｜企业\n正文", + wantCat: []string{"组织", "企业"}, + wantRest: "正文", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cat, rest := splitCategoryLine(tt.content) + if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") { + t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat) + } + if rest != tt.wantRest { + t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest) + } + }) + } +} + +func TestFormatExistingTaxonomyForPrompt(t *testing.T) { + got := formatExistingTaxonomyForPrompt([][]string{ + {"春节", "传统习俗"}, + {"春节", "文化习俗", "节日习俗"}, + {"春节习俗"}, + {"产品定位"}, + }) + want := "产品定位\n" + + "春节\n" + + " 传统习俗\n" + + " 文化习俗\n" + + " 节日习俗\n" + + "春节习俗" + if got != want { + t.Fatalf("formatExistingTaxonomyForPrompt():\n%q\nwant:\n%q", got, want) + } +} + +func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) { + if got := formatExistingTaxonomyForPrompt(nil); got != "" { + t.Fatalf("formatExistingTaxonomyForPrompt(nil) = %q, want empty", got) + } +} + +func TestDedupeWikiCategoryPaths(t *testing.T) { + got := dedupeWikiCategoryPaths([][]string{ + {"春节", "传统习俗"}, + {" 春节 ", "传统习俗"}, + {"春节", "文化习俗"}, + {"", "空"}, + {}, + }, 10) + if len(got) != 3 { + t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got) + } + if got[0][0] != "春节" || got[0][1] != "传统习俗" { + t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0]) + } + if got[1][1] != "文化习俗" { + t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1]) + } +} + +func TestResolveExistingTaxonomyForPrompt(t *testing.T) { + t.Run("nil batch context", func(t *testing.T) { + got := resolveExistingTaxonomyForPrompt(t.Context(), nil) + if got == "" { + t.Fatal("expected fallback text for nil batch context") + } + }) + t.Run("batch context with taxonomy", func(t *testing.T) { + batchCtx := &WikiBatchContext{ + ExistingTaxonomy: func(ctx context.Context) string { + return "春节\n 传统习俗" + }, + } + got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx) + if got != "春节\n 传统习俗" { + t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got) + } + }) +} + +func TestRemapCategoryPath(t *testing.T) { + remap := map[string][]string{ + categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"}, + categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"}, + } + + tests := []struct { + name string + current []string + want []string + wantHit bool + }{ + { + name: "single-label synonym remapped", + current: []string{"春节习俗"}, + want: []string{"春节", "传统习俗"}, + wantHit: true, + }, + { + name: "document-role path collapsed to coarse folder", + current: []string{"组织单位", "致谢人员"}, + want: []string{"组织"}, + wantHit: true, + }, + { + name: "noise normalized before matching", + current: []string{"实体/春节习俗"}, + want: []string{"春节", "传统习俗"}, + wantHit: true, + }, + { + name: "unrelated path untouched", + current: []string{"人物"}, + wantHit: false, + }, + { + name: "empty path untouched", + current: nil, + wantHit: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, hit := remapCategoryPath(tt.current, remap) + if hit != tt.wantHit { + t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit) + } + if !hit { + return + } + if len(got) != len(tt.want) { + t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got) + } + } + }) + } +} diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index de71a4f62..a6750ae1e 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -67,6 +67,7 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage) // Parse outbound links from content page.OutLinks = s.parseOutLinks(page.Content) + normalizeWikiHierarchy(page) now := time.Now() page.CreatedAt = now @@ -115,6 +116,11 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage) existing.SourceRefs = page.SourceRefs existing.ChunkRefs = page.ChunkRefs existing.PageMetadata = page.PageMetadata + existing.ParentSlug = page.ParentSlug + existing.CategoryPath = page.CategoryPath + existing.WikiPath = page.WikiPath + existing.Depth = page.Depth + existing.SortOrder = page.SortOrder existing.Status = page.Status existing.UpdatedAt = time.Now() @@ -122,11 +128,18 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage) // when content shifts. Re-parse unconditionally to stay consistent with // the stored body. existing.OutLinks = s.parseOutLinks(existing.Content) + normalizeWikiHierarchy(existing) if contentChanged { if err := s.repo.Update(ctx, existing); err != nil { return nil, fmt.Errorf("update wiki page: %w", err) } + // GORM's struct Updates path skips zero values, so persist hierarchy + // metadata through the explicit map path as well. This keeps clearing + // parent/category fields deterministic without changing version again. + if err := s.repo.UpdateMeta(ctx, existing); err != nil { + return nil, fmt.Errorf("update wiki page hierarchy meta: %w", err) + } } else { // No user-visible change — persist bookkeeping fields but preserve // the version so downstream consumers can rely on it. @@ -145,6 +158,7 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage) // UpdatePageMeta updates only metadata (status, source_refs) without version bump or link re-parse. func (s *wikiPageService) UpdatePageMeta(ctx context.Context, page *types.WikiPage) error { + normalizeWikiHierarchy(page) page.UpdatedAt = time.Now() return s.repo.UpdateMeta(ctx, page) } @@ -192,6 +206,9 @@ func (s *wikiPageService) ListPages(ctx context.Context, req *types.WikiPageList if err != nil { return nil, err } + for _, page := range pages { + normalizeWikiHierarchy(page) + } pageSize := req.PageSize if pageSize < 1 { @@ -323,6 +340,9 @@ func (s *wikiPageService) GetIndexView( if entries == nil { entries = []types.WikiIndexEntry{} } + for i := range entries { + normalizeWikiIndexEntryHierarchy(&entries[i], pt) + } next := "" // Only emit a cursor when a full page was returned AND more rows // remain past `offset + limit`. A short page or one that exactly @@ -811,6 +831,20 @@ func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, que return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit) } +// ListDistinctCategoryPaths returns unique category_path folders already used +// by entity/concept pages. Used by wiki ingest to ground extraction prompts. +func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) { + return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths) +} + +// ListDistinctCategoryPathsByType returns unique category_path folders across +// the given page types. The browser uses this to render directory nodes before +// article pages are paged in; passing several types lets one tab (e.g. the +// merged knowledge tab) build a single directory skeleton server-side. +func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) { + return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths) +} + // CountByType is a service-layer pass-through over the repo. Used by // the index intro path to frame the LLM prompt's "showing N of M" hint. func (s *wikiPageService) CountByType(ctx context.Context, kbID string) (map[string]int64, error) { @@ -917,6 +951,7 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl Summary: title, Version: 1, } + normalizeWikiHierarchy(page) if err := s.repo.Create(ctx, page); err != nil { return nil, fmt.Errorf("create default %s page: %w", slug, err) @@ -924,6 +959,115 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl return page, nil } +func normalizeWikiHierarchy(page *types.WikiPage) { + if page == nil { + return + } + page.ParentSlug = strings.TrimSpace(page.ParentSlug) + + cleanPath := make(types.StringArray, 0, len(page.CategoryPath)) + for _, part := range page.CategoryPath { + for _, cleanPart := range cleanWikiCategoryPart(part) { + if !containsString(cleanPath, cleanPart) { + cleanPath = append(cleanPath, cleanPart) + } + if len(cleanPath) >= 3 { + break + } + } + if len(cleanPath) >= 3 { + break + } + } + page.CategoryPath = cleanPath + page.Depth = len(cleanPath) + + parts := make([]string, 0, len(cleanPath)+2) + if page.PageType != "" { + parts = append(parts, page.PageType) + } + for _, part := range cleanPath { + parts = append(parts, part) + } + display := strings.TrimSpace(page.Title) + if display == "" { + display = strings.TrimSpace(page.Slug) + } + if display != "" { + parts = append(parts, display) + } + page.WikiPath = strings.Join(parts, "/") +} + +func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) { + if entry == nil { + return + } + + cleanPath := make(types.StringArray, 0, len(entry.CategoryPath)) + for _, part := range entry.CategoryPath { + for _, cleanPart := range cleanWikiCategoryPart(part) { + if !containsString(cleanPath, cleanPart) { + cleanPath = append(cleanPath, cleanPart) + } + if len(cleanPath) >= 3 { + break + } + } + if len(cleanPath) >= 3 { + break + } + } + entry.CategoryPath = cleanPath + entry.Depth = len(cleanPath) + + parts := make([]string, 0, len(cleanPath)+2) + if strings.TrimSpace(pageType) != "" { + parts = append(parts, strings.TrimSpace(pageType)) + } + parts = append(parts, cleanPath...) + display := strings.TrimSpace(entry.Title) + if display == "" { + display = strings.TrimSpace(entry.Slug) + } + if display != "" { + parts = append(parts, display) + } + entry.WikiPath = strings.Join(parts, "/") +} + +func cleanWikiCategoryPart(part string) []string { + part = strings.TrimSpace(part) + if part == "" { + return nil + } + + part = strings.NewReplacer("／", "/", "｜", "/", "|", "/").Replace(part) + rawParts := strings.Split(part, "/") + cleaned := make([]string, 0, len(rawParts)) + for _, raw := range rawParts { + label := strings.TrimSpace(raw) + label = strings.Trim(label, `"'“”‘’[]（）()`) + label = strings.TrimSpace(label) + if label == "" || isWikiTypeCategoryLabel(label) { + continue + } + cleaned = append(cleaned, label) + } + return cleaned +} + +func isWikiTypeCategoryLabel(label string) bool { + normalized := strings.ToLower(strings.TrimSpace(label)) + normalized = strings.TrimSuffix(normalized, "s") + switch normalized { + case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面": + return true + default: + return false + } +} + // containsString checks if a string slice contains a given string func containsString(slice []string, s string) bool { for _, v := range slice { diff --git a/internal/application/service/wiki_page_test.go b/internal/application/service/wiki_page_test.go index 0bb071c49..99e3e54e0 100644 --- a/internal/application/service/wiki_page_test.go +++ b/internal/application/service/wiki_page_test.go @@ -99,6 +99,56 @@ func TestNormalizeSlug(t *testing.T) { } } +func TestNormalizeWikiHierarchyCleansModelCategoryNoise(t *testing.T) { + page := &types.WikiPage{ + Slug: "concept/ai-knowledge", + Title: "AI时代的知识困境", + PageType: types.WikiPageTypeConcept, + CategoryPath: types.StringArray{"概念/爱护花草", " 实体/标牌 ", "摘要", "生态/平台", "多余层级"}, + } + + normalizeWikiHierarchy(page) + + want := types.StringArray{"爱护花草", "标牌", "生态"} + if len(page.CategoryPath) != len(want) { + t.Fatalf("CategoryPath = %v, want %v", page.CategoryPath, want) + } + for i := range want { + if page.CategoryPath[i] != want[i] { + t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, page.CategoryPath[i], want[i], page.CategoryPath) + } + } + if page.Depth != 3 { + t.Fatalf("Depth = %d, want 3", page.Depth) + } + if page.WikiPath != "concept/爱护花草/标牌/生态/AI时代的知识困境" { + t.Fatalf("WikiPath = %q", page.WikiPath) + } +} + +func TestNormalizeWikiIndexEntryHierarchyCleansModelCategoryNoise(t *testing.T) { + entry := &types.WikiIndexEntry{ + Slug: "entity/sign", + Title: "爱护花草标牌", + CategoryPath: types.StringArray{"实体/标牌", "概念", "生态/行为倡导"}, + } + + normalizeWikiIndexEntryHierarchy(entry, types.WikiPageTypeEntity) + + want := types.StringArray{"标牌", "生态", "行为倡导"} + if len(entry.CategoryPath) != len(want) { + t.Fatalf("CategoryPath = %v, want %v", entry.CategoryPath, want) + } + for i := range want { + if entry.CategoryPath[i] != want[i] { + t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, entry.CategoryPath[i], want[i], entry.CategoryPath) + } + } + if entry.WikiPath != "entity/标牌/生态/行为倡导/爱护花草标牌" { + t.Fatalf("WikiPath = %q", entry.WikiPath) + } +} + func TestContainsString(t *testing.T) { slice := []string{"a", "b", "c"} if !containsString(slice, "b") { diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index 3b626aef8..065f0f073 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -77,7 +77,7 @@ func getSlugParam(c *gin.Context) string { // @Tags Wiki // @Produce json // @Param kb_id path string true "Knowledge base ID" -// @Param page_type query string false "Filter by page type" +// @Param page_type query string false "Filter by page type; comma-separated for multiple (e.g. entity,concept)" // @Param status query string false "Filter by status" // @Param query query string false "Full-text search" // @Param page query int false "Page number" @@ -97,12 +97,21 @@ func (h *WikiPageHandler) ListPages(c *gin.Context) { page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20")) + categoryPath := parseWikiCategoryPath(c.Query("category_path")) + var categoryDepth *int + if raw := c.Query("category_depth"); raw != "" { + if depth, parseErr := strconv.Atoi(raw); parseErr == nil && depth >= 0 { + categoryDepth = &depth + } + } req := &types.WikiPageListRequest{ KnowledgeBaseID: kbID, PageType: c.Query("page_type"), Status: c.Query("status"), Query: c.Query("query"), + CategoryPath: types.StringArray(categoryPath), + CategoryDepth: categoryDepth, Page: page, PageSize: pageSize, SortBy: c.DefaultQuery("sort_by", "updated_at"), @@ -118,6 +127,106 @@ func (h *WikiPageHandler) ListPages(c *gin.Context) { c.JSON(http.StatusOK, resp) } +// ListCategories godoc +// @Summary List wiki category paths +// @Description Retrieve direct child directories for a wiki page type and optional parent_path +// @Tags Wiki +// @Produce json +// @Param kb_id path string true "Knowledge base ID" +// @Param page_type query string true "Wiki page type; comma-separated for multiple (e.g. entity,concept)" +// @Param parent_path query string false "Parent category path, slash-separated" +// @Param page query int false "Page number (1-based)" +// @Param page_size query int false "Page size" +// @Success 200 {object} types.WikiCategoryPathListResponse +// @Failure 400 {object} errors.AppError +// @Security Bearer +// @Router /knowledgebase/{kb_id}/wiki/categories [get] +func (h *WikiPageHandler) ListCategories(c *gin.Context) { + kbID, _, err := h.validateWikiKB(c) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + pageTypes := parseWikiPageTypes(c.Query("page_type")) + if len(pageTypes) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "page_type is required"}) + return + } + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + if page < 1 { + page = 1 + } + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "2000")) + if pageSize <= 0 { + pageSize = 2000 + } + parentPath := parseWikiCategoryPath(c.Query("parent_path")) + paths, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, pageSize) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + total := len(paths) + offset := (page - 1) * pageSize + if offset > total { + offset = total + } + end := offset + pageSize + if end > total { + end = total + } + totalPages := 0 + if pageSize > 0 { + totalPages = (total + pageSize - 1) / pageSize + } + + c.JSON(http.StatusOK, types.WikiCategoryPathListResponse{ + Paths: paths[offset:end], + Total: total, + Page: page, + PageSize: pageSize, + TotalPages: totalPages, + }) +} + +// parseWikiPageTypes splits a comma-separated page_type query value (e.g. +// "entity,concept") into a deduplicated slice, dropping blanks. +func parseWikiPageTypes(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + seen := make(map[string]struct{}) + out := make([]string, 0, 4) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if _, ok := seen[part]; ok { + continue + } + seen[part] = struct{}{} + out = append(out, part) + } + return out +} + +func parseWikiCategoryPath(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + parts := strings.Split(raw, "/") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + // CreatePage godoc // @Summary Create a wiki page // @Description Create a new wiki page in the knowledge base diff --git a/internal/router/router.go b/internal/router/router.go index 7b72dcd76..ce197fe24 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -1748,6 +1748,7 @@ func RegisterWikiPageRoutes(r *gin.RouterGroup, wikiHandler *handler.WikiPageHan { // Page CRUD wiki.GET("/pages", g.Viewer(), wikiHandler.ListPages) + wiki.GET("/categories", g.Viewer(), wikiHandler.ListCategories) wiki.POST("/pages", g.OwnedWikiKBOrAdmin(), wikiHandler.CreatePage) wiki.GET("/pages/*slug", g.Viewer(), wikiHandler.GetPage) wiki.PUT("/pages/*slug", g.OwnedWikiKBOrAdmin(), wikiHandler.UpdatePage) diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 170459dbc..47b9af839 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -156,6 +156,16 @@ type WikiPageService interface { // merge targets server-side. FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) + // ListDistinctCategoryPaths returns unique non-empty category_path + // values from entity/concept pages. Used by wiki ingest to inject + // established folder taxonomy into extraction prompts. + ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) + + // ListDistinctCategoryPathsByType returns unique non-empty category_path + // values across the given page types. Used by the wiki browser to render a + // stable directory skeleton independently of page pagination. + ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) + // CountByType returns page counts grouped by type for a knowledge // base. Re-exposed at the service layer so the index intro // generation path can frame the LLM prompt with "showing N of M" @@ -264,6 +274,14 @@ type WikiPageRepository interface { // surface candidate merge targets server-side. FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) + // ListDistinctCategoryPaths scans entity/concept pages and returns + // unique non-empty category_path values for wiki ingest prompts. + ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) + + // ListDistinctCategoryPathsByType returns unique non-empty category_path + // values across the given page types. + ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) + // ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation). ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error) diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index 3b254db41..a2386ab5e 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -65,6 +65,22 @@ type WikiPage struct { Summary string `json:"summary" gorm:"type:text"` // Alternate names, abbreviations, acronyms or translated names Aliases StringArray `json:"aliases" gorm:"type:json"` + // ParentSlug optionally points at the wiki page that should act as this + // page's semantic parent in the directory tree. The parent may be empty + // when the page is grouped only by CategoryPath. + ParentSlug string `json:"parent_slug,omitempty" gorm:"type:varchar(255);index"` + // CategoryPath is the directory breadcrumb that groups this page in the + // wiki browser, e.g. ["AI", "LLM 应用", "RAG"]. It is intentionally + // label-based so intermediate directory nodes do not need to be real pages. + CategoryPath StringArray `json:"category_path,omitempty" gorm:"type:json"` + // WikiPath is a normalized, sortable path derived from page_type, + // category_path, and title. It keeps large directory listings cheap to sort. + WikiPath string `json:"wiki_path,omitempty" gorm:"type:varchar(1024);index"` + // Depth is len(CategoryPath), cached for filtering / display. + Depth int `json:"depth,omitempty" gorm:"default:0;index"` + // SortOrder allows generated or manually edited pages to control sibling + // ordering before falling back to title. + SortOrder int `json:"sort_order,omitempty" gorm:"default:0;index"` // References to source knowledge IDs that contributed to this page. // Format matches the legacy "|" convention used // across the ingest pipeline, so retract / display code can split on `|` @@ -226,14 +242,16 @@ func (c *WikiConfig) Scan(value interface{}) error { // WikiPageListRequest represents a request to list wiki pages with filtering type WikiPageListRequest struct { - KnowledgeBaseID string `json:"knowledge_base_id"` - PageType string `json:"page_type,omitempty"` // filter by type - Status string `json:"status,omitempty"` // filter by status - Query string `json:"query,omitempty"` // full-text search - Page int `json:"page,omitempty"` // pagination page (1-based) - PageSize int `json:"page_size,omitempty"` // pagination size - SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title" - SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc" + KnowledgeBaseID string `json:"knowledge_base_id"` + PageType string `json:"page_type,omitempty"` // filter by type + Status string `json:"status,omitempty"` // filter by status + Query string `json:"query,omitempty"` // full-text search + CategoryPath StringArray `json:"category_path,omitempty"` // exact directory path + CategoryDepth *int `json:"category_depth,omitempty"` // exact directory depth, including 0 for root + Page int `json:"page,omitempty"` // pagination page (1-based) + PageSize int `json:"page_size,omitempty"` // pagination size + SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title" + SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc" } // WikiPageListResponse represents a paginated list of wiki pages @@ -245,6 +263,21 @@ type WikiPageListResponse struct { TotalPages int `json:"total_pages"` } +// WikiCategoryPathListResponse returns the directory skeleton for wiki pages. +type WikiCategoryPathListResponse struct { + Paths []WikiCategoryPath `json:"paths"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + TotalPages int `json:"total_pages"` +} + +// WikiCategoryPath is one directory node in the wiki sidebar skeleton. +type WikiCategoryPath struct { + Path []string `json:"path"` + Count int64 `json:"count"` +} + // WikiGraphMode enumerates the graph query modes exposed to the API. const ( // WikiGraphModeOverview returns the top-N most-connected pages as an @@ -345,9 +378,14 @@ func (WikiPageIssue) TableName() string { // carried — the backend projects SELECT slug, title, summary so a 40k- // page KB does not pay for TEXT content transport on every index open. type WikiIndexEntry struct { - Slug string `json:"slug"` - Title string `json:"title"` - Summary string `json:"summary"` + Slug string `json:"slug"` + Title string `json:"title"` + Summary string `json:"summary"` + ParentSlug string `json:"parent_slug,omitempty"` + CategoryPath StringArray `json:"category_path,omitempty"` + WikiPath string `json:"wiki_path,omitempty"` + Depth int `json:"depth,omitempty"` + SortOrder int `json:"sort_order,omitempty"` } // WikiIndexGroup bundles the entries for one page_type into a page-sized @@ -397,4 +435,3 @@ type WikiPageLite struct { Aliases StringArray `json:"aliases,omitempty"` OutLinks StringArray `json:"out_links,omitempty"` } - diff --git a/migrations/versioned/000037_wiki_and_indexing.up.sql b/migrations/versioned/000037_wiki_and_indexing.up.sql index 3b1dfecb2..36bae6ace 100644 --- a/migrations/versioned/000037_wiki_and_indexing.up.sql +++ b/migrations/versioned/000037_wiki_and_indexing.up.sql @@ -22,6 +22,11 @@ CREATE TABLE IF NOT EXISTS wiki_pages ( status VARCHAR(32) NOT NULL DEFAULT 'published', content TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', + parent_slug VARCHAR(255) NOT NULL DEFAULT '', + category_path JSONB DEFAULT '[]'::JSONB, + wiki_path VARCHAR(1024) NOT NULL DEFAULT '', + depth INT NOT NULL DEFAULT 0, + sort_order INT NOT NULL DEFAULT 0, source_refs JSONB DEFAULT '[]'::JSONB, chunk_refs JSONB DEFAULT '[]'::JSONB, in_links JSONB DEFAULT '[]'::JSONB, @@ -45,6 +50,12 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_kb_id CREATE INDEX IF NOT EXISTS idx_wiki_pages_page_type ON wiki_pages (knowledge_base_id, page_type); +CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug + ON wiki_pages (knowledge_base_id, parent_slug); + +CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree + ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title); + CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id ON wiki_pages (tenant_id); diff --git a/migrations/versioned/000061_wiki_page_hierarchy.down.sql b/migrations/versioned/000061_wiki_page_hierarchy.down.sql new file mode 100644 index 000000000..ee9168aa2 --- /dev/null +++ b/migrations/versioned/000061_wiki_page_hierarchy.down.sql @@ -0,0 +1,10 @@ +-- Migration: 000061_wiki_page_hierarchy (rollback) + +DROP INDEX IF EXISTS idx_wiki_pages_tree; +DROP INDEX IF EXISTS idx_wiki_pages_parent_slug; + +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS sort_order; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS depth; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS wiki_path; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_path; +ALTER TABLE wiki_pages DROP COLUMN IF EXISTS parent_slug; diff --git a/migrations/versioned/000061_wiki_page_hierarchy.up.sql b/migrations/versioned/000061_wiki_page_hierarchy.up.sql new file mode 100644 index 000000000..b10f29798 --- /dev/null +++ b/migrations/versioned/000061_wiki_page_hierarchy.up.sql @@ -0,0 +1,29 @@ +-- Migration: 000061_wiki_page_hierarchy +-- Description: Add structured directory hierarchy fields to wiki_pages. + +DO $$ BEGIN RAISE NOTICE '[Migration 000061] Applying wiki page hierarchy schema'; END $$; + +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS parent_slug VARCHAR(255) NOT NULL DEFAULT ''; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_path JSONB DEFAULT '[]'::JSONB; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS wiki_path VARCHAR(1024) NOT NULL DEFAULT ''; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS depth INT NOT NULL DEFAULT 0; +ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS sort_order INT NOT NULL DEFAULT 0; + +UPDATE wiki_pages +SET + category_path = COALESCE(category_path, '[]'::JSONB), + depth = COALESCE(depth, 0), + wiki_path = CASE + WHEN COALESCE(wiki_path, '') <> '' THEN wiki_path + WHEN page_type IN ('index', 'log') THEN page_type || '/' || COALESCE(NULLIF(title, ''), slug) + ELSE page_type || '/' || COALESCE(NULLIF(title, ''), slug) + END +WHERE wiki_path = '' OR wiki_path IS NULL OR category_path IS NULL OR depth IS NULL; + +CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug + ON wiki_pages (knowledge_base_id, parent_slug); + +CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree + ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title); + +DO $$ BEGIN RAISE NOTICE '[Migration 000061] wiki page hierarchy schema applied successfully'; END $$;