feat(wiki): add page hierarchy/category metadata and retrieval APIs

Persist category and hierarchy metadata on wiki pages (new migration), expose
category-path retrieval and related repository/service/handler/router APIs, and
update wiki ingestion (batch, cite, taxonomy) plus agent wiki prompts to build
and consume the taxonomy. Regenerate swagger docs for the new endpoints.
This commit is contained in:
wizardchen
2026-06-16 20:26:06 +08:00
committed by lyingbug
parent ec66a3dac7
commit fe33db9493
20 changed files with 1893 additions and 25 deletions
+153 -1
View File
@@ -6350,6 +6350,71 @@ const docTemplate = `{
}
}
},
"/knowledgebase/{kb_id}/wiki/categories": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Retrieve direct child directories for a wiki page type and optional parent_path",
"produces": [
"application/json"
],
"tags": [
"Wiki"
],
"summary": "List wiki category paths",
"parameters": [
{
"type": "string",
"description": "Knowledge base ID",
"name": "kb_id",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Wiki page type; comma-separated for multiple (e.g. entity,concept)",
"name": "page_type",
"in": "query",
"required": true
},
{
"type": "string",
"description": "Parent category path, slash-separated",
"name": "parent_path",
"in": "query"
},
{
"type": "integer",
"description": "Page number (1-based)",
"name": "page",
"in": "query"
},
{
"type": "integer",
"description": "Page size",
"name": "page_size",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
}
}
}
},
"/knowledgebase/{kb_id}/wiki/graph": {
"get": {
"security": [
@@ -6682,7 +6747,7 @@ const docTemplate = `{
},
{
"type": "string",
"description": "Filter by page type",
"description": "Filter by page type; comma-separated for multiple (e.g. entity,concept)",
"name": "page_type",
"in": "query"
},
@@ -13730,6 +13795,9 @@ const docTemplate = `{
"dimension": {
"type": "integer"
},
"supports_dimension_override": {
"type": "boolean"
},
"truncate_prompt_tokens": {
"type": "integer"
}
@@ -17095,6 +17163,43 @@ const docTemplate = `{
"WebSearchProviderTypeSearxng"
]
},
"github_com_Tencent_WeKnora_internal_types.WikiCategoryPath": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"path": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse": {
"type": "object",
"properties": {
"page": {
"type": "integer"
},
"page_size": {
"type": "integer"
},
"paths": {
"type": "array",
"items": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath"
}
},
"total": {
"type": "integer"
},
"total_pages": {
"type": "integer"
}
}
},
"github_com_Tencent_WeKnora_internal_types.WikiConfig": {
"type": "object",
"properties": {
@@ -17223,14 +17328,32 @@ const docTemplate = `{
"github_com_Tencent_WeKnora_internal_types.WikiIndexEntry": {
"type": "object",
"properties": {
"category_path": {
"type": "array",
"items": {
"type": "string"
}
},
"depth": {
"type": "integer"
},
"parent_slug": {
"type": "string"
},
"slug": {
"type": "string"
},
"sort_order": {
"type": "integer"
},
"summary": {
"type": "string"
},
"title": {
"type": "string"
},
"wiki_path": {
"type": "string"
}
}
},
@@ -17350,6 +17473,13 @@ const docTemplate = `{
"type": "string"
}
},
"category_path": {
"description": "CategoryPath is the directory breadcrumb that groups this page in the\nwiki browser, e.g. [\"AI\", \"LLM 应用\", \"RAG\"]. It is intentionally\nlabel-based so intermediate directory nodes do not need to be real pages.",
"type": "array",
"items": {
"type": "string"
}
},
"chunk_refs": {
"description": "ChunkRefs records the specific source-document chunks this page was\nbuilt from — one UUID per cited chunk. Populated during ingest from\nthe chunk-citation pass; refreshed wholesale whenever the page is\nre-materialized. Empty for summary pages (they are document-level\nsynopses and don't carry chunk-level citations). Use this when you\nneed to surface the underlying evidence for a wiki page, or to\nretract citations when a source document is deleted.",
"type": "array",
@@ -17373,6 +17503,10 @@ const docTemplate = `{
}
]
},
"depth": {
"description": "Depth is len(CategoryPath), cached for filtering / display.",
"type": "integer"
},
"id": {
"description": "Unique identifier (UUID)",
"type": "string"
@@ -17406,10 +17540,18 @@ const docTemplate = `{
"description": "Page type: summary, entity, concept, index, log, synthesis, comparison",
"type": "string"
},
"parent_slug": {
"description": "ParentSlug optionally points at the wiki page that should act as this\npage's semantic parent in the directory tree. The parent may be empty\nwhen the page is grouped only by CategoryPath.",
"type": "string"
},
"slug": {
"description": "URL-friendly slug for addressing, e.g. \"entity/acme-corp\", \"concept/rag\"\nUnique within a knowledge base",
"type": "string"
},
"sort_order": {
"description": "SortOrder allows generated or manually edited pages to control sibling\nordering before falling back to title.",
"type": "integer"
},
"source_refs": {
"description": "References to source knowledge IDs that contributed to this page.\nFormat matches the legacy \"\u003cknowledge_id\u003e|\u003cdoc_title\u003e\" convention used\nacross the ingest pipeline, so retract / display code can split on ` + "`" + `|` + "`" + `\nto recover the title. Document-level granularity.",
"type": "array",
@@ -17440,6 +17582,10 @@ const docTemplate = `{
"version": {
"description": "Version number. Incremented only when a user-visible content field\n(title, content, summary, page_type, status) actually changes; pure\nbookkeeping writes (link maintenance, same-content re-ingest, status\nsync from background jobs) leave it untouched so it can be used as a\nreal \"the page was edited\" signal.",
"type": "integer"
},
"wiki_path": {
"description": "WikiPath is a normalized, sortable path derived from page_type,\ncategory_path, and title. It keeps large directory listings cheap to sort.",
"type": "string"
}
}
},
@@ -18203,6 +18349,9 @@ const docTemplate = `{
"source": {
"description": "为空时按需默认为 \"remote\"",
"type": "string"
},
"supportsDimensionOverride": {
"type": "boolean"
}
}
},
@@ -18437,6 +18586,9 @@ const docTemplate = `{
"source": {
"description": "为空时按需默认为 \"remote\"",
"type": "string"
},
"supportsDimensionOverride": {
"type": "boolean"
}
}
},
+153 -1
View File
@@ -6343,6 +6343,71 @@
}
}
},
"/knowledgebase/{kb_id}/wiki/categories": {
"get": {
"security": [
{
"Bearer": []
}
],
"description": "Retrieve direct child directories for a wiki page type and optional parent_path",
"produces": [
"application/json"
],
"tags": [
"Wiki"
],
"summary": "List wiki category paths",
"parameters": [
{
"type": "string",
"description": "Knowledge base ID",
"name": "kb_id",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Wiki page type; comma-separated for multiple (e.g. entity,concept)",
"name": "page_type",
"in": "query",
"required": true
},
{
"type": "string",
"description": "Parent category path, slash-separated",
"name": "parent_path",
"in": "query"
},
{
"type": "integer",
"description": "Page number (1-based)",
"name": "page",
"in": "query"
},
{
"type": "integer",
"description": "Page size",
"name": "page_size",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
}
}
}
},
"/knowledgebase/{kb_id}/wiki/graph": {
"get": {
"security": [
@@ -6675,7 +6740,7 @@
},
{
"type": "string",
"description": "Filter by page type",
"description": "Filter by page type; comma-separated for multiple (e.g. entity,concept)",
"name": "page_type",
"in": "query"
},
@@ -13723,6 +13788,9 @@
"dimension": {
"type": "integer"
},
"supports_dimension_override": {
"type": "boolean"
},
"truncate_prompt_tokens": {
"type": "integer"
}
@@ -17088,6 +17156,43 @@
"WebSearchProviderTypeSearxng"
]
},
"github_com_Tencent_WeKnora_internal_types.WikiCategoryPath": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"path": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse": {
"type": "object",
"properties": {
"page": {
"type": "integer"
},
"page_size": {
"type": "integer"
},
"paths": {
"type": "array",
"items": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath"
}
},
"total": {
"type": "integer"
},
"total_pages": {
"type": "integer"
}
}
},
"github_com_Tencent_WeKnora_internal_types.WikiConfig": {
"type": "object",
"properties": {
@@ -17216,14 +17321,32 @@
"github_com_Tencent_WeKnora_internal_types.WikiIndexEntry": {
"type": "object",
"properties": {
"category_path": {
"type": "array",
"items": {
"type": "string"
}
},
"depth": {
"type": "integer"
},
"parent_slug": {
"type": "string"
},
"slug": {
"type": "string"
},
"sort_order": {
"type": "integer"
},
"summary": {
"type": "string"
},
"title": {
"type": "string"
},
"wiki_path": {
"type": "string"
}
}
},
@@ -17343,6 +17466,13 @@
"type": "string"
}
},
"category_path": {
"description": "CategoryPath is the directory breadcrumb that groups this page in the\nwiki browser, e.g. [\"AI\", \"LLM 应用\", \"RAG\"]. It is intentionally\nlabel-based so intermediate directory nodes do not need to be real pages.",
"type": "array",
"items": {
"type": "string"
}
},
"chunk_refs": {
"description": "ChunkRefs records the specific source-document chunks this page was\nbuilt from — one UUID per cited chunk. Populated during ingest from\nthe chunk-citation pass; refreshed wholesale whenever the page is\nre-materialized. Empty for summary pages (they are document-level\nsynopses and don't carry chunk-level citations). Use this when you\nneed to surface the underlying evidence for a wiki page, or to\nretract citations when a source document is deleted.",
"type": "array",
@@ -17366,6 +17496,10 @@
}
]
},
"depth": {
"description": "Depth is len(CategoryPath), cached for filtering / display.",
"type": "integer"
},
"id": {
"description": "Unique identifier (UUID)",
"type": "string"
@@ -17399,10 +17533,18 @@
"description": "Page type: summary, entity, concept, index, log, synthesis, comparison",
"type": "string"
},
"parent_slug": {
"description": "ParentSlug optionally points at the wiki page that should act as this\npage's semantic parent in the directory tree. The parent may be empty\nwhen the page is grouped only by CategoryPath.",
"type": "string"
},
"slug": {
"description": "URL-friendly slug for addressing, e.g. \"entity/acme-corp\", \"concept/rag\"\nUnique within a knowledge base",
"type": "string"
},
"sort_order": {
"description": "SortOrder allows generated or manually edited pages to control sibling\nordering before falling back to title.",
"type": "integer"
},
"source_refs": {
"description": "References to source knowledge IDs that contributed to this page.\nFormat matches the legacy \"\u003cknowledge_id\u003e|\u003cdoc_title\u003e\" convention used\nacross the ingest pipeline, so retract / display code can split on `|`\nto recover the title. Document-level granularity.",
"type": "array",
@@ -17433,6 +17575,10 @@
"version": {
"description": "Version number. Incremented only when a user-visible content field\n(title, content, summary, page_type, status) actually changes; pure\nbookkeeping writes (link maintenance, same-content re-ingest, status\nsync from background jobs) leave it untouched so it can be used as a\nreal \"the page was edited\" signal.",
"type": "integer"
},
"wiki_path": {
"description": "WikiPath is a normalized, sortable path derived from page_type,\ncategory_path, and title. It keeps large directory listings cheap to sort.",
"type": "string"
}
}
},
@@ -18196,6 +18342,9 @@
"source": {
"description": "为空时按需默认为 \"remote\"",
"type": "string"
},
"supportsDimensionOverride": {
"type": "boolean"
}
}
},
@@ -18430,6 +18579,9 @@
"source": {
"description": "为空时按需默认为 \"remote\"",
"type": "string"
},
"supportsDimensionOverride": {
"type": "boolean"
}
}
},
+113 -1
View File
@@ -850,6 +850,8 @@ definitions:
properties:
dimension:
type: integer
supports_dimension_override:
type: boolean
truncate_prompt_tokens:
type: integer
type: object
@@ -3345,6 +3347,30 @@ definitions:
- WebSearchProviderTypeOllama
- WebSearchProviderTypeBaidu
- WebSearchProviderTypeSearxng
github_com_Tencent_WeKnora_internal_types.WikiCategoryPath:
properties:
count:
type: integer
path:
items:
type: string
type: array
type: object
github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse:
properties:
page:
type: integer
page_size:
type: integer
paths:
items:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPath'
type: array
total:
type: integer
total_pages:
type: integer
type: object
github_com_Tencent_WeKnora_internal_types.WikiConfig:
properties:
extraction_granularity:
@@ -3450,12 +3476,24 @@ definitions:
type: object
github_com_Tencent_WeKnora_internal_types.WikiIndexEntry:
properties:
category_path:
items:
type: string
type: array
depth:
type: integer
parent_slug:
type: string
slug:
type: string
sort_order:
type: integer
summary:
type: string
title:
type: string
wiki_path:
type: string
type: object
github_com_Tencent_WeKnora_internal_types.WikiIndexGroup:
properties:
@@ -3549,6 +3587,14 @@ definitions:
items:
type: string
type: array
category_path:
description: |-
CategoryPath is the directory breadcrumb that groups this page in the
wiki browser, e.g. ["AI", "LLM 应用", "RAG"]. It is intentionally
label-based so intermediate directory nodes do not need to be real pages.
items:
type: string
type: array
chunk_refs:
description: |-
ChunkRefs records the specific source-document chunks this page was
@@ -3571,6 +3617,9 @@ definitions:
allOf:
- $ref: '#/definitions/gorm.DeletedAt'
description: Soft delete
depth:
description: Depth is len(CategoryPath), cached for filtering / display.
type: integer
id:
description: Unique identifier (UUID)
type: string
@@ -3596,11 +3645,22 @@ definitions:
description: 'Page type: summary, entity, concept, index, log, synthesis,
comparison'
type: string
parent_slug:
description: |-
ParentSlug optionally points at the wiki page that should act as this
page's semantic parent in the directory tree. The parent may be empty
when the page is grouped only by CategoryPath.
type: string
slug:
description: |-
URL-friendly slug for addressing, e.g. "entity/acme-corp", "concept/rag"
Unique within a knowledge base
type: string
sort_order:
description: |-
SortOrder allows generated or manually edited pages to control sibling
ordering before falling back to title.
type: integer
source_refs:
description: |-
References to source knowledge IDs that contributed to this page.
@@ -3633,6 +3693,11 @@ definitions:
sync from background jobs) leave it untouched so it can be used as a
real "the page was edited" signal.
type: integer
wiki_path:
description: |-
WikiPath is a normalized, sortable path derived from page_type,
category_path, and title. It keeps large directory listings cheap to sort.
type: string
type: object
github_com_Tencent_WeKnora_internal_types.WikiPageIssue:
properties:
@@ -4159,6 +4224,8 @@ definitions:
source:
description: 为空时按需默认为 "remote"
type: string
supportsDimensionOverride:
type: boolean
required:
- modelName
type: object
@@ -4325,6 +4392,8 @@ definitions:
source:
description: 为空时按需默认为 "remote"
type: string
supportsDimensionOverride:
type: boolean
required:
- modelName
type: object
@@ -8834,6 +8903,49 @@ paths:
summary: Auto-fix wiki issues
tags:
- Wiki
/knowledgebase/{kb_id}/wiki/categories:
get:
description: Retrieve direct child directories for a wiki page type and optional
parent_path
parameters:
- description: Knowledge base ID
in: path
name: kb_id
required: true
type: string
- description: Wiki page type; comma-separated for multiple (e.g. entity,concept)
in: query
name: page_type
required: true
type: string
- description: Parent category path, slash-separated
in: query
name: parent_path
type: string
- description: Page number (1-based)
in: query
name: page
type: integer
- description: Page size
in: query
name: page_size
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.WikiCategoryPathListResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
security:
- Bearer: []
summary: List wiki category paths
tags:
- Wiki
/knowledgebase/{kb_id}/wiki/graph:
get:
description: |-
@@ -9050,7 +9162,7 @@ paths:
name: kb_id
required: true
type: string
- description: Filter by page type
- description: Filter by page type; comma-separated for multiple (e.g. entity,concept)
in: query
name: page_type
type: string
+75 -2
View File
@@ -1,5 +1,25 @@
package agent
// wikiExistingTaxonomyXML is injected into the page-modify prompt so each page's
// CATEGORY line can reuse folder labels already present in the knowledge base
// instead of inventing synonyms. Category assignment happens at page-write
// (reduce) time, where the LLM sees the whole entity — not per source document.
const wikiExistingTaxonomyXML = `
<existing_taxonomy>
{{.ExistingTaxonomy}}
</existing_taxonomy>
`
// wikiExistingTaxonomyContinuityRules extends Directory Taxonomy Rules with
// cross-document folder reuse requirements.
const wikiExistingTaxonomyContinuityRules = `
- If <existing_taxonomy> above lists folders already used in this knowledge base, you MUST reuse those **exact folder labels** (character-for-character) when the new item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节" already has a "传统习俗" subfolder and that path fits semantically).
- Prefer nesting under an existing parent folder over creating a new top-level folder with a similar name.
- When multiple existing paths could fit, pick the most specific existing path whose labels match the document's section hierarchy.
- Only introduce a new folder label when no existing folder is a reasonable match.
- You may extend an existing path with one new trailing folder only when the document clearly introduces a genuinely new sub-section.
`
// Wiki ingest prompt templates for LLM-powered wiki page generation.
// These prompts are used by the wiki ingest pipeline to extract structured
// knowledge from raw documents and build/update wiki pages.
@@ -313,8 +333,22 @@ The <new_information> block above is assembled from VERBATIM source chunks that
{{.AvailableSlugs}}
</valid_wiki_links>
<current_category>
{{.CurrentCategory}}
</current_category>
` + wikiExistingTaxonomyXML + `
<instructions>
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
1b. The SECOND line of your output MUST be: CATEGORY: {a navigation taxonomy path for this page, folders from broad to narrow separated by " / ", e.g. "CATEGORY: 人物" or "CATEGORY: 组织 / 企业"}. Follow the Directory Taxonomy Rules below. Almost every page can be filed under at least one broad top-level folder, so leaving it empty should be rare.
### Directory Taxonomy Rules (for the CATEGORY line)
The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label that stays true across the whole knowledge base — never the role it happens to play in one document.
- Litmus test: keep a label only if it would still fit when this same {{.PageType}} shows up in a completely unrelated document. If a label only describes how it relates to THIS document's topic (who organized, sponsored, acknowledged, authored, or took part in something), that is a ROLE, not a category — leave it out and let the page body and [[wiki-links]] carry the relationship.
- Always assign at least the single broadest top-level folder that fits (think "which shelf would this always sit on in a library"). A coarse one-level category is expected for most pages; reserve a second level for a genuinely durable sub-domain.
- Do NOT use the item type itself as a folder (e.g. "实体", "概念", "摘要").
- Do NOT put slashes inside a single label. Use at most 2 levels.
- Treat <current_category> as the default and keep it unless new evidence clearly points to a better stable classification.
- Only output an empty "CATEGORY:" in the rare case where the page genuinely cannot be classified under any broad, durable folder.` + wikiExistingTaxonomyContinuityRules + `
{{if .HasRetractions}}
2. REMOVE facts/claims that were ONLY sourced from the <deleted_documents> and are NOT present in any <remaining_source_documents> or <new_information>.
{{end}}
@@ -332,12 +366,51 @@ The <new_information> block above is assembled from VERBATIM source chunks that
6. Maintain the existing page structure and formatting style. Use "# {{.PageTitle}}" as the top-level heading if the page does not already have one. Do NOT introduce new heading levels beyond what the source or existing page justifies.
7. **Image rule**: Include relevant images using Markdown syntax: ![caption](url) from new information if applicable. The URL inside ![caption](url) is an opaque token; reproduce it EXACTLY and VERBATIM, do not alter, shorten, or normalize it.
{{if .HasRetractions}}
8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\nCATEGORY:\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
{{end}}
9. Write in {{.Language}}.
</instructions>
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
Output the SUMMARY line first, then the CATEGORY line, then the updated Markdown content. Do not include any other preamble.`
// WikiTaxonomyReconcilePrompt is a per-knowledge-base maintenance prompt. It is
// given the full set of distinct category_path folders currently in use and asks
// the LLM to collapse synonymous folders onto a single canonical path. The
// result is applied in Go by re-pointing matching pages' category_path — slugs
// never change. This runs occasionally (after a batch introduces new folders, or
// on manual trigger), NOT per page, so its cost is O(folder count).
const WikiTaxonomyReconcilePrompt = `You are a taxonomy curator for a wiki knowledge base. Below is the complete list of directory paths (category_path) currently used to file wiki pages. Some of them are synonyms or near-duplicates that should share ONE canonical path.
<current_taxonomy>
{{.CurrentTaxonomy}}
</current_taxonomy>
<instructions>
Identify groups of paths that mean the SAME thing and pick the single best canonical path for each group, then output a remap from every non-canonical path to its canonical path.
Rules:
- Only merge paths that are genuinely synonymous or where one is clearly a less-canonical spelling/wording of another (e.g. "春节习俗" → "春节 / 传统习俗"; "AI厂商" and "AI行业 / 厂商" → one of them).
- Prefer the more standard, concise, and reusable label as the canonical target. Prefer an existing path over inventing a new one.
- Do NOT merge paths that refer to DIFFERENT things, even if they look similar.
- Do NOT merge across clearly different domains.
- Do NOT output an entry whose "from" equals its "to".
- A canonical "to" path may itself be one of the listed paths; never map a path onto a synonym chain (every "to" must be a final canonical path, not another "from").
- Keep canonical paths to at most 2 levels.
- Each path is an array of folder labels from broad to narrow. Write ALL labels in {{.Language}}.
- If nothing should be merged, return {"remap": []}.
### JSON Formatting Rules
- Output ONLY valid JSON, no preamble.
- Do NOT use literal newlines inside JSON string values.
</instructions>
Output format:
{
"remap": [
{"from": ["春节习俗"], "to": ["春节", "传统习俗"]},
{"from": ["AI厂商"], "to": ["AI行业", "厂商"]}
]
}`
// WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only).
const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page.
+265 -6
View File
@@ -28,6 +28,13 @@ func NewWikiPageRepository(db *gorm.DB) interfaces.WikiPageRepository {
return &wikiPageRepository{db: db}
}
func (r *wikiPageRepository) wikiCategoryRankOrder() string {
if r.db != nil && r.db.Dialector != nil && r.db.Dialector.Name() == "sqlite" {
return "CASE WHEN COALESCE(json_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC"
}
return "CASE WHEN COALESCE(jsonb_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC"
}
// Create inserts a new wiki page record
func (r *wikiPageRepository) Create(ctx context.Context, page *types.WikiPage) error {
return r.db.WithContext(ctx).Create(page).Error
@@ -101,6 +108,11 @@ func (r *wikiPageRepository) UpdateMeta(ctx context.Context, page *types.WikiPag
"source_refs": page.SourceRefs,
"chunk_refs": page.ChunkRefs,
"page_metadata": page.PageMetadata,
"parent_slug": page.ParentSlug,
"category_path": page.CategoryPath,
"wiki_path": page.WikiPath,
"depth": page.Depth,
"sort_order": page.SortOrder,
"updated_at": page.UpdatedAt,
})
if result.Error != nil {
@@ -138,13 +150,38 @@ func (r *wikiPageRepository) GetBySlug(ctx context.Context, kbID string, slug st
return &page, nil
}
// splitWikiPageTypes parses a page_type filter that may carry several
// comma-separated types (e.g. "entity,concept") into a deduplicated slice.
// An empty/whitespace-only input yields nil, meaning "no type filter".
func splitWikiPageTypes(raw string) []string {
if strings.TrimSpace(raw) == "" {
return nil
}
seen := make(map[string]struct{})
out := make([]string, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if _, ok := seen[part]; ok {
continue
}
seen[part] = struct{}{}
out = append(out, part)
}
return out
}
// List retrieves wiki pages with filtering and pagination
func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRequest) ([]*types.WikiPage, int64, error) {
query := r.db.WithContext(ctx).Model(&types.WikiPage{}).
Where("knowledge_base_id = ?", req.KnowledgeBaseID)
if req.PageType != "" {
query = query.Where("page_type = ?", req.PageType)
if pageTypes := splitWikiPageTypes(req.PageType); len(pageTypes) == 1 {
query = query.Where("page_type = ?", pageTypes[0])
} else if len(pageTypes) > 1 {
query = query.Where("page_type IN ?", pageTypes)
}
if req.Status != "" {
query = query.Where("status = ?", req.Status)
@@ -157,6 +194,24 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
"%"+req.Query+"%",
)
}
// Directory filters are pushed to SQL so the DB does the counting and
// pagination instead of loading every page of the type into memory. `depth`
// is a cached column (= len(category_path)); `category_path` is a JSON column
// whose stored text is json.Marshal of the cleaned path, so we compare
// against the same encoding. Postgres needs an explicit jsonb cast for array
// equality; SQLite stores JSON as TEXT and compares directly.
if req.CategoryDepth != nil {
query = query.Where("depth = ?", *req.CategoryDepth)
}
if wantPath := cleanWikiRepositoryCategoryPath(req.CategoryPath); len(wantPath) > 0 {
if encoded, err := json.Marshal([]string(wantPath)); err == nil {
if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" {
query = query.Where("category_path::jsonb = ?::jsonb", string(encoded))
} else {
query = query.Where("category_path = ?", string(encoded))
}
}
}
var total int64
if err := query.Count(&total).Error; err != nil {
@@ -167,7 +222,7 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
sortBy := "updated_at"
if req.SortBy != "" {
switch req.SortBy {
case "title", "created_at", "updated_at", "page_type":
case "title", "created_at", "updated_at", "page_type", "wiki_path", "sort_order", "depth":
sortBy = req.SortBy
}
}
@@ -175,9 +230,15 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
if req.SortOrder == "asc" {
sortOrder = "ASC"
}
query = query.Order(fmt.Sprintf("%s %s", sortBy, sortOrder))
if sortBy == "wiki_path" {
query = query.Order(r.wikiCategoryRankOrder()).
Order(fmt.Sprintf("wiki_path %s", sortOrder)).
Order("sort_order ASC").
Order("title ASC")
} else {
query = query.Order(fmt.Sprintf("%s %s", sortBy, sortOrder))
}
// Pagination
page := req.Page
if page < 1 {
page = 1
@@ -187,6 +248,8 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
pageSize = 20
}
offset := (page - 1) * pageSize
// Pagination
query = query.Offset(offset).Limit(pageSize)
var pages []*types.WikiPage
@@ -249,7 +312,10 @@ func (r *wikiPageRepository) ListByTypeLight(
var entries []types.WikiIndexEntry
if err := base.
Select("slug", "title", "summary").
Select("slug", "title", "summary", "parent_slug", "category_path", "wiki_path", "depth", "sort_order").
Order(r.wikiCategoryRankOrder()).
Order("wiki_path ASC").
Order("sort_order ASC").
Order("title ASC").
Limit(limit).
Offset(offset).
@@ -368,6 +434,199 @@ func (r *wikiPageRepository) ListBySlugs(
return out, nil
}
// ListDistinctCategoryPaths scans entity/concept pages and returns unique
// non-empty category_path values. Rows are read in wiki_path order so the
// prompt tree is stable; deduplication happens in Go with a maxPaths cap.
func (r *wikiPageRepository) ListDistinctCategoryPaths(
ctx context.Context,
kbID string,
maxPaths int,
) ([][]string, error) {
if maxPaths <= 0 {
maxPaths = 150
}
type row struct {
CategoryPath types.StringArray `gorm:"column:category_path"`
}
var rows []row
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Select("category_path").
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
kbID,
[]string{types.WikiPageTypeEntity, types.WikiPageTypeConcept},
types.WikiPageStatusArchived,
).
Order(r.wikiCategoryRankOrder()).
Order("wiki_path ASC").
Scan(&rows).Error; err != nil {
return nil, err
}
seen := make(map[string]struct{}, len(rows))
out := make([][]string, 0, len(rows))
for _, row := range rows {
if len(row.CategoryPath) == 0 {
continue
}
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
if len(clean) == 0 {
continue
}
key := strings.Join(clean, "\x00")
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, clean)
if len(out) >= maxPaths {
break
}
}
return out, nil
}
// ListDistinctCategoryPathsByType scans one page type and returns unique
// non-empty category_path values. Unlike ListDistinctCategoryPaths, this is
// used by the UI and must work for every visible page_type bucket.
func (r *wikiPageRepository) ListDistinctCategoryPathsByType(
ctx context.Context,
kbID string,
pageTypes []string,
parentPath []string,
maxPaths int,
) ([]types.WikiCategoryPath, error) {
if maxPaths <= 0 {
maxPaths = 2000
}
if len(pageTypes) == 0 {
return nil, nil
}
type row struct {
CategoryPath types.StringArray `gorm:"column:category_path"`
}
var rows []row
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
Select("category_path").
Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
kbID,
pageTypes,
types.WikiPageStatusArchived,
).
Order(r.wikiCategoryRankOrder()).
Order("wiki_path ASC").
Scan(&rows).Error; err != nil {
return nil, err
}
parent := cleanWikiRepositoryCategoryPath(parentPath)
counts := make(map[string]int64, len(rows))
pathsByKey := make(map[string][]string, len(rows))
order := make([]string, 0, len(rows))
for _, row := range rows {
if len(row.CategoryPath) == 0 {
continue
}
clean := cleanWikiRepositoryCategoryPath(row.CategoryPath)
if len(clean) == 0 {
continue
}
if len(clean) <= len(parent) || !wikiPathHasPrefix(clean, parent) {
continue
}
path := clean[:len(parent)+1]
key := strings.Join(path, "\x00")
if _, ok := pathsByKey[key]; !ok {
cp := append([]string(nil), path...)
pathsByKey[key] = cp
order = append(order, key)
}
counts[key]++
}
out := make([]types.WikiCategoryPath, 0, len(order))
for _, key := range order {
out = append(out, types.WikiCategoryPath{
Path: pathsByKey[key],
Count: counts[key],
})
}
return out, nil
}
func cleanWikiRepositoryCategoryPath(parts []string) []string {
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
for _, label := range cleanWikiRepositoryCategoryPart(part) {
if containsWikiRepositoryString(cleaned, label) {
continue
}
cleaned = append(cleaned, label)
}
}
return cleaned
}
func cleanWikiRepositoryCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiRepositoryTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
func isWikiRepositoryTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
}
func containsWikiRepositoryString(slice []string, s string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}
func wikiPathHasPrefix(path []string, prefix []string) bool {
if len(prefix) == 0 {
return true
}
if len(path) < len(prefix) {
return false
}
for i, part := range prefix {
if path[i] != part {
return false
}
}
return true
}
// ListSummariesByKnowledgeIDs returns summary-page content keyed by the
// knowledge id that authored it. The page_type filter is applied first
// (only summary pages have content suitable for retract framing); within
@@ -29,6 +29,11 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
status VARCHAR(32) NOT NULL DEFAULT 'published',
content TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
parent_slug VARCHAR(255) NOT NULL DEFAULT '',
category_path TEXT DEFAULT '[]',
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
depth INTEGER NOT NULL DEFAULT 0,
sort_order INTEGER NOT NULL DEFAULT 0,
source_refs TEXT DEFAULT '[]',
chunk_refs TEXT DEFAULT '[]',
in_links TEXT DEFAULT '[]',
@@ -68,12 +73,57 @@ func makeWikiPage(kbID, slug, pageType, status string) *types.WikiPage {
Status: status,
Content: "body of " + slug,
Summary: "summary of " + slug,
WikiPath: pageType + "/" + title,
Version: 1,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
}
func makeCategorizedWikiPage(kbID, slug, pageType, status string, categoryPath ...string) *types.WikiPage {
page := makeWikiPage(kbID, slug, pageType, status)
page.CategoryPath = types.StringArray(categoryPath)
if len(categoryPath) > 0 {
page.WikiPath = pageType + "/" + strings.Join(categoryPath, "/") + "/" + page.Title
page.Depth = len(categoryPath)
}
return page
}
// TestList_WikiPathSortReturnsCategorizedPagesFirst protects the sidebar's
// IDE-like tree contract. Pagination happens in the repository, so the DB
// must return pages with category_path before loose root pages; otherwise the
// frontend cannot know about directories hiding on later pages.
func TestList_WikiPathSortReturnsCategorizedPagesFirst(t *testing.T) {
db := setupWikiPagesTestDB(t)
repo := NewWikiPageRepository(db)
ctx := context.Background()
pages := []*types.WikiPage{
makeWikiPage("kb-a", "entity/000-root", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
makeCategorizedWikiPage("kb-a", "entity/999-child", types.WikiPageTypeEntity, types.WikiPageStatusPublished, "zzz-folder"),
makeWikiPage("kb-a", "entity/001-root", types.WikiPageTypeEntity, types.WikiPageStatusPublished),
}
for _, p := range pages {
require.NoError(t, repo.Create(ctx, p))
}
got, total, err := repo.List(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: "kb-a",
PageType: types.WikiPageTypeEntity,
Page: 1,
PageSize: 10,
SortBy: "wiki_path",
SortOrder: "asc",
})
require.NoError(t, err)
assert.Equal(t, int64(3), total)
require.Len(t, got, 3)
assert.Equal(t, "entity/999-child", got[0].Slug)
assert.Equal(t, "entity/000-root", got[1].Slug)
assert.Equal(t, "entity/001-root", got[2].Slug)
}
// TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies
// that the index-view projection only emits the slug/title/summary
// triples and respects the archived-status filter. This is the whole
+170
View File
@@ -658,6 +658,12 @@ type WikiBatchContext struct {
// Already Normalize()'d — consumers can assume it is one of the
// three valid values.
ExtractionGranularity types.WikiExtractionGranularity
// ExistingTaxonomy returns a formatted directory tree of category_path
// folders already used in this KB. Loaded once per batch and injected
// into entity/concept extraction prompts so new pages reuse established
// folder labels instead of inventing synonyms.
ExistingTaxonomy func(ctx context.Context) string
}
// SlugUpdate represents a single update operation for a specific slug
@@ -1197,6 +1203,126 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef {
return refs
}
// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when
// building the existing-taxonomy prompt block. Distinct paths are deduped
// in Go; this bound keeps the query cheap on large KBs.
const wikiTaxonomyScanRowLimit = 2000
// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes
// are rendered into the LLM prompt to control token usage.
const wikiTaxonomyPromptMaxPaths = 150
func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string {
if batchCtx == nil || batchCtx.ExistingTaxonomy == nil {
return "(none — this knowledge base has no established folders yet)"
}
text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx))
if text == "" {
return "(none — this knowledge base has no established folders yet)"
}
return text
}
type wikiTaxonomyNode struct {
children map[string]*wikiTaxonomyNode
}
func insertWikiTaxonomyPath(root *wikiTaxonomyNode, path []string) {
if root == nil || len(path) == 0 {
return
}
cur := root
for _, part := range path {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if cur.children == nil {
cur.children = make(map[string]*wikiTaxonomyNode)
}
child := cur.children[part]
if child == nil {
child = &wikiTaxonomyNode{}
cur.children[part] = child
}
cur = child
}
}
func appendWikiTaxonomyNode(buf *strings.Builder, label string, node *wikiTaxonomyNode, depth int) {
if label != "" {
fmt.Fprintf(buf, "%s%s\n", strings.Repeat(" ", depth), label)
}
if node == nil || len(node.children) == 0 {
return
}
keys := make([]string, 0, len(node.children))
for k := range node.children {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
appendWikiTaxonomyNode(buf, k, node.children[k], depth+1)
}
}
// formatExistingTaxonomyForPrompt renders distinct category_path values as an
// indented folder tree for LLM extraction prompts.
func formatExistingTaxonomyForPrompt(paths [][]string) string {
if len(paths) == 0 {
return ""
}
root := &wikiTaxonomyNode{}
for _, path := range paths {
insertWikiTaxonomyPath(root, path)
}
if len(root.children) == 0 {
return ""
}
var buf strings.Builder
keys := make([]string, 0, len(root.children))
for k := range root.children {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
appendWikiTaxonomyNode(&buf, k, root.children[k], 0)
}
return strings.TrimSpace(buf.String())
}
// dedupeWikiCategoryPaths returns unique non-empty category_path slices,
// preserving first-seen order up to maxPaths.
func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string {
if maxPaths <= 0 {
maxPaths = wikiTaxonomyPromptMaxPaths
}
seen := make(map[string]struct{}, len(paths))
out := make([][]string, 0, len(paths))
for _, raw := range paths {
clean := make([]string, 0, len(raw))
for _, part := range raw {
part = strings.TrimSpace(part)
if part != "" {
clean = append(clean, part)
}
}
if len(clean) == 0 {
continue
}
key := strings.Join(clean, "\x00")
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, clean)
if len(out) >= maxPaths {
break
}
}
return out
}
// getExistingPageSlugsForKnowledge returns all page slugs that currently
// reference a given knowledge ID in their source_refs. Used to snapshot
// state before re-ingest so the reduce phase can reconcile additions vs
@@ -1435,6 +1561,50 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content
// (the page body AFTER the SUMMARY line has been stripped) and returns the
// parsed folder labels plus the remaining content with that line removed.
//
// Category assignment lives in the page-modify output (reduce time) rather
// than per-document extraction so it reflects WHAT the whole entity IS, not
// the role it played in one source document. An empty/absent CATEGORY line
// yields nil so callers keep the page's existing category instead of wiping it.
func splitCategoryLine(content string) (category []string, rest string) {
trimmed := strings.TrimLeft(content, " \t\r\n")
if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY:") {
return nil, content
}
var line, remainder string
if idx := strings.IndexByte(trimmed, '\n'); idx < 0 {
line = trimmed
} else {
line = trimmed[:idx]
remainder = trimmed[idx+1:]
}
line = strings.TrimPrefix(line, "CATEGORY:")
line = strings.TrimPrefix(line, "CATEGORY:")
return parseCategoryPath(line), strings.TrimSpace(remainder)
}
// parseCategoryPath splits a "a / b" breadcrumb string into clean folder
// labels. Separator variants and wrapping quotes/brackets are normalized; the
// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is
// left to normalizeWikiHierarchy on the page.
func parseCategoryPath(line string) []string {
line = strings.NewReplacer("/", "/", "|", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line)
parts := strings.Split(line, "/")
out := make([]string, 0, len(parts))
for _, p := range parts {
label := strings.TrimSpace(p)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label != "" {
out = append(out, label)
}
}
return out
}
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
// pure (no DB access) so callers can accumulate entries cheaply under their
// lock and flush them in a single AppendBatch call at the end of the batch.
@@ -357,6 +357,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ExtractionGranularity: granularity,
}
var taxonomyOnce sync.Once
var taxonomyText string
batchCtx.ExistingTaxonomy = func(ctx context.Context) string {
taxonomyOnce.Do(func() {
if s.wikiService == nil {
return
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths)
if err != nil {
logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err)
return
}
taxonomyText = formatExistingTaxonomyForPrompt(paths)
})
return taxonomyText
}
// Snapshot the KB's category folders before this batch runs so we can tell
// afterwards whether new (possibly synonymous) folders were introduced and
// a taxonomy reconcile pass is worth running.
startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID)
// 1. MAP PHASE (Parallel extraction and generation of updates)
var mapMu sync.Mutex
var failedOps []WikiPendingOp
@@ -628,6 +650,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
}
// Collapse synonymous category folders introduced by this batch (cold
// start, intra-batch races, long-term drift). Runs only when a genuinely
// new folder appeared vs the pre-batch snapshot; re-points pages' category
// without touching slugs, then refreshes the index.
if len(ingestPagesAffected) > 0 {
s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys)
}
// Clean dead [[slug]] references whenever ANY page was touched this
// batch (not just retracts). Reduce-phase failures can leave stale
// references in pages we just rewrote (e.g. summary pages cite
@@ -960,7 +990,7 @@ func (s *wikiIngestService) mapOneDocument(
return
}
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang, batchCtx)
s.tracker().EndSpan(ctx, classifySpan, types.JSONMap{
"cited_slugs": len(citations),
"new_slugs": len(newSlugs),
@@ -1585,6 +1615,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
}
pageAliases := strings.Join(page.Aliases, ", ")
currentCategory := strings.Join(page.CategoryPath, " / ")
if strings.TrimSpace(currentCategory) == "" {
currentCategory = "(none yet)"
}
var updatedContent string
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
"HasAdditions": hasAdditionsStr,
@@ -1598,11 +1633,22 @@ func (s *wikiIngestService) reduceSlugUpdates(
"DeletedContent": deletedContent.String(),
"RemainingSourcesContent": remainingSourcesContent.String(),
"AvailableSlugs": relatedSlugs.String(),
"CurrentCategory": currentCategory,
"ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx),
"Language": language,
})
if err == nil && updatedContent != "" {
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
// CATEGORY is the page-level intrinsic classification produced by
// the editor model. Only overwrite when the model actually emitted
// a non-empty path so a lazy/empty line never wipes a good category.
category, bodyAfterCategory := splitCategoryLine(updatedBody)
updatedBody = bodyAfterCategory
if len(category) > 0 {
page.CategoryPath = types.StringArray(category)
page.WikiPath = ""
}
if updatedBody != "" {
page.Content = updatedBody
} else {
@@ -279,6 +279,7 @@ func (s *wikiIngestService) classifyChunkCitations(
candidatesXML string,
chunks []*types.Chunk,
lang string,
batchCtx *WikiBatchContext,
) (map[string][]string, []newSlugFromCitation, int) {
batches := splitChunksIntoCitationBatches(chunks)
if len(batches) == 0 || strings.TrimSpace(candidatesXML) == "" {
@@ -0,0 +1,259 @@
package service
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass
// will rewrite, so an over-eager remap cannot fan out into an unbounded write
// storm on a very large KB. Pages beyond the cap are picked up by the next run.
const wikiTaxonomyReconcileMaxPages = 5000
// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the
// KB to apply a category remap.
const wikiTaxonomyReconcilePageSize = 500
type wikiTaxonomyRemapEntry struct {
From []string `json:"from"`
To []string `json:"to"`
}
type wikiTaxonomyRemapResult struct {
Remap []wikiTaxonomyRemapEntry `json:"remap"`
}
// ReconcileTaxonomy is the manual entry point for collapsing synonymous
// category folders across a wiki knowledge base. It resolves the KB's synthesis
// chat model itself so it can be invoked outside an ingest batch (e.g. from a
// future maintenance handler).
func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err)
}
if !kb.IsWikiEnabled() {
return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID)
}
synthesisModelID := ""
if kb.WikiConfig != nil {
synthesisModelID = kb.WikiConfig.SynthesisModelID
}
if synthesisModelID == "" {
synthesisModelID = kb.SummaryModelID
}
if synthesisModelID == "" {
return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID)
}
chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
if err != nil {
return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err)
}
return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx))
}
// reconcileTaxonomy collapses synonymous category_path folders across the KB
// onto a single canonical path. It is per-KB (cost O(folder count), not page
// count): one LLM call clusters the distinct folders, then matching pages are
// re-pointed in Go. Slugs are NEVER touched — only the navigation category.
func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error {
if s.wikiService == nil || chatModel == nil {
return nil
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil {
return fmt.Errorf("list distinct category paths: %w", err)
}
if len(paths) < 2 {
return nil // nothing to merge
}
remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang)
if len(remap) == 0 {
return nil
}
updated := s.applyTaxonomyRemap(ctx, kbID, remap)
if updated > 0 {
logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID)
if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err)
}
}
return nil
}
// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced
// at least one category folder that was not present in startKeys (the snapshot
// taken before the batch ran). This keeps the extra LLM call off the hot path
// for batches that only reuse established folders.
func (s *wikiIngestService) maybeReconcileTaxonomy(
ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool,
) {
if s.wikiService == nil {
return
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil || len(paths) < 2 {
return
}
introducedNewFolder := false
for _, p := range paths {
if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] {
introducedNewFolder = true
break
}
}
if !introducedNewFolder {
return
}
if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err)
}
}
// snapshotCategoryKeys returns the set of canonical category keys currently in
// use, used as the "before" baseline for maybeReconcileTaxonomy.
func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool {
out := map[string]bool{}
if s.wikiService == nil {
return out
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil {
return out
}
for _, p := range paths {
out[categoryPathKey(cleanCategoryPathParts(p))] = true
}
return out
}
// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a
// map keyed by the canonical key of each non-canonical path to its replacement.
func (s *wikiIngestService) buildTaxonomyRemap(
ctx context.Context, chatModel chat.Chat, paths [][]string, lang string,
) map[string][]string {
taxonomyText := formatExistingTaxonomyForPrompt(paths)
if strings.TrimSpace(taxonomyText) == "" {
return nil
}
raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{
"CurrentTaxonomy": taxonomyText,
"Language": lang,
})
if err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err)
return nil
}
raw = cleanLLMJSON(raw)
var parsed wikiTaxonomyRemapResult
if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw)
return nil
}
out := make(map[string][]string, len(parsed.Remap))
for _, e := range parsed.Remap {
from := cleanCategoryPathParts(e.From)
to := cleanCategoryPathParts(e.To)
if len(from) == 0 || len(to) == 0 {
continue
}
fromKey := categoryPathKey(from)
if fromKey == categoryPathKey(to) {
continue // no-op rule
}
out[fromKey] = to
}
return out
}
// applyTaxonomyRemap scans entity/concept pages and re-points any whose
// category matches a remap rule. Returns the number of pages updated.
func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int {
var cursor string
var updated int
for updated < wikiTaxonomyReconcileMaxPages {
pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize)
if err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err)
break
}
if len(pages) == 0 {
break
}
for _, p := range pages {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap)
if !ok {
continue
}
p.CategoryPath = types.StringArray(newPath)
p.WikiPath = "" // force recompute in normalizeWikiHierarchy
if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err)
continue
}
updated++
if updated >= wikiTaxonomyReconcileMaxPages {
break
}
}
if next == "" {
break
}
cursor = next
}
return updated
}
// remapCategoryPath returns the canonical category for current when it matches
// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can
// be unit-tested in isolation.
func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) {
cleaned := cleanCategoryPathParts(current)
if len(cleaned) == 0 {
return nil, false
}
to, ok := remap[categoryPathKey(cleaned)]
if !ok {
return nil, false
}
if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) {
return nil, false
}
return append([]string(nil), to...), true
}
// cleanCategoryPathParts normalizes a category path the same way the page
// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
// depth cap) so remap keys match what is stored on pages.
func cleanCategoryPathParts(parts []string) []string {
out := make([]string, 0, len(parts))
for _, part := range parts {
for _, c := range cleanWikiCategoryPart(part) {
out = append(out, c)
if len(out) >= 3 {
return out
}
}
}
return out
}
// categoryPathKey builds a stable comparison key for a cleaned category path.
func categoryPathKey(parts []string) string {
return strings.Join(parts, "\u0000")
}
@@ -0,0 +1,185 @@
package service
import (
"context"
"strings"
"testing"
)
func TestSplitCategoryLine(t *testing.T) {
tests := []struct {
name string
content string
wantCat []string
wantRest string
}{
{
name: "two level category",
content: "CATEGORY: 组织 / 企业\n# 标题\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "# 标题\n正文",
},
{
name: "single level category",
content: "CATEGORY: 人物\n# 陈洋",
wantCat: []string{"人物"},
wantRest: "# 陈洋",
},
{
name: "empty category keeps existing (nil) and strips line",
content: "CATEGORY:\n# 标题",
wantCat: nil,
wantRest: "# 标题",
},
{
name: "no category line left untouched",
content: "# 标题\n正文",
wantCat: nil,
wantRest: "# 标题\n正文",
},
{
name: "fullwidth colon and separators",
content: "CATEGORY:组织|企业\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "正文",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cat, rest := splitCategoryLine(tt.content)
if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") {
t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat)
}
if rest != tt.wantRest {
t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest)
}
})
}
}
func TestFormatExistingTaxonomyForPrompt(t *testing.T) {
got := formatExistingTaxonomyForPrompt([][]string{
{"春节", "传统习俗"},
{"春节", "文化习俗", "节日习俗"},
{"春节习俗"},
{"产品定位"},
})
want := "产品定位\n" +
"春节\n" +
" 传统习俗\n" +
" 文化习俗\n" +
" 节日习俗\n" +
"春节习俗"
if got != want {
t.Fatalf("formatExistingTaxonomyForPrompt():\n%q\nwant:\n%q", got, want)
}
}
func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) {
if got := formatExistingTaxonomyForPrompt(nil); got != "" {
t.Fatalf("formatExistingTaxonomyForPrompt(nil) = %q, want empty", got)
}
}
func TestDedupeWikiCategoryPaths(t *testing.T) {
got := dedupeWikiCategoryPaths([][]string{
{"春节", "传统习俗"},
{" 春节 ", "传统习俗"},
{"春节", "文化习俗"},
{"", "空"},
{},
}, 10)
if len(got) != 3 {
t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got)
}
if got[0][0] != "春节" || got[0][1] != "传统习俗" {
t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0])
}
if got[1][1] != "文化习俗" {
t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1])
}
}
func TestResolveExistingTaxonomyForPrompt(t *testing.T) {
t.Run("nil batch context", func(t *testing.T) {
got := resolveExistingTaxonomyForPrompt(t.Context(), nil)
if got == "" {
t.Fatal("expected fallback text for nil batch context")
}
})
t.Run("batch context with taxonomy", func(t *testing.T) {
batchCtx := &WikiBatchContext{
ExistingTaxonomy: func(ctx context.Context) string {
return "春节\n 传统习俗"
},
}
got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx)
if got != "春节\n 传统习俗" {
t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got)
}
})
}
func TestRemapCategoryPath(t *testing.T) {
remap := map[string][]string{
categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"},
categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"},
}
tests := []struct {
name string
current []string
want []string
wantHit bool
}{
{
name: "single-label synonym remapped",
current: []string{"春节习俗"},
want: []string{"春节", "传统习俗"},
wantHit: true,
},
{
name: "document-role path collapsed to coarse folder",
current: []string{"组织单位", "致谢人员"},
want: []string{"组织"},
wantHit: true,
},
{
name: "noise normalized before matching",
current: []string{"实体/春节习俗"},
want: []string{"春节", "传统习俗"},
wantHit: true,
},
{
name: "unrelated path untouched",
current: []string{"人物"},
wantHit: false,
},
{
name: "empty path untouched",
current: nil,
wantHit: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, hit := remapCategoryPath(tt.current, remap)
if hit != tt.wantHit {
t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit)
}
if !hit {
return
}
if len(got) != len(tt.want) {
t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want)
}
for i := range tt.want {
if got[i] != tt.want[i] {
t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got)
}
}
})
}
}
+144
View File
@@ -67,6 +67,7 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage)
// Parse outbound links from content
page.OutLinks = s.parseOutLinks(page.Content)
normalizeWikiHierarchy(page)
now := time.Now()
page.CreatedAt = now
@@ -115,6 +116,11 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
existing.SourceRefs = page.SourceRefs
existing.ChunkRefs = page.ChunkRefs
existing.PageMetadata = page.PageMetadata
existing.ParentSlug = page.ParentSlug
existing.CategoryPath = page.CategoryPath
existing.WikiPath = page.WikiPath
existing.Depth = page.Depth
existing.SortOrder = page.SortOrder
existing.Status = page.Status
existing.UpdatedAt = time.Now()
@@ -122,11 +128,18 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
// when content shifts. Re-parse unconditionally to stay consistent with
// the stored body.
existing.OutLinks = s.parseOutLinks(existing.Content)
normalizeWikiHierarchy(existing)
if contentChanged {
if err := s.repo.Update(ctx, existing); err != nil {
return nil, fmt.Errorf("update wiki page: %w", err)
}
// GORM's struct Updates path skips zero values, so persist hierarchy
// metadata through the explicit map path as well. This keeps clearing
// parent/category fields deterministic without changing version again.
if err := s.repo.UpdateMeta(ctx, existing); err != nil {
return nil, fmt.Errorf("update wiki page hierarchy meta: %w", err)
}
} else {
// No user-visible change — persist bookkeeping fields but preserve
// the version so downstream consumers can rely on it.
@@ -145,6 +158,7 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
// UpdatePageMeta updates only metadata (status, source_refs) without version bump or link re-parse.
func (s *wikiPageService) UpdatePageMeta(ctx context.Context, page *types.WikiPage) error {
normalizeWikiHierarchy(page)
page.UpdatedAt = time.Now()
return s.repo.UpdateMeta(ctx, page)
}
@@ -192,6 +206,9 @@ func (s *wikiPageService) ListPages(ctx context.Context, req *types.WikiPageList
if err != nil {
return nil, err
}
for _, page := range pages {
normalizeWikiHierarchy(page)
}
pageSize := req.PageSize
if pageSize < 1 {
@@ -323,6 +340,9 @@ func (s *wikiPageService) GetIndexView(
if entries == nil {
entries = []types.WikiIndexEntry{}
}
for i := range entries {
normalizeWikiIndexEntryHierarchy(&entries[i], pt)
}
next := ""
// Only emit a cursor when a full page was returned AND more rows
// remain past `offset + limit`. A short page or one that exactly
@@ -811,6 +831,20 @@ func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, que
return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit)
}
// ListDistinctCategoryPaths returns unique category_path folders already used
// by entity/concept pages. Used by wiki ingest to ground extraction prompts.
func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) {
return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths)
}
// ListDistinctCategoryPathsByType returns unique category_path folders across
// the given page types. The browser uses this to render directory nodes before
// article pages are paged in; passing several types lets one tab (e.g. the
// merged knowledge tab) build a single directory skeleton server-side.
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) {
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths)
}
// CountByType is a service-layer pass-through over the repo. Used by
// the index intro path to frame the LLM prompt's "showing N of M" hint.
func (s *wikiPageService) CountByType(ctx context.Context, kbID string) (map[string]int64, error) {
@@ -917,6 +951,7 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
Summary: title,
Version: 1,
}
normalizeWikiHierarchy(page)
if err := s.repo.Create(ctx, page); err != nil {
return nil, fmt.Errorf("create default %s page: %w", slug, err)
@@ -924,6 +959,115 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
return page, nil
}
func normalizeWikiHierarchy(page *types.WikiPage) {
if page == nil {
return
}
page.ParentSlug = strings.TrimSpace(page.ParentSlug)
cleanPath := make(types.StringArray, 0, len(page.CategoryPath))
for _, part := range page.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
page.CategoryPath = cleanPath
page.Depth = len(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if page.PageType != "" {
parts = append(parts, page.PageType)
}
for _, part := range cleanPath {
parts = append(parts, part)
}
display := strings.TrimSpace(page.Title)
if display == "" {
display = strings.TrimSpace(page.Slug)
}
if display != "" {
parts = append(parts, display)
}
page.WikiPath = strings.Join(parts, "/")
}
func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) {
if entry == nil {
return
}
cleanPath := make(types.StringArray, 0, len(entry.CategoryPath))
for _, part := range entry.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
entry.CategoryPath = cleanPath
entry.Depth = len(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if strings.TrimSpace(pageType) != "" {
parts = append(parts, strings.TrimSpace(pageType))
}
parts = append(parts, cleanPath...)
display := strings.TrimSpace(entry.Title)
if display == "" {
display = strings.TrimSpace(entry.Slug)
}
if display != "" {
parts = append(parts, display)
}
entry.WikiPath = strings.Join(parts, "/")
}
func cleanWikiCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
func isWikiTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
}
// containsString checks if a string slice contains a given string
func containsString(slice []string, s string) bool {
for _, v := range slice {
@@ -99,6 +99,56 @@ func TestNormalizeSlug(t *testing.T) {
}
}
func TestNormalizeWikiHierarchyCleansModelCategoryNoise(t *testing.T) {
page := &types.WikiPage{
Slug: "concept/ai-knowledge",
Title: "AI时代的知识困境",
PageType: types.WikiPageTypeConcept,
CategoryPath: types.StringArray{"概念/爱护花草", " 实体/标牌 ", "摘要", "生态/平台", "多余层级"},
}
normalizeWikiHierarchy(page)
want := types.StringArray{"爱护花草", "标牌", "生态"}
if len(page.CategoryPath) != len(want) {
t.Fatalf("CategoryPath = %v, want %v", page.CategoryPath, want)
}
for i := range want {
if page.CategoryPath[i] != want[i] {
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, page.CategoryPath[i], want[i], page.CategoryPath)
}
}
if page.Depth != 3 {
t.Fatalf("Depth = %d, want 3", page.Depth)
}
if page.WikiPath != "concept/爱护花草/标牌/生态/AI时代的知识困境" {
t.Fatalf("WikiPath = %q", page.WikiPath)
}
}
func TestNormalizeWikiIndexEntryHierarchyCleansModelCategoryNoise(t *testing.T) {
entry := &types.WikiIndexEntry{
Slug: "entity/sign",
Title: "爱护花草标牌",
CategoryPath: types.StringArray{"实体/标牌", "概念", "生态/行为倡导"},
}
normalizeWikiIndexEntryHierarchy(entry, types.WikiPageTypeEntity)
want := types.StringArray{"标牌", "生态", "行为倡导"}
if len(entry.CategoryPath) != len(want) {
t.Fatalf("CategoryPath = %v, want %v", entry.CategoryPath, want)
}
for i := range want {
if entry.CategoryPath[i] != want[i] {
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, entry.CategoryPath[i], want[i], entry.CategoryPath)
}
}
if entry.WikiPath != "entity/标牌/生态/行为倡导/爱护花草标牌" {
t.Fatalf("WikiPath = %q", entry.WikiPath)
}
}
func TestContainsString(t *testing.T) {
slice := []string{"a", "b", "c"}
if !containsString(slice, "b") {
+110 -1
View File
@@ -77,7 +77,7 @@ func getSlugParam(c *gin.Context) string {
// @Tags Wiki
// @Produce json
// @Param kb_id path string true "Knowledge base ID"
// @Param page_type query string false "Filter by page type"
// @Param page_type query string false "Filter by page type; comma-separated for multiple (e.g. entity,concept)"
// @Param status query string false "Filter by status"
// @Param query query string false "Full-text search"
// @Param page query int false "Page number"
@@ -97,12 +97,21 @@ func (h *WikiPageHandler) ListPages(c *gin.Context) {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
categoryPath := parseWikiCategoryPath(c.Query("category_path"))
var categoryDepth *int
if raw := c.Query("category_depth"); raw != "" {
if depth, parseErr := strconv.Atoi(raw); parseErr == nil && depth >= 0 {
categoryDepth = &depth
}
}
req := &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageType: c.Query("page_type"),
Status: c.Query("status"),
Query: c.Query("query"),
CategoryPath: types.StringArray(categoryPath),
CategoryDepth: categoryDepth,
Page: page,
PageSize: pageSize,
SortBy: c.DefaultQuery("sort_by", "updated_at"),
@@ -118,6 +127,106 @@ func (h *WikiPageHandler) ListPages(c *gin.Context) {
c.JSON(http.StatusOK, resp)
}
// ListCategories godoc
// @Summary List wiki category paths
// @Description Retrieve direct child directories for a wiki page type and optional parent_path
// @Tags Wiki
// @Produce json
// @Param kb_id path string true "Knowledge base ID"
// @Param page_type query string true "Wiki page type; comma-separated for multiple (e.g. entity,concept)"
// @Param parent_path query string false "Parent category path, slash-separated"
// @Param page query int false "Page number (1-based)"
// @Param page_size query int false "Page size"
// @Success 200 {object} types.WikiCategoryPathListResponse
// @Failure 400 {object} errors.AppError
// @Security Bearer
// @Router /knowledgebase/{kb_id}/wiki/categories [get]
func (h *WikiPageHandler) ListCategories(c *gin.Context) {
kbID, _, err := h.validateWikiKB(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
pageTypes := parseWikiPageTypes(c.Query("page_type"))
if len(pageTypes) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "page_type is required"})
return
}
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
if page < 1 {
page = 1
}
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "2000"))
if pageSize <= 0 {
pageSize = 2000
}
parentPath := parseWikiCategoryPath(c.Query("parent_path"))
paths, err := h.wikiService.ListDistinctCategoryPathsByType(c.Request.Context(), kbID, pageTypes, parentPath, pageSize)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
total := len(paths)
offset := (page - 1) * pageSize
if offset > total {
offset = total
}
end := offset + pageSize
if end > total {
end = total
}
totalPages := 0
if pageSize > 0 {
totalPages = (total + pageSize - 1) / pageSize
}
c.JSON(http.StatusOK, types.WikiCategoryPathListResponse{
Paths: paths[offset:end],
Total: total,
Page: page,
PageSize: pageSize,
TotalPages: totalPages,
})
}
// parseWikiPageTypes splits a comma-separated page_type query value (e.g.
// "entity,concept") into a deduplicated slice, dropping blanks.
func parseWikiPageTypes(raw string) []string {
if strings.TrimSpace(raw) == "" {
return nil
}
seen := make(map[string]struct{})
out := make([]string, 0, 4)
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if _, ok := seen[part]; ok {
continue
}
seen[part] = struct{}{}
out = append(out, part)
}
return out
}
func parseWikiCategoryPath(raw string) []string {
if strings.TrimSpace(raw) == "" {
return nil
}
parts := strings.Split(raw, "/")
out := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
// CreatePage godoc
// @Summary Create a wiki page
// @Description Create a new wiki page in the knowledge base
+1
View File
@@ -1748,6 +1748,7 @@ func RegisterWikiPageRoutes(r *gin.RouterGroup, wikiHandler *handler.WikiPageHan
{
// Page CRUD
wiki.GET("/pages", g.Viewer(), wikiHandler.ListPages)
wiki.GET("/categories", g.Viewer(), wikiHandler.ListCategories)
wiki.POST("/pages", g.OwnedWikiKBOrAdmin(), wikiHandler.CreatePage)
wiki.GET("/pages/*slug", g.Viewer(), wikiHandler.GetPage)
wiki.PUT("/pages/*slug", g.OwnedWikiKBOrAdmin(), wikiHandler.UpdatePage)
+18
View File
@@ -156,6 +156,16 @@ type WikiPageService interface {
// merge targets server-side.
FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error)
// ListDistinctCategoryPaths returns unique non-empty category_path
// values from entity/concept pages. Used by wiki ingest to inject
// established folder taxonomy into extraction prompts.
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
// ListDistinctCategoryPathsByType returns unique non-empty category_path
// values across the given page types. Used by the wiki browser to render a
// stable directory skeleton independently of page pagination.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
// CountByType returns page counts grouped by type for a knowledge
// base. Re-exposed at the service layer so the index intro
// generation path can frame the LLM prompt with "showing N of M"
@@ -264,6 +274,14 @@ type WikiPageRepository interface {
// surface candidate merge targets server-side.
FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error)
// ListDistinctCategoryPaths scans entity/concept pages and returns
// unique non-empty category_path values for wiki ingest prompts.
ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error)
// ListDistinctCategoryPathsByType returns unique non-empty category_path
// values across the given page types.
ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error)
// ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation).
ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error)
+49 -12
View File
@@ -65,6 +65,22 @@ type WikiPage struct {
Summary string `json:"summary" gorm:"type:text"`
// Alternate names, abbreviations, acronyms or translated names
Aliases StringArray `json:"aliases" gorm:"type:json"`
// ParentSlug optionally points at the wiki page that should act as this
// page's semantic parent in the directory tree. The parent may be empty
// when the page is grouped only by CategoryPath.
ParentSlug string `json:"parent_slug,omitempty" gorm:"type:varchar(255);index"`
// CategoryPath is the directory breadcrumb that groups this page in the
// wiki browser, e.g. ["AI", "LLM 应用", "RAG"]. It is intentionally
// label-based so intermediate directory nodes do not need to be real pages.
CategoryPath StringArray `json:"category_path,omitempty" gorm:"type:json"`
// WikiPath is a normalized, sortable path derived from page_type,
// category_path, and title. It keeps large directory listings cheap to sort.
WikiPath string `json:"wiki_path,omitempty" gorm:"type:varchar(1024);index"`
// Depth is len(CategoryPath), cached for filtering / display.
Depth int `json:"depth,omitempty" gorm:"default:0;index"`
// SortOrder allows generated or manually edited pages to control sibling
// ordering before falling back to title.
SortOrder int `json:"sort_order,omitempty" gorm:"default:0;index"`
// References to source knowledge IDs that contributed to this page.
// Format matches the legacy "<knowledge_id>|<doc_title>" convention used
// across the ingest pipeline, so retract / display code can split on `|`
@@ -226,14 +242,16 @@ func (c *WikiConfig) Scan(value interface{}) error {
// WikiPageListRequest represents a request to list wiki pages with filtering
type WikiPageListRequest struct {
KnowledgeBaseID string `json:"knowledge_base_id"`
PageType string `json:"page_type,omitempty"` // filter by type
Status string `json:"status,omitempty"` // filter by status
Query string `json:"query,omitempty"` // full-text search
Page int `json:"page,omitempty"` // pagination page (1-based)
PageSize int `json:"page_size,omitempty"` // pagination size
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
KnowledgeBaseID string `json:"knowledge_base_id"`
PageType string `json:"page_type,omitempty"` // filter by type
Status string `json:"status,omitempty"` // filter by status
Query string `json:"query,omitempty"` // full-text search
CategoryPath StringArray `json:"category_path,omitempty"` // exact directory path
CategoryDepth *int `json:"category_depth,omitempty"` // exact directory depth, including 0 for root
Page int `json:"page,omitempty"` // pagination page (1-based)
PageSize int `json:"page_size,omitempty"` // pagination size
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
}
// WikiPageListResponse represents a paginated list of wiki pages
@@ -245,6 +263,21 @@ type WikiPageListResponse struct {
TotalPages int `json:"total_pages"`
}
// WikiCategoryPathListResponse returns the directory skeleton for wiki pages.
type WikiCategoryPathListResponse struct {
Paths []WikiCategoryPath `json:"paths"`
Total int `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalPages int `json:"total_pages"`
}
// WikiCategoryPath is one directory node in the wiki sidebar skeleton.
type WikiCategoryPath struct {
Path []string `json:"path"`
Count int64 `json:"count"`
}
// WikiGraphMode enumerates the graph query modes exposed to the API.
const (
// WikiGraphModeOverview returns the top-N most-connected pages as an
@@ -345,9 +378,14 @@ func (WikiPageIssue) TableName() string {
// carried — the backend projects SELECT slug, title, summary so a 40k-
// page KB does not pay for TEXT content transport on every index open.
type WikiIndexEntry struct {
Slug string `json:"slug"`
Title string `json:"title"`
Summary string `json:"summary"`
Slug string `json:"slug"`
Title string `json:"title"`
Summary string `json:"summary"`
ParentSlug string `json:"parent_slug,omitempty"`
CategoryPath StringArray `json:"category_path,omitempty"`
WikiPath string `json:"wiki_path,omitempty"`
Depth int `json:"depth,omitempty"`
SortOrder int `json:"sort_order,omitempty"`
}
// WikiIndexGroup bundles the entries for one page_type into a page-sized
@@ -397,4 +435,3 @@ type WikiPageLite struct {
Aliases StringArray `json:"aliases,omitempty"`
OutLinks StringArray `json:"out_links,omitempty"`
}
@@ -22,6 +22,11 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
status VARCHAR(32) NOT NULL DEFAULT 'published',
content TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT '',
parent_slug VARCHAR(255) NOT NULL DEFAULT '',
category_path JSONB DEFAULT '[]'::JSONB,
wiki_path VARCHAR(1024) NOT NULL DEFAULT '',
depth INT NOT NULL DEFAULT 0,
sort_order INT NOT NULL DEFAULT 0,
source_refs JSONB DEFAULT '[]'::JSONB,
chunk_refs JSONB DEFAULT '[]'::JSONB,
in_links JSONB DEFAULT '[]'::JSONB,
@@ -45,6 +50,12 @@ CREATE INDEX IF NOT EXISTS idx_wiki_pages_kb_id
CREATE INDEX IF NOT EXISTS idx_wiki_pages_page_type
ON wiki_pages (knowledge_base_id, page_type);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
ON wiki_pages (knowledge_base_id, parent_slug);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id
ON wiki_pages (tenant_id);
@@ -0,0 +1,10 @@
-- Migration: 000061_wiki_page_hierarchy (rollback)
DROP INDEX IF EXISTS idx_wiki_pages_tree;
DROP INDEX IF EXISTS idx_wiki_pages_parent_slug;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS sort_order;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS depth;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS wiki_path;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS category_path;
ALTER TABLE wiki_pages DROP COLUMN IF EXISTS parent_slug;
@@ -0,0 +1,29 @@
-- Migration: 000061_wiki_page_hierarchy
-- Description: Add structured directory hierarchy fields to wiki_pages.
DO $$ BEGIN RAISE NOTICE '[Migration 000061] Applying wiki page hierarchy schema'; END $$;
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS parent_slug VARCHAR(255) NOT NULL DEFAULT '';
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS category_path JSONB DEFAULT '[]'::JSONB;
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS wiki_path VARCHAR(1024) NOT NULL DEFAULT '';
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS depth INT NOT NULL DEFAULT 0;
ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS sort_order INT NOT NULL DEFAULT 0;
UPDATE wiki_pages
SET
category_path = COALESCE(category_path, '[]'::JSONB),
depth = COALESCE(depth, 0),
wiki_path = CASE
WHEN COALESCE(wiki_path, '') <> '' THEN wiki_path
WHEN page_type IN ('index', 'log') THEN page_type || '/' || COALESCE(NULLIF(title, ''), slug)
ELSE page_type || '/' || COALESCE(NULLIF(title, ''), slug)
END
WHERE wiki_path = '' OR wiki_path IS NULL OR category_path IS NULL OR depth IS NULL;
CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
ON wiki_pages (knowledge_base_id, parent_slug);
CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
DO $$ BEGIN RAISE NOTICE '[Migration 000061] wiki page hierarchy schema applied successfully'; END $$;