opensource/bisheng
3
0
Fork 0
mirror of https://github.com/dataelement/bisheng.git synced 2026-09-21 12:43:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

update coding 0723_922764

Browse Source
This commit is contained in:
xuhl2
2026-07-23 20:28:28 +08:00
parent 7d0bb4664e
commit 901d23711b
5 changed files with 158 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
debug-knowledge-title-task-not-executed.md
+35 -1
View File
@@ -36,6 +36,34 @@
| D | Task executed but skipped because preconditions not met. | Medium | Low | Worker log shows `title extraction skipped, ...` with reason (status, missing object_name, no title extracted). |
| E | Redis broker contains stale messages or API/worker code versions are inconsistent. | Low | Medium | Old error `module 'bisheng.worker.knowledge.file_worker' has no attribute 'extract_knowledge_file_title_celery'` still appears in worker logs after API restart. |
**Code Changes Made (Instrumentation / Logging)**:
To make root-cause identification easier in the testing environment, the following logging-only changes were applied (no business logic changed):
1. [src/backend/bisheng/worker/knowledge/file_title_worker.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/worker/knowledge/file_title_worker.py)
- Added `title extraction preparing ...` log showing `status`, `object_name`, `tenant_id`.
- Added `title extraction skipped, file status=... is not WAITING` guard (precondition check).
- Added `title extraction downloaded ... local_path=... exists=...` log.
- Added `title extraction result ... raw_title=...` log.
- Added `alias generation result ... alias_name=...` log.
2. [src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py)
- Added `alias generation config ... file_alias_model_id=...` log.
- Upgraded `file_alias_model_id not configured` from `debug` to `warning`.
- Added `alias generation llm response ... content=...` log.
- Added `alias generation parsed raw_alias=...` log.
3. [src/backend/bisheng/knowledge/domain/services/file_title_extractor.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py)
- Added `title extraction dispatch ... extension=... extractor=...` log.
- Added `title extraction done ... title=...` log.
4. [src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py) — robustness & fallback improvements
- Changed `_JSON_BLOCK_RE` from greedy `{.*}` to non-greedy `{.*?}` so it does not swallow trailing explanation text.
- Added `_CODE_BLOCK_RE` to support JSON wrapped in markdown code blocks (e.g. ```json {...} ```).
- Refactored `_parse_llm_json` to try: code block -> direct JSON -> first JSON object.
- Added detailed logs in `_extract_alias_from_dict` and `_normalize_alias_name`.
- **Added fallback logic**: when `file_alias_model_id` is empty, use `extract_title_model_id` instead. Only return `None` when both are missing.
**Verification Commands (run in testing env)**:
1. Check task registration:
@@ -58,7 +86,11 @@
4. Check worker logs for title extraction logs after uploading a file:
- Look for `extract_knowledge_file_title_celery start file_id=...`
- Look for `title extraction skipped, ...`
- Look for `title extraction preparing ...`
- Look for `title extraction dispatch ... extension=... extractor=...`
- Look for `title extraction result ... raw_title=...`
- Look for `alias generation config ... file_alias_model_id=...`
- Look for `alias generation llm response ...`
- Look for `file alias generated file_id=... alias_name=...`
- Look for any traceback after the start line.
@@ -68,3 +100,5 @@
FROM knowledge_file
WHERE id = <file_id>;
```
**Mandatory Deployment Note**: Because new files (`file_title_worker.py`, `file_title_extractor.py`, `file_alias_name_generator.py`, `gen_title.yaml`) were added, the Docker image **must be rebuilt** and the API + Celery Worker containers **must be restarted** for any of these changes to take effect in the testing environment.
src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py
+70 -23
View File
@@ -52,7 +52,11 @@ class FileAliasNameGeneratorService:
}
)
_JSON_BLOCK_RE = re.compile(r"\{.*\}", re.DOTALL)
# Extract a JSON object from free-form LLM output. Non-greedy so it stops
# at the first closing brace and avoids swallowing trailing explanation text.
_JSON_BLOCK_RE = re.compile(r"\{.*?\}", re.DOTALL)
# Extract JSON that is wrapped in a markdown code block.
_CODE_BLOCK_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL)
@classmethod
def generate_alias_name(
@@ -66,15 +70,30 @@ class FileAliasNameGeneratorService:
"""Return an LLM-generated alias (with original extension) or ``None``."""
try:
knowledge_llm = LLMService.get_knowledge_llm(tenant_id=tenant_id)
if not knowledge_llm or not knowledge_llm.file_alias_model_id:
logger.debug(
"file_alias_model_id not configured tenant_id={}",
file_alias_model_id = (
knowledge_llm.file_alias_model_id
if knowledge_llm and knowledge_llm.file_alias_model_id
else None
)
# Fallback to the extract-title model when no alias model is configured.
if not file_alias_model_id and knowledge_llm and knowledge_llm.extract_title_model_id:
file_alias_model_id = knowledge_llm.extract_title_model_id
logger.info(
"alias generation config tenant_id={} file_alias_model_id={} extract_title_model_id={} resolved_model_id={}",
tenant_id,
getattr(knowledge_llm, "file_alias_model_id", None) if knowledge_llm else None,
getattr(knowledge_llm, "extract_title_model_id", None) if knowledge_llm else None,
file_alias_model_id,
)
if not file_alias_model_id:
logger.warning(
"file_alias_model_id not configured and no extract_title_model_id fallback tenant_id={}",
tenant_id,
)
return None
llm = LLMService.get_bisheng_llm_sync(
model_id=knowledge_llm.file_alias_model_id,
model_id=file_alias_model_id,
app_id=ApplicationTypeEnum.KNOWLEDGE_BASE.value,
app_name=ApplicationTypeEnum.KNOWLEDGE_BASE.value,
app_type=ApplicationTypeEnum.KNOWLEDGE_BASE,
@@ -101,11 +120,21 @@ class FileAliasNameGeneratorService:
]
response = llm.invoke(messages)
content = response.content.strip() if response.content else ""
logger.info(
"alias generation llm response file_name={} content={}",
file_name,
content,
)
if not content:
logger.warning("LLM returned empty alias generation response")
return None
raw_alias = cls._parse_llm_json(content)
logger.info(
"alias generation parsed raw_alias={} file_name={}",
raw_alias,
file_name,
)
if not raw_alias:
return None
@@ -138,21 +167,29 @@ class FileAliasNameGeneratorService:
@classmethod
def _parse_llm_json(cls, content: str) -> str | None:
"""Parse the LLM JSON response and return the raw new_file_name."""
# Try direct JSON parsing first.
try:
data = json.loads(content)
return cls._extract_alias_from_dict(data)
except json.JSONDecodeError:
pass
# Strategy: markdown code block -> direct JSON -> first JSON object in text.
candidates = []
# Fall back to extracting the first JSON object with a regex.
match = cls._JSON_BLOCK_RE.search(content)
if match:
code_match = cls._CODE_BLOCK_RE.search(content)
if code_match:
candidates.append(code_match.group(1).strip())
candidates.append(content.strip())
json_match = cls._JSON_BLOCK_RE.search(content)
if json_match:
candidates.append(json_match.group(0))
for candidate in candidates:
if not candidate:
continue
try:
data = json.loads(match.group(0))
return cls._extract_alias_from_dict(data)
data = json.loads(candidate)
alias = cls._extract_alias_from_dict(data)
if alias is not None:
return alias
except json.JSONDecodeError:
pass
continue
logger.warning("failed to parse alias JSON from LLM response: {}", content)
return None
@@ -160,11 +197,14 @@ class FileAliasNameGeneratorService:
@classmethod
def _extract_alias_from_dict(cls, data: dict) -> str | None:
"""Validate the parsed JSON dict and return the new file name."""
if data.get("status") != "success":
logger.debug("LLM returned status=%s", data.get("status"))
return None
status = data.get("status")
new_name = data.get("new_file_name")
logger.info("alias extract from dict status=%s new_file_name=%s", status, new_name)
if status != "success":
logger.info("LLM returned non-success status=%s", status)
return None
if not isinstance(new_name, str) or not new_name.strip():
logger.info("LLM returned empty or invalid new_file_name")
return None
return new_name.strip()
@@ -174,13 +214,20 @@ class FileAliasNameGeneratorService:
original_ext = os.path.splitext(original_file_name)[1].lower()
alias_base, alias_ext = os.path.splitext(raw_alias)
alias_base = alias_base.strip()
logger.info(
"alias normalize raw_alias=%s original_ext=%s alias_base=%s",
raw_alias,
original_ext,
alias_base,
)
if not alias_base:
logger.info("alias normalize skipped, empty base")
return None
# Always keep the original extension for consistency with file_name.
ext = original_ext if original_ext else alias_ext.lower()
max_base_length = 200 - len(ext)
safe_base = sanitize_file_name(alias_base, max_length=max(max_base_length, 1))
if not safe_base:
return None
return f"{safe_base}{ext}"
result = f"{safe_base}{ext}" if safe_base else None
logger.info("alias normalize result=%s", result)
return result
src/backend/bisheng/knowledge/domain/services/file_title_extractor.py
+11 -2
View File
@@ -585,14 +585,23 @@ class FileTitleExtractorService:
file name.
"""
if not file_path or not os.path.exists(file_path):
logger.info("title extraction skipped, file missing file_path={}", file_path)
return None
ext = os.path.splitext(file_path)[1].lower().lstrip(".")
extractor = cls._EXTRACTORS.get(ext)
logger.info("title extraction dispatch file_path={} extension={} extractor={}", file_path, ext, type(extractor).__name__ if extractor else None)
if extractor is None:
logger.debug("no title extractor for extension: {}", ext)
logger.info("no title extractor for extension: {}", ext)
return None
try:
return extractor.extract(file_path)
title = extractor.extract(file_path)
logger.info(
"title extraction done file_path={} extension={} title={}",
file_path,
ext,
title,
)
return title
except Exception as e:
# Title extraction is best-effort; a parse failure should not block parsing.
logger.warning("title extraction failed for {}: {}", file_path, e)
src/backend/bisheng/knowledge/domain/services/knowledge_space_service.py
+5 -2
View File
@@ -12311,13 +12311,16 @@ class KnowledgeSpaceService(KnowledgeUtils):
) -> None:
"""Enqueue title extraction and AI alias generation before formal parsing."""
from bisheng.worker.knowledge.file_title_worker import (
extract_and_generate_alias,
extract_knowledge_file_title_celery,
)
if len(process_files) != len(preview_cache_keys):
raise ValueError("process_files and preview_cache_keys length mismatch")
for index, knowledge_file in enumerate(process_files):
extract_and_generate_alias(knowledge_file.id)
extract_knowledge_file_title_celery.delay(
knowledge_file.id,
preview_cache_keys[index],
)
@staticmethod
def enqueue_file_processing(
src/backend/bisheng/worker/knowledge/file_title_worker.py
+37
View File
@@ -27,6 +27,25 @@ def extract_and_generate_alias(file_id: int) -> str | None:
logger.warning("title extraction skipped, file not found file_id={}", file_id)
return None
logger.info(
"title extraction preparing file_id={} file_name={} status={} object_name={} tenant_id={}",
file_id,
db_file.file_name,
db_file.status,
db_file.object_name,
db_file.tenant_id,
)
if db_file.status != KnowledgeFileStatus.WAITING.value:
logger.info(
"title extraction skipped, file status={} is not WAITING file_id={}",
db_file.status,
file_id,
)
return None
if not db_file.object_name:
logger.warning("title extraction skipped, missing object_name file_id={}", file_id)
return None
try:
with tempfile.TemporaryDirectory() as tmp_dir:
local_path, _ = download_minio_file(
@@ -34,11 +53,23 @@ def extract_and_generate_alias(file_id: int) -> str | None:
root_dir=tmp_dir,
calc_sha256=False,
)
logger.info(
"title extraction downloaded file_id={} local_path={} exists={}",
file_id,
local_path,
os.path.exists(local_path) if local_path else False,
)
if not local_path or not os.path.exists(local_path):
logger.warning("title extraction skipped, download failed file_id={}", file_id)
return None
raw_title = FileTitleExtractorService.extract_title(local_path)
logger.info(
"title extraction result file_id={} file_name={} raw_title={}",
file_id,
db_file.file_name,
raw_title,
)
if not raw_title:
logger.info("no title extracted file_id={} file_name={}", file_id, db_file.file_name)
return None
@@ -50,6 +81,12 @@ def extract_and_generate_alias(file_id: int) -> str | None:
invoke_user_id=db_file.user_id or 0,
tenant_id=db_file.tenant_id or 1,
)
logger.info(
"alias generation result file_id={} file_name={} alias_name={}",
file_id,
db_file.file_name,
alias_name,
)
if alias_name and alias_name != db_file.alias_name:
db_file.alias_name = alias_name
KnowledgeFileDao.update(db_file)