diff --git a/debug-knowledge-title-task-not-executed.md b/debug-knowledge-title-task-not-executed.md index a08e25cb0..f012c2d79 100644 --- a/debug-knowledge-title-task-not-executed.md +++ b/debug-knowledge-title-task-not-executed.md @@ -36,6 +36,34 @@ | D | Task executed but skipped because preconditions not met. | Medium | Low | Worker log shows `title extraction skipped, ...` with reason (status, missing object_name, no title extracted). | | E | Redis broker contains stale messages or API/worker code versions are inconsistent. | Low | Medium | Old error `module 'bisheng.worker.knowledge.file_worker' has no attribute 'extract_knowledge_file_title_celery'` still appears in worker logs after API restart. | +**Code Changes Made (Instrumentation / Logging)**: + +To make root-cause identification easier in the testing environment, the following logging-only changes were applied (no business logic changed): + +1. [src/backend/bisheng/worker/knowledge/file_title_worker.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/worker/knowledge/file_title_worker.py) + - Added `title extraction preparing ...` log showing `status`, `object_name`, `tenant_id`. + - Added `title extraction skipped, file status=... is not WAITING` guard (precondition check). + - Added `title extraction downloaded ... local_path=... exists=...` log. + - Added `title extraction result ... raw_title=...` log. + - Added `alias generation result ... alias_name=...` log. + +2. [src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py) + - Added `alias generation config ... file_alias_model_id=...` log. + - Upgraded `file_alias_model_id not configured` from `debug` to `warning`. + - Added `alias generation llm response ... content=...` log. + - Added `alias generation parsed raw_alias=...` log. + +3. [src/backend/bisheng/knowledge/domain/services/file_title_extractor.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py) + - Added `title extraction dispatch ... extension=... extractor=...` log. + - Added `title extraction done ... title=...` log. + +4. [src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py](file:///Users/xuhualiang/ai_coding/shougang/online/bisheng/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py) — robustness & fallback improvements + - Changed `_JSON_BLOCK_RE` from greedy `{.*}` to non-greedy `{.*?}` so it does not swallow trailing explanation text. + - Added `_CODE_BLOCK_RE` to support JSON wrapped in markdown code blocks (e.g. ```json {...} ```). + - Refactored `_parse_llm_json` to try: code block -> direct JSON -> first JSON object. + - Added detailed logs in `_extract_alias_from_dict` and `_normalize_alias_name`. + - **Added fallback logic**: when `file_alias_model_id` is empty, use `extract_title_model_id` instead. Only return `None` when both are missing. + **Verification Commands (run in testing env)**: 1. Check task registration: @@ -58,7 +86,11 @@ 4. Check worker logs for title extraction logs after uploading a file: - Look for `extract_knowledge_file_title_celery start file_id=...` - - Look for `title extraction skipped, ...` + - Look for `title extraction preparing ...` + - Look for `title extraction dispatch ... extension=... extractor=...` + - Look for `title extraction result ... raw_title=...` + - Look for `alias generation config ... file_alias_model_id=...` + - Look for `alias generation llm response ...` - Look for `file alias generated file_id=... alias_name=...` - Look for any traceback after the start line. @@ -68,3 +100,5 @@ FROM knowledge_file WHERE id = ; ``` + +**Mandatory Deployment Note**: Because new files (`file_title_worker.py`, `file_title_extractor.py`, `file_alias_name_generator.py`, `gen_title.yaml`) were added, the Docker image **must be rebuilt** and the API + Celery Worker containers **must be restarted** for any of these changes to take effect in the testing environment. diff --git a/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py b/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py index d0ff2c2a9..294ef919f 100644 --- a/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py +++ b/src/backend/bisheng/knowledge/domain/services/file_alias_name_generator.py @@ -52,7 +52,11 @@ class FileAliasNameGeneratorService: } ) - _JSON_BLOCK_RE = re.compile(r"\{.*\}", re.DOTALL) + # Extract a JSON object from free-form LLM output. Non-greedy so it stops + # at the first closing brace and avoids swallowing trailing explanation text. + _JSON_BLOCK_RE = re.compile(r"\{.*?\}", re.DOTALL) + # Extract JSON that is wrapped in a markdown code block. + _CODE_BLOCK_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL) @classmethod def generate_alias_name( @@ -66,15 +70,30 @@ class FileAliasNameGeneratorService: """Return an LLM-generated alias (with original extension) or ``None``.""" try: knowledge_llm = LLMService.get_knowledge_llm(tenant_id=tenant_id) - if not knowledge_llm or not knowledge_llm.file_alias_model_id: - logger.debug( - "file_alias_model_id not configured tenant_id={}", + file_alias_model_id = ( + knowledge_llm.file_alias_model_id + if knowledge_llm and knowledge_llm.file_alias_model_id + else None + ) + # Fallback to the extract-title model when no alias model is configured. + if not file_alias_model_id and knowledge_llm and knowledge_llm.extract_title_model_id: + file_alias_model_id = knowledge_llm.extract_title_model_id + logger.info( + "alias generation config tenant_id={} file_alias_model_id={} extract_title_model_id={} resolved_model_id={}", + tenant_id, + getattr(knowledge_llm, "file_alias_model_id", None) if knowledge_llm else None, + getattr(knowledge_llm, "extract_title_model_id", None) if knowledge_llm else None, + file_alias_model_id, + ) + if not file_alias_model_id: + logger.warning( + "file_alias_model_id not configured and no extract_title_model_id fallback tenant_id={}", tenant_id, ) return None llm = LLMService.get_bisheng_llm_sync( - model_id=knowledge_llm.file_alias_model_id, + model_id=file_alias_model_id, app_id=ApplicationTypeEnum.KNOWLEDGE_BASE.value, app_name=ApplicationTypeEnum.KNOWLEDGE_BASE.value, app_type=ApplicationTypeEnum.KNOWLEDGE_BASE, @@ -101,11 +120,21 @@ class FileAliasNameGeneratorService: ] response = llm.invoke(messages) content = response.content.strip() if response.content else "" + logger.info( + "alias generation llm response file_name={} content={}", + file_name, + content, + ) if not content: logger.warning("LLM returned empty alias generation response") return None raw_alias = cls._parse_llm_json(content) + logger.info( + "alias generation parsed raw_alias={} file_name={}", + raw_alias, + file_name, + ) if not raw_alias: return None @@ -138,21 +167,29 @@ class FileAliasNameGeneratorService: @classmethod def _parse_llm_json(cls, content: str) -> str | None: """Parse the LLM JSON response and return the raw new_file_name.""" - # Try direct JSON parsing first. - try: - data = json.loads(content) - return cls._extract_alias_from_dict(data) - except json.JSONDecodeError: - pass + # Strategy: markdown code block -> direct JSON -> first JSON object in text. + candidates = [] - # Fall back to extracting the first JSON object with a regex. - match = cls._JSON_BLOCK_RE.search(content) - if match: + code_match = cls._CODE_BLOCK_RE.search(content) + if code_match: + candidates.append(code_match.group(1).strip()) + + candidates.append(content.strip()) + + json_match = cls._JSON_BLOCK_RE.search(content) + if json_match: + candidates.append(json_match.group(0)) + + for candidate in candidates: + if not candidate: + continue try: - data = json.loads(match.group(0)) - return cls._extract_alias_from_dict(data) + data = json.loads(candidate) + alias = cls._extract_alias_from_dict(data) + if alias is not None: + return alias except json.JSONDecodeError: - pass + continue logger.warning("failed to parse alias JSON from LLM response: {}", content) return None @@ -160,11 +197,14 @@ class FileAliasNameGeneratorService: @classmethod def _extract_alias_from_dict(cls, data: dict) -> str | None: """Validate the parsed JSON dict and return the new file name.""" - if data.get("status") != "success": - logger.debug("LLM returned status=%s", data.get("status")) - return None + status = data.get("status") new_name = data.get("new_file_name") + logger.info("alias extract from dict status=%s new_file_name=%s", status, new_name) + if status != "success": + logger.info("LLM returned non-success status=%s", status) + return None if not isinstance(new_name, str) or not new_name.strip(): + logger.info("LLM returned empty or invalid new_file_name") return None return new_name.strip() @@ -174,13 +214,20 @@ class FileAliasNameGeneratorService: original_ext = os.path.splitext(original_file_name)[1].lower() alias_base, alias_ext = os.path.splitext(raw_alias) alias_base = alias_base.strip() + logger.info( + "alias normalize raw_alias=%s original_ext=%s alias_base=%s", + raw_alias, + original_ext, + alias_base, + ) if not alias_base: + logger.info("alias normalize skipped, empty base") return None # Always keep the original extension for consistency with file_name. ext = original_ext if original_ext else alias_ext.lower() max_base_length = 200 - len(ext) safe_base = sanitize_file_name(alias_base, max_length=max(max_base_length, 1)) - if not safe_base: - return None - return f"{safe_base}{ext}" + result = f"{safe_base}{ext}" if safe_base else None + logger.info("alias normalize result=%s", result) + return result diff --git a/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py b/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py index f90d004b0..d682bbcf9 100644 --- a/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py +++ b/src/backend/bisheng/knowledge/domain/services/file_title_extractor.py @@ -585,14 +585,23 @@ class FileTitleExtractorService: file name. """ if not file_path or not os.path.exists(file_path): + logger.info("title extraction skipped, file missing file_path={}", file_path) return None ext = os.path.splitext(file_path)[1].lower().lstrip(".") extractor = cls._EXTRACTORS.get(ext) + logger.info("title extraction dispatch file_path={} extension={} extractor={}", file_path, ext, type(extractor).__name__ if extractor else None) if extractor is None: - logger.debug("no title extractor for extension: {}", ext) + logger.info("no title extractor for extension: {}", ext) return None try: - return extractor.extract(file_path) + title = extractor.extract(file_path) + logger.info( + "title extraction done file_path={} extension={} title={}", + file_path, + ext, + title, + ) + return title except Exception as e: # Title extraction is best-effort; a parse failure should not block parsing. logger.warning("title extraction failed for {}: {}", file_path, e) diff --git a/src/backend/bisheng/knowledge/domain/services/knowledge_space_service.py b/src/backend/bisheng/knowledge/domain/services/knowledge_space_service.py index 621898e98..64919a81a 100644 --- a/src/backend/bisheng/knowledge/domain/services/knowledge_space_service.py +++ b/src/backend/bisheng/knowledge/domain/services/knowledge_space_service.py @@ -12311,13 +12311,16 @@ class KnowledgeSpaceService(KnowledgeUtils): ) -> None: """Enqueue title extraction and AI alias generation before formal parsing.""" from bisheng.worker.knowledge.file_title_worker import ( - extract_and_generate_alias, + extract_knowledge_file_title_celery, ) if len(process_files) != len(preview_cache_keys): raise ValueError("process_files and preview_cache_keys length mismatch") for index, knowledge_file in enumerate(process_files): - extract_and_generate_alias(knowledge_file.id) + extract_knowledge_file_title_celery.delay( + knowledge_file.id, + preview_cache_keys[index], + ) @staticmethod def enqueue_file_processing( diff --git a/src/backend/bisheng/worker/knowledge/file_title_worker.py b/src/backend/bisheng/worker/knowledge/file_title_worker.py index d90d9eee6..51e2ae71a 100644 --- a/src/backend/bisheng/worker/knowledge/file_title_worker.py +++ b/src/backend/bisheng/worker/knowledge/file_title_worker.py @@ -27,6 +27,25 @@ def extract_and_generate_alias(file_id: int) -> str | None: logger.warning("title extraction skipped, file not found file_id={}", file_id) return None + logger.info( + "title extraction preparing file_id={} file_name={} status={} object_name={} tenant_id={}", + file_id, + db_file.file_name, + db_file.status, + db_file.object_name, + db_file.tenant_id, + ) + if db_file.status != KnowledgeFileStatus.WAITING.value: + logger.info( + "title extraction skipped, file status={} is not WAITING file_id={}", + db_file.status, + file_id, + ) + return None + if not db_file.object_name: + logger.warning("title extraction skipped, missing object_name file_id={}", file_id) + return None + try: with tempfile.TemporaryDirectory() as tmp_dir: local_path, _ = download_minio_file( @@ -34,11 +53,23 @@ def extract_and_generate_alias(file_id: int) -> str | None: root_dir=tmp_dir, calc_sha256=False, ) + logger.info( + "title extraction downloaded file_id={} local_path={} exists={}", + file_id, + local_path, + os.path.exists(local_path) if local_path else False, + ) if not local_path or not os.path.exists(local_path): logger.warning("title extraction skipped, download failed file_id={}", file_id) return None raw_title = FileTitleExtractorService.extract_title(local_path) + logger.info( + "title extraction result file_id={} file_name={} raw_title={}", + file_id, + db_file.file_name, + raw_title, + ) if not raw_title: logger.info("no title extracted file_id={} file_name={}", file_id, db_file.file_name) return None @@ -50,6 +81,12 @@ def extract_and_generate_alias(file_id: int) -> str | None: invoke_user_id=db_file.user_id or 0, tenant_id=db_file.tenant_id or 1, ) + logger.info( + "alias generation result file_id={} file_name={} alias_name={}", + file_id, + db_file.file_name, + alias_name, + ) if alias_name and alias_name != db_file.alias_name: db_file.alias_name = alias_name KnowledgeFileDao.update(db_file)