mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-10-28 02:36:45 +08:00
Add global log level configuration support across multiple modules
This commit is contained in:
@@ -94,9 +94,10 @@ OCR 模式通过每次 `POST /v1/parse/jobs` 请求的 `ocr_mode` 设置,可
|
||||
### 服务日志级别
|
||||
|
||||
`mineru-kit api-server`、`mineru-api` 和 `python -m mineru.parser.api_server` 均支持 `--log-level`,
|
||||
可选 `critical`、`error`、`warning`、`info`、`debug`、`trace`,大小写不敏感,直接启动时默认 `info`。
|
||||
该参数控制 Uvicorn 的启停、HTTP 访问、ASGI 日志及 API 服务自身日志。模型加载、解析阶段的模型日志、
|
||||
模型原生日志和 tqdm 进度条保留各自原有设置。
|
||||
可选 `critical`、`error`、`warning`、`info`、`debug`、`trace`,大小写不敏感。未显式传入时使用全局
|
||||
`log.level`(默认 `info`,可被 `MINERU_LOG_LEVEL` 覆盖)。该参数控制 Uvicorn 的启停、HTTP 访问、
|
||||
ASGI 日志、API 服务自身日志,以及模型加载/解析阶段 Loguru 默认 stderr sink。宿主程序显式添加的
|
||||
Loguru sink 和 tqdm 进度条保留各自原有设置。
|
||||
|
||||
```bash
|
||||
mineru-kit api-server --log-level warning
|
||||
|
||||
+7
-1
@@ -118,6 +118,11 @@ MinerU VLM 的跨页单元格检测保持关闭。任一功能启用时,`api_k
|
||||
- 隐私优先:任何配置都不能导致静默上传文档。
|
||||
- 质量优先:PDF/image 主动阅读未指定 tier 时使用默认选择策略,不能静默降级到 `flash`;OFD/EPUB/Office/HTML/CSV 按实际能力归一为 `flash`;其它 text 直接读取。
|
||||
|
||||
全局日志级别由 `log.level` 控制,默认 `info`,可选 `trace`、`debug`、`info`、
|
||||
`warning`、`error`、`critical`。环境变量 `MINERU_LOG_LEVEL` 可覆盖 YAML;
|
||||
该配置作用于当前进程的 Loguru 默认 stderr sink,不会移除宿主程序显式添加的
|
||||
自定义 sink。`doclib.log.level` 与 api-server 启动参数 `--log-level` 是局部覆盖。
|
||||
|
||||
## 2. 两阶段配置模型
|
||||
|
||||
### 2.1 启动前配置
|
||||
@@ -128,6 +133,7 @@ MinerU VLM 的跨页单元格检测保持关闭。任一功能启用时,`api_k
|
||||
|
||||
| 分组 | 字段 | 默认值 | 说明 |
|
||||
|------|------|--------|------|
|
||||
| 全局日志 | `log.level` | `info` | Loguru 默认 stderr sink 级别;可用 `MINERU_LOG_LEVEL` 覆盖 |
|
||||
| UDS | `doclib.uds.enabled` | `auto` | `auto` / `true` / `false`;`auto` 时当前 Python runtime 支持 UDS 则启用,否则关闭 |
|
||||
| UDS | `doclib.uds.path` | `~/.mineru/doclib.sock` | CLI / doclib 通信 socket,默认取 `$MINERU_HOME/doclib.sock` |
|
||||
| UDS | `doclib.uds.permission` | `0o600` | socket 权限 |
|
||||
@@ -145,7 +151,7 @@ MinerU VLM 的跨页单元格检测保持关闭。任一功能启用时,`api_k
|
||||
| log | `doclib.log.stderr_path` | unset → `<dir>/doclib.stderr.log` | server 子进程 stderr fallback 日志路径 |
|
||||
| log | `doclib.log.parse_server_stdout_path` | unset → `<dir>/doclib.parse-server.stdout.log` | managed parse-server stdout 日志路径 |
|
||||
| log | `doclib.log.parse_server_stderr_path` | unset → `<dir>/doclib.parse-server.stderr.log` | managed parse-server stderr 日志路径 |
|
||||
| log | `doclib.log.level` | `info` | 日志级别 |
|
||||
| log | `doclib.log.level` | unset → `log.level` | Doclib 进程日志级别局部覆盖 |
|
||||
| doclib | `doclib.endpoint_path` | `~/.mineru/doclib.endpoint.json` | 当前 server 实际可用 endpoint discovery 文件 |
|
||||
| doclib | `doclib.data_dir` | `~/.mineru/doclib` | 数据目录,默认取 `$MINERU_HOME/doclib`,但仍可通过配置文件或环境变量覆盖 |
|
||||
| doclib | `doclib.managed_parse_server.host` | `127.0.0.1` | managed parse-server 监听地址 |
|
||||
|
||||
@@ -13,6 +13,7 @@ from ..config import VlmConfig, config
|
||||
from ..integrations.docvortex import build_metadata, read_source_properties
|
||||
from ..types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson
|
||||
from ..utils.async_utils import run_sync
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..version import __version__ as mineru_version
|
||||
from .analysis.contracts import AnalysisResult, AnalyzeEffort, OfficeSuffix, ParseMode
|
||||
|
||||
@@ -40,6 +41,7 @@ def doc_analyze(
|
||||
source_properties: DocumentProperties | None = None,
|
||||
) -> tuple[MiddleJson, ModelJson]:
|
||||
"""生产严格 ModelJson,并在统一边界构造严格 MiddleJson。"""
|
||||
configure_global_log_level()
|
||||
_validate_analyze(effort, file_suffix, page_index_map)
|
||||
|
||||
if source_properties is None:
|
||||
@@ -98,6 +100,7 @@ async def aio_doc_analyze(
|
||||
source_properties: DocumentProperties | None = None,
|
||||
) -> tuple[MiddleJson, ModelJson]:
|
||||
"""vLLM/HTTP 的 PDF 分析使用原生异步编排,其余路径保持受控线程回退。"""
|
||||
configure_global_log_level()
|
||||
_validate_analyze(effort, file_suffix, page_index_map)
|
||||
native_async = False
|
||||
if file_suffix == "pdf" and effort in {"high", "xhigh"}:
|
||||
|
||||
@@ -6,6 +6,7 @@ import typer
|
||||
from click.core import Context
|
||||
from typer.core import TyperGroup
|
||||
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..utils.stdio import configure_standard_streams
|
||||
from .commands import cleanup, config, list_resources, server, show, telemetry, usage, watch
|
||||
from .commands.forget import forget_cmd
|
||||
@@ -93,6 +94,7 @@ app.command("version")(version_cmd)
|
||||
|
||||
def main() -> None:
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
app()
|
||||
|
||||
|
||||
|
||||
+30
-1
@@ -28,6 +28,7 @@ MINERU_ENV_PREFIX = "MINERU_"
|
||||
AutoBool = Literal["auto"] | bool
|
||||
ConfigSource = Literal["default", "file", "env"]
|
||||
ModelSource = Literal["auto", "huggingface", "modelscope", "local"]
|
||||
LogLevel = Literal["trace", "debug", "info", "warning", "error", "critical"]
|
||||
|
||||
_INTERPOLATION_RE = re.compile(r"\$\{(\w+)(?::-([^${}]*))?\}")
|
||||
|
||||
@@ -43,6 +44,13 @@ def _default_path(path1: str, /, *paths: str) -> str:
|
||||
return os.path.join(_mineru_home(), path1, *paths)
|
||||
|
||||
|
||||
def _normalize_log_level(value: LogLevel | str | None) -> LogLevel | str | None:
|
||||
"""归一化日志级别输入;None 表示未设置局部覆盖。"""
|
||||
if value is None or not isinstance(value, str):
|
||||
return value
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def _uds_available() -> bool:
|
||||
try:
|
||||
socket.socket(socket.AF_UNIX, socket.SOCK_STREAM).close()
|
||||
@@ -302,6 +310,18 @@ class TCPConfig(BaseModel):
|
||||
timeout: int = 600
|
||||
|
||||
|
||||
class GlobalLogConfig(BaseModel):
|
||||
"""MinerU 全局日志配置,控制当前进程的 Loguru 默认输出。"""
|
||||
|
||||
level: LogLevel = "info"
|
||||
|
||||
@field_validator("level", mode="before")
|
||||
@classmethod
|
||||
def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
|
||||
"""允许配置文件和环境变量使用大小写混写,并忽略首尾空白。"""
|
||||
return _normalize_log_level(value)
|
||||
|
||||
|
||||
class LogConfig(BaseModel):
|
||||
dir: str = _default_path("logs")
|
||||
app_path: str | None = None
|
||||
@@ -310,7 +330,13 @@ class LogConfig(BaseModel):
|
||||
stderr_path: str | None = None
|
||||
parse_server_stdout_path: str | None = None
|
||||
parse_server_stderr_path: str | None = None
|
||||
level: str = "info"
|
||||
level: LogLevel | None = None
|
||||
|
||||
@field_validator("level", mode="before")
|
||||
@classmethod
|
||||
def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
|
||||
"""归一化 Doclib 的局部日志级别覆盖。"""
|
||||
return _normalize_log_level(value)
|
||||
|
||||
@property
|
||||
def resolved_app_path(self) -> str:
|
||||
@@ -532,6 +558,7 @@ class DoclibConfig(BaseModel):
|
||||
class Config(BaseModel):
|
||||
"""Top-level MinerU startup configuration."""
|
||||
|
||||
log: GlobalLogConfig = Field(default_factory=GlobalLogConfig)
|
||||
doclib: DoclibConfig = Field(default_factory=DoclibConfig)
|
||||
render: RenderConfig = Field(default_factory=RenderConfig)
|
||||
llm_aided: LLMAidedConfig = Field(default_factory=LLMAidedConfig)
|
||||
@@ -551,6 +578,8 @@ __all__ = [
|
||||
"AutoBool",
|
||||
"ConfigSource",
|
||||
"LoadedConfig",
|
||||
"LogLevel",
|
||||
"GlobalLogConfig",
|
||||
"ModelConfig",
|
||||
"VlmConfig",
|
||||
"ModelSource",
|
||||
|
||||
+11
-2
@@ -21,6 +21,7 @@ from loguru import logger as loguru_logger
|
||||
|
||||
from ..config import Config, LogConfig, _mineru_home, config, get_config_source
|
||||
from ..errors import MineruError, error_response, http_status_for
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..utils.stdio import configure_standard_streams
|
||||
from .endpoint import EndpointTransport, read_endpoint_file, remove_endpoint_file, uds_available, write_endpoint_file
|
||||
from .instance_lock import DoclibLockUnavailable, build_doclib_home_owned_message, doclib_home_lock
|
||||
@@ -442,6 +443,7 @@ def _bind_tcp_socket(host: str, port: int, *, strict_port: bool, port_probe_coun
|
||||
def main() -> None:
|
||||
"""Entry point: python -m mineru.doclib.app"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
try:
|
||||
with doclib_home_lock():
|
||||
_run_server(config)
|
||||
@@ -466,7 +468,13 @@ def _run_server(cfg: Config) -> None:
|
||||
|
||||
app = create_app(cfg)
|
||||
server_id = app.state.doclib_state.server_id
|
||||
log_source = get_config_source("doclib.log.level") if cfg is config else "provided"
|
||||
log_source = (
|
||||
get_config_source("doclib.log.level")
|
||||
if cfg is config and cfg.doclib.log.level is not None
|
||||
else get_config_source("log.level")
|
||||
if cfg is config
|
||||
else "provided"
|
||||
)
|
||||
logger.info(
|
||||
"Doclib logging configured level=%s source=%s path=%s",
|
||||
logging.getLevelName(_resolve_log_level(cfg.doclib.log)),
|
||||
@@ -630,7 +638,8 @@ def _remove_owned_uds_path(uds_path: str, expected_identity: tuple[int, int]) ->
|
||||
|
||||
|
||||
def _resolve_log_level(log_cfg: LogConfig) -> int:
|
||||
return getattr(logging, log_cfg.level.upper(), logging.INFO)
|
||||
level = log_cfg.level or config.log.level
|
||||
return getattr(logging, level.upper(), logging.INFO)
|
||||
|
||||
|
||||
def _setup_logging(log_cfg: LogConfig) -> None:
|
||||
|
||||
@@ -8,6 +8,7 @@ import typer
|
||||
from ...parser import api_server as parser_api_server
|
||||
from ...types import SERVER_TIERS, ServerTier
|
||||
from ...model.ocr.language import PUBLIC_OCR_LANGUAGES, validate_public_ocr_lang
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from ..errors import exit_with_message
|
||||
|
||||
@@ -62,10 +63,13 @@ def api_server_cmd(
|
||||
vlm_max_concurrency: int | None = typer.Option(
|
||||
None, "--vlm-max-concurrency", min=1, help="VLM inference concurrency (default: 100)"
|
||||
),
|
||||
log_level: str = typer.Option(
|
||||
"info",
|
||||
log_level: str | None = typer.Option(
|
||||
None,
|
||||
"--log-level",
|
||||
help="API service log level: critical, error, warning, info, debug, trace; preserves model logs and progress bars",
|
||||
help=(
|
||||
"API service log level: critical, error, warning, info, debug, trace; "
|
||||
"default: global log.level. Also filters the Loguru default model-log sink"
|
||||
),
|
||||
),
|
||||
) -> None:
|
||||
"""转发显式启动参数,启动 self-hosted MinerU 解析 API 服务。"""
|
||||
@@ -92,8 +96,7 @@ def api_server_cmd(
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
"--log-level",
|
||||
log_level,
|
||||
*(["--log-level", log_level] if log_level is not None else []),
|
||||
"--concurrency",
|
||||
str(concurrency),
|
||||
"--url-timeout",
|
||||
@@ -126,6 +129,7 @@ def api_server_cmd(
|
||||
def main() -> None:
|
||||
"""配置标准流后,以独立命令运行新版 API 服务入口。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
typer.run(api_server_cmd)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from ...model.registry import (
|
||||
from ...model.runtime.device import resolve_small_model_backend
|
||||
from ...model.vlm.selector import resolve_vlm_engine
|
||||
from ...types import DEPLOYMENT_TIERS
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from ..errors import exit_with_message
|
||||
from ..output import print_info, print_success
|
||||
@@ -202,6 +203,7 @@ def verify_cmd(
|
||||
def download_main() -> None:
|
||||
"""配置标准流后直接运行模型下载命令,无需再输入 download 子命令。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
typer.run(download_cmd)
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import typer
|
||||
from loguru import logger
|
||||
|
||||
from ...model.runtime.platform import is_mac_os_version_supported
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from ..errors import exit_with_message
|
||||
|
||||
@@ -119,6 +120,7 @@ def vlm_server_cmd(
|
||||
def main() -> None:
|
||||
"""以独立命令启动 VLM 服务,复用 kit 的未知引擎参数透传规则。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
app = typer.Typer(add_completion=False)
|
||||
app.command(context_settings=FORWARD_CONTEXT_SETTINGS)(vlm_server_cmd)
|
||||
app()
|
||||
|
||||
@@ -12,6 +12,7 @@ from packaging.specifiers import SpecifierSet
|
||||
|
||||
from ...model.ocr.language import validate_public_ocr_lang
|
||||
from ...types import SERVER_TIERS, ServerTier
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from ..errors import exit_with_message
|
||||
|
||||
@@ -125,6 +126,7 @@ def webui_cmd(
|
||||
def main() -> None:
|
||||
"""以独立 console script 运行Web UI 命令。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
typer.run(webui_cmd)
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from urllib.parse import quote
|
||||
from ...errors import MineruError
|
||||
from ...filetypes import FLASH_ONLY_PARSE_EXTENSIONS, IMAGE_EXTENSIONS, OFFICE_EXTENSIONS, PARSEABLE_EXTENSIONS, PDF_EXTENSIONS
|
||||
from ...types import TIERS, Tier
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from .artifacts import RunArtifacts, persist_parse_result, render_download, render_html_preview
|
||||
from .client import (
|
||||
@@ -992,6 +993,7 @@ def launch_gradio(
|
||||
) -> None:
|
||||
"""启动 Gradio,并在外部服务缺少 Flash 时托管本地 Flash V1 服务。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
validate_max_pages(max_pages)
|
||||
resolved_api_key = api_key if api_key is not None else os.environ.get("MINERU_API_KEY")
|
||||
output_root = Path(output_dir).expanduser().resolve()
|
||||
|
||||
@@ -7,6 +7,7 @@ from click.core import Context
|
||||
from typer.core import TyperGroup
|
||||
|
||||
from ..cli.version_command import show_version, version_cmd
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..utils.stdio import configure_standard_streams
|
||||
from .commands import api_server, models, parse, router, vlm_server, webui
|
||||
|
||||
@@ -60,6 +61,7 @@ app.command("version")(version_cmd)
|
||||
|
||||
def main() -> None:
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
app()
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import typer
|
||||
import uvicorn
|
||||
|
||||
from ...types import SERVER_TIERS, ServerTier
|
||||
from ...utils.logger import configure_global_log_level
|
||||
from ...utils.stdio import configure_standard_streams
|
||||
from ..errors import exit_with_message
|
||||
from .app import create_app
|
||||
@@ -64,6 +65,7 @@ def router_cmd(
|
||||
def main() -> None:
|
||||
"""配置标准流后运行过渡 `mineru-router` 单命令入口。"""
|
||||
configure_standard_streams()
|
||||
configure_global_log_level()
|
||||
typer.run(router_cmd)
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Literal
|
||||
from docvortex.document.contracts import HtmlSourceContext
|
||||
|
||||
from ..config import VlmConfig
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..types import Tier
|
||||
from .api_client import ApiJobStatus, MinerUApiParser
|
||||
from .base import MIDDLE_JSON_SCHEMA_VERSION, DocumentParser, ParseResult
|
||||
@@ -38,6 +39,7 @@ def parse(
|
||||
vlm_config: VlmConfig | None = None,
|
||||
) -> ParseResult:
|
||||
"""同步解析文档;source_context 仅供保留 HTML 原始来源的内部调用方使用。"""
|
||||
configure_global_log_level()
|
||||
parser = MinerUParser(tier=tier, parse_mode=ocr_mode, image_analysis=image_analysis, vlm_config=vlm_config)
|
||||
return parser.parse(path, page_range=page_range, source_context=source_context)
|
||||
|
||||
@@ -53,5 +55,6 @@ async def parse_async(
|
||||
vlm_config: VlmConfig | None = None,
|
||||
) -> ParseResult:
|
||||
"""异步解析文档;source_context 仅供保留 HTML 原始来源的内部调用方使用。"""
|
||||
configure_global_log_level()
|
||||
parser = MinerUParser(tier=tier, parse_mode=ocr_mode, image_analysis=image_analysis, vlm_config=vlm_config)
|
||||
return await parser.parse_async(path, page_range=page_range, source_context=source_context)
|
||||
|
||||
@@ -54,6 +54,7 @@ from ..model.ocr.language import PUBLIC_OCR_LANGUAGES, validate_public_ocr_lang
|
||||
from ..types import SERVER_TIERS, TIERS_BY_SERVER_TIER, DeploymentTier, PageInfo, ServerTier, Tier, select_default_quality_tier
|
||||
from ..utils.async_utils import drain_future, run_sync
|
||||
from ..model.vlm.async_runtime import RuntimeOwner, runtime_owner
|
||||
from ..utils.logger import configure_global_log_level
|
||||
from ..utils.stdio import configure_standard_streams
|
||||
from ..version import __version__
|
||||
from . import parse_async
|
||||
@@ -2439,10 +2440,12 @@ def _build_server_log_config(log_level: str) -> dict[str, Any]:
|
||||
@click.option("--port", default=8000, type=int, help="Server port")
|
||||
@click.option(
|
||||
"--log-level",
|
||||
default="info",
|
||||
default=None,
|
||||
type=click.Choice(["critical", "error", "warning", "info", "debug", "trace"], case_sensitive=False),
|
||||
show_default=True,
|
||||
help="API service log level; model logs and progress bars keep their existing settings.",
|
||||
help=(
|
||||
"API service log level: critical, error, warning, info, debug, trace; "
|
||||
"default: global log.level. Also filters the Loguru default model-log sink."
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--upload-dir",
|
||||
@@ -2550,10 +2553,11 @@ def main(
|
||||
vlm_model: str | None,
|
||||
vlm_http_timeout: int | None,
|
||||
vlm_max_concurrency: int | None,
|
||||
log_level: str,
|
||||
log_level: str | None,
|
||||
) -> None:
|
||||
"""合并显式 VLM 参数后启动 MinerU v1 REST API 服务,不修改全局配置。"""
|
||||
configure_standard_streams()
|
||||
effective_log_level = configure_global_log_level(log_level)
|
||||
shutdown_requested = threading.Event()
|
||||
server_ref: list[uvicorn.Server | None] = [None]
|
||||
|
||||
@@ -2615,8 +2619,8 @@ def main(
|
||||
application,
|
||||
host=host,
|
||||
port=port,
|
||||
log_level=log_level,
|
||||
log_config=_build_server_log_config(log_level),
|
||||
log_level=effective_log_level,
|
||||
log_config=_build_server_log_config(effective_log_level),
|
||||
)
|
||||
server = uvicorn.Server(config)
|
||||
server_ref[0] = server
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""MinerU 进程内 Loguru 日志级别配置。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from ..config import LogLevel
|
||||
|
||||
_managed_sink_ids: set[int] = set()
|
||||
_configured_level: LogLevel | None = None
|
||||
_explicit_level: LogLevel | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _active_managed_sink_ids() -> set[int]:
|
||||
"""返回仍然存在的 MinerU 托管 sink,避免其它入口清理后使用失效 ID。"""
|
||||
# Loguru 未公开枚举 handler 的稳定 API;这里只做只读检查,不修改私有状态。
|
||||
from loguru import logger
|
||||
|
||||
return _managed_sink_ids.intersection(logger._core.handlers)
|
||||
|
||||
|
||||
def configure_global_log_level(level: LogLevel | str | None = None) -> LogLevel:
|
||||
"""配置当前进程的 Loguru 默认 stderr sink,并返回归一化后的级别。
|
||||
|
||||
``level`` 为 ``None`` 时使用 ``config.log.level``;显式传入后作为进程级
|
||||
局部覆盖,后续默认调用不会把它改回全局配置。函数只移除 Loguru 初始
|
||||
sink 和 MinerU 自己创建的 sink,不会影响宿主程序显式添加的自定义 sink。
|
||||
"""
|
||||
from loguru import logger
|
||||
|
||||
from ..config import GlobalLogConfig, config
|
||||
|
||||
global _configured_level, _explicit_level
|
||||
|
||||
with _lock:
|
||||
if level is None:
|
||||
normalized = _explicit_level or config.log.level
|
||||
else:
|
||||
normalized = GlobalLogConfig(level=level).level
|
||||
_explicit_level = normalized
|
||||
active_ids = _active_managed_sink_ids()
|
||||
has_initial_sink = 0 in logger._core.handlers
|
||||
if _configured_level == normalized and (active_ids or not has_initial_sink):
|
||||
return normalized
|
||||
if not active_ids and not has_initial_sink:
|
||||
# 宿主或 Doclib 可能已刻意移除默认 stderr sink;此时尊重现有配置,不重新添加。
|
||||
_configured_level = normalized
|
||||
return normalized
|
||||
|
||||
for sink_id in active_ids:
|
||||
logger.remove(sink_id)
|
||||
_managed_sink_ids.clear()
|
||||
|
||||
# Loguru 的初始 sink 固定使用 ID 0;如果宿主已经移除它,则忽略即可。
|
||||
try:
|
||||
logger.remove(0)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
managed_sink_id = logger.add(sys.stderr, level=normalized.upper())
|
||||
_managed_sink_ids.add(managed_sink_id)
|
||||
_configured_level = normalized
|
||||
return normalized
|
||||
|
||||
|
||||
__all__ = ["LogLevel", "configure_global_log_level"]
|
||||
@@ -37,14 +37,19 @@ def test_log_level_reaches_server(monkeypatch: pytest.MonkeyPatch, entrypoint: s
|
||||
application = object()
|
||||
create_app = MagicMock(return_value=application)
|
||||
config = MagicMock(return_value=object())
|
||||
expected = (level or "debug").lower()
|
||||
configure_global = MagicMock(return_value=expected)
|
||||
server = SimpleNamespace(run=MagicMock(), should_exit=False)
|
||||
monkeypatch.setattr(api_server.mineru_config.log, "level", "debug")
|
||||
monkeypatch.setattr(api_server, "create_app", create_app)
|
||||
monkeypatch.setattr(api_server, "configure_global_log_level", configure_global)
|
||||
monkeypatch.setattr(kit_api, "configure_global_log_level", MagicMock(return_value="debug"))
|
||||
monkeypatch.setattr(api_server.uvicorn, "Config", config)
|
||||
monkeypatch.setattr(api_server.uvicorn, "Server", MagicMock(return_value=server))
|
||||
monkeypatch.setattr(api_server.ManagedProcessControlWatcher, "from_environment", lambda callback: None)
|
||||
|
||||
assert _invoke_entrypoint(monkeypatch, entrypoint, [] if level is None else ["--log-level", level]) == 0
|
||||
expected = (level or "info").lower()
|
||||
configure_global.assert_called_once_with(None if level is None else expected)
|
||||
config.assert_called_once()
|
||||
assert config.call_args.args == (application,)
|
||||
assert config.call_args.kwargs["log_level"] == expected
|
||||
@@ -69,8 +74,8 @@ def test_invalid_log_level_fails_before_initialization(monkeypatch: pytest.Monke
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", ["info", "warning", "error"])
|
||||
def test_service_log_output_is_filtered_without_changing_model_logs(level: str) -> None:
|
||||
"""在独立进程验证真实日志输出、重复配置以及 root、模型日志和 tqdm 的隔离。"""
|
||||
def test_service_log_output_is_filtered_while_preserving_explicit_model_sink(level: str) -> None:
|
||||
"""在独立进程验证真实日志输出、重复配置以及 root、显式模型日志和 tqdm 的隔离。"""
|
||||
script = r"""
|
||||
import io
|
||||
import json
|
||||
|
||||
@@ -296,6 +296,14 @@ def test_bind_tcp_socket_strict_port_does_not_probe(monkeypatch: pytest.MonkeyPa
|
||||
assert bind_calls == [("127.0.0.1", 15980)]
|
||||
|
||||
|
||||
def test_doclib_log_level_inherits_global_and_accepts_local_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""验证 Doclib 未设置级别时继承全局配置,显式设置时局部覆盖。"""
|
||||
monkeypatch.setattr(doclib_app.config.log, "level", "debug")
|
||||
|
||||
assert doclib_app._resolve_log_level(LogConfig()) == logging.DEBUG
|
||||
assert doclib_app._resolve_log_level(LogConfig(level="warning")) == logging.WARNING
|
||||
|
||||
|
||||
def test_setup_logging_routes_application_logs_to_rotating_file_without_stderr_duplication(tmp_path: Path) -> None:
|
||||
_clear_test_loggers()
|
||||
log_path = tmp_path / "doclib.log"
|
||||
|
||||
@@ -278,6 +278,52 @@ def test_legacy_llm_aided_root_key_is_ignored() -> None:
|
||||
assert cfg.llm_aided.features.title_leveling is False
|
||||
|
||||
|
||||
def test_default_global_and_doclib_log_levels() -> None:
|
||||
"""验证全局日志默认 info,且 Doclib 默认不设置局部覆盖。"""
|
||||
cfg = Config()
|
||||
|
||||
assert cfg.log.level == "info"
|
||||
assert cfg.doclib.log.level is None
|
||||
|
||||
|
||||
def test_global_log_level_reads_yaml_and_tracks_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""验证 YAML 可配置全局日志级别并正确记录配置来源。"""
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(
|
||||
"""
|
||||
log:
|
||||
level: " DEBUG "
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("MINERU_CONFIG", str(config_file))
|
||||
|
||||
loaded = _load_effective_config()
|
||||
|
||||
assert loaded.config.log.level == "debug"
|
||||
assert loaded.sources[("log", "level")] == "file"
|
||||
|
||||
|
||||
def test_global_log_level_env_overrides_yaml_and_normalizes_value(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""验证环境变量覆盖 YAML,并归一化大小写与空白。"""
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text("log:\n level: debug\n", encoding="utf-8")
|
||||
monkeypatch.setenv("MINERU_CONFIG", str(config_file))
|
||||
monkeypatch.setenv("MINERU_LOG_LEVEL", " WARNING ")
|
||||
|
||||
loaded = _load_effective_config()
|
||||
|
||||
assert loaded.config.log.level == "warning"
|
||||
assert loaded.sources[("log", "level")] == "env"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", ["", "silent", "success"])
|
||||
def test_global_log_level_rejects_invalid_values(level: str) -> None:
|
||||
"""验证非法全局日志级别在配置模型边界被拒绝。"""
|
||||
with pytest.raises(ValueError):
|
||||
Config(log={"level": level})
|
||||
|
||||
|
||||
def test_default_doclib_data_dir_uses_doclib_directory() -> None:
|
||||
cfg = Config()
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""验证全局 Loguru 日志级别配置和自定义 sink 兼容行为。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import pytest
|
||||
|
||||
_SCRIPT = r"""
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from contextlib import redirect_stderr
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from mineru.utils.logger import configure_global_log_level
|
||||
|
||||
mode = sys.argv[1]
|
||||
if mode == "filter":
|
||||
level = sys.argv[2]
|
||||
with redirect_stderr(io.StringIO()) as stderr:
|
||||
configure_global_log_level(level)
|
||||
logger.debug("DEBUG_MARKER")
|
||||
logger.info("INFO_MARKER")
|
||||
print(json.dumps({"stderr": stderr.getvalue()}))
|
||||
elif mode == "custom-sink":
|
||||
custom_stream = io.StringIO()
|
||||
custom_sink = logger.add(custom_stream, level="INFO", format="{message}")
|
||||
with redirect_stderr(io.StringIO()) as stderr:
|
||||
configure_global_log_level("ERROR")
|
||||
logger.info("CUSTOM_BEFORE")
|
||||
configure_global_log_level("ERROR")
|
||||
logger.info("CUSTOM_AFTER")
|
||||
logger.error("DEFAULT_ERROR")
|
||||
logger.remove(custom_sink)
|
||||
print(json.dumps({"custom": custom_stream.getvalue(), "stderr": stderr.getvalue()}))
|
||||
else:
|
||||
with redirect_stderr(io.StringIO()) as stderr:
|
||||
configure_global_log_level("ERROR")
|
||||
configure_global_log_level()
|
||||
logger.info("INFO_SHOULD_NOT_APPEAR")
|
||||
logger.error("ERROR_SHOULD_APPEAR")
|
||||
print(json.dumps({"stderr": stderr.getvalue()}))
|
||||
"""
|
||||
|
||||
|
||||
def _run_script(mode: str, *args: str) -> dict[str, str]:
|
||||
"""在独立进程中执行日志行为脚本,避免污染当前测试进程的全局 Loguru 状态。"""
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", _SCRIPT, mode, *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("level", "expected_debug"), [("info", False), ("debug", True)])
|
||||
def test_global_log_level_filters_default_stderr_sink(level: str, expected_debug: bool) -> None:
|
||||
"""验证显式级别会过滤 Loguru 默认 stderr sink。"""
|
||||
output = _run_script("filter", level)
|
||||
|
||||
assert ("DEBUG_MARKER" in output["stderr"]) is expected_debug
|
||||
assert "INFO_MARKER" in output["stderr"]
|
||||
|
||||
|
||||
def test_repeated_configuration_preserves_custom_sink() -> None:
|
||||
"""验证幂等配置不会移除宿主显式添加的自定义 sink。"""
|
||||
output = _run_script("custom-sink")
|
||||
|
||||
assert output["custom"].count("CUSTOM_BEFORE") == 1
|
||||
assert output["custom"].count("CUSTOM_AFTER") == 1
|
||||
assert "DEFAULT_ERROR" in output["stderr"]
|
||||
|
||||
|
||||
def test_default_call_reads_global_environment_config() -> None:
|
||||
"""验证未显式传级别时读取 MINERU_LOG_LEVEL 对应的全局配置。"""
|
||||
script = (
|
||||
"import io, json; from contextlib import redirect_stderr; "
|
||||
"from loguru import logger; "
|
||||
"from mineru.utils.logger import configure_global_log_level; "
|
||||
"stream = io.StringIO(); "
|
||||
"redirect = redirect_stderr(stream); redirect.__enter__(); "
|
||||
"configure_global_log_level(); logger.info('HIDDEN'); logger.error('VISIBLE'); "
|
||||
"redirect.__exit__(None, None, None); "
|
||||
"print(json.dumps({'stderr': stream.getvalue()}))"
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
env={**os.environ, "MINERU_LOG_LEVEL": "error"},
|
||||
)
|
||||
output = json.loads(completed.stdout)
|
||||
|
||||
assert "HIDDEN" not in output["stderr"]
|
||||
assert "VISIBLE" in output["stderr"]
|
||||
|
||||
|
||||
def test_explicit_level_persists_across_default_calls() -> None:
|
||||
"""验证局部显式级别不会被后续默认配置回退。"""
|
||||
output = _run_script("explicit-persist")
|
||||
|
||||
assert "INFO_SHOULD_NOT_APPEAR" not in output["stderr"]
|
||||
assert "ERROR_SHOULD_APPEAR" in output["stderr"]
|
||||
|
||||
|
||||
def test_helper_module_has_no_runtime_side_effect() -> None:
|
||||
"""验证导入日志工具模块不会修改 Loguru sink。"""
|
||||
script = (
|
||||
"from mineru.utils import logger as module; from loguru import logger; print(json.dumps(list(logger._core.handlers)))"
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", f"import json; {script}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
assert json.loads(completed.stdout) == [0]
|
||||
Reference in New Issue
Block a user