mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-10-25 09:35:17 +08:00
Refactor model download storage
Add model repo registry and tier-based model downloads. Move model source/base_dir handling into config.yaml and remove legacy mineru.json model path handling. Update runtime model path resolution, Docker/docs, and tests.
This commit is contained in:
jinzhenj
@@ -21,7 +21,7 @@ RUN python3 -m pip install -U 'mineru[core]>=3.4.0' -i https://mirrors.aliyun.co
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -21,7 +21,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
@@ -25,7 +25,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
@@ -24,7 +24,7 @@ RUN python3 -m pip install "mineru[core]>=3.4.0" \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -24,7 +24,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -32,7 +32,7 @@ RUN /opt/conda/bin/python3 -m pip install -U pip -i https://mirrors.aliyun.com/p
|
||||
/opt/conda/bin/python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "/opt/conda/bin/mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "/opt/conda/bin/mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -36,7 +36,7 @@ RUN /bin/bash -c '\
|
||||
if [ "$BACKEND" = "vllm" ]; then \
|
||||
source /torch/venv3/pytorch_infer/bin/activate; \
|
||||
fi && \
|
||||
mineru-kit models download all -s modelscope'
|
||||
mineru-kit models download --tier high -s modelscope'
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -29,7 +29,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN TORCH_DEVICE_BACKEND_AUTOLOAD=0 /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN TORCH_DEVICE_BACKEND_AUTOLOAD=0 /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -28,7 +28,7 @@ RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s modelscope"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s modelscope"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -21,7 +21,7 @@ RUN python3 -m pip install -U 'mineru[core]>=3.4.0' --break-system-packages && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
# Download models and update the configuration file
|
||||
RUN /bin/bash -c "mineru-kit models download all -s huggingface"
|
||||
RUN /bin/bash -c "mineru-kit models download --tier high -s huggingface"
|
||||
|
||||
# Set the entry point to activate the virtual environment and run the command line tool
|
||||
ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
|
||||
|
||||
@@ -9,7 +9,7 @@ MinerU uses `HuggingFace` and `ModelScope` as model repositories. Users can swit
|
||||
## Methods to Switch Model Sources
|
||||
|
||||
### Configure via Environment Variables
|
||||
MinerU configures model sources through the `MINERU_MODEL_SOURCE` environment variable. This applies to all command line tools and API calls. Supported values are `huggingface`, `modelscope`, and `local`. The environment variable has higher priority than `model-source` in `mineru.json`. Do not set this environment variable to `auto`; unset it if you want MinerU to choose a source automatically.
|
||||
MinerU configures model sources through the `MINERU_MODEL_SOURCE` environment variable. This applies to all command line tools and API calls. Supported values are `auto`, `huggingface`, `modelscope`, and `local`. The environment variable has higher priority than `model.source` in `config.yaml`.
|
||||
```bash
|
||||
export MINERU_MODEL_SOURCE=modelscope
|
||||
mineru -p <input_path> -o <output_path>
|
||||
@@ -23,11 +23,11 @@ os.environ["MINERU_MODEL_SOURCE"] = "modelscope"
|
||||
> MinerU no longer provides a CLI flag for model source selection. Model sources set through environment variables take effect in the current terminal session until the terminal is closed or the environment variable is modified.
|
||||
|
||||
### Configure via Configuration File
|
||||
If `MINERU_MODEL_SOURCE` is not set, MinerU reads the `model-source` field from `mineru.json` in the user directory. `model-source` supports fixed values `huggingface` and `modelscope`, and also supports the template's first-run placeholder value `auto`. When the value is `auto` or the field is missing, MinerU probes the actual source first. After the first auto probe resolves an actual source, MinerU writes `model-source` back as `huggingface` or `modelscope` to avoid switching sources on later startups due to network fluctuations.
|
||||
```json
|
||||
{
|
||||
"model-source": "auto"
|
||||
}
|
||||
If `MINERU_MODEL_SOURCE` is not set, MinerU reads `model.source` from `config.yaml`. `model.source` supports `auto`, `huggingface`, `modelscope`, and `local`. When the value is `auto` or the field is missing, MinerU probes the actual source first. If the value came from the config file or built-in default, MinerU writes the resolved source back as `huggingface` or `modelscope` to avoid switching sources on later startups due to network fluctuations.
|
||||
```yaml
|
||||
model:
|
||||
source: auto
|
||||
base_dir: ~/.mineru/models
|
||||
```
|
||||
|
||||
## Using Local Models
|
||||
@@ -38,15 +38,14 @@ mineru-kit models download --help
|
||||
```
|
||||
or download all built-in model bundles:
|
||||
```bash
|
||||
mineru-kit models download all
|
||||
mineru-kit models download --tier high
|
||||
```
|
||||
> [!NOTE]
|
||||
>- After download completion, the model path will be output in the current terminal window and automatically written to `mineru.json` in the user directory. The `model-source` field records the actual remote source used for this download, either `huggingface` or `modelscope`.
|
||||
>- You can also create it by copying the [configuration template file](https://github.com/opendatalab/MinerU/blob/master/mineru.template.json) to your user directory and renaming it to `mineru.json`. The template sets `model-source` to `auto`, so MinerU auto-detects once and writes back the resolved source on first use.
|
||||
>- After downloading models locally, you can freely move the model folder to other locations while updating the model path in `mineru.json`.
|
||||
>- If you deploy the model folder to another server, please ensure you move the `mineru.json` file to the user directory of the new device and configure the model path correctly.
|
||||
>- If you need to update model files, you can run `mineru-kit models download all` again. Model updates do not support custom paths currently - if you haven't moved the local model folder, model files will be incrementally updated; if you have moved the model folder, model files will be re-downloaded to the default location and `mineru.json` will be updated.
|
||||
>- `mineru-kit models download` must use a remote model source to perform a real download. If your current shell already sets `MINERU_MODEL_SOURCE=local`, this command will temporarily ignore that value for this invocation and use your selected `auto`, `huggingface`, or `modelscope` source instead.
|
||||
>- Models are downloaded under `config.model.base_dir`. By default, this is `~/.mineru/models`.
|
||||
>- `mineru-kit models download` does not write model paths to `mineru.json`.
|
||||
>- If you need a custom model directory, set `model.base_dir` in `config.yaml` before downloading.
|
||||
>- If you need to update model files, run `mineru-kit models download --tier high` again. Existing files in the same `model.base_dir` are incrementally reused by the provider SDK.
|
||||
>- `mineru-kit models download` must use a remote model source to perform a real download. If your current config sets `model.source: local`, this command temporarily treats it as `auto` for this invocation.
|
||||
|
||||
### 2. Use Local Models for Parsing
|
||||
|
||||
|
||||
@@ -116,10 +116,7 @@ If you need to adjust parsing options through custom parameters, you can also ch
|
||||
|
||||
## Extending MinerU Functionality with Configuration Files
|
||||
|
||||
MinerU is now ready to use out of the box, but also supports extending functionality through configuration files. You can edit `mineru.json` file in your user directory to add custom configurations.
|
||||
|
||||
>[!IMPORTANT]
|
||||
>The `mineru.json` file will be automatically generated when you use the built-in model download command `mineru-kit models download all`, or you can create it by copying the [configuration template file](https://github.com/opendatalab/MinerU/blob/master/mineru.template.json) to your user directory and renaming it to `mineru.json`.
|
||||
MinerU is now ready to use out of the box, but also supports extending functionality through configuration files. Legacy tool options such as LaTeX delimiters and LLM-aided title hierarchy still use `mineru.json` in your user directory. Model storage and model source settings use `config.yaml`; see [Model Source Documentation](./model_source.md).
|
||||
|
||||
Here are some available configuration options:
|
||||
|
||||
@@ -152,7 +149,3 @@ Here are some available configuration options:
|
||||
}
|
||||
```
|
||||
|
||||
- `models-dir`:
|
||||
* Used to specify local model storage directory
|
||||
* Please specify model directories for the local lightweight model bundle (`models-dir.pipeline`) and the VLM bundle (`models-dir.vlm`) separately.
|
||||
* After specifying the directory, you can use local models by configuring the environment variable `export MINERU_MODEL_SOURCE=local`.
|
||||
|
||||
@@ -17,7 +17,7 @@ Chat Completions 和 Responses 面向细粒度文档理解场景，例如单张
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"model": "MinerU2.5-Pro-2605-1.2B",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
@@ -97,7 +97,7 @@ Chat Completions 和 Responses 面向细粒度文档理解场景，例如单张
|
||||
"id": "chatcmpl-B9MBs8CjcvOU2jLn4n570S5qMJKcT",
|
||||
"object": "chat.completion",
|
||||
"created": 1741569952,
|
||||
"model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"model": "MinerU2.5-Pro-2605-1.2B",
|
||||
"system_fingerprint": "fp_44709d6fcb",
|
||||
"choices": [
|
||||
{
|
||||
@@ -137,9 +137,9 @@ Chat Completions 和 Responses 面向细粒度文档理解场景，例如单张
|
||||
`stream=true` 时返回 SSE，每行 `data:` 后是一个 `chat.completion.chunk` JSON 对象，最后以 `data: [DONE]` 结束。
|
||||
|
||||
```text
|
||||
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"MinerU2.5-Pro-2604-1.2B","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}
|
||||
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"MinerU2.5-Pro-2605-1.2B","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}
|
||||
|
||||
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"MinerU2.5-Pro-2604-1.2B","choices":[{"index":0,"delta":{"content":"# 文本"},"finish_reason":null}]}
|
||||
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1694268190,"model":"MinerU2.5-Pro-2605-1.2B","choices":[{"index":0,"delta":{"content":"# 文本"},"finish_reason":null}]}
|
||||
|
||||
data: [DONE]
|
||||
```
|
||||
@@ -178,7 +178,7 @@ Chat 错误:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"model": "MinerU2.5-Pro-2605-1.2B",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
@@ -236,7 +236,7 @@ Content part:
|
||||
"object": "response",
|
||||
"created_at": 1741476542,
|
||||
"status": "completed",
|
||||
"model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"model": "MinerU2.5-Pro-2605-1.2B",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
|
||||
@@ -214,7 +214,7 @@ OCR 策略和图片分析能力由 `tier` 与服务端实际引擎自动决定
|
||||
"page_range": "1~12",
|
||||
"status": "completed",
|
||||
"parse": {
|
||||
"model_used": "MinerU2.5-Pro-2604-1.2B",
|
||||
"model_used": "MinerU2.5-Pro-2605-1.2B",
|
||||
"duration_ms": 8234,
|
||||
"parser_version": "3.1.14"
|
||||
},
|
||||
|
||||
@@ -3,76 +3,97 @@
|
||||
状态: Draft
|
||||
读者: 服务部署者、批处理开发者、解析内核开发者
|
||||
范围: `mineru-kit models` 的模型下载、查看和校验
|
||||
非目标: parse-server 生命周期；新配置体系迁移；模型清理与回收
|
||||
来源: 由根目录旧 CLI 底稿迁移整理而来
|
||||
非目标: parse-server 生命周期；模型清理与回收
|
||||
来源: ADR-0025
|
||||
|
||||
## 1. 定位
|
||||
|
||||
`mineru-kit models` 是 `mineru-kit` 的模型管理命令组。
|
||||
|
||||
第一阶段目标只有三个：
|
||||
第一阶段目标只有三个:
|
||||
|
||||
- 下载模型
|
||||
- 查看当前模型配置
|
||||
- 轻量校验模型配置和关键路径
|
||||
- 按 tier 或模型仓库下载模型
|
||||
- 查看当前模型配置和 readiness
|
||||
- 校验模型仓库关键路径
|
||||
|
||||
第一阶段不引入更重的模型管理能力，例如删除、清理、GC 或手工目录登记。
|
||||
|
||||
## 2. 配置文件
|
||||
|
||||
第一阶段继续使用旧配置文件体系：
|
||||
模型配置使用 `config.yaml`:
|
||||
|
||||
- 默认：`~/mineru.json`
|
||||
- 可由环境变量 `MINERU_TOOLS_CONFIG_JSON` 指定其它路径
|
||||
- 默认路径: `${MINERU_HOME:-~/.mineru}/config.yaml`
|
||||
- 可由环境变量 `MINERU_CONFIG` 指定其它路径
|
||||
|
||||
命令组默认读写其中的：
|
||||
模型相关配置:
|
||||
|
||||
- `models-dir.pipeline`
|
||||
- `models-dir.vlm`
|
||||
```yaml
|
||||
model:
|
||||
base_dir: ${MINERU_HOME:-~/.mineru}/models
|
||||
source: auto
|
||||
```
|
||||
|
||||
当前不切换到新的全局配置体系。
|
||||
`model.base_dir` 是所有 MinerU 模型仓库的根目录。当前模型仓库会落在:
|
||||
|
||||
- `{model.base_dir}/PDF-Extract-Kit-1.0`
|
||||
- `{model.base_dir}/MinerU2.5-Pro-2605-1.2B`
|
||||
|
||||
`model.source` 支持:
|
||||
|
||||
- `auto`
|
||||
- `huggingface`
|
||||
- `modelscope`
|
||||
- `local`
|
||||
|
||||
`MINERU_MODEL_SOURCE` 会覆盖 `model.source`。当环境变量覆盖为 `auto` 时，不会把自动探测结果写回配置文件。
|
||||
|
||||
## 3. 子命令
|
||||
|
||||
### 3.1 `mineru-kit models download`
|
||||
|
||||
下载指定模型 bundle，并在完成后更新配置文件。
|
||||
下载指定模型仓库，或下载某个 tier 需要的模型仓库。
|
||||
|
||||
```bash
|
||||
mineru-kit models download <pipeline|vlm|all> [flags]
|
||||
mineru-kit models download <repo> [flags]
|
||||
mineru-kit models download --tier <flash|medium|high|xhigh> [flags]
|
||||
```
|
||||
|
||||
参数：
|
||||
参数:
|
||||
|
||||
| Flag | 简写 | 类型 | 默认 | 说明 |
|
||||
|------|------|------|------|------|
|
||||
| `--source` | `-s` | `auto \| huggingface \| modelscope` | `auto` | 下载源 |
|
||||
| `--verbose` | `-v` | bool | false | 输出详细日志 |
|
||||
| `--tier` | - | `flash \| medium \| high \| xhigh` | - | 按 tier 下载所需模型 |
|
||||
| `--source` | `-s` | `auto \| huggingface \| modelscope` | 配置值 | 本次下载源 |
|
||||
| `--verbose` | `-v` | bool | false | 输出详细路径 |
|
||||
|
||||
规则：
|
||||
规则:
|
||||
|
||||
- `bundle` 使用位置参数，必须显式给出
|
||||
- `--source auto` 会先探测 Hugging Face 是否可访问，并在本次下载中使用解析出的真实远端源
|
||||
- 不支持交互式 prompt
|
||||
- 下载完成后默认更新配置文件
|
||||
- 不支持 `--no-config`
|
||||
- 不支持 `--config`
|
||||
- 不支持显式设置模型下载目录
|
||||
- 保持当前下载行为：由 Hugging Face / ModelScope 下载器自身决定缓存落点，MinerU 只记录最终模型目录
|
||||
- repo 位置参数与 `--tier` 互斥
|
||||
- 不带 repo 且不带 `--tier` 会报错
|
||||
- `--tier flash` 不需要模型，返回成功
|
||||
- `--source` 不支持 `local`
|
||||
- 如果当前配置为 `model.source: local`，显式 download 会临时按 `auto` 解析远端源，不改写配置
|
||||
- 下载目标固定为 `config.model.base_dir` 下的 repo local dir
|
||||
|
||||
`bundle` 的含义：
|
||||
支持的 repo 名:
|
||||
|
||||
- `pipeline`: 下载 Pipeline 解析所需模型集合
|
||||
- `vlm`: 下载 VLM 解析所需模型仓
|
||||
- `all`: 同时下载 `pipeline` 与 `vlm`
|
||||
- `PDF-Extract-Kit-1.0`
|
||||
- `MinerU2.5-Pro-2605-1.2B`
|
||||
|
||||
示例：
|
||||
Tier 到 repo 的映射:
|
||||
|
||||
- `flash`: 不需要模型
|
||||
- `medium`: `PDF-Extract-Kit-1.0`
|
||||
- `high`: `PDF-Extract-Kit-1.0` + `MinerU2.5-Pro-2605-1.2B`
|
||||
- `xhigh`: `PDF-Extract-Kit-1.0` + `MinerU2.5-Pro-2605-1.2B`
|
||||
|
||||
示例:
|
||||
|
||||
```bash
|
||||
mineru-kit models download pipeline
|
||||
mineru-kit models download vlm --source modelscope
|
||||
mineru-kit models download all --source auto
|
||||
mineru-kit models download all
|
||||
mineru-kit models download --tier medium
|
||||
mineru-kit models download --tier high --source modelscope
|
||||
mineru-kit models download PDF-Extract-Kit-1.0
|
||||
mineru-kit models download MinerU2.5-Pro-2605-1.2B --source huggingface
|
||||
```
|
||||
|
||||
### 3.2 `mineru-kit models show`
|
||||
@@ -83,67 +104,46 @@ mineru-kit models download all
|
||||
mineru-kit models show
|
||||
```
|
||||
|
||||
建议输出内容：
|
||||
输出内容:
|
||||
|
||||
- 当前实际使用的配置文件路径
|
||||
- `model-source`
|
||||
- `models-dir.pipeline`
|
||||
- `models-dir.vlm`
|
||||
- 当前 `MINERU_MODEL_SOURCE` 环境值
|
||||
- 上述目录是否存在
|
||||
- 当前实际使用的 `config.yaml` 路径
|
||||
- `MINERU_MODEL_SOURCE`
|
||||
- `model.base_dir` 和来源
|
||||
- `model.source` 和来源
|
||||
- 每个 repo 的 local dir 和 readiness
|
||||
- 每个 tier 需要的 repo 集合
|
||||
|
||||
第一阶段不支持 `--json`。
|
||||
|
||||
### 3.3 `mineru-kit models verify`
|
||||
|
||||
轻量校验模型配置与关键路径。
|
||||
|
||||
```bash
|
||||
mineru-kit models verify [pipeline|vlm|all]
|
||||
```
|
||||
|
||||
规则：
|
||||
|
||||
- 默认校验全部已配置 bundle
|
||||
- 也可显式指定 `pipeline`、`vlm` 或 `all`
|
||||
- 不是单纯目录存在性检查，还会检查关键子路径是否存在
|
||||
- 第一阶段不做 hash 级完整性校验
|
||||
|
||||
示例：
|
||||
校验模型仓库关键路径。
|
||||
|
||||
```bash
|
||||
mineru-kit models verify
|
||||
mineru-kit models verify pipeline
|
||||
mineru-kit models verify vlm
|
||||
mineru-kit models verify <repo>
|
||||
mineru-kit models verify --tier <flash|medium|high|xhigh>
|
||||
```
|
||||
|
||||
## 4. 当前 bundle 定义
|
||||
规则:
|
||||
|
||||
当前项目的模型 bundle 划分如下：
|
||||
- 默认校验全部 repo
|
||||
- repo 位置参数与 `--tier` 互斥
|
||||
- `--tier flash` 直接成功
|
||||
- 不是单纯目录存在性检查，还会检查 registry 中声明的关键路径
|
||||
- 第一阶段不做 hash 级完整性校验
|
||||
|
||||
- `pipeline`
|
||||
- `vlm`
|
||||
- `all = pipeline + vlm`
|
||||
示例:
|
||||
|
||||
其中：
|
||||
```bash
|
||||
mineru-kit models verify
|
||||
mineru-kit models verify PDF-Extract-Kit-1.0
|
||||
mineru-kit models verify --tier high
|
||||
```
|
||||
|
||||
- `vlm` 对应单个 VLM 模型仓
|
||||
- `pipeline` 对应一组版面、OCR、公式和表格相关模型路径
|
||||
|
||||
命令层不把这些内部模型再拆成更多顶级下载对象。
|
||||
|
||||
## 5. 非目标
|
||||
|
||||
第一阶段暂不支持：
|
||||
|
||||
- 模型目录显式自定义
|
||||
- `--json`
|
||||
- 模型删除 / 清理 / GC
|
||||
- 通过命令设置多个模型目录
|
||||
- 切换到新的全局配置文件体系
|
||||
|
||||
## 6. 相关文档
|
||||
## 4. 相关文档
|
||||
|
||||
- [ADR-0019](../decisions/0019-mineru-kit-models-command.md)
|
||||
- [ADR-0025](../decisions/0025-model-download-local-dir.md)
|
||||
- [mineru-kit](mineru-kit.md)
|
||||
- [mineru-kit parse](mineru-kit-parse.md)
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
- ../cli/mineru-kit-models.md
|
||||
- ../cli/README.md
|
||||
|
||||
更新: ADR-0025 已取代本文中关于模型配置落点、`pipeline` / `vlm` / `all` bundle 参数，以及下载后写入 `mineru.json` 的决策。本文保留为 `mineru-kit models` 命令组建立背景记录。
|
||||
|
||||
## 背景
|
||||
|
||||
旧 CLI 提供 `mineru-models-download`，用于下载 Pipeline 或 VLM 模型，并在下载完成后更新 `mineru.json` 中的 `models-dir` 配置。
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
# ADR-0025: 模型下载与本地模型目录
|
||||
|
||||
状态: Accepted
|
||||
日期: 2026-07-13
|
||||
相关文档:
|
||||
- ../cli/mineru-kit-models.md
|
||||
- ../config.md
|
||||
- 0019-mineru-kit-models-command.md
|
||||
|
||||
本 ADR supersedes ADR-0019 中关于模型配置落点、`pipeline` / `vlm` / `all` bundle 命令语义，以及下载后写入 `mineru.json` 的决策。ADR-0019 中关于建立 `mineru-kit models` 命令组的总体方向仍保留。
|
||||
|
||||
## 背景
|
||||
|
||||
MinerU 当前模型下载逻辑仍依赖旧 `mineru.json` 中的 `models-dir` 和 `model-source` 字段。下载完成后，MinerU 会记录 Hugging Face 或 ModelScope 返回的模型根目录。这个设计有几个问题:
|
||||
|
||||
- 模型路径/source 配置与新的 `config.yaml` 配置体系割裂。
|
||||
- 下载结果被写入配置文件，配置文件既表达用户意图，又混入运行时探测结果。
|
||||
- Hugging Face 和 ModelScope 默认 cache 目录结构不同，难以形成统一的本地模型库语义。
|
||||
- 首次使用本地 managed parser 时，模型是否已准备好不容易在配置阶段判断。
|
||||
- `mineru-kit models verify`、managed tier readiness 检查、parse 懒下载需要共享一套模型定位和校验规则。
|
||||
- `pipeline` / `vlm` 是旧实现分组名，不是用户真正关心的模型库或解析能力；继续暴露这些 bundle 会让模型下载命令和 tier 语义脱节。
|
||||
|
||||
经过验证，Hugging Face 和 ModelScope 都支持 `local_dir`，并且可以把同一模型仓库展开到同一个目录。两者的 provider 元数据不会互相覆盖:
|
||||
|
||||
- Hugging Face 使用 `.cache/huggingface/...`
|
||||
- ModelScope 使用 `.msc`、`.mv`、`._____temp`
|
||||
|
||||
在不并发写同一目录的前提下，两个 provider 可以顺序 materialize 到同一个 `local_dir`。payload 文件最终只存储一份；但切换 provider 时仍可能重复下载大文件，因此不能把该目录视为 provider 共享 cache。
|
||||
|
||||
## 决策
|
||||
|
||||
### 1. 模型配置迁移到 `config.yaml`
|
||||
|
||||
在顶层 `Config` 中增加 `model` 配置，与 `doclib` 同级:
|
||||
|
||||
```yaml
|
||||
model:
|
||||
base_dir: ${MINERU_HOME:-~/.mineru}/models
|
||||
source: auto
|
||||
```
|
||||
|
||||
`model.source` 支持以下取值:
|
||||
|
||||
| 值 | 语义 |
|
||||
|----|------|
|
||||
| `auto` | 探测网络可达性后选择 `huggingface` 或 `modelscope` |
|
||||
| `huggingface` | 强制使用 Hugging Face 下载 |
|
||||
| `modelscope` | 强制使用 ModelScope 下载 |
|
||||
| `local` | 禁止 parse 懒下载和自动下载，只使用已经存在且 ready 的本地模型；显式 `mineru-kit models download` 例外 |
|
||||
|
||||
`MINERU_MODEL_SOURCE` 作为 `config.model.source` 的环境变量覆盖项，沿用 `config.py` 现有环境变量合并规则。
|
||||
|
||||
### 2. `auto` 探测结果写回当前配置文件
|
||||
|
||||
当 `model.source=auto` 时，MinerU 会探测当前环境应使用 Hugging Face 还是 ModelScope。探测成功后，可以把实际来源写回当前上下文中的配置文件:
|
||||
|
||||
- 如果 `MINERU_CONFIG` 指向自定义配置文件，则写入该文件。
|
||||
- 否则写入默认 `~/.mineru/config.yaml`。
|
||||
|
||||
写回只更新:
|
||||
|
||||
```yaml
|
||||
model:
|
||||
source: huggingface
|
||||
```
|
||||
|
||||
或:
|
||||
|
||||
```yaml
|
||||
model:
|
||||
source: modelscope
|
||||
```
|
||||
|
||||
下载结果、模型路径和 provider 返回的 snapshot/cache 路径不写入配置文件。
|
||||
|
||||
当前进程应继续使用本次解析出的 source，不依赖重新 import `config` 后再生效。
|
||||
|
||||
### 2.1 配置文件原子写入与来源追踪
|
||||
|
||||
`config.py` 需要提供通用配置文件写入和来源查询能力，而不是只为 `model.source` 写专用逻辑。
|
||||
|
||||
新增能力:
|
||||
|
||||
```python
|
||||
ConfigSource = Literal["default", "file", "env"]
|
||||
|
||||
def get_config_source(path: str | Sequence[str]) -> ConfigSource:
|
||||
...
|
||||
|
||||
def get_config_file_path() -> str:
|
||||
...
|
||||
|
||||
def get_config_file_exists() -> bool:
|
||||
...
|
||||
|
||||
def update_config_file(patch: dict[str, Any]) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
来源追踪规则:
|
||||
|
||||
- 内置默认值来源为 `default`。
|
||||
- YAML 文件中出现的叶子字段来源为 `file`。
|
||||
- 环境变量覆盖的叶子字段来源为 `env`。
|
||||
|
||||
例如 `get_config_source("model.source")` 返回 `default`、`file` 或 `env`。
|
||||
|
||||
未设置 `MINERU_CONFIG` 且默认配置文件不存在时，默认路径仍是当前配置上下文路径，读取结果等价于空配置文件。`get_config_file_path()` 返回默认路径，`get_config_file_exists()` 返回 `False`。如果显式设置 `MINERU_CONFIG` 且文件不存在，仍应报错，避免把拼写错误静默变成新配置文件。
|
||||
|
||||
`update_config_file` 必须原子写入:
|
||||
|
||||
- 对目标配置文件加锁，例如 `{config_path}.lock`。
|
||||
- 读取现有 YAML；缺失或为空时按 `{}` 处理。
|
||||
- deep merge patch。
|
||||
- 写到同目录临时文件。
|
||||
- 使用 `os.replace(tmp, config_path)` 原子替换。
|
||||
- 保留未知字段。
|
||||
|
||||
写入目标:
|
||||
|
||||
- 如果 `MINERU_CONFIG` 指向自定义配置文件，写该文件。
|
||||
- 否则写默认 `~/.mineru/config.yaml`。
|
||||
|
||||
`auto` 探测后的写回规则:
|
||||
|
||||
- 只有 `model.source` 的来源是 `default` 或 `file`，且有效值为 `auto` 时，才写回 resolved source。
|
||||
- 如果 `model.source` 来源是 `env`，即使环境变量值是 `auto`，也不写回配置文件。
|
||||
- 如果 `model.source=local`，显式 download 命令可临时按 auto 下载，但不得把 `local` 覆盖掉。
|
||||
|
||||
### 3. 模型目录由 MinerU 管理
|
||||
|
||||
所有 MinerU 使用的模型仓库都展开到:
|
||||
|
||||
```text
|
||||
{config.model.base_dir}/{spec.local_name}
|
||||
```
|
||||
|
||||
例如:
|
||||
|
||||
```text
|
||||
~/.mineru/models/PDF-Extract-Kit-1.0
|
||||
~/.mineru/models/MinerU2.5-Pro-2605-1.2B
|
||||
```
|
||||
|
||||
Hugging Face 和 ModelScope 共用同一个 `local_dir`。该目录语义是 MinerU 管理的模型展开目录，不是 Hugging Face / ModelScope 的共享 cache。
|
||||
|
||||
### 4. 移除模型路径/source 的 `mineru.json` 依赖
|
||||
|
||||
新模型下载功能不再读取或写入 `mineru.json` 中的:
|
||||
|
||||
- `models-dir`
|
||||
- `model-source`
|
||||
|
||||
`mineru.json` 不再提供任何与模型路径或模型 source 有关的配置。仍需要保留的旧工具配置应与模型下载逻辑解耦。
|
||||
|
||||
### 5. 增加模型 registry，并移除 bundle 概念
|
||||
|
||||
新增:
|
||||
|
||||
```text
|
||||
mineru/utils/model_registry.py
|
||||
```
|
||||
|
||||
用于集中描述模型仓库和 repo 内部路径。
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class ModelPath:
|
||||
repo: ModelRepo
|
||||
name: str
|
||||
relative_path: str
|
||||
|
||||
def ensure(self, *, source: ModelSource | None = None) -> Path:
|
||||
...
|
||||
|
||||
def local_path(self) -> Path:
|
||||
...
|
||||
|
||||
def path(self, relative_path: str, /, *children: str) -> ModelPath:
|
||||
...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelRepo:
|
||||
name: str
|
||||
local_name: str
|
||||
repos: dict[str, str]
|
||||
paths: dict[str, str]
|
||||
|
||||
def ensure(self, *, source: ModelSource | None = None) -> Path:
|
||||
...
|
||||
|
||||
def path(self, name: str) -> ModelPath:
|
||||
...
|
||||
|
||||
def __getattr__(self, name: str) -> ModelPath:
|
||||
...
|
||||
```
|
||||
|
||||
不再使用 `pipeline` / `vlm` 作为用户可见 bundle 名。registry 使用模型仓库原名:
|
||||
|
||||
- `PDF-Extract-Kit-1.0`
|
||||
- `MinerU2.5-Pro-2605-1.2B`
|
||||
|
||||
`PIPELINE_MODEL_PATHS` 和 `VLM_MODEL_MARKERS` 迁入 registry，避免 core 下载逻辑依赖 `mineru/kit/common.py`。旧 `ModelPath` 类中的模型仓库 root 和 repo 内路径也迁入 registry；`mineru/utils/enum_class.py` 不再承载模型路径配置。
|
||||
|
||||
示例:
|
||||
|
||||
```python
|
||||
PDF_EXTRACT_KIT = ModelRepo(
|
||||
name="PDF-Extract-Kit-1.0",
|
||||
local_name="PDF-Extract-Kit-1.0",
|
||||
repos={
|
||||
"huggingface": "opendatalab/PDF-Extract-Kit-1.0",
|
||||
"modelscope": "OpenDataLab/PDF-Extract-Kit-1.0",
|
||||
},
|
||||
paths={
|
||||
"pp_doclayout_v2": "models/Layout/PP-DocLayoutV2",
|
||||
"unimernet_small": "models/MFR/unimernet_hf_small_2503",
|
||||
"pp_formulanet_plus_m": "models/MFR/pp_formulanet_plus_m",
|
||||
"pytorch_paddle": "models/OCR/paddleocr_torch",
|
||||
"slanet_plus": "models/TabRec/SlanetPlus/slanet-plus.onnx",
|
||||
"unet_structure": "models/TabRec/UnetStructure/unet.onnx",
|
||||
"paddle_table_cls": "models/TabCls/paddle_table_cls/PP-LCNet_x1_0_table_cls.onnx",
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
调用方可以使用绑定 repo 的路径对象:
|
||||
|
||||
```python
|
||||
PDF_EXTRACT_KIT.ensure()
|
||||
PDF_EXTRACT_KIT.pp_doclayout_v2.ensure()
|
||||
PDF_EXTRACT_KIT.path("pp_doclayout_v2").ensure()
|
||||
```
|
||||
|
||||
`PDF_EXTRACT_KIT.pp_doclayout_v2` 不提供静态 type hint。为保持实现简单，第一版接受 `__getattr__` 形式，并同时提供 `path(name)` 作为显式入口。
|
||||
|
||||
`ModelRepo.path()` 和 `ModelPath.path()` 都返回绑定同一 repo 的 `ModelPath`，用于支持更细粒度的局部下载。`path()` 参数语义是相对路径片段，不混用 registry path key。
|
||||
|
||||
示例:
|
||||
|
||||
```python
|
||||
PDF_EXTRACT_KIT.path("models/OCR/paddleocr_torch").ensure()
|
||||
PDF_EXTRACT_KIT.pytorch_paddle.path(det_model_name).ensure()
|
||||
PDF_EXTRACT_KIT.pytorch_paddle.path(rec_model_name).ensure()
|
||||
```
|
||||
|
||||
### 6. Tier 到模型仓库的映射
|
||||
|
||||
模型下载命令不再使用 `pipeline` / `vlm` bundle。需要下载哪些模型由 tier 决定:
|
||||
|
||||
```python
|
||||
REPOS_FOR_TIER = {
|
||||
"flash": (),
|
||||
"medium": (PDF_EXTRACT_KIT,),
|
||||
"high": (PDF_EXTRACT_KIT, MINERU_2_5_PRO_2605_1_2B),
|
||||
"xhigh": (PDF_EXTRACT_KIT, MINERU_2_5_PRO_2605_1_2B),
|
||||
}
|
||||
```
|
||||
|
||||
语义:
|
||||
|
||||
- `flash` 不需要本地模型下载。
|
||||
- `medium` 需要 `PDF-Extract-Kit-1.0`。
|
||||
- `high` 和 `xhigh` 需要 `PDF-Extract-Kit-1.0` 与 `MinerU2.5-Pro-2605-1.2B`。
|
||||
|
||||
repo registry 不反向依赖 tier。tier 只选择需要的 repo 集合，repo 自身仍只描述模型仓库。
|
||||
|
||||
### 7. 下载 API
|
||||
|
||||
提供两个公开下载 API:
|
||||
|
||||
```python
|
||||
def download_model_repo(repo: ModelRepo, *, source: ModelSource | None = None) -> Path:
|
||||
...
|
||||
|
||||
def download_model_files(repo: ModelRepo, paths: Sequence[str | ModelPath], *, source: ModelSource | None = None) -> Path:
|
||||
...
|
||||
```
|
||||
|
||||
内部使用一个统一实现，避免整仓下载和局部下载逻辑分叉:
|
||||
|
||||
```python
|
||||
def _download_model(
|
||||
repo: ModelRepo,
|
||||
*,
|
||||
paths: Sequence[str | ModelPath] | None,
|
||||
source: ModelSource | None,
|
||||
) -> Path:
|
||||
...
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- `download_model_repo` 下载整仓，并验证 repo 的完整 required paths。
|
||||
- `download_model_files` 只下载指定子集，并只验证本次请求的 paths。
|
||||
- 文件路径直接加入 `allow_patterns`。
|
||||
- 目录路径展开为 `path` 和 `path/*`。
|
||||
- Hugging Face 与 ModelScope 都通过 `snapshot_download(..., local_dir=...)` 实现。
|
||||
- 第一版不为单文件下载分叉到 `hf_hub_download` 或 `model_file_download`。
|
||||
|
||||
### 8. 移除旧兼容 wrapper
|
||||
|
||||
不再保留旧函数:
|
||||
|
||||
```python
|
||||
auto_download_and_get_model_root_path(relative_path: str, repo_mode: str = "pipeline") -> str
|
||||
```
|
||||
|
||||
原因:
|
||||
|
||||
- 当前内部调用点已迁移到 registry-aware API。
|
||||
- `repo_mode="pipeline"` / `repo_mode="vlm"` 会继续泄漏旧 bundle 名。
|
||||
- 旧 wrapper 返回 repo root，调用方再拼接相对路径，容易重新引入路径拼接错误。
|
||||
|
||||
代码应直接使用:
|
||||
|
||||
```python
|
||||
PDF_EXTRACT_KIT.pp_doclayout_v2.ensure()
|
||||
PDF_EXTRACT_KIT.ensure()
|
||||
```
|
||||
|
||||
### 9. Readiness 校验
|
||||
|
||||
模型是否 ready 由 MinerU 自己判断，不依赖 provider 元数据。
|
||||
|
||||
提供校验函数:
|
||||
|
||||
```python
|
||||
def verify_model_repo(repo: ModelRepo) -> ModelReadyResult:
|
||||
...
|
||||
|
||||
def is_model_repo_ready(repo: ModelRepo) -> bool:
|
||||
...
|
||||
|
||||
def verify_model_tier(tier: Tier) -> ModelReadyResult:
|
||||
...
|
||||
```
|
||||
|
||||
校验规则:
|
||||
|
||||
- `model.source=local` 时禁止下载，只做 ready 检查。
|
||||
- `mineru-kit models verify <repo>` 检查指定 repo 的完整 required paths。
|
||||
- `mineru-kit models verify --tier <tier>` 检查该 tier 需要的 repo 集合。
|
||||
- managed local parse server 的 tier readiness 检查基于 `REPOS_FOR_TIER`。
|
||||
- partial download 只验证本次请求 paths，不要求整仓 ready。
|
||||
|
||||
### 10. 并发安全
|
||||
|
||||
引入第三方依赖 `filelock`，并写入项目 dependencies。
|
||||
|
||||
每个模型目录对应一个锁文件:
|
||||
|
||||
```text
|
||||
{config.model.base_dir}/.locks/{safe_local_name}.lock
|
||||
```
|
||||
|
||||
锁保护整个模型 `local_dir`。以下操作必须在锁内执行:
|
||||
|
||||
- 整仓下载
|
||||
- 局部文件下载
|
||||
- 下载后的校验
|
||||
- 需要防止与下载并发冲突的模型删除或清理操作
|
||||
|
||||
锁文件不放在模型 `local_dir` 内，避免与 provider 写入的 payload 和元数据混在一起。
|
||||
|
||||
等待策略:
|
||||
|
||||
- `mineru-kit models download` 默认无限等待锁。
|
||||
- parse 懒下载默认等待锁，并记录正在等待模型下载锁。
|
||||
- 第一版不新增锁 timeout 配置。
|
||||
|
||||
文件锁只能保护遵守 MinerU 锁协议的进程。用户手工修改模型目录或外部脚本直接写入同一目录，不在该锁的保护范围内。
|
||||
|
||||
### 11. `mineru-kit models` 命令语义
|
||||
|
||||
模型管理命令继续挂在 `mineru-kit models` 下，不新增 `mineru models` 子命令。
|
||||
|
||||
`mineru-kit models download` 支持按 tier 或按模型仓库下载:
|
||||
|
||||
```bash
|
||||
mineru-kit models download --tier medium
|
||||
mineru-kit models download --tier high
|
||||
mineru-kit models download --tier xhigh
|
||||
mineru-kit models download PDF-Extract-Kit-1.0
|
||||
mineru-kit models download MinerU2.5-Pro-2605-1.2B
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
- repo 参数和 `--tier` 二选一。
|
||||
- 两者都不传时报错，不默认下载大模型。
|
||||
- 两者同时出现时报错。
|
||||
- `--tier flash` 成功 no-op，并提示 flash 不需要本地模型下载。
|
||||
- 不支持 repo alias；repo 参数只接受 registry 中的模型仓库主名。错误时列出可用 repo 名。
|
||||
- 继续保留 `--source`。
|
||||
- `--source` 只接受 `auto`、`huggingface`、`modelscope`；不接受 `local`。
|
||||
- `--source` 未传时为 `None`，表示使用 `config.model.source` 的有效值。
|
||||
|
||||
source 解析规则:
|
||||
|
||||
| 场景 | 行为 |
|
||||
|------|------|
|
||||
| `--source huggingface` | 本次强制使用 Hugging Face |
|
||||
| `--source modelscope` | 本次强制使用 ModelScope |
|
||||
| `--source auto` | 本次自动探测 |
|
||||
| 未传 `--source` 且 `config.model.source=huggingface` | 使用 Hugging Face |
|
||||
| 未传 `--source` 且 `config.model.source=modelscope` | 使用 ModelScope |
|
||||
| 未传 `--source` 且 `config.model.source=auto` | 自动探测，并可按 2.1 规则写回 |
|
||||
| 未传 `--source` 且 `config.model.source=local` | 显式 download 命令临时按 auto 下载，但不写回覆盖 `local` |
|
||||
|
||||
除 `local` 外，配置或环境变量中的无效 `model.source` 值按 `auto` 处理并记录 warning。CLI `--source` 的无效值必须报错，避免用户 typo 被静默解释为 auto。
|
||||
|
||||
`MINERU_MODEL_SOURCE=auto` 是合法环境覆盖，但由于来源是 `env`，不会触发写回配置文件。
|
||||
|
||||
`mineru-kit models show` 展示:
|
||||
|
||||
- 当前配置文件路径
|
||||
- `model.base_dir`
|
||||
- `model.source`
|
||||
- `MINERU_MODEL_SOURCE`
|
||||
- 每个模型仓库的 expected local dir
|
||||
- 每个 tier 需要的模型仓库
|
||||
- readiness 状态
|
||||
|
||||
`mineru-kit models verify` 使用与 download 相同的目标选择语义:
|
||||
|
||||
```bash
|
||||
mineru-kit models verify --tier high
|
||||
mineru-kit models verify PDF-Extract-Kit-1.0
|
||||
mineru-kit models verify
|
||||
```
|
||||
|
||||
不带 repo 且不带 `--tier` 时，`verify` 检查全部 registry repo。`verify` 不读取 `mineru.json`。
|
||||
|
||||
## 替代方案
|
||||
|
||||
### 方案 A: 继续记录 provider 返回的 cache/snapshot 路径
|
||||
|
||||
未采用。
|
||||
|
||||
原因:
|
||||
|
||||
- Hugging Face 与 ModelScope 的 cache 结构不同。
|
||||
- 配置文件会继续混入运行时下载结果。
|
||||
- readiness 检查必须理解 provider cache 细节，不利于稳定。
|
||||
|
||||
### 方案 B: HF 和 ModelScope 使用不同本地目录
|
||||
|
||||
未采用。
|
||||
|
||||
原因:
|
||||
|
||||
- 同一模型会长期占用两份空间。
|
||||
- 用户看到的本地模型库结构更复杂。
|
||||
- 已验证顺序写同一个 `local_dir` 基本可行，只需加锁和 readiness 校验。
|
||||
|
||||
### 方案 C: 使用 provider 共享 `cache_dir`
|
||||
|
||||
未采用。
|
||||
|
||||
原因:
|
||||
|
||||
- HF 和 ModelScope 的 cache 元数据、目录布局和一致性协议不同。
|
||||
- 共享 `cache_dir` 不是两边 SDK 的稳定契约。
|
||||
- 本 ADR 只共享最终展开的 `local_dir`，不共享 provider cache。
|
||||
|
||||
### 方案 D: 自己实现跨平台锁
|
||||
|
||||
未采用。
|
||||
|
||||
原因:
|
||||
|
||||
- `fcntl` 与 `msvcrt` 需要分别处理，超时和异常释放逻辑容易出错。
|
||||
- `filelock` 已是成熟小依赖，且 Hugging Face 生态中已有类似依赖使用经验。
|
||||
- 模型下载锁不是 MinerU 的核心差异化能力，不值得维护自研跨平台锁。
|
||||
|
||||
### 方案 E: 彻底移除 `auto_download_and_get_model_root_path`
|
||||
|
||||
采用。
|
||||
|
||||
原因:
|
||||
|
||||
- 内部调用点已经迁移到 registry-aware API。
|
||||
- 继续保留 wrapper 会保留 `pipeline` / `vlm` 旧概念。
|
||||
- 删除 wrapper 后，模型路径解析只有 `ModelRepo` / `ModelPath` 一条主路径。
|
||||
|
||||
### 方案 F: 继续使用 `pipeline` / `vlm` bundle 作为下载目标
|
||||
|
||||
未采用。
|
||||
|
||||
原因:
|
||||
|
||||
- `pipeline` / `vlm` 是旧实现分组名，不是模型仓库名。
|
||||
- 用户关心的是准备某个 tier 所需模型，或下载某个具体模型仓库。
|
||||
- 继续暴露 bundle 会让 `medium/high/xhigh` 与模型准备动作之间多一层间接概念。
|
||||
- 当前实际远端模型仓库只有两个，用真实仓库名表达更直接。
|
||||
|
||||
### 方案 G: 为每个 repo 编写强类型路径类或 decorator 语法
|
||||
|
||||
未采用。
|
||||
|
||||
候选形式包括:
|
||||
|
||||
```python
|
||||
PDF_EXTRACT_KIT.pp_doclayout_v2.ensure()
|
||||
```
|
||||
|
||||
并为 `pp_doclayout_v2` 提供完整静态 type hint，或通过 decorator 把类定义转换为 repo spec。
|
||||
|
||||
未采用的原因:
|
||||
|
||||
- 强类型 repo class 会让 registry 定义明显变复杂。
|
||||
- decorator 会引入运行时转换，不符合本项目“确定性优先”的编码原则。
|
||||
- 当前只有两个 repo，路径数量有限，使用 `ModelRepo` + `ModelPath` 已能显著降低调用端冗余。
|
||||
- 第一版接受 `PDF_EXTRACT_KIT.pp_doclayout_v2` 无静态属性 type hint，并保留 `PDF_EXTRACT_KIT.path("pp_doclayout_v2")` 作为显式入口。
|
||||
|
||||
## 影响
|
||||
|
||||
### 对配置
|
||||
|
||||
- 模型路径和 source 迁移到 `config.yaml`。
|
||||
- `MINERU_MODEL_SOURCE` 成为 `config.model.source` 的自然环境变量覆盖项。
|
||||
- `mineru.json` 不再承载模型路径/source。
|
||||
|
||||
### 对下载行为
|
||||
|
||||
- HF/MS 下载到同一个 MinerU-owned `local_dir`。
|
||||
- 最终模型 payload 只存储一份。
|
||||
- 切换 provider 时可能重复下载大文件，这是可接受的 provider fallback 成本。
|
||||
- 下载结果不再写入配置文件。
|
||||
|
||||
### 对 managed local parser
|
||||
|
||||
- 设置 `parse_server.local.mode=managed` 或 `parse_server.local.managed_tier` 时，可以基于 `config.model.base_dir` 做 readiness 检查。
|
||||
- 如果模型未 ready，应提示用户显式执行 `mineru-kit models download ...`。
|
||||
- 不改变 parse server 启动流程，不在 parse 阶段新增额外 readiness gate。
|
||||
|
||||
### 对实现边界
|
||||
|
||||
- `mineru/utils/model_registry.py` 成为模型仓库、repo 内路径和 tier-to-repo 映射的定义边界。
|
||||
- `models_download_utils.py` 负责 source 解析、锁、下载和 ready 检查。
|
||||
- `mineru/kit/common.py` 不再持有模型 required path 常量。
|
||||
- `mineru/utils/enum_class.py` 不再持有 `ModelPath`；该文件暂时只保留仍被非模型下载逻辑使用的枚举/常量。
|
||||
|
||||
### 对用户体验
|
||||
|
||||
- 本地模型目录稳定、可解释。
|
||||
- `mineru-kit models show/verify` 能直接展示模型仓库、tier 需求、expected local dir 和 readiness。
|
||||
- `mineru-kit models download --tier high` 直接表达“准备 high tier”，不要求用户理解旧 `pipeline` / `vlm` 分组。
|
||||
- 使用 `model.source=local` 时，离线部署行为更明确。
|
||||
- 首次模型准备可以从“parse 时静默下载”转为“配置/下载命令显式准备”。
|
||||
|
||||
## 后续动作
|
||||
|
||||
1. 在 `mineru/config.py` 增加 `ModelConfig`。
|
||||
2. 在 `mineru/config.py` 增加字段级来源查询和原子配置写入函数。
|
||||
3. 增加 `mineru/utils/model_registry.py`，定义 `PDF_EXTRACT_KIT`、`MINERU_2_5_PRO_2605_1_2B`、repo 内 `ModelPath` 和 `REPOS_FOR_TIER`。
|
||||
4. 重构 `mineru/utils/models_download_utils.py`，移除 `mineru.json` 模型路径/source 读写。
|
||||
5. 引入 `filelock` 依赖。
|
||||
6. 改造 `mineru-kit models download/show/verify`，支持 repo 参数和 `--tier` 选择器，移除 `pipeline` / `vlm` bundle 参数。
|
||||
7. 移除旧 `auto_download_and_get_model_root_path` wrapper。
|
||||
8. 在 managed local parser 配置变更时接入 readiness 检查。
|
||||
9. 将 API server tier metadata 中的模型名同步为 registry 中的 repo name。
|
||||
10. 更新 CLI 与配置文档，说明 `model.base_dir`、`model.source` 和 `local` source。
|
||||
11. 增加测试覆盖:
|
||||
- `MINERU_MODEL_SOURCE` 覆盖 `config.model.source`
|
||||
- `get_config_source("model.source")` 区分 `default`、`file`、`env`
|
||||
- `update_config_file` 并发/原子写入
|
||||
- `auto` 探测写回当前配置文件
|
||||
- `MINERU_MODEL_SOURCE=auto` 不写回配置文件
|
||||
- HF/MS 共用 `local_dir` 时锁路径稳定
|
||||
- `model.source=local` 缺失模型时报错
|
||||
- partial download 只验证请求路径
|
||||
- `ModelRepo.path()` 和 `ModelPath.path()` 支持更细粒度 partial download
|
||||
- `mineru-kit models download` 的 repo 参数与 `--tier` 互斥
|
||||
- `mineru-kit models download --tier flash` 为 no-op
|
||||
- `mineru-kit models verify` 检查 repo required paths 和 tier 所需 repo 集合
|
||||
@@ -52,6 +52,7 @@
|
||||
| [ADR-0022](0022-doclib-file-type-tier-remote-semantics.md) | Accepted | Doclib 文件类型、Tier 与 Remote 语义 |
|
||||
| [ADR-0023](0023-cli-runtime-contract.md) | Accepted | CLI Runtime 输出、错误与退出码集中契约 |
|
||||
| [ADR-0024](0024-file-type-tier-normalization.md) | Accepted | 文件类型与 Tier 归一化语义 |
|
||||
| [ADR-0025](0025-model-download-local-dir.md) | Accepted | 模型下载与本地模型目录 |
|
||||
|
||||
## 模板
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ MinerU使用 `HuggingFace` 和 `ModelScope` 作为模型仓库，用户可以根
|
||||
## 模型源的切换方法
|
||||
|
||||
### 通过环境变量切换
|
||||
MinerU 通过 `MINERU_MODEL_SOURCE` 环境变量配置模型源，这适用于所有命令行工具和 API 调用。支持的取值为 `huggingface`、`modelscope` 和 `local`，环境变量优先级高于 `mineru.json` 中的 `model-source`。请不要将环境变量设置为 `auto`；如需自动选择来源，请删除该环境变量。
|
||||
MinerU 通过 `MINERU_MODEL_SOURCE` 环境变量配置模型源，这适用于所有命令行工具和 API 调用。支持的取值为 `auto`、`huggingface`、`modelscope` 和 `local`，环境变量优先级高于 `config.yaml` 中的 `model.source`。
|
||||
```bash
|
||||
export MINERU_MODEL_SOURCE=modelscope
|
||||
mineru -p <input_path> -o <output_path>
|
||||
@@ -23,11 +23,11 @@ os.environ["MINERU_MODEL_SOURCE"] = "modelscope"
|
||||
> MinerU 已不再提供用于切换模型源的命令行参数。通过环境变量设置的模型源会在当前终端会话中生效，直到终端关闭或环境变量被修改。
|
||||
|
||||
### 通过配置文件切换
|
||||
如果未设置 `MINERU_MODEL_SOURCE`，MinerU 会读取用户目录下 `mineru.json` 中的 `model-source` 字段。`model-source` 支持固定值 `huggingface`、`modelscope`，也支持模板中的首次解析占位值 `auto`。当值为 `auto` 或字段缺失时，会先自动探测实际来源；首次自动探测完成后，会将 `model-source` 写回为 `huggingface` 或 `modelscope`，避免后续启动时因网络波动反复切换来源。
|
||||
```json
|
||||
{
|
||||
"model-source": "auto"
|
||||
}
|
||||
如果未设置 `MINERU_MODEL_SOURCE`，MinerU 会读取 `config.yaml` 中的 `model.source` 字段。`model.source` 支持 `auto`、`huggingface`、`modelscope` 和 `local`。当值为 `auto` 或字段缺失时，会先自动探测实际来源；如果该值来自配置文件或内置默认值，首次自动探测完成后会将 `model.source` 写回为 `huggingface` 或 `modelscope`，避免后续启动时因网络波动反复切换来源。
|
||||
```yaml
|
||||
model:
|
||||
source: auto
|
||||
base_dir: ~/.mineru/models
|
||||
```
|
||||
|
||||
|
||||
@@ -39,15 +39,14 @@ mineru-kit models download --help
|
||||
```
|
||||
或下载全部内置模型包：
|
||||
```bash
|
||||
mineru-kit models download all
|
||||
mineru-kit models download --tier high
|
||||
```
|
||||
> [!NOTE]
|
||||
>- 下载完成后，模型路径会在当前终端窗口输出，并自动写入用户目录下的 `mineru.json`。配置文件中的 `model-source` 会记录本次实际使用的远端来源，即 `huggingface` 或 `modelscope`。
|
||||
>- 您也可以通过将[配置模板文件](https://github.com/opendatalab/MinerU/blob/master/mineru.template.json)复制到用户目录下并重命名为 `mineru.json` 来创建配置文件；模板中的 `model-source` 默认为 `auto`，首次使用时会自动探测并写回实际来源。
|
||||
>- 模型下载到本地后，您可以自由移动模型文件夹到其他位置，同时需要在 `mineru.json` 中更新模型路径。
|
||||
>- 如您将模型文件夹部署到其他服务器上，请确保将 `mineru.json`文件一同移动到新设备的用户目录中并正确配置模型路径。
|
||||
>- 如您需要更新模型文件，可以再次运行 `mineru-kit models download all` 命令，模型更新暂不支持自定义路径，如您没有移动本地模型文件夹，模型文件会增量更新；如您移动了模型文件夹，模型文件会重新下载到默认位置并更新`mineru.json`。
|
||||
>- `mineru-kit models download` 必须使用远端模型源执行真实下载；如果当前终端已设置 `MINERU_MODEL_SOURCE=local`，该命令会仅在本次执行中临时忽略该值，并改用您选择的 `auto`、`huggingface` 或 `modelscope` 下载模型。
|
||||
>- 模型会下载到 `config.model.base_dir` 下，默认路径为 `~/.mineru/models`。
|
||||
>- `mineru-kit models download` 不会把模型路径写入 `mineru.json`。
|
||||
>- 如需自定义模型目录，请先在 `config.yaml` 中设置 `model.base_dir`，再执行下载。
|
||||
>- 如需更新模型文件，可以再次运行 `mineru-kit models download --tier high`；在同一个 `model.base_dir` 下，provider SDK 会尽量复用已有文件做增量更新。
|
||||
>- `mineru-kit models download` 必须使用远端模型源执行真实下载；如果当前配置为 `model.source: local`，该命令会仅在本次执行中临时按 `auto` 处理。
|
||||
|
||||
### 2. 使用本地模型进行解析
|
||||
|
||||
|
||||
@@ -116,10 +116,7 @@ mineru -p <input_path> -o <output_path>
|
||||
|
||||
## 基于配置文件扩展 MinerU 功能
|
||||
|
||||
MinerU 现已实现开箱即用，但也支持通过配置文件扩展功能。您可通过编辑用户目录下的 `mineru.json` 文件，添加自定义配置。
|
||||
|
||||
>[!IMPORTANT]
|
||||
>`mineru.json` 文件会在您使用内置模型下载命令 `mineru-kit models download all` 时自动生成，也可以通过将[配置模板文件](https://github.com/opendatalab/MinerU/blob/master/mineru.template.json)复制到用户目录下并重命名为 `mineru.json` 来创建。
|
||||
MinerU 现已实现开箱即用，但也支持通过配置文件扩展功能。LaTeX 分隔符、LLM 辅助标题分级等旧工具配置仍使用用户目录下的 `mineru.json`。模型存储目录和模型源配置使用 `config.yaml`，详见[模型源说明](./model_source.md)。
|
||||
|
||||
以下是一些可用的配置选项：
|
||||
|
||||
@@ -152,6 +149,3 @@ MinerU 现已实现开箱即用，但也支持通过配置文件扩展功能。
|
||||
}
|
||||
```
|
||||
|
||||
- `models-dir`：
|
||||
* 用于指定本地模型存储目录，请分别指定本地小模型 bundle（`models-dir.pipeline`）和 VLM bundle（`models-dir.vlm`）的模型目录，
|
||||
* 指定目录后您可通过配置环境变量`export MINERU_MODEL_SOURCE=local`来使用本地模型。
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
{
|
||||
"bucket_info":{
|
||||
"bucket-name-1":["ak", "sk", "endpoint"],
|
||||
"bucket-name-2":["ak", "sk", "endpoint"]
|
||||
},
|
||||
"latex-delimiter-config": {
|
||||
"display": {
|
||||
"left": "$$",
|
||||
@@ -22,10 +18,5 @@
|
||||
"enable": false
|
||||
}
|
||||
},
|
||||
"models-dir": {
|
||||
"pipeline": "",
|
||||
"vlm": ""
|
||||
},
|
||||
"model-source": "auto",
|
||||
"config_version": "1.3.2"
|
||||
}
|
||||
|
||||
@@ -20,8 +20,7 @@ from ..model.table.cls.paddle_table_cls import PaddleTableClsModel
|
||||
from ..model.table.rec.slanet_plus.main import PaddleTableModel
|
||||
from ..model.table.rec.unet_table.main import UnetTableModel
|
||||
from ..utils.config_reader import get_device
|
||||
from ..utils.enum_class import ModelPath
|
||||
from ..utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ..utils.model_registry import PDF_EXTRACT_KIT, ModelPath
|
||||
from ..utils.ocr_language import normalize_ocr_model_lang
|
||||
|
||||
|
||||
@@ -98,12 +97,12 @@ else:
|
||||
MFR_MODEL = "unimernet_small"
|
||||
|
||||
|
||||
def _resolve_mfr_model_path() -> str:
|
||||
def _resolve_mfr_model_path() -> ModelPath:
|
||||
"""解析当前公式识别模型路径，集中维护 MFR_MODEL 到模型目录枚举的映射关系。"""
|
||||
if MFR_MODEL == "unimernet_small":
|
||||
return ModelPath.unimernet_small
|
||||
return PDF_EXTRACT_KIT.unimernet_small
|
||||
if MFR_MODEL == "pp_formulanet_plus_m":
|
||||
return ModelPath.pp_formulanet_plus_m
|
||||
return PDF_EXTRACT_KIT.pp_formulanet_plus_m
|
||||
logger.error("MFR model name not allow")
|
||||
exit(1)
|
||||
|
||||
@@ -353,12 +352,7 @@ class HybridLocalModelContext:
|
||||
"""获取 Layout 原子模型，供 Hybrid 本地 layout、标题拆分和公式框检测复用。"""
|
||||
return self.atom_model_manager.get_atom_model(
|
||||
atom_model_name=AtomicModel.Layout,
|
||||
pp_doclayout_v2_weights=str(
|
||||
os.path.join(
|
||||
auto_download_and_get_model_root_path(ModelPath.pp_doclayout_v2),
|
||||
ModelPath.pp_doclayout_v2,
|
||||
)
|
||||
),
|
||||
pp_doclayout_v2_weights=str(PDF_EXTRACT_KIT.pp_doclayout_v2.ensure()),
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
@@ -367,6 +361,6 @@ class HybridLocalModelContext:
|
||||
mfr_model_path = _resolve_mfr_model_path()
|
||||
return self.atom_model_manager.get_atom_model(
|
||||
atom_model_name=AtomicModel.MFR,
|
||||
mfr_weight_dir=str(os.path.join(auto_download_and_get_model_root_path(mfr_model_path), mfr_model_path)),
|
||||
mfr_weight_dir=str(mfr_model_path.ensure()),
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
@@ -8,7 +8,6 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import click
|
||||
import typer
|
||||
|
||||
from ...doclib.client import DoclibClient
|
||||
@@ -80,13 +79,6 @@ def _validate_page_range_input(page_range: str | None) -> None:
|
||||
raise MineruError("page_range_invalid", f"Invalid page range: {page_range}", "pages") from None
|
||||
|
||||
|
||||
def _option_was_explicit(option_name: str) -> bool:
|
||||
ctx = click.get_current_context(silent=True)
|
||||
if ctx is None:
|
||||
return False
|
||||
return ctx.get_parameter_source(option_name) == click.core.ParameterSource.COMMANDLINE
|
||||
|
||||
|
||||
def parse_cmd(
|
||||
path: str = typer.Argument(..., help="Path to the document file"),
|
||||
tier: Tier | None = typer.Option(
|
||||
@@ -96,7 +88,7 @@ def parse_cmd(
|
||||
),
|
||||
pages: str = typer.Option(None, "-p", "--pages", help="Page range, e.g. '1~5' or 'all'"),
|
||||
after: str = typer.Option(None, "--after", help="Continue reading after a content cursor"),
|
||||
limit: int = typer.Option(30000, "--limit", help="Soft character limit for STDOUT content"),
|
||||
limit: int | None = typer.Option(None, "--limit", help="Soft character limit for STDOUT content"),
|
||||
format: Literal["markdown"] = typer.Option("markdown", "-f", "--format", help="Output format: markdown"),
|
||||
force: bool = typer.Option(False, "--force", help="Force re-parse, ignore cache"),
|
||||
remote: bool = typer.Option(False, "--remote", help="Use remote parse-server"),
|
||||
@@ -109,6 +101,7 @@ def parse_cmd(
|
||||
) -> None:
|
||||
"""Parse a document file."""
|
||||
ctx = CliContext(json_mode=json_mode, verbose=verbose)
|
||||
effective_limit = 30000 if limit is None else limit
|
||||
run_cli(
|
||||
ctx,
|
||||
lambda: _parse(
|
||||
@@ -116,7 +109,7 @@ def parse_cmd(
|
||||
tier=tier,
|
||||
pages=pages,
|
||||
after=after,
|
||||
limit=limit,
|
||||
limit=effective_limit,
|
||||
format=format,
|
||||
force=force,
|
||||
remote=remote,
|
||||
@@ -126,6 +119,9 @@ def parse_cmd(
|
||||
no_marker=no_marker,
|
||||
json_mode=json_mode,
|
||||
verbose=verbose,
|
||||
explicit_tier=tier is not None,
|
||||
explicit_limit=limit is not None,
|
||||
explicit_remote=remote,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -146,6 +142,9 @@ def _parse(
|
||||
no_marker: bool,
|
||||
json_mode: bool,
|
||||
verbose: bool,
|
||||
explicit_tier: bool,
|
||||
explicit_limit: bool,
|
||||
explicit_remote: bool,
|
||||
) -> None:
|
||||
file_path = normalize_cli_path(path)
|
||||
|
||||
@@ -170,9 +169,9 @@ def _parse(
|
||||
req_tier = result.tier
|
||||
status = result.status
|
||||
wait_parse_ids = result.wait_parse_ids
|
||||
next_marker_tier = tier if _option_was_explicit("tier") else None
|
||||
next_marker_limit = limit if _option_was_explicit("limit") else None
|
||||
next_marker_remote = remote if _option_was_explicit("remote") else False
|
||||
next_marker_tier = tier if explicit_tier else None
|
||||
next_marker_limit = limit if explicit_limit else None
|
||||
next_marker_remote = remote if explicit_remote else False
|
||||
|
||||
# cached
|
||||
if status == "done":
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
from contextlib import contextmanager
|
||||
import os
|
||||
import sys
|
||||
import click
|
||||
from loguru import logger
|
||||
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.models_download_utils import (
|
||||
CONFIG_TEMPLATE_URL,
|
||||
auto_download_and_get_model_root_path,
|
||||
download_and_modify_json,
|
||||
get_tools_config_file_path,
|
||||
resolve_model_source,
|
||||
)
|
||||
|
||||
MODEL_SOURCE_ENV_VAR = 'MINERU_MODEL_SOURCE'
|
||||
REMOTE_MODEL_SOURCES = ('auto', 'huggingface', 'modelscope')
|
||||
|
||||
|
||||
def configure_model(model_dir, model_type, model_source):
|
||||
"""配置模型"""
|
||||
config_file = get_tools_config_file_path()
|
||||
|
||||
json_mods = {
|
||||
'models-dir': {
|
||||
f'{model_type}': model_dir
|
||||
},
|
||||
'model-source': model_source,
|
||||
}
|
||||
|
||||
download_and_modify_json(CONFIG_TEMPLATE_URL, config_file, json_mods)
|
||||
logger.info(f'The configuration file has been successfully configured, the path is: {config_file}')
|
||||
|
||||
|
||||
def download_pipeline_models(model_source):
|
||||
"""下载Pipeline模型"""
|
||||
model_paths = [
|
||||
ModelPath.pp_doclayout_v2,
|
||||
ModelPath.unimernet_small,
|
||||
ModelPath.pytorch_paddle,
|
||||
ModelPath.slanet_plus,
|
||||
ModelPath.unet_structure,
|
||||
ModelPath.paddle_table_cls,
|
||||
ModelPath.pp_formulanet_plus_m,
|
||||
]
|
||||
download_finish_path = ""
|
||||
for model_path in model_paths:
|
||||
logger.info(f"Downloading model: {model_path}")
|
||||
download_finish_path = auto_download_and_get_model_root_path(model_path, repo_mode='pipeline')
|
||||
logger.info(f"Pipeline models downloaded successfully to: {download_finish_path}")
|
||||
configure_model(download_finish_path, "pipeline", model_source)
|
||||
|
||||
|
||||
def download_vlm_models(model_source):
|
||||
"""下载VLM模型"""
|
||||
download_finish_path = auto_download_and_get_model_root_path("/", repo_mode='vlm')
|
||||
logger.info(f"VLM models downloaded successfully to: {download_finish_path}")
|
||||
configure_model(download_finish_path, "vlm", model_source)
|
||||
|
||||
|
||||
def get_effective_download_model_source(requested_model_source):
|
||||
"""获取本次下载命令实际使用的模型源。"""
|
||||
current_model_source = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
if current_model_source == 'local':
|
||||
logger.warning(
|
||||
f"{MODEL_SOURCE_ENV_VAR}=local means using pre-downloaded local models. "
|
||||
f"`mineru-models-download` will temporarily use '{requested_model_source}' "
|
||||
f"to perform a real download."
|
||||
)
|
||||
return resolve_model_source(requested_model_source, allow_auto=True)
|
||||
|
||||
if current_model_source is None:
|
||||
return resolve_model_source(requested_model_source, allow_auto=True)
|
||||
|
||||
return resolve_model_source(current_model_source)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def temporary_model_source(model_source):
|
||||
"""在命令执行期间临时设置模型源，并在结束后恢复。"""
|
||||
original_model_source = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
os.environ[MODEL_SOURCE_ENV_VAR] = model_source
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if original_model_source is None:
|
||||
os.environ.pop(MODEL_SOURCE_ENV_VAR, None)
|
||||
else:
|
||||
os.environ[MODEL_SOURCE_ENV_VAR] = original_model_source
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
'-s',
|
||||
'--source',
|
||||
'model_source',
|
||||
type=click.Choice(REMOTE_MODEL_SOURCES),
|
||||
help="""
|
||||
The source of the model repository.
|
||||
""",
|
||||
default=None,
|
||||
)
|
||||
@click.option(
|
||||
'-m',
|
||||
'--model_type',
|
||||
'model_type',
|
||||
type=click.Choice(['pipeline', 'vlm', 'all']),
|
||||
help="""
|
||||
The type of the model to download.
|
||||
""",
|
||||
default=None,
|
||||
)
|
||||
def download_models(model_source, model_type):
|
||||
"""Download MinerU model files.
|
||||
|
||||
Supports downloading pipeline or VLM models from ModelScope or HuggingFace.
|
||||
"""
|
||||
# 如果未显式指定则交互式输入下载来源
|
||||
if model_source is None:
|
||||
model_source = click.prompt(
|
||||
"Please select the model download source: ",
|
||||
type=click.Choice(REMOTE_MODEL_SOURCES),
|
||||
default='auto'
|
||||
)
|
||||
|
||||
effective_model_source = get_effective_download_model_source(model_source)
|
||||
|
||||
# 如果未显式指定则交互式输入模型类型
|
||||
if model_type is None:
|
||||
model_type = click.prompt(
|
||||
"Please select the model type to download: ",
|
||||
type=click.Choice(['pipeline', 'vlm', 'all']),
|
||||
default='all'
|
||||
)
|
||||
|
||||
logger.info(f"Downloading {model_type} model from {effective_model_source}...")
|
||||
|
||||
try:
|
||||
with temporary_model_source(effective_model_source):
|
||||
if model_type == 'pipeline':
|
||||
download_pipeline_models(effective_model_source)
|
||||
elif model_type == 'vlm':
|
||||
download_vlm_models(effective_model_source)
|
||||
elif model_type == 'all':
|
||||
download_pipeline_models(effective_model_source)
|
||||
download_vlm_models(effective_model_source)
|
||||
else:
|
||||
click.echo(f"Unsupported model type: {model_type}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"An error occurred while downloading models: {str(e)}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
download_models()
|
||||
+151
-27
@@ -7,9 +7,13 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
from typing import Any, Literal
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, get_args, get_origin
|
||||
|
||||
import yaml
|
||||
from filelock import FileLock
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic_core import to_jsonable_python
|
||||
|
||||
@@ -20,6 +24,8 @@ MINERU_CONFIG_ENV = "MINERU_CONFIG"
|
||||
MINERU_ENV_PREFIX = "MINERU_"
|
||||
|
||||
AutoBool = Literal["auto"] | bool
|
||||
ConfigSource = Literal["default", "file", "env"]
|
||||
ModelSource = Literal["auto", "huggingface", "modelscope", "local"]
|
||||
|
||||
_INTERPOLATION_RE = re.compile(r"\$\{(\w+)(?::-([^${}]*))?\}")
|
||||
|
||||
@@ -91,24 +97,23 @@ def _load_config(config_file: str) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def _read_config() -> dict[str, Any]:
|
||||
def _resolve_config_file() -> tuple[str, bool]:
|
||||
config_file = os.getenv(MINERU_CONFIG_ENV)
|
||||
if config_file and not os.path.isfile(config_file):
|
||||
raise FileNotFoundError(f"MinerU config file [{config_file}] does not exist.")
|
||||
if config_file not in (None, ""):
|
||||
config_file = os.path.expanduser(config_file)
|
||||
if not os.path.isfile(config_file):
|
||||
raise FileNotFoundError(f"MinerU config file [{config_file}] does not exist.")
|
||||
return config_file, True
|
||||
|
||||
default_config_file = _default_path("config.yaml")
|
||||
if not config_file and os.path.isfile(default_config_file):
|
||||
config_file = default_config_file
|
||||
|
||||
if not config_file:
|
||||
exists = os.path.isfile(default_config_file)
|
||||
if not exists:
|
||||
_logger.debug(
|
||||
"MinerU config file not found. Default path is %s. Use %s to specify a custom path.",
|
||||
default_config_file,
|
||||
MINERU_CONFIG_ENV,
|
||||
)
|
||||
return {}
|
||||
|
||||
return _load_config(config_file)
|
||||
return default_config_file, exists
|
||||
|
||||
|
||||
def _collect_path(remaining: str, model_class: type[BaseModel]) -> list[str] | None:
|
||||
@@ -138,19 +143,10 @@ def _deep_merge(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, An
|
||||
return result
|
||||
|
||||
|
||||
def _apply_env_overrides(cfg: "Config", prefix: str = MINERU_ENV_PREFIX) -> "Config":
|
||||
"""Return a new Config with matching environment variables merged in.
|
||||
|
||||
Environment variable names use the prefix plus a greedy field path joined by
|
||||
underscores. Values remain strings here and are converted by Pydantic when
|
||||
rebuilding Config.
|
||||
|
||||
Examples:
|
||||
MINERU_DOCLIB_TCP_PORT=15981 -> config.doclib.tcp.port
|
||||
MINERU_DOCLIB_SQLITE_MMAP_SIZE=0 -> config.doclib.sqlite.mmap_size
|
||||
"""
|
||||
def _collect_env_overrides(prefix: str = MINERU_ENV_PREFIX) -> tuple[dict[str, Any], set[tuple[str, ...]]]:
|
||||
prefix_upper = prefix.upper()
|
||||
overrides: dict[str, Any] = {}
|
||||
paths: set[tuple[str, ...]] = set()
|
||||
|
||||
for key, value in os.environ.items():
|
||||
if not key.startswith(prefix_upper):
|
||||
@@ -165,12 +161,125 @@ def _apply_env_overrides(cfg: "Config", prefix: str = MINERU_ENV_PREFIX) -> "Con
|
||||
for part in path[:-1]:
|
||||
node = node.setdefault(part, {})
|
||||
node[path[-1]] = value
|
||||
paths.add(tuple(path))
|
||||
|
||||
if not overrides:
|
||||
return cfg
|
||||
return overrides, paths
|
||||
|
||||
merged = _deep_merge(to_jsonable_python(cfg), overrides)
|
||||
return Config(**merged)
|
||||
|
||||
def _model_annotation(annotation: Any) -> type[BaseModel] | None:
|
||||
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
||||
return annotation
|
||||
origin = get_origin(annotation)
|
||||
if origin is None:
|
||||
return None
|
||||
for arg in get_args(annotation):
|
||||
if isinstance(arg, type) and issubclass(arg, BaseModel):
|
||||
return arg
|
||||
return None
|
||||
|
||||
|
||||
def _default_source_paths(model_class: type[BaseModel], prefix: tuple[str, ...] = ()) -> set[tuple[str, ...]]:
|
||||
paths: set[tuple[str, ...]] = set()
|
||||
for field_name, field in model_class.model_fields.items():
|
||||
sub_model = _model_annotation(field.annotation)
|
||||
current = (*prefix, field_name)
|
||||
if sub_model is not None:
|
||||
paths.update(_default_source_paths(sub_model, current))
|
||||
else:
|
||||
paths.add(current)
|
||||
return paths
|
||||
|
||||
|
||||
def _configured_source_paths(
|
||||
data: dict[str, Any],
|
||||
model_class: type[BaseModel],
|
||||
prefix: tuple[str, ...] = (),
|
||||
) -> set[tuple[str, ...]]:
|
||||
paths: set[tuple[str, ...]] = set()
|
||||
for key, value in data.items():
|
||||
if key not in model_class.model_fields:
|
||||
continue
|
||||
field = model_class.model_fields[key]
|
||||
sub_model = _model_annotation(field.annotation)
|
||||
current = (*prefix, key)
|
||||
if sub_model is not None and isinstance(value, dict):
|
||||
paths.update(_configured_source_paths(value, sub_model, current))
|
||||
else:
|
||||
paths.add(current)
|
||||
return paths
|
||||
|
||||
|
||||
def _normalize_config_source_path(path: str | Sequence[str]) -> tuple[str, ...]:
|
||||
if isinstance(path, str):
|
||||
return tuple(part for part in path.split(".") if part)
|
||||
return tuple(path)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoadedConfig:
|
||||
config: "Config"
|
||||
sources: dict[tuple[str, ...], ConfigSource]
|
||||
config_file: str
|
||||
config_file_exists: bool
|
||||
|
||||
|
||||
def _load_effective_config() -> LoadedConfig:
|
||||
config_file, config_file_exists = _resolve_config_file()
|
||||
raw_config = _load_config(config_file) if config_file_exists else {}
|
||||
sources: dict[tuple[str, ...], ConfigSource] = dict.fromkeys(_default_source_paths(Config), "default")
|
||||
for path in _configured_source_paths(raw_config, Config):
|
||||
sources[path] = "file"
|
||||
|
||||
base_config = Config(**raw_config)
|
||||
overrides, env_paths = _collect_env_overrides()
|
||||
if overrides:
|
||||
base_config = Config(**_deep_merge(to_jsonable_python(base_config), overrides))
|
||||
for path in env_paths:
|
||||
sources[path] = "env"
|
||||
return LoadedConfig(
|
||||
config=base_config,
|
||||
sources=sources,
|
||||
config_file=config_file,
|
||||
config_file_exists=config_file_exists,
|
||||
)
|
||||
|
||||
|
||||
def get_config_source(path: str | Sequence[str]) -> ConfigSource:
|
||||
return _loaded_config.sources.get(_normalize_config_source_path(path), "default")
|
||||
|
||||
|
||||
def get_config_file_path() -> str:
|
||||
return _loaded_config.config_file
|
||||
|
||||
|
||||
def get_config_file_exists() -> bool:
|
||||
return _loaded_config.config_file_exists
|
||||
|
||||
|
||||
def update_config_file(patch: dict[str, Any]) -> None:
|
||||
config_file = get_config_file_path()
|
||||
config_dir = os.path.dirname(config_file)
|
||||
if config_dir:
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
|
||||
lock = FileLock(f"{config_file}.lock")
|
||||
with lock:
|
||||
if os.path.exists(config_file):
|
||||
with open(config_file, encoding="utf-8") as file:
|
||||
loaded = yaml.safe_load(file)
|
||||
current = loaded if isinstance(loaded, dict) else {}
|
||||
else:
|
||||
current = {}
|
||||
|
||||
updated = _deep_merge(current, patch)
|
||||
fd, tmp_path = tempfile.mkstemp(prefix=".config.", suffix=".tmp", dir=config_dir or None)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as file:
|
||||
yaml.safe_dump(updated, file, allow_unicode=True, sort_keys=False)
|
||||
os.replace(tmp_path, config_file)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
class UDSConfig(BaseModel):
|
||||
@@ -244,6 +353,11 @@ class ManagedParseServerConfig(BaseModel):
|
||||
port_probe_count: int = Field(default=100, ge=1)
|
||||
|
||||
|
||||
class ModelConfig(BaseModel):
|
||||
base_dir: str = _default_path("models")
|
||||
source: str = "auto"
|
||||
|
||||
|
||||
class DoclibConfig(BaseModel):
|
||||
"""Doclib startup configuration.
|
||||
|
||||
@@ -289,10 +403,12 @@ class Config(BaseModel):
|
||||
"""Top-level MinerU startup configuration."""
|
||||
|
||||
doclib: DoclibConfig = Field(default_factory=DoclibConfig)
|
||||
model: ModelConfig = Field(default_factory=ModelConfig)
|
||||
# render: RenderConfig
|
||||
|
||||
|
||||
config = _apply_env_overrides(Config(**_read_config()))
|
||||
_loaded_config = _load_effective_config()
|
||||
config = _loaded_config.config
|
||||
|
||||
|
||||
def PatchedConfig(**kwargs: Any) -> Config:
|
||||
@@ -302,6 +418,10 @@ def PatchedConfig(**kwargs: Any) -> Config:
|
||||
|
||||
__all__ = [
|
||||
"AutoBool",
|
||||
"ConfigSource",
|
||||
"LoadedConfig",
|
||||
"ModelConfig",
|
||||
"ModelSource",
|
||||
"config",
|
||||
"Config",
|
||||
"DoclibConfig",
|
||||
@@ -310,6 +430,10 @@ __all__ = [
|
||||
"SQLiteConfig",
|
||||
"UDSConfig",
|
||||
"PatchedConfig",
|
||||
"get_config_source",
|
||||
"get_config_file_path",
|
||||
"get_config_file_exists",
|
||||
"update_config_file",
|
||||
"MINERU_HOME_ENV",
|
||||
"MINERU_CONFIG_ENV",
|
||||
"MINERU_ENV_PREFIX",
|
||||
|
||||
+96
-153
@@ -1,198 +1,141 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from ...utils.enum_class import ModelPath
|
||||
from ...config import config, get_config_file_exists, get_config_file_path, get_config_source
|
||||
from ...types import validate_tier
|
||||
from ...utils.model_registry import (
|
||||
MODEL_REPOS,
|
||||
ModelRepo,
|
||||
get_model_repo,
|
||||
model_repo_names,
|
||||
model_repos_for_tier,
|
||||
)
|
||||
from ...utils.models_download_utils import (
|
||||
MINERU_CONFIG_VERSION,
|
||||
auto_download_and_get_model_root_path,
|
||||
is_config_version_outdated,
|
||||
merge_config_dict,
|
||||
resolve_model_source,
|
||||
DOWNLOAD_MODEL_SOURCES,
|
||||
MODEL_SOURCE_ENV_VAR,
|
||||
download_model_repo,
|
||||
verify_model_repo,
|
||||
)
|
||||
from ..common import PIPELINE_MODEL_PATHS, VLM_MODEL_MARKERS, read_json_file, resolve_models_config_path, write_json_file
|
||||
from ..errors import exit_with_message
|
||||
from ..output import print_info, print_success
|
||||
|
||||
app = typer.Typer(help="Download, inspect, and verify local MinerU models.", no_args_is_help=True)
|
||||
|
||||
MODEL_SOURCE_ENV_VAR = "MINERU_MODEL_SOURCE"
|
||||
REMOTE_MODEL_SOURCES = ("auto", "huggingface", "modelscope")
|
||||
|
||||
def _validate_download_source(source: str | None) -> str | None:
|
||||
if source is None:
|
||||
return None
|
||||
normalized = source.strip().lower()
|
||||
if normalized not in DOWNLOAD_MODEL_SOURCES:
|
||||
expected = ", ".join(DOWNLOAD_MODEL_SOURCES)
|
||||
exit_with_message("invalid_request", f"Unsupported source '{source}'. Expected one of: {expected}.", "source")
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_or_template_config() -> dict:
|
||||
config_path = resolve_models_config_path()
|
||||
payload = read_json_file(config_path)
|
||||
template_path = Path(__file__).resolve().parents[3] / "mineru.template.json"
|
||||
template_payload = read_json_file(template_path) or {"models-dir": {}, "config_version": MINERU_CONFIG_VERSION}
|
||||
if payload is not None:
|
||||
if not is_config_version_outdated(payload.get("config_version", "0.0.0")):
|
||||
return payload
|
||||
return merge_config_dict(template_payload, payload, skip_keys={"config_version"})
|
||||
return template_payload
|
||||
def _select_target_repos(repo_name: str | None, tier: str | None) -> tuple[ModelRepo, ...]:
|
||||
if repo_name and tier:
|
||||
exit_with_message("invalid_request", "Pass either a model repo name or --tier, not both.")
|
||||
if not repo_name and tier is None:
|
||||
exit_with_message("invalid_request", "Pass a model repo name or --tier.")
|
||||
|
||||
if tier is not None:
|
||||
try:
|
||||
resolved_tier = validate_tier(tier)
|
||||
except ValueError as exc:
|
||||
exit_with_message("invalid_request", str(exc), "tier")
|
||||
return model_repos_for_tier(resolved_tier)
|
||||
|
||||
try:
|
||||
return (get_model_repo(repo_name or ""),)
|
||||
except ValueError as exc:
|
||||
exit_with_message("invalid_request", str(exc), "repo")
|
||||
|
||||
|
||||
def _update_models_dir(bundle: str, model_dir: str) -> Path:
|
||||
config_path = resolve_models_config_path()
|
||||
payload = _load_or_template_config()
|
||||
models_dir = payload.get("models-dir")
|
||||
if not isinstance(models_dir, dict):
|
||||
models_dir = {}
|
||||
payload["models-dir"] = models_dir
|
||||
models_dir[bundle] = model_dir
|
||||
model_source = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
if model_source in {"huggingface", "modelscope"}:
|
||||
payload["model-source"] = model_source
|
||||
write_json_file(config_path, payload)
|
||||
return config_path
|
||||
|
||||
|
||||
def _get_effective_download_model_source(requested_model_source: str) -> str:
|
||||
current_model_source = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
if current_model_source == "local":
|
||||
logger.warning(
|
||||
f"{MODEL_SOURCE_ENV_VAR}=local means using pre-downloaded local models. "
|
||||
f"`mineru-kit models download` will temporarily use '{requested_model_source}' to perform a real download."
|
||||
)
|
||||
return resolve_model_source(requested_model_source, allow_auto=True)
|
||||
if current_model_source is None:
|
||||
return resolve_model_source(requested_model_source, allow_auto=True)
|
||||
return resolve_model_source(current_model_source)
|
||||
|
||||
|
||||
def _download_pipeline_models() -> str:
|
||||
download_finish_path = ""
|
||||
for model_path in PIPELINE_MODEL_PATHS:
|
||||
logger.info(f"Downloading model: {model_path}")
|
||||
download_finish_path = auto_download_and_get_model_root_path(model_path, repo_mode="pipeline")
|
||||
return download_finish_path
|
||||
|
||||
|
||||
def _download_vlm_models() -> str:
|
||||
return auto_download_and_get_model_root_path("/", repo_mode="vlm")
|
||||
|
||||
|
||||
def _with_temporary_model_source(model_source: str):
|
||||
class _ModelSourceContext:
|
||||
def __enter__(self_inner) -> None:
|
||||
self_inner.original = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
os.environ[MODEL_SOURCE_ENV_VAR] = model_source
|
||||
|
||||
def __exit__(self_inner, exc_type, exc, tb) -> None:
|
||||
if self_inner.original is None:
|
||||
os.environ.pop(MODEL_SOURCE_ENV_VAR, None)
|
||||
else:
|
||||
os.environ[MODEL_SOURCE_ENV_VAR] = self_inner.original
|
||||
|
||||
return _ModelSourceContext()
|
||||
def _format_repo_status(repo: ModelRepo) -> str:
|
||||
result = verify_model_repo(repo)
|
||||
status = "ready" if result.ready else "missing"
|
||||
return f"{repo.name}: {status} ({repo.local_dir()})"
|
||||
|
||||
|
||||
@app.command("download")
|
||||
def download_cmd(
|
||||
bundle: str = typer.Argument(..., help="Model bundle: pipeline, vlm, all"),
|
||||
source: str = typer.Option("auto", "--source", "-s", help="Model source: auto, huggingface, or modelscope"),
|
||||
repo: str | None = typer.Argument(None, help="Model repo: PDF-Extract-Kit-1.0 or MinerU2.5-Pro-2605-1.2B"),
|
||||
tier: str | None = typer.Option(None, "--tier", help="Tier to prepare: flash, medium, high, or xhigh"),
|
||||
source: str | None = typer.Option(None, "--source", "-s", help="Model source: auto, huggingface, or modelscope"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose output"),
|
||||
) -> None:
|
||||
"""Download a model bundle and update mineru.json."""
|
||||
if bundle not in {"pipeline", "vlm", "all"}:
|
||||
exit_with_message("invalid_request", f"Unsupported bundle '{bundle}'. Expected pipeline, vlm, or all.", "bundle")
|
||||
if source not in REMOTE_MODEL_SOURCES:
|
||||
exit_with_message("invalid_request", f"Unsupported source '{source}'.", "source")
|
||||
effective_source = _get_effective_download_model_source(source)
|
||||
try:
|
||||
with _with_temporary_model_source(effective_source):
|
||||
if bundle in {"pipeline", "all"}:
|
||||
pipeline_dir = _download_pipeline_models()
|
||||
config_path = _update_models_dir("pipeline", pipeline_dir)
|
||||
if verbose:
|
||||
print_info(f"Configured pipeline models in {config_path}")
|
||||
if bundle in {"vlm", "all"}:
|
||||
vlm_dir = _download_vlm_models()
|
||||
config_path = _update_models_dir("vlm", vlm_dir)
|
||||
if verbose:
|
||||
print_info(f"Configured vlm models in {config_path}")
|
||||
except Exception as exc:
|
||||
exit_with_message("api_error", f"Failed to download models: {exc}")
|
||||
print_success(f"Downloaded {bundle} models from {effective_source}.")
|
||||
"""Download a model repo or the repos required by a tier."""
|
||||
normalized_source = _validate_download_source(source)
|
||||
repos = _select_target_repos(repo, tier)
|
||||
if tier is not None and not repos:
|
||||
print_success("No model download required for tier flash.")
|
||||
return
|
||||
|
||||
for target_repo in repos:
|
||||
try:
|
||||
root = download_model_repo(target_repo, source=normalized_source, local_as_auto=True)
|
||||
except Exception as exc:
|
||||
exit_with_message("api_error", f"Failed to download {target_repo.name}: {exc}")
|
||||
if verbose:
|
||||
print_info(f"{target_repo.name}: {root}")
|
||||
|
||||
label = f"tier {tier}" if tier is not None else repos[0].name
|
||||
print_success(f"Downloaded models for {label}.")
|
||||
|
||||
|
||||
@app.command("show")
|
||||
def show_cmd() -> None:
|
||||
"""Show current MinerU model configuration."""
|
||||
config_path = resolve_models_config_path()
|
||||
payload = read_json_file(config_path)
|
||||
if payload is None:
|
||||
print_info(f"Config: {config_path} (missing)")
|
||||
return
|
||||
models_dir = payload.get("models-dir", {})
|
||||
pipeline = models_dir.get("pipeline", "") if isinstance(models_dir, dict) else ""
|
||||
vlm = models_dir.get("vlm", "") if isinstance(models_dir, dict) else ""
|
||||
config_file = get_config_file_path()
|
||||
lines = [
|
||||
f"Config: {config_path}",
|
||||
f"Config: {config_file}",
|
||||
f"Config exists: {str(get_config_file_exists()).lower()}",
|
||||
f"MINERU_MODEL_SOURCE={os.getenv(MODEL_SOURCE_ENV_VAR, '') or '(unset)'}",
|
||||
f"model-source: {payload.get('model-source', '') or '(unset)'}",
|
||||
f"pipeline: {pipeline or '(unset)'}",
|
||||
f"pipeline.exists: {Path(pipeline).exists() if pipeline else False}",
|
||||
f"vlm: {vlm or '(unset)'}",
|
||||
f"vlm.exists: {Path(vlm).exists() if vlm else False}",
|
||||
f"model.base_dir: {config.model.base_dir}",
|
||||
f"model.base_dir.source: {get_config_source('model.base_dir')}",
|
||||
f"model.source: {config.model.source}",
|
||||
f"model.source.source: {get_config_source('model.source')}",
|
||||
"Repos:",
|
||||
]
|
||||
for line in lines:
|
||||
print_info(line)
|
||||
|
||||
for repo in MODEL_REPOS:
|
||||
print_info(f" {_format_repo_status(repo)}")
|
||||
|
||||
def _verify_pipeline(root: Path) -> list[str]:
|
||||
missing: list[str] = []
|
||||
for rel in PIPELINE_MODEL_PATHS:
|
||||
if not (root / rel).exists():
|
||||
missing.append(rel)
|
||||
return missing
|
||||
|
||||
|
||||
def _verify_vlm(root: Path) -> list[str]:
|
||||
missing: list[str] = []
|
||||
for rel in VLM_MODEL_MARKERS:
|
||||
if not (root / rel).exists():
|
||||
missing.append(rel)
|
||||
return missing
|
||||
print_info("Tiers:")
|
||||
for tier in ("flash", "medium", "high", "xhigh"):
|
||||
repos = model_repos_for_tier(validate_tier(tier))
|
||||
names = ", ".join(repo.name for repo in repos) or "(none)"
|
||||
print_info(f" {tier}: {names}")
|
||||
|
||||
|
||||
@app.command("verify")
|
||||
def verify_cmd(
|
||||
bundle: str | None = typer.Argument(None, help="Optional bundle: pipeline, vlm, all"),
|
||||
repo: str | None = typer.Argument(None, help="Optional model repo name"),
|
||||
tier: str | None = typer.Option(None, "--tier", help="Optional tier: flash, medium, high, or xhigh"),
|
||||
) -> None:
|
||||
"""Verify configured model directories and key paths."""
|
||||
if bundle is not None and bundle not in {"pipeline", "vlm", "all"}:
|
||||
exit_with_message("invalid_request", f"Unsupported bundle '{bundle}'. Expected pipeline, vlm, or all.", "bundle")
|
||||
config_path = resolve_models_config_path()
|
||||
payload = read_json_file(config_path)
|
||||
if payload is None:
|
||||
exit_with_message("file_not_found", f"Config file not found: {config_path}")
|
||||
models_dir = payload.get("models-dir")
|
||||
if not isinstance(models_dir, dict):
|
||||
exit_with_message("invalid_request", f"'models-dir' not found in {config_path}")
|
||||
"""Verify local model repos and required paths."""
|
||||
repos = MODEL_REPOS if repo is None and tier is None else _select_target_repos(repo, tier)
|
||||
if tier == "flash":
|
||||
print_success("flash: ok")
|
||||
return
|
||||
|
||||
targets = ("pipeline", "vlm") if bundle is None or bundle == "all" else (bundle,)
|
||||
failures = 0
|
||||
for target in targets:
|
||||
raw = models_dir.get(target)
|
||||
if not raw:
|
||||
print_info(f"{target}: missing config")
|
||||
failures += 1
|
||||
for target_repo in repos:
|
||||
result = verify_model_repo(target_repo)
|
||||
if result.ready:
|
||||
print_success(f"{target_repo.name}: ok")
|
||||
continue
|
||||
root = Path(raw)
|
||||
if not root.exists():
|
||||
print_info(f"{target}: missing directory {root}")
|
||||
failures += 1
|
||||
continue
|
||||
missing = _verify_pipeline(root) if target == "pipeline" else _verify_vlm(root)
|
||||
if missing:
|
||||
print_info(f"{target}: missing key paths: {', '.join(missing)}")
|
||||
failures += 1
|
||||
else:
|
||||
print_success(f"{target}: ok")
|
||||
failures += 1
|
||||
missing = ", ".join(result.missing_paths)
|
||||
print_info(f"{target_repo.name}: missing key paths: {missing}")
|
||||
|
||||
if failures:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
__all__ = ["app", "download_cmd", "show_cmd", "verify_cmd", "model_repo_names"]
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Literal, TypeAlias
|
||||
@@ -14,7 +12,6 @@ from ..render.writer import FileBasedDataWriter
|
||||
from ..types import Tier
|
||||
from ..utils.image_payload import validate_image_sidecar_path
|
||||
|
||||
SupportedBundle = Literal["pipeline", "vlm", "all"]
|
||||
KitFormat = Literal["markdown", "middle_json", "zip"]
|
||||
LocalTier: TypeAlias = Tier
|
||||
|
||||
@@ -25,17 +22,6 @@ OUTPUT_FILE_SUFFIXES = {
|
||||
"zip": ".zip",
|
||||
}
|
||||
|
||||
PIPELINE_MODEL_PATHS = (
|
||||
"models/Layout/PP-DocLayoutV2",
|
||||
"models/MFR/unimernet_hf_small_2503",
|
||||
"models/MFR/pp_formulanet_plus_m",
|
||||
"models/OCR/paddleocr_torch",
|
||||
"models/TabRec/SlanetPlus/slanet-plus.onnx",
|
||||
"models/TabRec/UnetStructure/unet.onnx",
|
||||
"models/TabCls/paddle_table_cls/PP-LCNet_x1_0_table_cls.onnx",
|
||||
)
|
||||
VLM_MODEL_MARKERS = ("config.json", "preprocessor_config.json", "tokenizer_config.json")
|
||||
|
||||
|
||||
def expand_input_paths(inputs: list[str]) -> list[Path]:
|
||||
paths = [Path(raw).expanduser() for raw in inputs]
|
||||
@@ -187,22 +173,3 @@ def parse_result_payload(path: Path, dest: Path, format: KitFormat) -> dict[str,
|
||||
"output": str(dest),
|
||||
"format": format,
|
||||
}
|
||||
|
||||
|
||||
def resolve_models_config_path() -> Path:
|
||||
config_name = os.getenv("MINERU_TOOLS_CONFIG_JSON", "mineru.json")
|
||||
config_path = Path(config_name)
|
||||
if config_path.is_absolute():
|
||||
return config_path
|
||||
return Path.home() / config_path
|
||||
|
||||
|
||||
def read_json_file(path: Path) -> dict | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_json_file(path: Path, payload: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=4), encoding="utf-8")
|
||||
|
||||
@@ -4,11 +4,11 @@ import sys
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from mineru.model.vlm.engine_utils import set_lmdeploy_backend
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ...model.vlm.engine_utils import set_lmdeploy_backend
|
||||
from ...utils.model_registry import MINERU_2_5_PRO_2605_1_2B
|
||||
|
||||
|
||||
def main():
|
||||
def main() -> None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
has_port_arg = False
|
||||
@@ -73,7 +73,7 @@ def main():
|
||||
args.extend(["--device", device_type])
|
||||
args.extend(["--backend", lm_backend])
|
||||
|
||||
model_path = auto_download_and_get_model_root_path("/", "vlm")
|
||||
model_path = str(MINERU_2_5_PRO_2605_1_2B.ensure())
|
||||
|
||||
# logger.debug(args)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import uvicorn
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from mineru_vl_utils.mlx_compat import load_mlx_model
|
||||
|
||||
from ...utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ...utils.model_registry import MINERU_2_5_PRO_2605_1_2B
|
||||
|
||||
|
||||
async def _resolve_result(value: Any) -> Any:
|
||||
@@ -21,7 +21,7 @@ async def _resolve_result(value: Any) -> Any:
|
||||
def _default_model_id(configured_model: str | None) -> str:
|
||||
if configured_model:
|
||||
return configured_model
|
||||
return auto_download_and_get_model_root_path("/", "vlm")
|
||||
return str(MINERU_2_5_PRO_2605_1_2B.ensure())
|
||||
|
||||
|
||||
def _load_mlx_model_for_server(path_or_hf_repo: str, adapter_path: str | None = None, **kwargs: Any) -> Any:
|
||||
|
||||
@@ -2,14 +2,17 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from mineru.model.vlm.engine_utils import set_default_gpu_memory_utilization, enable_custom_logits_processors, \
|
||||
mod_kwargs_by_device_type
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ...model.vlm.engine_utils import (
|
||||
enable_custom_logits_processors,
|
||||
mod_kwargs_by_device_type,
|
||||
set_default_gpu_memory_utilization,
|
||||
)
|
||||
from ...utils.model_registry import MINERU_2_5_PRO_2605_1_2B
|
||||
|
||||
from vllm.entrypoints.cli.main import main as vllm_main
|
||||
|
||||
|
||||
def main():
|
||||
def main() -> None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
has_port_arg = False
|
||||
@@ -48,7 +51,7 @@ def main():
|
||||
gpu_memory_utilization = str(set_default_gpu_memory_utilization())
|
||||
args.extend(["--gpu-memory-utilization", gpu_memory_utilization])
|
||||
if not model_path:
|
||||
model_path = auto_download_and_get_model_root_path("/", "vlm")
|
||||
model_path = str(MINERU_2_5_PRO_2605_1_2B.ensure())
|
||||
if (not has_logits_processors_arg) and custom_logits_processors:
|
||||
args.extend(["--logits-processors", "mineru_vl_utils:MinerULogitsProcessor"])
|
||||
|
||||
|
||||
@@ -1744,11 +1744,9 @@ if __name__ == "__main__":
|
||||
args.device = get_device()
|
||||
|
||||
if args.model is None:
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
args.model = str(
|
||||
os.path.join(auto_download_and_get_model_root_path(ModelPath.pp_doclayout_v2), ModelPath.pp_doclayout_v2)
|
||||
)
|
||||
from ...utils.model_registry import PDF_EXTRACT_KIT
|
||||
|
||||
args.model = str(PDF_EXTRACT_KIT.pp_doclayout_v2.ensure())
|
||||
|
||||
args.image = "/Users/myhloli/pdf/png/index.png"
|
||||
|
||||
|
||||
@@ -12,8 +12,7 @@ from loguru import logger
|
||||
|
||||
from mineru.model.ocr.seal_crop import CropByPolys, SortPolyBoxes
|
||||
from mineru.utils.config_reader import get_device
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ...utils.model_registry import PDF_EXTRACT_KIT
|
||||
from mineru.utils.ocr_language import normalize_ocr_model_lang
|
||||
from mineru.utils.ocr_utils import (
|
||||
check_img,
|
||||
@@ -67,12 +66,8 @@ class PytorchPaddleOCR(TextSystem):
|
||||
supported_langs=config['lang'],
|
||||
)
|
||||
det, rec, dict_file = get_model_params(self.lang, config)
|
||||
ocr_models_dir = ModelPath.pytorch_paddle
|
||||
|
||||
det_model_path = f"{ocr_models_dir}/{det}"
|
||||
det_model_path = os.path.join(auto_download_and_get_model_root_path(det_model_path), det_model_path)
|
||||
rec_model_path = f"{ocr_models_dir}/{rec}"
|
||||
rec_model_path = os.path.join(auto_download_and_get_model_root_path(rec_model_path), rec_model_path)
|
||||
det_model_path = str(PDF_EXTRACT_KIT.pytorch_paddle.path(det).ensure())
|
||||
rec_model_path = str(PDF_EXTRACT_KIT.pytorch_paddle.path(rec).ensure())
|
||||
kwargs['det_model_path'] = det_model_path
|
||||
kwargs['rec_model_path'] = rec_model_path
|
||||
kwargs['rec_char_dict_path'] = os.path.join(root_dir, 'pytorchocr', 'utils', 'resources', 'dict', dict_file)
|
||||
|
||||
@@ -1,24 +1,18 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
import cv2
|
||||
import numpy as np
|
||||
import onnxruntime
|
||||
from loguru import logger
|
||||
from tqdm import tqdm
|
||||
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ....utils.model_registry import PDF_EXTRACT_KIT
|
||||
|
||||
|
||||
class PaddleTableClsModel:
|
||||
def __init__(self):
|
||||
from mineru.backend.local_model_runtime import AtomicModel
|
||||
|
||||
self.sess = onnxruntime.InferenceSession(
|
||||
os.path.join(auto_download_and_get_model_root_path(ModelPath.paddle_table_cls), ModelPath.paddle_table_cls)
|
||||
)
|
||||
self.sess = onnxruntime.InferenceSession(str(PDF_EXTRACT_KIT.paddle_table_cls.ensure()))
|
||||
self.less_length = 256
|
||||
self.cw, self.ch = 224, 224
|
||||
self.std = [0.229, 0.224, 0.225]
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
import os
|
||||
import copy
|
||||
import time
|
||||
import html
|
||||
@@ -12,10 +11,9 @@ import numpy as np
|
||||
from loguru import logger
|
||||
from tqdm import tqdm
|
||||
|
||||
from .....utils.model_registry import PDF_EXTRACT_KIT
|
||||
from .matcher import TableMatch
|
||||
from .table_structure import TableStructurer
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -152,13 +150,8 @@ def escape_html(input_string):
|
||||
|
||||
class PaddleTableModel(object):
|
||||
def __init__(self, ocr_engine):
|
||||
slanet_plus_model_path = os.path.join(
|
||||
auto_download_and_get_model_root_path(ModelPath.slanet_plus),
|
||||
ModelPath.slanet_plus,
|
||||
)
|
||||
input_args = PaddleTableInput(
|
||||
model_type="slanet_plus", model_path=slanet_plus_model_path
|
||||
)
|
||||
slanet_plus_model_path = str(PDF_EXTRACT_KIT.slanet_plus.ensure())
|
||||
input_args = PaddleTableInput(model_type="slanet_plus", model_path=slanet_plus_model_path)
|
||||
self.table_model = PaddleTable(input_args)
|
||||
self.ocr_engine = ocr_engine
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import asdict, dataclass
|
||||
@@ -13,13 +12,12 @@ from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
from PIL import Image
|
||||
|
||||
from mineru.utils.enum_class import ModelPath
|
||||
from mineru.utils.image_utils import calculate_contrast
|
||||
from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from .....utils.model_registry import PDF_EXTRACT_KIT
|
||||
|
||||
from .table_recover import TableRecover
|
||||
from .table_structure_unet import TSRUnet
|
||||
from .utils import InputType, LoadImage, VisTable
|
||||
from .utils import InputType, LoadImage
|
||||
from .utils_table_recover import (
|
||||
box_4_2_poly_to_box_4_1,
|
||||
gather_ocr_list_by_row,
|
||||
@@ -212,7 +210,12 @@ class WiredTableRecognition:
|
||||
# ocr_result = [[]]
|
||||
# for crop_img in img_crop_list:
|
||||
# tmp_ocr_result = self.ocr_engine.ocr(crop_img)
|
||||
# if tmp_ocr_result[0] and len(tmp_ocr_result[0]) > 0 and isinstance(tmp_ocr_result[0], list) and len(tmp_ocr_result[0][0]) == 2:
|
||||
# if (
|
||||
# tmp_ocr_result[0]
|
||||
# and len(tmp_ocr_result[0]) > 0
|
||||
# and isinstance(tmp_ocr_result[0], list)
|
||||
# and len(tmp_ocr_result[0][0]) == 2
|
||||
# ):
|
||||
# ocr_result[0].append(tmp_ocr_result[0][0][1])
|
||||
# else:
|
||||
# ocr_result[0].append(("", 0.0))
|
||||
@@ -260,7 +263,7 @@ def count_table_cells_physical(html_code):
|
||||
|
||||
class UnetTableModel:
|
||||
def __init__(self, ocr_engine):
|
||||
model_path = os.path.join(auto_download_and_get_model_root_path(ModelPath.unet_structure), ModelPath.unet_structure)
|
||||
model_path = str(PDF_EXTRACT_KIT.unet_structure.ensure())
|
||||
wired_input_args = WiredTableInput(model_path=model_path)
|
||||
self.wired_table_model = WiredTableRecognition(wired_input_args, ocr_engine)
|
||||
self.ocr_engine = ocr_engine
|
||||
@@ -311,7 +314,10 @@ class UnetTableModel:
|
||||
wireless_text_count += 1
|
||||
if ocr_res[1] in wired_html_code:
|
||||
wired_text_count += 1
|
||||
# logger.debug(f"wireless table ocr text count: {wireless_text_count}, wired table ocr text count: {wired_text_count}")
|
||||
# logger.debug(
|
||||
# f"wireless table ocr text count: {wireless_text_count}, "
|
||||
# f"wired table ocr text count: {wired_text_count}"
|
||||
# )
|
||||
|
||||
# 使用HTML解析器计算空单元格数量
|
||||
wireless_soup = (
|
||||
@@ -321,7 +327,10 @@ class UnetTableModel:
|
||||
# 计算空单元格数量(没有文本内容或只有空白字符)
|
||||
wireless_blank_count = sum(1 for cell in wireless_soup.find_all(["td", "th"]) if not cell.text.strip())
|
||||
wired_blank_count = sum(1 for cell in wired_soup.find_all(["td", "th"]) if not cell.text.strip())
|
||||
# logger.debug(f"wireless table blank cell count: {wireless_blank_count}, wired table blank cell count: {wired_blank_count}")
|
||||
# logger.debug(
|
||||
# f"wireless table blank cell count: {wireless_blank_count}, "
|
||||
# f"wired table blank cell count: {wired_blank_count}"
|
||||
# )
|
||||
|
||||
# 计算非空单元格数量
|
||||
wireless_non_blank_count = wireless_len - wireless_blank_count
|
||||
@@ -331,7 +340,11 @@ class UnetTableModel:
|
||||
if wireless_non_blank_count > wired_non_blank_count:
|
||||
# 假设非空表格是接近正方表，使用非空单元格数量开平方作为表格规模的估计
|
||||
wired_table_scale = round(wired_non_blank_count**0.5)
|
||||
# logger.debug(f"wireless non-blank cell count: {wireless_non_blank_count}, wired non-blank cell count: {wired_non_blank_count}, wired table scale: {wired_table_scale}")
|
||||
# logger.debug(
|
||||
# f"wireless non-blank cell count: {wireless_non_blank_count}, "
|
||||
# f"wired non-blank cell count: {wired_non_blank_count}, "
|
||||
# f"wired table scale: {wired_table_scale}"
|
||||
# )
|
||||
# 如果无线表非空格的数量比有线表多一列或以上，需要切换到无线表
|
||||
wired_scale_plus_2_cols = wired_non_blank_count + (wired_table_scale * 2)
|
||||
wired_scale_squared_plus_2_rows = wired_table_scale * (wired_table_scale + 2)
|
||||
|
||||
@@ -17,7 +17,7 @@ from packaging import version
|
||||
|
||||
from ...utils.check_sys_env import is_mac_os_version_supported
|
||||
from ...utils.config_reader import get_device
|
||||
from ...utils.models_download_utils import auto_download_and_get_model_root_path
|
||||
from ...utils.model_registry import MINERU_2_5_PRO_2605_1_2B
|
||||
from .engine_utils import (
|
||||
enable_custom_logits_processors,
|
||||
mod_kwargs_by_device_type,
|
||||
@@ -79,7 +79,7 @@ class ModelSingleton:
|
||||
if param in kwargs:
|
||||
del kwargs[param]
|
||||
if backend not in ["http-client"] and not model_path:
|
||||
model_path = auto_download_and_get_model_root_path("/", "vlm")
|
||||
model_path = str(MINERU_2_5_PRO_2605_1_2B.ensure())
|
||||
if backend == "transformers":
|
||||
try:
|
||||
from transformers import (
|
||||
@@ -417,4 +417,3 @@ async def aio_predictor_execution_guard(predictor: MinerUClient) -> AsyncIterato
|
||||
yield
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
|
||||
@@ -1993,19 +1993,19 @@ def _model_ids_and_tiers_for_server_tier(tier: Tier) -> tuple[list[str], list[Ti
|
||||
},
|
||||
]
|
||||
if tier == "xhigh":
|
||||
return ["MinerU2.5-Pro-2604-1.2B", "MinerU-HTML"], [
|
||||
return ["MinerU2.5-Pro-2605-1.2B", "MinerU-HTML"], [
|
||||
{
|
||||
"id": "xhigh",
|
||||
"description": "Hybrid maximum-accuracy parsing.",
|
||||
"current_model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"current_model": "MinerU2.5-Pro-2605-1.2B",
|
||||
},
|
||||
]
|
||||
|
||||
return ["MinerU2.5-Pro-2604-1.2B", "MinerU-HTML"], [
|
||||
return ["MinerU2.5-Pro-2605-1.2B", "MinerU-HTML"], [
|
||||
{
|
||||
"id": "high",
|
||||
"description": "Hybrid high-accuracy parsing.",
|
||||
"current_model": "MinerU2.5-Pro-2604-1.2B",
|
||||
"current_model": "MinerU2.5-Pro-2605-1.2B",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -29,32 +29,6 @@ def read_config() -> dict[str, Any] | None:
|
||||
return config
|
||||
|
||||
|
||||
def get_configured_model_source(default: str | None = None) -> str | None:
|
||||
"""读取配置文件中的固定模型来源配置，auto 或缺失时返回默认值。"""
|
||||
supported_sources = {"huggingface", "modelscope"}
|
||||
config = read_config()
|
||||
if config is None:
|
||||
return default
|
||||
|
||||
model_source = config.get("model-source")
|
||||
if model_source is None:
|
||||
return default
|
||||
if not isinstance(model_source, str):
|
||||
logger.warning(f"'model-source' in {get_tools_config_file_path()} must be a string, use {default} as default")
|
||||
return default
|
||||
|
||||
normalized_model_source = model_source.strip().lower()
|
||||
if not normalized_model_source:
|
||||
return default
|
||||
if normalized_model_source == "auto":
|
||||
return default
|
||||
if normalized_model_source in supported_sources:
|
||||
return normalized_model_source
|
||||
|
||||
logger.warning(f"Unsupported 'model-source' in {get_tools_config_file_path()}: {model_source}, use {default} as default")
|
||||
return default
|
||||
|
||||
|
||||
def get_device() -> str:
|
||||
device_mode = os.getenv("MINERU_DEVICE_MODE", None)
|
||||
if device_mode is not None:
|
||||
@@ -159,12 +133,3 @@ def get_llm_aided_config() -> dict[str, Any] | None:
|
||||
else:
|
||||
return llm_aided_config
|
||||
|
||||
|
||||
def get_local_models_dir() -> dict[str, str] | None:
|
||||
config = read_config()
|
||||
if config is None:
|
||||
return None
|
||||
models_dir = config.get("models-dir")
|
||||
if models_dir is None:
|
||||
logger.warning(f"'models-dir' not found in {CONFIG_FILE_NAME}, use None as default")
|
||||
return models_dir
|
||||
|
||||
@@ -1,20 +1,6 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
|
||||
|
||||
class ModelPath:
|
||||
vlm_root_hf = "opendatalab/MinerU2.5-Pro-2605-1.2B"
|
||||
vlm_root_modelscope = "OpenDataLab/MinerU2.5-Pro-2605-1.2B"
|
||||
pipeline_root_modelscope = "OpenDataLab/PDF-Extract-Kit-1.0"
|
||||
pipeline_root_hf = "opendatalab/PDF-Extract-Kit-1.0"
|
||||
pp_doclayout_v2 = "models/Layout/PP-DocLayoutV2"
|
||||
unimernet_small = "models/MFR/unimernet_hf_small_2503"
|
||||
pp_formulanet_plus_m = "models/MFR/pp_formulanet_plus_m"
|
||||
pytorch_paddle = "models/OCR/paddleocr_torch"
|
||||
slanet_plus = "models/TabRec/SlanetPlus/slanet-plus.onnx"
|
||||
unet_structure = "models/TabRec/UnetStructure/unet.onnx"
|
||||
paddle_table_cls = "models/TabCls/paddle_table_cls/PP-LCNet_x1_0_table_cls.onnx"
|
||||
|
||||
|
||||
class ImageType:
|
||||
PIL = "pil_img"
|
||||
BASE64 = "base64_img"
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..config import config
|
||||
from ..types import Tier, validate_tier
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config import ModelSource
|
||||
|
||||
|
||||
def _join_model_path(*parts: str) -> str:
|
||||
cleaned: list[str] = []
|
||||
for raw in parts:
|
||||
value = str(raw).replace("\\", "/").strip("/")
|
||||
if not value:
|
||||
continue
|
||||
cleaned.extend(part for part in value.split("/") if part and part != ".")
|
||||
if any(part == ".." for part in cleaned):
|
||||
raise ValueError("Model paths must not contain '..' segments.")
|
||||
return "/".join(cleaned)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelPath:
|
||||
repo: "ModelRepo"
|
||||
name: str
|
||||
relative_path: str
|
||||
|
||||
def path(self, relative_path: str, /, *children: str) -> "ModelPath":
|
||||
joined = _join_model_path(self.relative_path, relative_path, *children)
|
||||
name = _join_model_path(self.name, relative_path, *children)
|
||||
return ModelPath(repo=self.repo, name=name, relative_path=joined)
|
||||
|
||||
def local_path(self) -> Path:
|
||||
return self.repo.local_dir() / self.relative_path
|
||||
|
||||
def ensure(self, *, source: "ModelSource | None" = None) -> Path:
|
||||
from .models_download_utils import download_model_files
|
||||
|
||||
download_model_files(self.repo, [self], source=source)
|
||||
return self.local_path()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelRepo:
|
||||
name: str
|
||||
local_name: str
|
||||
repos: dict[str, str]
|
||||
paths: dict[str, str]
|
||||
|
||||
def __getattr__(self, name: str) -> ModelPath:
|
||||
try:
|
||||
return self.named_path(name)
|
||||
except KeyError as exc:
|
||||
raise AttributeError(name) from exc
|
||||
|
||||
def named_path(self, name: str) -> ModelPath:
|
||||
return ModelPath(repo=self, name=name, relative_path=self.paths[name])
|
||||
|
||||
def path(self, relative_path: str, /, *children: str) -> ModelPath:
|
||||
joined = _join_model_path(relative_path, *children)
|
||||
return ModelPath(repo=self, name=joined, relative_path=joined)
|
||||
|
||||
def required_paths(self) -> tuple[ModelPath, ...]:
|
||||
return tuple(self.named_path(name) for name in self.paths)
|
||||
|
||||
def local_dir(self) -> Path:
|
||||
return Path(config.model.base_dir).expanduser() / self.local_name
|
||||
|
||||
def lock_path(self) -> Path:
|
||||
safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", self.local_name).strip("._") or "model"
|
||||
return Path(config.model.base_dir).expanduser() / ".locks" / f"{safe_name}.lock"
|
||||
|
||||
def ensure(self, *, source: "ModelSource | None" = None) -> Path:
|
||||
from .models_download_utils import download_model_repo
|
||||
|
||||
return download_model_repo(self, source=source)
|
||||
|
||||
|
||||
PDF_EXTRACT_KIT = ModelRepo(
|
||||
name="PDF-Extract-Kit-1.0",
|
||||
local_name="PDF-Extract-Kit-1.0",
|
||||
repos={
|
||||
"huggingface": "opendatalab/PDF-Extract-Kit-1.0",
|
||||
"modelscope": "OpenDataLab/PDF-Extract-Kit-1.0",
|
||||
},
|
||||
paths={
|
||||
"pp_doclayout_v2": "models/Layout/PP-DocLayoutV2",
|
||||
"unimernet_small": "models/MFR/unimernet_hf_small_2503",
|
||||
"pp_formulanet_plus_m": "models/MFR/pp_formulanet_plus_m",
|
||||
"pytorch_paddle": "models/OCR/paddleocr_torch",
|
||||
"slanet_plus": "models/TabRec/SlanetPlus/slanet-plus.onnx",
|
||||
"unet_structure": "models/TabRec/UnetStructure/unet.onnx",
|
||||
"paddle_table_cls": "models/TabCls/paddle_table_cls/PP-LCNet_x1_0_table_cls.onnx",
|
||||
},
|
||||
)
|
||||
|
||||
MINERU_2_5_PRO_2605_1_2B = ModelRepo(
|
||||
name="MinerU2.5-Pro-2605-1.2B",
|
||||
local_name="MinerU2.5-Pro-2605-1.2B",
|
||||
repos={
|
||||
"huggingface": "opendatalab/MinerU2.5-Pro-2605-1.2B",
|
||||
"modelscope": "OpenDataLab/MinerU2.5-Pro-2605-1.2B",
|
||||
},
|
||||
paths={
|
||||
"config_json": "config.json",
|
||||
"preprocessor_config_json": "preprocessor_config.json",
|
||||
"tokenizer_config_json": "tokenizer_config.json",
|
||||
},
|
||||
)
|
||||
|
||||
MODEL_REPOS: tuple[ModelRepo, ...] = (
|
||||
PDF_EXTRACT_KIT,
|
||||
MINERU_2_5_PRO_2605_1_2B,
|
||||
)
|
||||
|
||||
MODEL_REPOS_BY_NAME: dict[str, ModelRepo] = {repo.name: repo for repo in MODEL_REPOS}
|
||||
|
||||
REPOS_FOR_TIER: dict[Tier, tuple[ModelRepo, ...]] = {
|
||||
"flash": (),
|
||||
"medium": (PDF_EXTRACT_KIT,),
|
||||
"high": (PDF_EXTRACT_KIT, MINERU_2_5_PRO_2605_1_2B),
|
||||
"xhigh": (PDF_EXTRACT_KIT, MINERU_2_5_PRO_2605_1_2B),
|
||||
}
|
||||
|
||||
|
||||
def get_model_repo(name: str) -> ModelRepo:
|
||||
try:
|
||||
return MODEL_REPOS_BY_NAME[name]
|
||||
except KeyError as exc:
|
||||
available = ", ".join(MODEL_REPOS_BY_NAME)
|
||||
raise ValueError(f"Unsupported model repo '{name}'. Available repos: {available}.") from exc
|
||||
|
||||
|
||||
def model_repos_for_tier(tier: Tier) -> tuple[ModelRepo, ...]:
|
||||
return REPOS_FOR_TIER[validate_tier(tier)]
|
||||
|
||||
|
||||
def model_repo_names() -> tuple[str, ...]:
|
||||
return tuple(MODEL_REPOS_BY_NAME)
|
||||
|
||||
|
||||
def model_path_exists(path: ModelPath) -> bool:
|
||||
local_path = path.local_path()
|
||||
if local_path.is_file():
|
||||
return True
|
||||
if not local_path.is_dir():
|
||||
return False
|
||||
return any(files for _root, _dirs, files in os.walk(local_path))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MINERU_2_5_PRO_2605_1_2B",
|
||||
"MODEL_REPOS",
|
||||
"MODEL_REPOS_BY_NAME",
|
||||
"ModelPath",
|
||||
"ModelRepo",
|
||||
"PDF_EXTRACT_KIT",
|
||||
"REPOS_FOR_TIER",
|
||||
"get_model_repo",
|
||||
"model_path_exists",
|
||||
"model_repo_names",
|
||||
"model_repos_for_tier",
|
||||
]
|
||||
@@ -1,205 +1,56 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
import json
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from collections.abc import Sequence
|
||||
from typing import Literal
|
||||
|
||||
from filelock import FileLock
|
||||
from huggingface_hub import snapshot_download as hf_snapshot_download
|
||||
from loguru import logger
|
||||
from modelscope import snapshot_download as ms_snapshot_download
|
||||
import requests
|
||||
|
||||
from .config_reader import get_configured_model_source, get_local_models_dir, get_tools_config_file_path
|
||||
from .enum_class import ModelPath
|
||||
from ..config import config, get_config_source, update_config_file
|
||||
from .model_registry import ModelPath, ModelRepo, model_path_exists
|
||||
|
||||
MODEL_SOURCE_ENV_VAR = "MINERU_MODEL_SOURCE"
|
||||
CONFIG_TEMPLATE_URL = "https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/mineru.template.json"
|
||||
MINERU_CONFIG_VERSION = "1.3.2"
|
||||
HUGGINGFACE_MODELS_PAGE_URL = "https://huggingface.co/models"
|
||||
HUGGINGFACE_MODELS_PAGE_TIMEOUT = 3
|
||||
HUGGINGFACE_MODELS_PAGE_MAX_ATTEMPTS = 2
|
||||
REMOTE_MODEL_SOURCES = ("huggingface", "modelscope")
|
||||
_HUGGINGFACE_MODELS_PAGE_URL = "https://huggingface.co/models"
|
||||
_HUGGINGFACE_MODELS_PAGE_TIMEOUT = 3
|
||||
_HUGGINGFACE_MODELS_PAGE_MAX_ATTEMPTS = 2
|
||||
_REMOTE_MODEL_SOURCES = ("huggingface", "modelscope")
|
||||
DOWNLOAD_MODEL_SOURCES = ("auto", *_REMOTE_MODEL_SOURCES)
|
||||
ResolvedRemoteModelSource = Literal["huggingface", "modelscope"]
|
||||
ResolvedModelSource = Literal["huggingface", "modelscope", "local"]
|
||||
|
||||
|
||||
def download_json(url: str) -> dict:
|
||||
"""下载 JSON 文件并返回解析后的内容。"""
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
@dataclass(frozen=True)
|
||||
class ModelReadyResult:
|
||||
ready: bool
|
||||
root: Path
|
||||
missing_paths: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def is_config_version_outdated(config_version: object) -> bool:
|
||||
"""判断本地配置版本是否低于当前模板版本。"""
|
||||
|
||||
def version_tuple(version: object) -> tuple[int, ...]:
|
||||
"""将版本号字符串转换为可比较的整数元组。"""
|
||||
parts = []
|
||||
for part in str(version).split("."):
|
||||
parts.append(int(part) if part.isdigit() else 0)
|
||||
return tuple(parts)
|
||||
|
||||
current_version = version_tuple(config_version)
|
||||
target_version = version_tuple(MINERU_CONFIG_VERSION)
|
||||
max_len = max(len(current_version), len(target_version))
|
||||
current_version += (0,) * (max_len - len(current_version))
|
||||
target_version += (0,) * (max_len - len(target_version))
|
||||
return current_version < target_version
|
||||
def _normalize_source(raw_source: object, *, strict: bool) -> str:
|
||||
source = str(raw_source or "auto").strip().lower()
|
||||
if source in {"auto", "huggingface", "modelscope", "local"}:
|
||||
return source
|
||||
if strict:
|
||||
expected = ", ".join(DOWNLOAD_MODEL_SOURCES)
|
||||
raise ValueError(f"Unsupported model source '{raw_source}'. Expected one of: {expected}.")
|
||||
logger.warning(f"Unsupported model source {raw_source!r}; falling back to auto.")
|
||||
return "auto"
|
||||
|
||||
|
||||
def merge_config_dict(base_config: dict, override_config: dict, skip_keys: set[str] | None = None) -> dict:
|
||||
"""递归合并配置字典，用 override_config 覆盖 base_config 并保留新模板字段。"""
|
||||
skip_keys = skip_keys or set()
|
||||
merged_config = dict(base_config)
|
||||
for key, value in override_config.items():
|
||||
if key in skip_keys:
|
||||
continue
|
||||
base_value = merged_config.get(key)
|
||||
if isinstance(base_value, dict) and isinstance(value, dict):
|
||||
merged_config[key] = merge_config_dict(base_value, value, skip_keys=skip_keys)
|
||||
else:
|
||||
merged_config[key] = value
|
||||
return merged_config
|
||||
|
||||
|
||||
def download_and_modify_json(url: str, local_filename: str, modifications: dict) -> None:
|
||||
"""下载或读取 JSON 配置，并按 modifications 合并更新后写回。"""
|
||||
if os.path.exists(local_filename):
|
||||
with open(local_filename, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
config_version = data.get("config_version", "0.0.0")
|
||||
if is_config_version_outdated(config_version):
|
||||
template_data = download_json(url)
|
||||
data = merge_config_dict(template_data, data, skip_keys={"config_version"})
|
||||
else:
|
||||
data = download_json(url)
|
||||
|
||||
data = merge_config_dict(data, modifications)
|
||||
|
||||
os.makedirs(os.path.dirname(local_filename) or ".", exist_ok=True)
|
||||
with open(local_filename, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
def persist_resolved_model_source(model_source: str) -> None:
|
||||
"""将 auto 解析出的实际模型来源写入配置文件，避免后续启动受网络波动影响。"""
|
||||
if model_source not in REMOTE_MODEL_SOURCES:
|
||||
return
|
||||
try:
|
||||
download_and_modify_json(
|
||||
CONFIG_TEMPLATE_URL,
|
||||
get_tools_config_file_path(),
|
||||
{"model-source": model_source},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to persist resolved model source '{model_source}': {exc}")
|
||||
|
||||
|
||||
def normalize_download_relative_path(relative_path: str, repo_mode: str) -> str:
|
||||
"""按仓库模式规范化下载相对路径，保持 pipeline 与 VLM 根路径语义。"""
|
||||
if repo_mode == "pipeline":
|
||||
return relative_path.strip("/")
|
||||
if repo_mode == "vlm":
|
||||
if relative_path == "/":
|
||||
return relative_path
|
||||
return relative_path.strip("/")
|
||||
raise ValueError(f"Unsupported repo_mode: {repo_mode}, must be 'pipeline' or 'vlm'")
|
||||
|
||||
|
||||
def read_existing_tools_config() -> dict | None:
|
||||
"""读取已存在的工具配置文件；缺失、读取失败或格式异常时返回 None。"""
|
||||
config_file = get_tools_config_file_path()
|
||||
if not os.path.exists(config_file):
|
||||
return None
|
||||
try:
|
||||
with open(config_file, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to read model config from {config_file}: {exc}")
|
||||
return None
|
||||
if not isinstance(config, dict):
|
||||
logger.warning(f"Model config in {config_file} must be a JSON object.")
|
||||
return None
|
||||
return config
|
||||
|
||||
|
||||
def get_configured_repo_model_root(config: dict, repo_mode: str) -> str | None:
|
||||
"""从配置中读取指定仓库模式的模型根目录，缺失或类型错误时返回 None。"""
|
||||
models_dir = config.get("models-dir")
|
||||
if not isinstance(models_dir, dict):
|
||||
return None
|
||||
model_root = models_dir.get(repo_mode)
|
||||
if not isinstance(model_root, str):
|
||||
return None
|
||||
model_root = model_root.strip()
|
||||
if not model_root:
|
||||
return None
|
||||
return os.path.expanduser(model_root)
|
||||
|
||||
|
||||
def build_configured_model_path(model_root: str, relative_path: str) -> str:
|
||||
"""根据模型根目录和下载相对路径拼出本地校验路径。"""
|
||||
if relative_path in ("", "/"):
|
||||
return model_root
|
||||
return os.path.join(model_root, relative_path)
|
||||
|
||||
|
||||
def is_existing_model_path_usable(model_path: str) -> bool:
|
||||
"""判断本地模型路径是否可复用：文件可直接用，目录必须包含实际文件。"""
|
||||
if os.path.isfile(model_path):
|
||||
return True
|
||||
if not os.path.isdir(model_path):
|
||||
return False
|
||||
for _root, _dirs, files in os.walk(model_path):
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_existing_configured_model_root(repo_mode: str, relative_path: str) -> str | None:
|
||||
"""如果配置中的模型根目录已包含本次所需模型，则返回该根目录以跳过远端下载。"""
|
||||
config = read_existing_tools_config()
|
||||
if config is None:
|
||||
return None
|
||||
|
||||
model_root = get_configured_repo_model_root(config, repo_mode)
|
||||
if model_root is None:
|
||||
return None
|
||||
|
||||
local_model_path = build_configured_model_path(model_root, relative_path)
|
||||
if is_existing_model_path_usable(local_model_path):
|
||||
logger.debug(f"Use configured local {repo_mode} model path: {local_model_path}")
|
||||
return model_root
|
||||
return None
|
||||
|
||||
|
||||
def persist_downloaded_model_config(model_source: str, repo_mode: str, model_root: str) -> None:
|
||||
"""远端模型下载成功后，创建或更新配置文件中的模型根目录和实际模型来源。"""
|
||||
if model_source not in REMOTE_MODEL_SOURCES:
|
||||
return
|
||||
config_file = get_tools_config_file_path()
|
||||
try:
|
||||
download_and_modify_json(
|
||||
CONFIG_TEMPLATE_URL,
|
||||
config_file,
|
||||
{
|
||||
"models-dir": {
|
||||
repo_mode: model_root,
|
||||
},
|
||||
"model-source": model_source,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to persist downloaded {repo_mode} model config to {config_file}: {exc}")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def resolve_auto_model_source() -> str:
|
||||
"""通过 Hugging Face 模型列表页探测 auto 应该使用的实际模型来源。"""
|
||||
def _resolve_auto_model_source() -> ResolvedRemoteModelSource:
|
||||
"""Resolve auto source by probing Hugging Face reachability."""
|
||||
last_error = None
|
||||
for _ in range(HUGGINGFACE_MODELS_PAGE_MAX_ATTEMPTS):
|
||||
for _ in range(_HUGGINGFACE_MODELS_PAGE_MAX_ATTEMPTS):
|
||||
try:
|
||||
response = requests.get(
|
||||
HUGGINGFACE_MODELS_PAGE_URL,
|
||||
timeout=HUGGINGFACE_MODELS_PAGE_TIMEOUT,
|
||||
_HUGGINGFACE_MODELS_PAGE_URL,
|
||||
timeout=_HUGGINGFACE_MODELS_PAGE_TIMEOUT,
|
||||
)
|
||||
if 200 <= response.status_code < 400:
|
||||
return "huggingface"
|
||||
@@ -207,126 +58,160 @@ def resolve_auto_model_source() -> str:
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
|
||||
logger.warning(f"Failed to access {HUGGINGFACE_MODELS_PAGE_URL}: {last_error}, fallback to modelscope.")
|
||||
logger.warning("Failed to access %s: %s, fallback to modelscope.", _HUGGINGFACE_MODELS_PAGE_URL, last_error)
|
||||
return "modelscope"
|
||||
|
||||
|
||||
def resolve_model_source(model_source: str | None = None, allow_auto: bool = False) -> str:
|
||||
"""将环境变量或配置文件中的模型来源解析为实际可下载的来源。"""
|
||||
if model_source is None:
|
||||
model_source = os.getenv(MODEL_SOURCE_ENV_VAR)
|
||||
if isinstance(model_source, str) and model_source.strip().lower() == "auto":
|
||||
raise ValueError(
|
||||
f"{MODEL_SOURCE_ENV_VAR}=auto is not supported. "
|
||||
f"Unset {MODEL_SOURCE_ENV_VAR} to use auto detection once, "
|
||||
"or set it to huggingface/modelscope/local."
|
||||
)
|
||||
if model_source is None:
|
||||
model_source = get_configured_model_source()
|
||||
if model_source is None:
|
||||
model_source = "auto"
|
||||
allow_auto = True
|
||||
def _persist_resolved_model_source(model_source: str) -> None:
|
||||
if model_source not in _REMOTE_MODEL_SOURCES:
|
||||
return
|
||||
try:
|
||||
update_config_file({"model": {"source": model_source}})
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to persist resolved model source {model_source!r}: {exc}")
|
||||
|
||||
if not isinstance(model_source, str):
|
||||
logger.warning(f"Unsupported model source type: {type(model_source)}, fallback to auto.")
|
||||
model_source = "auto"
|
||||
allow_auto = True
|
||||
|
||||
normalized_model_source = model_source.strip().lower()
|
||||
if normalized_model_source == "local":
|
||||
return "local"
|
||||
if normalized_model_source == "auto":
|
||||
if not allow_auto:
|
||||
raise ValueError(
|
||||
"model source auto is only supported for internal default detection "
|
||||
"or explicit download command selection."
|
||||
)
|
||||
resolved_model_source = resolve_auto_model_source()
|
||||
persist_resolved_model_source(resolved_model_source)
|
||||
return resolved_model_source
|
||||
if normalized_model_source in REMOTE_MODEL_SOURCES:
|
||||
return normalized_model_source
|
||||
def resolve_model_source(
|
||||
model_source: str | None = None,
|
||||
*,
|
||||
allow_auto: bool = False,
|
||||
local_as_auto: bool = False,
|
||||
) -> ResolvedModelSource:
|
||||
"""Resolve the effective model source.
|
||||
|
||||
logger.warning(f"Unsupported model source: {model_source}, fallback to auto.")
|
||||
resolved_model_source = resolve_auto_model_source()
|
||||
persist_resolved_model_source(resolved_model_source)
|
||||
Explicit ``model_source`` is CLI input and is validated strictly. When no
|
||||
explicit source is supplied, ``config.model.source`` may come from config
|
||||
file, environment, or defaults; invalid values are treated as auto.
|
||||
"""
|
||||
explicit_source = model_source is not None
|
||||
raw_source = model_source if explicit_source else config.model.source
|
||||
normalized_source = _normalize_source(raw_source, strict=explicit_source)
|
||||
|
||||
if explicit_source and normalized_source == "local":
|
||||
raise ValueError("--source does not support 'local'. Use auto, huggingface, or modelscope.")
|
||||
|
||||
if normalized_source == "local":
|
||||
if not local_as_auto:
|
||||
return "local"
|
||||
normalized_source = "auto"
|
||||
|
||||
if normalized_source in _REMOTE_MODEL_SOURCES:
|
||||
return normalized_source # type: ignore[return-value]
|
||||
|
||||
if normalized_source != "auto":
|
||||
raise ValueError(f"Unsupported model source: {raw_source}")
|
||||
if not allow_auto:
|
||||
raise ValueError("model source auto is only supported when auto resolution is explicitly allowed.")
|
||||
|
||||
resolved_model_source = _resolve_auto_model_source()
|
||||
if not explicit_source and raw_source == "auto" and get_config_source("model.source") in {"default", "file"}:
|
||||
_persist_resolved_model_source(resolved_model_source)
|
||||
return resolved_model_source
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _snapshot_download_cached(model_source: str, repo_mode: str, repo: str, relative_path: str) -> str:
|
||||
"""按进程缓存远端 snapshot_download 结果，减少重复缓存检查和 Fetching 日志。"""
|
||||
def _snapshot_download(model_source: ResolvedRemoteModelSource, repo: ModelRepo, patterns: list[str] | None) -> str:
|
||||
repo_id = repo.repos[model_source]
|
||||
local_dir = repo.local_dir()
|
||||
local_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||
kwargs = {"local_dir": str(local_dir)}
|
||||
if patterns is not None:
|
||||
kwargs["allow_patterns"] = patterns
|
||||
if model_source == "huggingface":
|
||||
snapshot_download = hf_snapshot_download
|
||||
elif model_source == "modelscope":
|
||||
snapshot_download = ms_snapshot_download
|
||||
else:
|
||||
raise ValueError(f"未知的仓库类型: {model_source}")
|
||||
|
||||
if repo_mode == "pipeline":
|
||||
return snapshot_download(repo, allow_patterns=[relative_path, relative_path + "/*"])
|
||||
|
||||
if repo_mode == "vlm":
|
||||
# VLM 整仓下载和局部路径下载都参与缓存，但保持原有 allow_patterns 行为。
|
||||
if relative_path == "/":
|
||||
return snapshot_download(repo)
|
||||
return snapshot_download(repo, allow_patterns=[relative_path, relative_path + "/*"])
|
||||
|
||||
raise ValueError(f"Unsupported repo_mode: {repo_mode}, must be 'pipeline' or 'vlm'")
|
||||
return hf_snapshot_download(repo_id, **kwargs)
|
||||
return ms_snapshot_download(repo_id, **kwargs)
|
||||
|
||||
|
||||
def auto_download_and_get_model_root_path(relative_path: str, repo_mode: str = "pipeline") -> str:
|
||||
"""
|
||||
支持文件或目录的可靠下载。
|
||||
- 如果输入文件: 返回本地文件绝对路径
|
||||
- 如果输入目录: 返回本地缓存下与 relative_path 同结构的相对路径字符串
|
||||
:param repo_mode: 指定仓库模式，'pipeline' 或 'vlm'
|
||||
:param relative_path: 文件或目录相对路径
|
||||
:return: 本地文件绝对路径或相对路径
|
||||
"""
|
||||
model_source = resolve_model_source()
|
||||
|
||||
if model_source == "local":
|
||||
local_models_config = get_local_models_dir()
|
||||
if local_models_config is None:
|
||||
raise ValueError("Local model paths are not configured.")
|
||||
root_path = local_models_config.get(repo_mode, None)
|
||||
if not root_path:
|
||||
raise ValueError(f"Local path for repo_mode '{repo_mode}' is not configured.")
|
||||
return root_path
|
||||
|
||||
# 建立仓库模式到路径的映射
|
||||
repo_mapping = {
|
||||
"pipeline": {
|
||||
"huggingface": ModelPath.pipeline_root_hf,
|
||||
"modelscope": ModelPath.pipeline_root_modelscope,
|
||||
},
|
||||
"vlm": {
|
||||
"huggingface": ModelPath.vlm_root_hf,
|
||||
"modelscope": ModelPath.vlm_root_modelscope,
|
||||
},
|
||||
}
|
||||
|
||||
if repo_mode not in repo_mapping:
|
||||
raise ValueError(f"Unsupported repo_mode: {repo_mode}, must be 'pipeline' or 'vlm'")
|
||||
|
||||
# model_source 已解析为实际远端来源后，再选择对应仓库。
|
||||
repo = repo_mapping[repo_mode][model_source]
|
||||
|
||||
relative_path = normalize_download_relative_path(relative_path, repo_mode)
|
||||
configured_model_root = get_existing_configured_model_root(repo_mode, relative_path)
|
||||
if configured_model_root is not None:
|
||||
return configured_model_root
|
||||
|
||||
cache_dir = _snapshot_download_cached(model_source, repo_mode, repo, relative_path)
|
||||
|
||||
if not cache_dir:
|
||||
raise FileNotFoundError(f"Failed to download model: {relative_path} from {repo}")
|
||||
persist_downloaded_model_config(model_source, repo_mode, cache_dir)
|
||||
return cache_dir
|
||||
def _path_patterns(paths: list[str]) -> list[str]:
|
||||
patterns: list[str] = []
|
||||
for path in paths:
|
||||
normalized = path.strip("/")
|
||||
if not normalized:
|
||||
continue
|
||||
patterns.extend([normalized, f"{normalized}/*"])
|
||||
return patterns
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
path1 = "models/README.md"
|
||||
root = auto_download_and_get_model_root_path(path1)
|
||||
print("本地文件绝对路径:", os.path.join(root, path1))
|
||||
def _relative_paths(repo: ModelRepo, paths: Sequence[str | ModelPath]) -> list[str]:
|
||||
relative_paths: list[str] = []
|
||||
for path in paths:
|
||||
if isinstance(path, ModelPath):
|
||||
if path.repo is not repo:
|
||||
raise ValueError(
|
||||
f"Model path {path.relative_path!r} belongs to repo {path.repo.name!r}, not {repo.name!r}."
|
||||
)
|
||||
relative_paths.append(path.relative_path)
|
||||
else:
|
||||
relative_paths.append(str(path).strip("/"))
|
||||
return relative_paths
|
||||
|
||||
|
||||
def _verify_paths(repo: ModelRepo, paths: Sequence[str | ModelPath]) -> ModelReadyResult:
|
||||
model_paths = [path if isinstance(path, ModelPath) else repo.path(path) for path in paths]
|
||||
missing = [path.relative_path for path in model_paths if not model_path_exists(path)]
|
||||
return ModelReadyResult(ready=not missing, root=repo.local_dir(), missing_paths=missing)
|
||||
|
||||
|
||||
def _raise_not_ready(repo: ModelRepo, result: ModelReadyResult) -> None:
|
||||
missing = ", ".join(result.missing_paths)
|
||||
raise FileNotFoundError(f"Model repo {repo.name} is not ready under {result.root}; missing: {missing}")
|
||||
|
||||
|
||||
def download_model_repo(repo: ModelRepo, *, source: str | None = None, local_as_auto: bool = False) -> Path:
|
||||
resolved_source = resolve_model_source(source, allow_auto=True, local_as_auto=local_as_auto)
|
||||
if resolved_source == "local":
|
||||
result = verify_model_repo(repo)
|
||||
if not result.ready:
|
||||
_raise_not_ready(repo, result)
|
||||
return result.root
|
||||
|
||||
lock_path = repo.lock_path()
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with FileLock(str(lock_path)):
|
||||
_snapshot_download(resolved_source, repo, None)
|
||||
result = verify_model_repo(repo)
|
||||
if not result.ready:
|
||||
_raise_not_ready(repo, result)
|
||||
return result.root
|
||||
|
||||
|
||||
def download_model_files(
|
||||
repo: ModelRepo,
|
||||
paths: Sequence[str | ModelPath],
|
||||
*,
|
||||
source: str | None = None,
|
||||
local_as_auto: bool = False,
|
||||
) -> Path:
|
||||
if not paths:
|
||||
return repo.local_dir()
|
||||
|
||||
resolved_source = resolve_model_source(source, allow_auto=True, local_as_auto=local_as_auto)
|
||||
if resolved_source == "local":
|
||||
result = _verify_paths(repo, paths)
|
||||
if not result.ready:
|
||||
_raise_not_ready(repo, result)
|
||||
return result.root
|
||||
|
||||
relative_paths = _relative_paths(repo, paths)
|
||||
lock_path = repo.lock_path()
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with FileLock(str(lock_path)):
|
||||
_snapshot_download(resolved_source, repo, _path_patterns(relative_paths))
|
||||
result = _verify_paths(repo, paths)
|
||||
if not result.ready:
|
||||
_raise_not_ready(repo, result)
|
||||
return result.root
|
||||
|
||||
|
||||
def verify_model_repo(repo: ModelRepo) -> ModelReadyResult:
|
||||
return _verify_paths(repo, list(repo.required_paths()))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DOWNLOAD_MODEL_SOURCES",
|
||||
"MODEL_SOURCE_ENV_VAR",
|
||||
"ModelReadyResult",
|
||||
"download_model_files",
|
||||
"download_model_repo",
|
||||
"resolve_model_source",
|
||||
"verify_model_repo",
|
||||
]
|
||||
|
||||
+6
-1
@@ -46,6 +46,7 @@ dependencies = [
|
||||
"pdftext>=0.6.3,<0.8.0",
|
||||
"modelscope>=1.26.0",
|
||||
"huggingface-hub>=0.32.4",
|
||||
"filelock>=3.0.0",
|
||||
"json-repair>=0.46.2",
|
||||
"opencv-python>=4.11.0.86",
|
||||
"fast-langdetect>=0.2.3,<0.3.0",
|
||||
@@ -114,6 +115,11 @@ all = [
|
||||
"mineru[gradio]",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
exclude-dependencies = [
|
||||
"datasets",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
homepage = "https://mineru.net/"
|
||||
documentation = "https://opendatalab.github.io/MinerU/"
|
||||
@@ -150,7 +156,6 @@ command_line = "-m pytest tests/unittest/test_e2e.py"
|
||||
source = ["mineru/"]
|
||||
omit = [
|
||||
"*/gradio_app.py",
|
||||
"*/models_download.py",
|
||||
"*/fast_api.py",
|
||||
"*/cli/client.py",
|
||||
"*/vlm_vllm_server.py",
|
||||
|
||||
@@ -6,12 +6,14 @@ import pytest
|
||||
|
||||
from mineru.config import (
|
||||
Config,
|
||||
LoadedConfig,
|
||||
LogConfig,
|
||||
PatchedConfig,
|
||||
_apply_env_overrides,
|
||||
_collect_env_overrides,
|
||||
_interpolate_env,
|
||||
_load_config,
|
||||
_read_config,
|
||||
_load_effective_config,
|
||||
update_config_file,
|
||||
)
|
||||
from mineru.doclib.config_defaults import CONFIG_DEFAULTS
|
||||
|
||||
@@ -90,8 +92,10 @@ def test_apply_env_overrides_uses_greedy_field_path_matching(monkeypatch: pytest
|
||||
monkeypatch.setenv("TEST_MINERU_UNKNOWN_FIELD", "ignored")
|
||||
monkeypatch.setenv("TEST_MINERU_CONFIG", "/tmp/ignored.yaml")
|
||||
|
||||
cfg = _apply_env_overrides(Config(), prefix=prefix)
|
||||
overrides, paths = _collect_env_overrides(prefix=prefix)
|
||||
cfg = Config(**overrides)
|
||||
|
||||
assert ("config",) not in paths
|
||||
assert cfg.doclib.tcp.enabled is True
|
||||
assert cfg.doclib.tcp.port == 15990
|
||||
assert cfg.doclib.compaction_interval_sec == 5
|
||||
@@ -125,10 +129,25 @@ doclib:
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
data = _read_config()
|
||||
loaded = _load_effective_config()
|
||||
|
||||
assert data["doclib"]["tcp"]["port"] == 18080
|
||||
assert data["doclib"]["data_dir"] == "/tmp/ignored-data-dir"
|
||||
assert loaded.config_file == str(mineru_home / "config.yaml")
|
||||
assert loaded.config_file_exists is True
|
||||
assert loaded.config.doclib.tcp.port == 18080
|
||||
assert loaded.config.doclib.data_dir == "/tmp/ignored-data-dir"
|
||||
|
||||
|
||||
def test_default_config_path_exists_even_when_file_is_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
mineru_home = tmp_path / "mineru-home"
|
||||
mineru_home.mkdir()
|
||||
monkeypatch.setenv("MINERU_HOME", str(mineru_home))
|
||||
monkeypatch.delenv("MINERU_CONFIG", raising=False)
|
||||
|
||||
loaded = _load_effective_config()
|
||||
|
||||
assert loaded.config_file == str(mineru_home / "config.yaml")
|
||||
assert loaded.config_file_exists is False
|
||||
assert loaded.config == Config()
|
||||
|
||||
|
||||
def test_apply_env_overrides_can_override_doclib_data_dir(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -136,7 +155,8 @@ def test_apply_env_overrides_can_override_doclib_data_dir(monkeypatch: pytest.Mo
|
||||
monkeypatch.setenv("MINERU_HOME", "/tmp/mineru-home")
|
||||
monkeypatch.setenv("TEST_MINERU_DOCLIB_DATA_DIR", "/tmp/ignored-data-dir")
|
||||
|
||||
cfg = _apply_env_overrides(Config(), prefix=prefix)
|
||||
overrides, _paths = _collect_env_overrides(prefix=prefix)
|
||||
cfg = Config(**overrides)
|
||||
|
||||
assert cfg.doclib.data_dir == "/tmp/ignored-data-dir"
|
||||
|
||||
@@ -147,6 +167,60 @@ def test_default_doclib_data_dir_uses_doclib_directory() -> None:
|
||||
assert cfg.doclib.data_dir.endswith(".mineru/doclib")
|
||||
|
||||
|
||||
def test_default_model_config_uses_mineru_model_directory() -> None:
|
||||
cfg = Config()
|
||||
|
||||
assert cfg.model.base_dir.endswith(".mineru/models")
|
||||
assert cfg.model.source == "auto"
|
||||
|
||||
|
||||
def test_load_effective_config_tracks_file_and_env_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(
|
||||
"""
|
||||
model:
|
||||
base_dir: /tmp/file-models
|
||||
source: huggingface
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("MINERU_CONFIG", str(config_file))
|
||||
monkeypatch.setenv("MINERU_MODEL_SOURCE", "modelscope")
|
||||
|
||||
loaded = _load_effective_config()
|
||||
|
||||
assert loaded.config.model.base_dir == "/tmp/file-models"
|
||||
assert loaded.config.model.source == "modelscope"
|
||||
assert loaded.sources[("model", "base_dir")] == "file"
|
||||
assert loaded.sources[("model", "source")] == "env"
|
||||
|
||||
|
||||
def test_update_config_file_deep_merges_with_lock(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(
|
||||
"""
|
||||
doclib:
|
||||
tcp:
|
||||
port: 18080
|
||||
model:
|
||||
base_dir: /tmp/models
|
||||
source: auto
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"mineru.config._loaded_config",
|
||||
LoadedConfig(config=Config(), sources={}, config_file=str(config_file), config_file_exists=True),
|
||||
)
|
||||
|
||||
update_config_file({"model": {"source": "huggingface"}})
|
||||
|
||||
data = _load_config(str(config_file))
|
||||
assert data["doclib"]["tcp"]["port"] == 18080
|
||||
assert data["model"] == {"base_dir": "/tmp/models", "source": "huggingface"}
|
||||
assert not list(tmp_path.glob(".config.*.tmp"))
|
||||
|
||||
|
||||
def test_default_transport_prefers_uds_when_available(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("mineru.config._uds_available", lambda: True)
|
||||
|
||||
@@ -191,7 +265,8 @@ def test_transport_enabled_accepts_auto_from_env(monkeypatch: pytest.MonkeyPatch
|
||||
monkeypatch.setenv("TEST_MINERU_DOCLIB_UDS_ENABLED", "auto")
|
||||
monkeypatch.setenv("TEST_MINERU_DOCLIB_TCP_ENABLED", "auto")
|
||||
|
||||
cfg = _apply_env_overrides(Config(doclib={"uds": {"enabled": False}, "tcp": {"enabled": True}}), prefix=prefix)
|
||||
overrides, _paths = _collect_env_overrides(prefix=prefix)
|
||||
cfg = Config(**overrides)
|
||||
|
||||
assert cfg.doclib.uds.enabled == "auto"
|
||||
assert cfg.doclib.tcp.enabled == "auto"
|
||||
@@ -300,7 +375,8 @@ def test_log_config_dir_override_derives_paths_in_deep_patches(monkeypatch: pyte
|
||||
prefix = "TEST_MINERU_"
|
||||
monkeypatch.setenv("TEST_MINERU_DOCLIB_LOG_DIR", "/tmp/env-logs")
|
||||
|
||||
env_cfg = _apply_env_overrides(Config(), prefix=prefix)
|
||||
overrides, _paths = _collect_env_overrides(prefix=prefix)
|
||||
env_cfg = Config(**overrides)
|
||||
patched_cfg = PatchedConfig(doclib={"log": {"dir": "/tmp/patched-logs"}})
|
||||
|
||||
assert env_cfg.doclib.log.app_path is None
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextlib import contextmanager
|
||||
|
||||
from mineru.doclib.core import file_io
|
||||
|
||||
@@ -9,23 +8,15 @@ from mineru.doclib.core import file_io
|
||||
def test_extract_pdf_meta_serializes_pdfium_calls(monkeypatch) -> None:
|
||||
assert not hasattr(file_io, "open_pdfium_document")
|
||||
|
||||
state = {"guard_depth": 0, "closed_doc": False}
|
||||
|
||||
@contextmanager
|
||||
def _fake_guard():
|
||||
state["guard_depth"] += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
state["guard_depth"] -= 1
|
||||
state = {"closed_doc": False}
|
||||
|
||||
class _FakePdf:
|
||||
def __len__(self) -> int:
|
||||
assert state["guard_depth"] > 0
|
||||
@property
|
||||
def page_count(self) -> int:
|
||||
return 7
|
||||
|
||||
def get_metadata_dict(self) -> dict[str, str]:
|
||||
assert state["guard_depth"] > 0
|
||||
@property
|
||||
def metadata(self) -> dict[str, str]:
|
||||
return {
|
||||
"Title": "Test Title",
|
||||
"Author": "Test Author",
|
||||
@@ -33,9 +24,10 @@ def test_extract_pdf_meta_serializes_pdfium_calls(monkeypatch) -> None:
|
||||
"Keywords": "alpha,beta",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(file_io, "pdfium_guard", _fake_guard)
|
||||
monkeypatch.setattr(file_io.pypdfium2, "PdfDocument", lambda filepath: _FakePdf())
|
||||
monkeypatch.setattr(file_io, "close_pdfium_document", lambda pdf: state.__setitem__("closed_doc", True))
|
||||
def close(self) -> None:
|
||||
state["closed_doc"] = True
|
||||
|
||||
monkeypatch.setattr(file_io, "PDFDocument", lambda filepath: _FakePdf())
|
||||
|
||||
result = {
|
||||
"page_count": None,
|
||||
@@ -59,7 +51,7 @@ def test_extract_pdf_metadata_open_failure_uses_open_failed(monkeypatch) -> None
|
||||
def _fail_open(filepath):
|
||||
raise RuntimeError("cannot open pdf")
|
||||
|
||||
monkeypatch.setattr(file_io.pypdfium2, "PdfDocument", _fail_open)
|
||||
monkeypatch.setattr(file_io, "PDFDocument", _fail_open)
|
||||
|
||||
try:
|
||||
asyncio.run(file_io.extract_metadata("dummy.pdf"))
|
||||
@@ -73,20 +65,19 @@ def test_extract_pdf_metadata_open_failure_uses_open_failed(monkeypatch) -> None
|
||||
def test_extract_pdf_metadata_read_failure_uses_read_metadata_failed(monkeypatch) -> None:
|
||||
state = {"closed_doc": False}
|
||||
|
||||
@contextmanager
|
||||
def _fake_guard():
|
||||
yield
|
||||
|
||||
class _FakePdf:
|
||||
def __len__(self) -> int:
|
||||
@property
|
||||
def page_count(self) -> int:
|
||||
return 7
|
||||
|
||||
def get_metadata_dict(self) -> dict[str, str]:
|
||||
@property
|
||||
def metadata(self) -> dict[str, str]:
|
||||
raise RuntimeError("cannot read title")
|
||||
|
||||
monkeypatch.setattr(file_io, "pdfium_guard", _fake_guard)
|
||||
monkeypatch.setattr(file_io.pypdfium2, "PdfDocument", lambda filepath: _FakePdf())
|
||||
monkeypatch.setattr(file_io, "close_pdfium_document", lambda pdf: state.__setitem__("closed_doc", True))
|
||||
def close(self) -> None:
|
||||
state["closed_doc"] = True
|
||||
|
||||
monkeypatch.setattr(file_io, "PDFDocument", lambda filepath: _FakePdf())
|
||||
|
||||
try:
|
||||
asyncio.run(file_io.extract_metadata("dummy.pdf"))
|
||||
|
||||
@@ -227,74 +227,67 @@ def test_upload_filename_helper_import_boundary_is_explicit() -> None:
|
||||
assert "normalize_upload_filename" in fast_api_upload_imports
|
||||
|
||||
|
||||
def test_models_download_pipeline(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(models, "_download_pipeline_models", lambda: "/tmp/pipeline")
|
||||
monkeypatch.setattr(models, "_download_vlm_models", lambda: "/tmp/vlm")
|
||||
monkeypatch.setattr(models, "_update_models_dir", lambda bundle, model_dir: Path(f"/tmp/{bundle}.json"))
|
||||
def test_models_download_tier_medium(monkeypatch: Any) -> None:
|
||||
captured: list[str] = []
|
||||
|
||||
result = runner.invoke(app, ["models", "download", "pipeline"])
|
||||
def fake_download_model_repo(repo: Any, *, source: str | None = None, local_as_auto: bool = False) -> Path:
|
||||
captured.append(repo.name)
|
||||
return Path("/tmp/models") / repo.local_name
|
||||
|
||||
monkeypatch.setattr(models, "download_model_repo", fake_download_model_repo)
|
||||
|
||||
result = runner.invoke(app, ["models", "download", "--tier", "medium"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Downloaded pipeline models" in result.output
|
||||
assert captured == ["PDF-Extract-Kit-1.0"]
|
||||
assert "Downloaded models for tier medium" in result.output
|
||||
|
||||
|
||||
def test_models_download_auto_source_resolves_before_download(monkeypatch: Any) -> None:
|
||||
captured: dict[str, str] = {}
|
||||
def test_models_download_repo_uses_explicit_source(monkeypatch: Any) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_resolve_model_source(model_source: str | None = None, allow_auto: bool = False) -> str:
|
||||
assert model_source == "auto"
|
||||
assert allow_auto is True
|
||||
return "modelscope"
|
||||
def fake_download_model_repo(repo: Any, *, source: str | None = None, local_as_auto: bool = False) -> Path:
|
||||
captured["repo"] = repo.name
|
||||
captured["source"] = source
|
||||
captured["local_as_auto"] = local_as_auto
|
||||
return Path("/tmp/models") / repo.local_name
|
||||
|
||||
def fake_update_models_dir(bundle: str, model_dir: str) -> Path:
|
||||
captured["bundle"] = bundle
|
||||
captured["model_dir"] = model_dir
|
||||
captured["effective_source"] = models.os.getenv(models.MODEL_SOURCE_ENV_VAR, "")
|
||||
return Path(f"/tmp/{bundle}.json")
|
||||
monkeypatch.setattr(models, "download_model_repo", fake_download_model_repo)
|
||||
|
||||
monkeypatch.delenv(models.MODEL_SOURCE_ENV_VAR, raising=False)
|
||||
monkeypatch.setattr(models, "resolve_model_source", fake_resolve_model_source, raising=False)
|
||||
monkeypatch.setattr(models, "_download_pipeline_models", lambda: "/tmp/pipeline")
|
||||
monkeypatch.setattr(models, "_update_models_dir", fake_update_models_dir)
|
||||
|
||||
result = runner.invoke(app, ["models", "download", "pipeline", "--source", "auto"])
|
||||
result = runner.invoke(app, ["models", "download", "PDF-Extract-Kit-1.0", "--source", "auto"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert captured == {
|
||||
"bundle": "pipeline",
|
||||
"model_dir": "/tmp/pipeline",
|
||||
"effective_source": "modelscope",
|
||||
"repo": "PDF-Extract-Kit-1.0",
|
||||
"source": "auto",
|
||||
"local_as_auto": True,
|
||||
}
|
||||
assert "Downloaded pipeline models from modelscope" in result.output
|
||||
assert "Downloaded models for PDF-Extract-Kit-1.0" in result.output
|
||||
|
||||
|
||||
def test_models_show_and_verify(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
config = tmp_path / "mineru.json"
|
||||
pipeline_root = tmp_path / "pipeline"
|
||||
vlm_root = tmp_path / "vlm"
|
||||
for rel in models.PIPELINE_MODEL_PATHS:
|
||||
target = pipeline_root / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("x", encoding="utf-8")
|
||||
for rel in models.VLM_MODEL_MARKERS:
|
||||
target = vlm_root / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("x", encoding="utf-8")
|
||||
config.write_text(
|
||||
(f'{{\n "models-dir": {{"pipeline": "{pipeline_root}", "vlm": "{vlm_root}"}},\n "config_version": "1.3.1"\n}}\n'),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("MINERU_TOOLS_CONFIG_JSON", str(config))
|
||||
base_dir = tmp_path / "models"
|
||||
monkeypatch.setattr(models.config.model, "base_dir", str(base_dir))
|
||||
for repo in models.MODEL_REPOS:
|
||||
for model_path in repo.required_paths():
|
||||
target = repo.local_dir() / model_path.relative_path
|
||||
if Path(model_path.relative_path).suffix:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("x", encoding="utf-8")
|
||||
continue
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "marker").write_text("x", encoding="utf-8")
|
||||
|
||||
show_result = runner.invoke(app, ["models", "show"])
|
||||
verify_result = runner.invoke(app, ["models", "verify"])
|
||||
|
||||
assert show_result.exit_code == 0
|
||||
assert "pipeline.exists: True" in show_result.output
|
||||
assert "vlm.exists: True" in show_result.output
|
||||
assert "Config exists:" in show_result.output
|
||||
assert "PDF-Extract-Kit-1.0: ready" in show_result.output
|
||||
assert "MinerU2.5-Pro-2605-1.2B: ready" in show_result.output
|
||||
assert verify_result.exit_code == 0
|
||||
assert "pipeline: ok" in verify_result.output
|
||||
assert "vlm: ok" in verify_result.output
|
||||
assert "PDF-Extract-Kit-1.0: ok" in verify_result.output
|
||||
assert "MinerU2.5-Pro-2605-1.2B: ok" in verify_result.output
|
||||
|
||||
|
||||
def test_api_server_rejects_backend_and_effort_options() -> None:
|
||||
@@ -562,7 +555,7 @@ def test_mlx_vlm_server_adapter_defaults_to_mineru_vlm_model(monkeypatch: Any) -
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "mlx_vlm", SimpleNamespace(server=fake_mlx_server))
|
||||
monkeypatch.setitem(sys.modules, "mlx_vlm.server", fake_mlx_server)
|
||||
monkeypatch.setattr(mlx_vlm_server, "auto_download_and_get_model_root_path", lambda path, repo_mode: "/models/mineru-vlm")
|
||||
monkeypatch.setattr(mlx_vlm_server, "_default_model_id", lambda configured_model: "/models/mineru-vlm")
|
||||
|
||||
client = TestClient(mlx_vlm_server.create_app())
|
||||
response = client.get("/v1/models")
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from mineru.utils import models_download_utils
|
||||
|
||||
|
||||
def test_resolve_model_source_does_not_persist_env_auto(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(models_download_utils.config.model, "source", "auto")
|
||||
monkeypatch.setattr(models_download_utils, "get_config_source", lambda path: "env")
|
||||
monkeypatch.setattr(models_download_utils, "_resolve_auto_model_source", lambda: "modelscope")
|
||||
|
||||
def fail_persist(_model_source: str) -> None:
|
||||
raise AssertionError("env-provided auto must not be persisted")
|
||||
|
||||
monkeypatch.setattr(models_download_utils, "_persist_resolved_model_source", fail_persist)
|
||||
|
||||
assert models_download_utils.resolve_model_source(allow_auto=True) == "modelscope"
|
||||
|
||||
|
||||
def test_resolve_model_source_can_treat_config_local_as_auto_for_explicit_download(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(models_download_utils.config.model, "source", "local")
|
||||
monkeypatch.setattr(models_download_utils, "_resolve_auto_model_source", lambda: "huggingface")
|
||||
|
||||
assert models_download_utils.resolve_model_source(allow_auto=True) == "local"
|
||||
assert models_download_utils.resolve_model_source(allow_auto=True, local_as_auto=True) == "huggingface"
|
||||
|
||||
|
||||
def test_resolve_model_source_rejects_invalid_explicit_source() -> None:
|
||||
with pytest.raises(ValueError, match="Unsupported model source"):
|
||||
models_download_utils.resolve_model_source("invalid-source", allow_auto=True)
|
||||
@@ -2241,6 +2241,12 @@ def test_api_server_logs_traceback_when_job_file_fails(
|
||||
)
|
||||
job_store = api_server.JobStore()
|
||||
rec = job_store.create(request, file_store)
|
||||
mineru_logger = logging.getLogger("mineru")
|
||||
api_logger = logging.getLogger("mineru.parser.api_server")
|
||||
monkeypatch.setattr(mineru_logger, "disabled", False)
|
||||
monkeypatch.setattr(mineru_logger, "propagate", True)
|
||||
monkeypatch.setattr(api_logger, "disabled", False)
|
||||
monkeypatch.setattr(api_logger, "propagate", True)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="mineru.parser.api_server"):
|
||||
asyncio.run(
|
||||
@@ -2368,7 +2374,7 @@ def test_api_server_multi_tier_state_and_metadata(tmp_path: Path, monkeypatch: p
|
||||
assert app.state.backend == "hybrid-engine"
|
||||
assert app.state.effort == "high"
|
||||
assert [tier["id"] for tier in app.state.tiers] == ["medium", "high", "xhigh"]
|
||||
assert app.state.model_ids == ["Hybrid-Medium", "MinerU-HTML", "MinerU2.5-Pro-2604-1.2B"]
|
||||
assert app.state.model_ids == ["Hybrid-Medium", "MinerU-HTML", "MinerU2.5-Pro-2605-1.2B"]
|
||||
assert app.state.tier_runtime_options["medium"].as_kwargs() == {
|
||||
"tier": "medium",
|
||||
"backend": "hybrid-engine",
|
||||
@@ -2402,7 +2408,7 @@ def test_api_server_multi_tier_http_metadata(tmp_path: Path, monkeypatch: pytest
|
||||
"MinerU-Flash",
|
||||
"Hybrid-Medium",
|
||||
"MinerU-HTML",
|
||||
"MinerU2.5-Pro-2604-1.2B",
|
||||
"MinerU2.5-Pro-2605-1.2B",
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user