build: enable hf-xet model downloads

This commit is contained in:
jinzhenj
2026-07-21 18:24:46 +08:00
parent 0df7ae8bb4
commit d138d20170
3 changed files with 19 additions and 1 deletions
+8
View File
@@ -6,6 +6,14 @@ MinerU使用 `HuggingFace` 和 `ModelScope` 作为模型仓库,用户可以根
- `HuggingFace` 在全球范围内提供了优异的加载速度和极高稳定性。
- `ModelScope` 是中国大陆地区用户的最佳选择,提供了无缝兼容的SDK模块,适用于无法访问`HuggingFace`的用户。
使用 `HuggingFace` 下载模型时,MinerU 默认通过 `hf_xet` 加速传输。如果当前网络或代理无法访问 Xet CAS,设置
`HF_HUB_DISABLE_XET=1` 可回退到普通 HTTP 下载。
```powershell
$env:HF_HUB_DISABLE_XET = "1"
mineru-kit models download --tier standard --source huggingface
```
## 模型源的切换方法
### 通过环境变量切换
+1 -1
View File
@@ -45,7 +45,7 @@ dependencies = [
"reportlab",
"pdftext>=0.6.3,<0.8.0",
"modelscope>=1.26.0",
"huggingface-hub>=0.32.4",
"huggingface-hub[hf_xet]>=0.32.4",
"filelock>=3.0.0",
"json-repair>=0.46.2",
"opencv-python>=4.11.0.86",
+10
View File
@@ -8,6 +8,7 @@ from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from loguru import logger as loguru_logger
from packaging.requirements import Requirement
from mineru.config import LogConfig, PatchedConfig, config as mineru_config
from mineru.doclib import app as doclib_app
@@ -55,6 +56,15 @@ def test_doclib_runtime_dependencies_are_in_base_install() -> None:
assert "watchfiles" in dependency_names
def test_huggingface_hub_base_dependency_enables_xet() -> None:
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
dependencies = (Requirement(raw_dependency) for raw_dependency in pyproject["project"]["dependencies"])
huggingface_hub = next(dependency for dependency in dependencies if dependency.name == "huggingface-hub")
assert huggingface_hub.extras == {"hf_xet"}
def test_basic_extra_includes_preflight_runtime_dependencies() -> None:
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))