mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-10-28 02:36:45 +08:00
build: enable hf-xet model downloads
This commit is contained in:
@@ -6,6 +6,14 @@ MinerU使用 `HuggingFace` 和 `ModelScope` 作为模型仓库,用户可以根
|
||||
- `HuggingFace` 在全球范围内提供了优异的加载速度和极高稳定性。
|
||||
- `ModelScope` 是中国大陆地区用户的最佳选择,提供了无缝兼容的SDK模块,适用于无法访问`HuggingFace`的用户。
|
||||
|
||||
使用 `HuggingFace` 下载模型时,MinerU 默认通过 `hf_xet` 加速传输。如果当前网络或代理无法访问 Xet CAS,设置
|
||||
`HF_HUB_DISABLE_XET=1` 可回退到普通 HTTP 下载。
|
||||
|
||||
```powershell
|
||||
$env:HF_HUB_DISABLE_XET = "1"
|
||||
mineru-kit models download --tier standard --source huggingface
|
||||
```
|
||||
|
||||
## 模型源的切换方法
|
||||
|
||||
### 通过环境变量切换
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"reportlab",
|
||||
"pdftext>=0.6.3,<0.8.0",
|
||||
"modelscope>=1.26.0",
|
||||
"huggingface-hub>=0.32.4",
|
||||
"huggingface-hub[hf_xet]>=0.32.4",
|
||||
"filelock>=3.0.0",
|
||||
"json-repair>=0.46.2",
|
||||
"opencv-python>=4.11.0.86",
|
||||
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from loguru import logger as loguru_logger
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
from mineru.config import LogConfig, PatchedConfig, config as mineru_config
|
||||
from mineru.doclib import app as doclib_app
|
||||
@@ -55,6 +56,15 @@ def test_doclib_runtime_dependencies_are_in_base_install() -> None:
|
||||
assert "watchfiles" in dependency_names
|
||||
|
||||
|
||||
def test_huggingface_hub_base_dependency_enables_xet() -> None:
|
||||
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
|
||||
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
|
||||
dependencies = (Requirement(raw_dependency) for raw_dependency in pyproject["project"]["dependencies"])
|
||||
huggingface_hub = next(dependency for dependency in dependencies if dependency.name == "huggingface-hub")
|
||||
|
||||
assert huggingface_hub.extras == {"hf_xet"}
|
||||
|
||||
|
||||
def test_basic_extra_includes_preflight_runtime_dependencies() -> None:
|
||||
pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
|
||||
pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
|
||||
|
||||
Reference in New Issue
Block a user