From c06f1d45bf63f67eb2d518071871ef3b15750687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:45:57 +0800 Subject: [PATCH] =?UTF-8?q?feat(dl):=20support=20new=20safetensors=20model?= =?UTF-8?q?=20&=20drop=20gitcode=20=F0=9F=91=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/checksum.yml | 2 +- ChatTTS/res/sha256_map.json | 9 ++-- ChatTTS/utils/dl.py | 79 +++++++++++++++++----------------- tools/checksum/tmpl.go | 14 ++++-- 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/.github/workflows/checksum.yml b/.github/workflows/checksum.yml index 55e6843..162c43e 100644 --- a/.github/workflows/checksum.yml +++ b/.github/workflows/checksum.yml @@ -13,7 +13,7 @@ jobs: - name: Run RVC-Models-Downloader run: | - wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.7/rvcmd_linux_amd64.deb + wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.8/rvcmd_linux_amd64.deb sudo apt -y install ./rvcmd_linux_amd64.deb rm -f ./rvcmd_linux_amd64.deb rvcmd -notrs -w 1 -notui assets/chtts diff --git a/ChatTTS/res/sha256_map.json b/ChatTTS/res/sha256_map.json index ef47afb..b58273e 100644 --- a/ChatTTS/res/sha256_map.json +++ b/ChatTTS/res/sha256_map.json @@ -1,8 +1,11 @@ { - "sha256_asset_Decoder_pt" : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38", + "sha256_asset_Decoder_pt" : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38", "sha256_asset_DVAE_full_pt" : "553eb75763511e23f3e5f86303e2163c5ca775489d637fb635d979c8ae58bbe5", - "sha256_asset_GPT_pt" : "d7d4ee6461ea097a2be23eb40d73fb94ad3b3d39cb64fbb50cb3357fd466cadb", - "sha256_asset_Vocos_pt" : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58", + "sha256_asset_Embed_safetensors" : "2ff0be7134934155741b643b74e32fb6bf3eec41257984459b2ed60cdb4c48b0", + "sha256_asset_Vocos_pt" : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58", + + "sha256_asset_gpt_config_json" : "0aaa1ecd96c49ad4f473459eb1982fa7ad79fa5de08cde2781bf6ad1f9a0c236", + "sha256_asset_gpt_model_safetensors" : "cd0806fd971f52f6a22c923ec64982b305e817bcc41ca83417fcf9141b984a0f", "sha256_asset_tokenizer_special_tokens_map_json": "bd0ac9d9bb1657996b5c5fbcaa7d80f8de530d01a283da97f89deae5b1b8d011", "sha256_asset_tokenizer_tokenizer_config_json" : "43e9d658b554fa5ee8d8e1d763349323bfef1ed7a89c0794220ab8861387d421", diff --git a/ChatTTS/utils/dl.py b/ChatTTS/utils/dl.py index da21daa..70b6bea 100644 --- a/ChatTTS/utils/dl.py +++ b/ChatTTS/utils/dl.py @@ -3,7 +3,7 @@ from pathlib import Path import hashlib import requests from io import BytesIO -from typing import Dict +from typing import Dict, Tuple, Optional from mmap import mmap, ACCESS_READ from .log import logger @@ -42,46 +42,54 @@ def check_model( os.remove(bakfile) return True +def check_folder(base_dir: Path, *innder_dirs: str, names: Tuple[str], sha256_map: Dict[str, str], update=False) -> bool: + key = "sha256_" + current_dir = base_dir + for d in innder_dirs: + current_dir /= d + key += f"{d}_" + + for model in names: + menv = model.replace(".", "_") + if not check_model( + current_dir, model, sha256_map[f"{key}{menv}"], update + ): + return False + return True def check_all_assets(base_dir: Path, sha256_map: Dict[str, str], update=False) -> bool: logger.get_logger().info("checking assets...") - current_dir = base_dir / "asset" - names = [ + if not check_folder(base_dir, "asset", names=( "Decoder.pt", "DVAE_full.pt", - "GPT.pt", + "Embed.safetensors", "Vocos.pt", - ] - for model in names: - menv = model.replace(".", "_") - if not check_model( - current_dir, model, sha256_map[f"sha256_asset_{menv}"], update - ): - return False + ), sha256_map=sha256_map, update=update): + return False - current_dir = base_dir / "asset" / "tokenizer" - names = [ + if not check_folder(base_dir, "asset", "gpt", names=( + "config.json", + "model.safetensors", + ), sha256_map=sha256_map, update=update): + return False + + if not check_folder(base_dir, "asset", "tokenizer", names=( "special_tokens_map.json", "tokenizer_config.json", "tokenizer.json", - ] - for model in names: - menv = model.replace(".", "_") - if not check_model( - current_dir, model, sha256_map[f"sha256_asset_tokenizer_{menv}"], update - ): - return False + ), sha256_map=sha256_map, update=update): + return False logger.get_logger().info("all assets are already latest.") return True -def download_and_extract_tar_gz(url: str, folder: str): +def download_and_extract_tar_gz(url: str, folder: str, headers: Optional[Dict[str, str]] = None): import tarfile logger.get_logger().info(f"downloading {url}") - response = requests.get(url, stream=True, timeout=(5, 10)) + response = requests.get(url, headers=headers, stream=True, timeout=(10, 3)) with BytesIO() as out_file: out_file.write(response.content) out_file.seek(0) @@ -91,11 +99,11 @@ def download_and_extract_tar_gz(url: str, folder: str): logger.get_logger().info(f"extracted into {folder}") -def download_and_extract_zip(url: str, folder: str): +def download_and_extract_zip(url: str, folder: str, headers: Optional[Dict[str, str]] = None): import zipfile logger.get_logger().info(f"downloading {url}") - response = requests.get(url, stream=True, timeout=(5, 10)) + response = requests.get(url, headers=headers, stream=True, timeout=(10, 3)) with BytesIO() as out_file: out_file.write(response.content) out_file.seek(0) @@ -105,15 +113,15 @@ def download_and_extract_zip(url: str, folder: str): logger.get_logger().info(f"extracted into {folder}") -def download_dns_yaml(url: str, folder: str): +def download_dns_yaml(url: str, folder: str, headers: Dict[str, str]): logger.get_logger().info(f"downloading {url}") - response = requests.get(url, stream=True, timeout=(5, 10)) + response = requests.get(url, headers=headers, stream=True, timeout=(100, 3)) with open(os.path.join(folder, "dns.yaml"), "wb") as out_file: out_file.write(response.content) logger.get_logger().info(f"downloaded into {folder}") -def download_all_assets(tmpdir: str, version="0.2.7"): +def download_all_assets(tmpdir: str, version="0.2.8"): import subprocess import platform @@ -150,20 +158,13 @@ def download_all_assets(tmpdir: str, version="0.2.7"): os.chmod(cmdfile, 0o755) subprocess.run([cmdfile, "-notui", "-w", "0", "assets/chtts"]) except Exception: - BASE_URL = "https://raw.gitcode.com/u011570312/RVC-Models-Downloader/assets/" - suffix = { - "darwin_amd64": "987", - "darwin_arm64": "988", - "linux_386": "989", - "linux_amd64": "990", - "linux_arm64": "991", - "windows_386": "992", - "windows_amd64": "993", - }[f"{system_type}_{architecture}"] - RVCMD_URL = BASE_URL + suffix + BASE_URL = "https://gitea.seku.su/fumiama/RVC-Models-Downloader/releases/download/" + suffix = "zip" if is_win else "tar.gz" + RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}" download_dns_yaml( - "https://raw.gitcode.com/u011570312/RVC-Models-Downloader/raw/main/dns.yaml", + "https://gitea.seku.su/fumiama/RVC-Models-Downloader/raw/branch/main/dns.yaml", tmpdir, + headers={"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0"}, ) if is_win: download_and_extract_zip(RVCMD_URL, tmpdir) diff --git a/tools/checksum/tmpl.go b/tools/checksum/tmpl.go index cfee203..984c770 100644 --- a/tools/checksum/tmpl.go +++ b/tools/checksum/tmpl.go @@ -3,19 +3,25 @@ package main var files = [...]string{ "asset/Decoder.pt", "asset/DVAE_full.pt", - "asset/GPT.pt", + "asset/Embed.safetensors", "asset/Vocos.pt", + "asset/gpt/config.json", + "asset/gpt/model.safetensors", + "asset/tokenizer/special_tokens_map.json", "asset/tokenizer/tokenizer_config.json", "asset/tokenizer/tokenizer.json", } const jsontmpl = `{ - "sha256_asset_Decoder_pt" : "%s", + "sha256_asset_Decoder_pt" : "%s", "sha256_asset_DVAE_full_pt" : "%s", - "sha256_asset_GPT_pt" : "%s", - "sha256_asset_Vocos_pt" : "%s", + "sha256_asset_Embed_safetensors" : "%s", + "sha256_asset_Vocos_pt" : "%s", + + "sha256_asset_gpt_config_json" : "%s", + "sha256_asset_gpt_model_safetensors" : "%s", "sha256_asset_tokenizer_special_tokens_map_json": "%s", "sha256_asset_tokenizer_tokenizer_config_json" : "%s",