From d2c7d953902c0a80c7f3549b282db29619609990 Mon Sep 17 00:00:00 2001 From: lich99 Date: Thu, 30 May 2024 20:54:15 +0800 Subject: [PATCH] update example & minor fix --- ChatTTS/core.py | 5 +- ChatTTS/utils/infer_utils.py | 2 +- infer.ipynb => example.ipynb | 330 ++++++++++++++++++++++++++--------- 3 files changed, 247 insertions(+), 90 deletions(-) rename infer.ipynb => example.ipynb (82%) diff --git a/ChatTTS/core.py b/ChatTTS/core.py index 2ab4b70..35a7231 100644 --- a/ChatTTS/core.py +++ b/ChatTTS/core.py @@ -16,8 +16,6 @@ from huggingface_hub import snapshot_download logging.basicConfig(level = logging.INFO) -torch._dynamo.config.suppress_errors = True -torch.set_float32_matmul_precision('medium') class Chat: def __init__(self, ): @@ -138,6 +136,9 @@ class Chat: assert self.check_model(use_decoder=use_decoder) + if not isinstance(text, list): + text = [text] + if do_text_normalization: for i, t in enumerate(text): _lang = detect_language(t) if lang is None else lang diff --git a/ChatTTS/utils/infer_utils.py b/ChatTTS/utils/infer_utils.py index 9445df8..9a62818 100644 --- a/ChatTTS/utils/infer_utils.py +++ b/ChatTTS/utils/infer_utils.py @@ -48,7 +48,7 @@ class CustomRepetitionPenaltyLogitsProcessor(): def count_invalid_characters(s): s = re.sub(r'\[uv_break\]|\[laugh\]|\[lbreak\]', '', s) - pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z，。,\. ]') + pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z，。、,\. ]') non_alphabetic_chinese_chars = pattern.findall(s) return set(non_alphabetic_chinese_chars) diff --git a/infer.ipynb b/example.ipynb similarity index 82% rename from infer.ipynb rename to example.ipynb index 4894ede..dc9027c 100644 --- a/infer.ipynb +++ b/example.ipynb @@ -7,80 +7,76 @@ "outputs": [], "source": [ "import torch\n", + "torch._dynamo.config.cache_size_limit = 64\n", + "torch._dynamo.config.suppress_errors = True\n", + "torch.set_float32_matmul_precision('high')\n", + "\n", "import ChatTTS\n", "from IPython.display import Audio" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Models" + ] + }, { "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d47abb2d74f4488685a4efb5da8fd791", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Fetching 10 files: 0%| | 0/10 [00:00\n", + " \n", + " Your browser does not support the audio element.\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from ChatTTS.experimental.llm import llm_api\n", - "\n", - "API_KEY = ''\n", - "client = llm_api(api_key=API_KEY,\n", - " base_url=\"https://api.deepseek.com\",\n", - " model=\"deepseek-chat\")" + "Audio(wav[0], rate=24_000, autoplay=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### fix random speaker" ] }, { @@ -154,47 +201,17 @@ "metadata": {}, "outputs": [], "source": [ - "user_question = '四川有哪些好吃的美食呢?'\n", - "text = client.call(user_question, prompt_version = 'deepseek')\n", - "text = client.call(text, prompt_version = 'deepseek_TN')" + "rand_spk = chat.sample_random_speaker()\n", + "params_infer_code = {'spk_emb' : rand_spk, }\n", + "\n", + "wav = chat.infer('四川美食确实以辣闻名，但也有不辣的选择。比如甜水面、赖汤圆、蛋烘糕、叶儿粑等，这些小吃口味温和，甜而不腻，也很受欢迎。', \\\n", + " params_refine_text=params_refine_text, params_infer_code=params_infer_code)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, - "outputs": [], - "source": [ - "spk_stat = torch.load('ChatTTS/asset/spk_stat.pt')\n", - "rand_spk = torch.randn(768) * spk_stat.chunk(2)[0] + spk_stat.chunk(2)[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:ChatTTS.core:All initialized.\n", - " 16%|█▋ | 63/384 [00:00<00:04, 77.87it/s]\n", - " 26%|██▌ | 530/2048 [00:06<00:19, 78.64it/s]\n" - ] - } - ], - "source": [ - "params_infer_code = {'spk_emb' : rand_spk, 'temperature':.3}\n", - "params_refine_text = {'prompt':'[oral_2][laugh_0][break_6]'}\n", - "# wav = chat.infer('四川美食可多了，有麻辣火锅、宫保鸡丁、麻婆豆腐、担担面、回锅肉、夫妻肺片等，每样都让人垂涎三尺。', params_refine_text=params_refine_text, params_infer_code=params_infer_code)\n", - "wav = chat.infer('四川美食确实以辣闻名，但也有不辣的选择。比如甜水面、赖汤圆、蛋烘糕、叶儿粑等，这些小吃口味温和，甜而不腻，也很受欢迎。', params_refine_text=params_refine_text, params_infer_code=params_infer_code)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, "outputs": [ { "data": { @@ -219,6 +236,145 @@ "Audio(wav[0], rate=24_000, autoplay=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Two stage control" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:ChatTTS.core:All initialized.\n", + " 23%|██▎ | 87/384 [00:00<00:01, 150.60it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "['so we found being competitive and collaborative [uv_break] was a huge way of staying [uv_break] motivated towards our goals, [uv_break] so [uv_break] one person to call [uv_break] when you fall off, [uv_break] one person who [uv_break] gets you back [uv_break] on then [uv_break] one person [uv_break] to actually do the activity with.']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text = \"So we found being competitive and collaborative was a huge way of staying motivated towards our goals, so one person to call when you fall off, one person who gets you back on then one person to actually do the activity with.\"\n", + "chat.infer(text, refine_text_only=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:ChatTTS.core:All initialized.\n", + " 49%|████▊ | 995/2048 [00:07<00:07, 141.85it/s]\n" + ] + } + ], + "source": [ + "text = 'so we found being competitive and collaborative [uv_break] was a huge way of staying [uv_break] motivated towards our goals, [uv_break] so [uv_break] one person to call [uv_break] when you fall off, [uv_break] one person who [uv_break] gets you back [uv_break] on then [uv_break] one person [uv_break] to actually do the activity with.'\n", + "wav = chat.infer(text, skip_refine_text=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LLM Call" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from ChatTTS.experimental.llm import llm_api\n", + "\n", + "API_KEY = ''\n", + "client = llm_api(api_key=API_KEY,\n", + " base_url=\"https://api.deepseek.com\",\n", + " model=\"deepseek-chat\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "四川美食可多了, 有麻辣火锅、宫保鸡丁、麻婆豆腐、担担面、回锅肉、夫妻肺片、串串香、龙抄手、宜宾燃面、乐山钵钵鸡等, 每样都让人垂涎三尺。\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "四川美食可多了，有麻辣火锅、宫保鸡丁、麻婆豆腐、担担面、回锅肉、夫妻肺片、串串香、龙抄手、宜宾燃面、乐山钵钵鸡等，每样都让人垂涎三尺。\n" + ] + } + ], + "source": [ + "user_question = '四川有哪些好吃的美食呢?'\n", + "text = client.call(user_question, prompt_version = 'deepseek')\n", + "print(text)\n", + "text = client.call(text, prompt_version = 'deepseek_TN')\n", + "print(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:ChatTTS.core:All initialized.\n", + " 20%|█▉ | 75/384 [00:00<00:02, 144.93it/s]\n", + " 32%|███▏ | 647/2048 [00:04<00:09, 140.27it/s]\n" + ] + } + ], + "source": [ + "params_infer_code = {'spk_emb' : rand_spk, 'temperature':.3}\n", + "\n", + "wav = chat.infer(text, params_infer_code=params_infer_code)" + ] + }, { "cell_type": "code", "execution_count": null,