chore(format): run black on dev (#994)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/checksum.yml

@@ -13,7 +13,7 @@ jobs:
 
       - name: Run RVC-Models-Downloader
         run: |
-          wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.10/rvcmd_linux_amd64.deb
+          wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.11/rvcmd_linux_amd64.deb
           sudo apt -y install ./rvcmd_linux_amd64.deb
           rm -f ./rvcmd_linux_amd64.deb
           rvcmd -notrs -w 1 -notui assets/chtts

.github/workflows/unitest.yml

@@ -20,6 +20,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
 
       - name: Install Dependents
         run: |

ChatTTS/core.py

@@ -141,10 +141,11 @@ class Chat:
         compile: bool = False,
         custom_path: Optional[FileLike] = None,
         device: Optional[torch.device] = None,
-        coef: Optional[torch.Tensor] = None,
+        coef: Optional[str] = None,
         use_flash_attn=False,
         use_vllm=False,
         experimental: bool = False,
+        enable_cache=True,
     ) -> bool:
         download_path = self.download_models(source, force_redownload, custom_path)
         if download_path is None:
@@ -156,6 +157,7 @@ class Chat:
             use_flash_attn=use_flash_attn,
             use_vllm=use_vllm,
             experimental=experimental,
+            enable_cache=enable_cache,
             **{
                 k: os.path.join(download_path, v)
                 for k, v in asdict(self.config.path).items()
@@ -287,6 +289,7 @@ class Chat:
         use_flash_attn=False,
         use_vllm=False,
         experimental: bool = False,
+        enable_cache=True,
     ):
         if device is None:
             device = select_device(experimental=experimental)
@@ -351,6 +354,7 @@ class Chat:
             device=device,
             device_gpt=self.device_gpt,
             logger=self.logger,
+            enable_cache=enable_cache,
         ).eval()
         assert gpt_ckpt_path, "gpt_ckpt_path should not be None"
         gpt.load_pretrained(gpt_ckpt_path, embed_path, experimental=experimental)
@@ -425,6 +429,7 @@ class Chat:
             text_tokens = refined.ids
             text_tokens = [i[i.less(self.tokenizer.break_0_ids)] for i in text_tokens]
             text = self.tokenizer.decode(text_tokens)
+            self.logger.debug("refined texts %s", str(text))
             refined.destroy()
             if refine_text_only:
                 if split_text and isinstance(text, list):

ChatTTS/model/embed.py

@@ -54,12 +54,15 @@ class Embed(nn.Module):
         get_emb
         """
         device = next(self.parameters()).device
+        input_ids_dev = input_ids.to(device)
+        text_mask_dev = text_mask.to(device)
+
         emb_text: torch.Tensor = self.emb_text(
-            input_ids[text_mask].narrow(1, 0, 1).squeeze_(1).to(device)
+            input_ids_dev[text_mask_dev].narrow(1, 0, 1).squeeze_(1)
         )
 
-        text_mask_inv = text_mask.logical_not().to(device)
-        masked_input_ids: torch.Tensor = input_ids[text_mask_inv].to(device)
+        text_mask_inv = text_mask_dev.logical_not()
+        masked_input_ids: torch.Tensor = input_ids_dev[text_mask_inv]
 
         emb_code = [
             self.emb_code[i](masked_input_ids[:, i]) for i in range(self.num_vq)
@@ -67,11 +70,11 @@ class Embed(nn.Module):
         emb_code = torch.stack(emb_code, 2).sum(2)
 
         emb = torch.zeros(
-            (input_ids.shape[:-1]) + (emb_text.shape[-1],),
+            (input_ids_dev.shape[:-1]) + (emb_text.shape[-1],),
             device=emb_text.device,
             dtype=emb_text.dtype,
         )
-        emb[text_mask] = emb_text
+        emb[text_mask_dev] = emb_text
         emb[text_mask_inv] = emb_code.to(emb.dtype)
 
         del emb_text, emb_code, text_mask_inv

ChatTTS/model/gpt.py

@@ -28,6 +28,7 @@ class GPT(nn.Module):
         device=torch.device("cpu"),
         device_gpt=torch.device("cpu"),
         logger=logging.getLogger(__name__),
+        enable_cache=True,
     ):
         super().__init__()
 
@@ -36,6 +37,8 @@ class GPT(nn.Module):
         self.device = device
         self.device_gpt = device_gpt
 
+        self.enable_cache = enable_cache
+
         self.generator = torch.Generator(device=device)
 
         self.num_vq = int(gpt_config["num_vq"])
@@ -142,7 +145,6 @@ class GPT(nn.Module):
     class _GenerationInputs:
         position_ids: torch.Tensor
         cache_position: torch.Tensor
-        use_cache: bool
         input_ids: Optional[torch.Tensor] = None
         past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
         attention_mask: Optional[torch.Tensor] = None
@@ -162,12 +164,11 @@ class GPT(nn.Module):
     def _prepare_generation_inputs(
         self,
         input_ids: torch.Tensor,
-        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        past_key_values: Optional[Union[Tuple[Tuple[torch.FloatTensor]], Cache]] = None,
         attention_mask: Optional[torch.Tensor] = None,
         inputs_embeds: Optional[torch.Tensor] = None,
         cache_position: Optional[torch.Tensor] = None,
         position_ids: Optional[torch.Tensor] = None,
-        use_cache=True,
     ) -> _GenerationInputs:
         # With static cache, the `past_key_values` is None
         # TODO joao: standardize interface for the different Cache classes and remove of this if
@@ -180,28 +181,30 @@ class GPT(nn.Module):
             has_static_cache = past_key_values is not None
 
         past_length = 0
+        max_cache_length = None
+        cache_length = 0
         if past_key_values is not None:
             if isinstance(past_key_values, Cache):
-                past_length = (
-                    int(cache_position[0])
-                    if cache_position is not None
-                    else past_key_values.get_seq_length()
-                )
-                try:
-                    max_cache_length = past_key_values.get_max_cache_shape()
-                except:
-                    max_cache_length = (
-                        past_key_values.get_max_length()
-                    )  # deprecated in transformers 4.48
-                cache_length = (
-                    past_length
-                    if max_cache_length is None
-                    else min(max_cache_length, past_length)
-                )
+                if past_key_values.layers and len(past_key_values.layers):
+                    past_length = (
+                        int(cache_position[0])
+                        if cache_position is not None
+                        else past_key_values.get_seq_length()
+                    )
+                    try:
+                        max_cache_length = past_key_values.get_max_cache_shape()
+                    except:
+                        max_cache_length = (
+                            past_key_values.get_max_length()
+                        )  # deprecated in transformers 4.48
+                    cache_length = (
+                        past_length
+                        if max_cache_length is None
+                        else min(max_cache_length, past_length)
+                    )
             # TODO joao: remove this `else` after `generate` prioritizes `Cache` objects
             else:
                 cache_length = past_length = past_key_values[0][0].shape[2]
-                max_cache_length = None
 
             # Keep only the unprocessed tokens:
             # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
@@ -224,6 +227,7 @@ class GPT(nn.Module):
             # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
             if (
                 max_cache_length is not None
+                and max_cache_length > 0
                 and attention_mask is not None
                 and cache_length + input_ids.shape[1] > max_cache_length
             ):
@@ -256,7 +260,6 @@ class GPT(nn.Module):
         model_inputs = self._GenerationInputs(
             position_ids=position_ids,
             cache_position=cache_position,
-            use_cache=use_cache,
         )
 
         # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
@@ -294,22 +297,27 @@ class GPT(nn.Module):
         hiddens: List[torch.Tensor],
         infer_text: bool,
     ) -> GenerationOutputs:
-        inputs_ids = [
-            inputs_ids[idx].narrow(0, start_idx, i) for idx, i in enumerate(end_idx)
+        end_idx_int = end_idx.int()
+
+        inputs_ids_lst = [
+            inputs_ids[idx].narrow(0, start_idx, int(i))
+            for idx, i in enumerate(end_idx_int)
         ]
         if infer_text:
-            inputs_ids = [i.narrow(1, 0, 1).squeeze_(1) for i in inputs_ids]
+            inputs_ids_lst = [i.narrow(1, 0, 1).squeeze_(1) for i in inputs_ids_lst]
 
+        hiddens_lst = []
         if len(hiddens) > 0:
-            hiddens = torch.stack(hiddens, 1)
-            hiddens = [
-                hiddens[idx].narrow(0, 0, i) for idx, i in enumerate(end_idx.int())
+            hiddens_lst = torch.stack(hiddens, 1)
+            hiddens_lst = [
+                hiddens_lst[idx].narrow(0, 0, int(i))
+                for idx, i in enumerate(end_idx_int)
             ]
 
         return self.GenerationOutputs(
-            ids=inputs_ids,
+            ids=inputs_ids_lst,
             attentions=attentions,
-            hiddens=hiddens,
+            hiddens=hiddens_lst,
         )
 
     @torch.no_grad()
@@ -336,6 +344,8 @@ class GPT(nn.Module):
         context=Context(),
     ):
 
+        self.logger.debug("start generate")
+
         attentions: List[Optional[Tuple[torch.FloatTensor, ...]]] = []
         hiddens = []
         stream_iter = 0
@@ -345,6 +355,10 @@ class GPT(nn.Module):
         )
         finish = torch.zeros(inputs_ids.shape[0], device=inputs_ids.device).bool()
 
+        self.logger.debug(
+            f"set start_idx: {start_idx}, end_idx and finish with all zeros, len {inputs_ids.shape[0]}"
+        )
+
         old_temperature = temperature
 
         temperature = (
@@ -354,6 +368,10 @@ class GPT(nn.Module):
             .view(-1, 1)
         )
 
+        self.logger.debug(
+            f"expand temperature from shape {old_temperature.shape} to {temperature.shape}"
+        )
+
         attention_mask_cache = torch.ones(
             (
                 inputs_ids.shape[0],
@@ -362,10 +380,14 @@ class GPT(nn.Module):
             dtype=torch.bool,
             device=inputs_ids.device,
         )
+        self.logger.debug(
+            f"init attention_mask_cache with shape {attention_mask_cache.shape}"
+        )
         if attention_mask is not None:
             attention_mask_cache.narrow(1, 0, attention_mask.shape[1]).copy_(
                 attention_mask
             )
+            self.logger.debug(f"copy attention_mask with shape {attention_mask.shape}")
 
         progress = inputs_ids.size(1)
         # pre-allocate inputs_ids
@@ -377,6 +399,9 @@ class GPT(nn.Module):
             device=inputs_ids.device,
         )
         inputs_ids_buf.narrow(1, 0, progress).copy_(inputs_ids)
+        self.logger.debug(
+            f"expand inputs_ids buf from shape {inputs_ids.shape} to {inputs_ids_buf.shape}"
+        )
         del inputs_ids
         inputs_ids = inputs_ids_buf.narrow(1, 0, progress)
 
@@ -393,38 +418,46 @@ class GPT(nn.Module):
 
         for i in range(max_new_token):
 
+            self.logger.debug("start _prepare_generation_inputs")
             model_input = self._prepare_generation_inputs(
                 inputs_ids,
                 past_key_values,
                 attention_mask_cache.narrow(1, 0, inputs_ids.shape[1]),
-                use_cache=not self.is_te_llama,
             )
+            self.logger.debug("finis _prepare_generation_inputs")
 
             if i > 0:
                 del emb
                 inputs_ids_emb = model_input.input_ids.to(self.device_gpt)
                 if infer_text:
+                    self.logger.debug("start emb_text")
                     emb: torch.Tensor = self.emb_text(inputs_ids_emb[:, :, 0])
+                    self.logger.debug("finis emb_text")
                 else:
+                    self.logger.debug("start code_emb")
                     code_emb = [
-                        self.emb_code[i](inputs_ids_emb[:, :, i])
+                        self.emb_code[i](inputs_ids_emb[:, :, i]).to(self.device)
                         for i in range(self.num_vq)
                     ]
                     emb = torch.stack(code_emb, 3).sum(3)
+                    self.logger.debug("finis code_emb")
                 del inputs_ids_emb, model_input.input_ids
             model_input.inputs_embeds = emb
 
+            self.logger.debug(f"move model_input to device_gpt: {str(self.device_gpt)}")
             model_input.to(self.device_gpt, self.gpt.dtype)
 
+            self.logger.debug("start gpt...")
             outputs: BaseModelOutputWithPast = self.gpt(
                 attention_mask=model_input.attention_mask,
                 position_ids=model_input.position_ids,
                 past_key_values=model_input.past_key_values,
                 inputs_embeds=model_input.inputs_embeds,
-                use_cache=model_input.use_cache,
+                use_cache=not self.is_te_llama and self.enable_cache,
                 output_attentions=return_attn,
                 cache_position=model_input.cache_position,
             )
+            self.logger.debug("finis gpt")
             del_all(model_input)
             attentions.append(outputs.attentions)
             hidden_states = outputs.last_hidden_state.to(
@@ -437,8 +470,11 @@ class GPT(nn.Module):
 
             with P.cached():
                 if infer_text:
+                    self.logger.debug("start head_text")
                     logits: torch.Tensor = self.head_text(hidden_states)
+                    self.logger.debug("finis head_text")
                 else:
+                    self.logger.debug("start head_code")
                     # logits = torch.stack([self.head_code[i](hidden_states) for i in range(self.num_vq)], 3)
                     logits = torch.empty(
                         hidden_states.size(0),
@@ -452,9 +488,11 @@ class GPT(nn.Module):
                         x: torch.Tensor = self.head_code[num_vq_iter](hidden_states)
                         logits[..., num_vq_iter] = x
                         del x
+                    self.logger.debug("finis head_code")
 
             del hidden_states
 
+            self.logger.debug("start logits")
             # logits = logits[:, -1].float()
             logits = logits.narrow(1, -1, 1).squeeze_(1).float()
 
@@ -498,6 +536,9 @@ class GPT(nn.Module):
 
             del logits
 
+            self.logger.debug("finis logits")
+
+            self.logger.debug("start seed")
             if manual_seed is None:
                 idx_next = torch.multinomial(scores, num_samples=1).to(finish.device)
             else:
@@ -509,6 +550,10 @@ class GPT(nn.Module):
 
             del scores
 
+            self.logger.debug("finis seed")
+
+            self.logger.debug("start finish")
+
             if not infer_text:
                 # idx_next = rearrange(idx_next, "(b n) 1 -> b n", n=self.num_vq)
                 idx_next = idx_next.view(-1, self.num_vq)
@@ -524,6 +569,8 @@ class GPT(nn.Module):
                     idx_next.unsqueeze_(-1).expand(-1, -1, self.num_vq),
                 )
 
+            self.logger.debug("finis finish")
+
             if i == 0 and finish.any():
                 self.logger.warning(
                     "unexpected end at index %s",
@@ -569,6 +616,8 @@ class GPT(nn.Module):
                     del inputs_ids
                 return
 
+            self.logger.debug("start output")
+
             del idx_next
             progress += 1
             inputs_ids = inputs_ids_buf.narrow(1, 0, progress)
@@ -589,6 +638,8 @@ class GPT(nn.Module):
                     )
             del not_finished
 
+            self.logger.debug("finis output")
+
             if finish.all() or context.get():
                 break
 

ChatTTS/model/tokenizer.py

@@ -53,7 +53,12 @@ class Tokenizer:
 
         # avoid random speaker embedding of tokenizer in the other dims
         for t in text:
-            x = self._tokenizer.encode_plus(
+            encode_plus = (
+                self._tokenizer.encode_plus
+                if hasattr(self._tokenizer, "encode_plus")
+                else self._tokenizer._encode_plus
+            )
+            x = encode_plus(
                 t, return_tensors="pt", add_special_tokens=False, padding=True
             )
             input_ids_lst.append(x["input_ids"].squeeze_(0))
@@ -125,7 +130,7 @@ class Tokenizer:
 
         return new_input_ids, attention_mask, text_mask
 
-    @torch.inference_mode
+    @torch.inference_mode()
     def decode(
         self,
         sequences: Union[List[int], List[List[int]]],

ChatTTS/model/velocity/block_manager.py

@@ -156,7 +156,7 @@ class BlockSpaceManager:
                 self.block_sliding_window
                 and len(block_table) >= self.block_sliding_window
             ):
-                # re-use a block
+                # reuse a block
                 block_table.append(
                     block_table[len(block_table) % self.block_sliding_window]
                 )

ChatTTS/model/velocity/configs.py

@@ -12,7 +12,6 @@ import argparse
 import dataclasses
 from dataclasses import dataclass
 
-
 logger = init_logger(__name__)
 
 _GB = 1 << 30

ChatTTS/model/velocity/llama.py

@@ -21,6 +21,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Inference-only LLaMA model compatible with HuggingFace weights."""
+
 from typing import Any, Dict, List, Optional, Tuple
 
 import torch

ChatTTS/model/velocity/llm_engine.py

@@ -741,7 +741,7 @@ class LLMEngine:
 
     def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
         """Decodes the new token for a sequence."""
-        (new_tokens, new_output_text, prefix_offset, read_offset) = (
+        new_tokens, new_output_text, prefix_offset, read_offset = (
             detokenize_incrementally(
                 self.tokenizer,
                 all_input_ids=seq.get_token_ids(),

ChatTTS/model/velocity/model_runner.py

@@ -360,11 +360,11 @@ class ModelRunner:
             is_prompt = seq_group_metadata_list[0].is_prompt
             # Prepare input tensors.
             if is_prompt:
-                (input_tokens, input_positions, input_metadata, prompt_lens) = (
+                input_tokens, input_positions, input_metadata, prompt_lens = (
                     self._prepare_prompt(seq_group_metadata_list)
                 )
             else:
-                (input_tokens, input_positions, input_metadata) = self._prepare_decode(
+                input_tokens, input_positions, input_metadata = self._prepare_decode(
                     seq_group_metadata_list
                 )
                 prompt_lens = []
@@ -401,9 +401,9 @@ class ModelRunner:
                 broadcast(input_metadata.block_tables, src=0)
             broadcast(sampling_metadata.selected_token_indices, src=0)
         else:
-            receving_list = [None]
-            broadcast_object_list(receving_list, src=0)
-            py_data = receving_list[0]
+            receiving_list = [None]
+            broadcast_object_list(receiving_list, src=0)
+            py_data = receiving_list[0]
             input_tokens = torch.empty(
                 *py_data["input_tokens_size"], dtype=torch.long, device="cuda"
             )
@@ -505,9 +505,9 @@ class ModelRunner:
             model_executable = self.model
 
         infer_text = sampling_metadata.seq_groups[0][1].infer_text
-        temperture = sampling_metadata.seq_groups[0][1].temperature
+        temperature = sampling_metadata.seq_groups[0][1].temperature
         if not infer_text:
-            temperture = torch.tensor(temperture).to(input_tokens.device)
+            temperature = torch.tensor(temperature).to(input_tokens.device)
         logits_processors, logits_warpers = sampling_metadata.seq_groups[0][
             1
         ].logits_processors
@@ -553,7 +553,7 @@ class ModelRunner:
             ),
             hidden_states=hidden_states,
             infer_text=infer_text,
-            temperature=temperture,
+            temperature=temperature,
             logits_processors=logits_processors,
             logits_warpers=logits_warpers,
             min_new_token=min_new_token,

ChatTTS/model/velocity/output.py

@@ -107,14 +107,14 @@ class RequestOutput:
                 # always has the logprobs of the sampled tokens even if the
                 # logprobs are not requested.
                 logprobs = None
-            finshed_reason = SequenceStatus.get_finished_reason(seq.status)
+            finished_reason = SequenceStatus.get_finished_reason(seq.status)
             output = CompletionOutput(
                 seqs.index(seq),
                 seq.output_text,
                 seq.get_output_token_ids(),
                 seq.get_cumulative_logprob(),
                 logprobs,
-                finshed_reason,
+                finished_reason,
                 seq.data.hidden_states,
             )
             outputs.append(output)

ChatTTS/model/velocity/scheduler.py

@@ -128,7 +128,7 @@ class Scheduler:
         return len(self.waiting) + len(self.running) + len(self.swapped)
 
     def _schedule(self) -> SchedulerOutputs:
-        # Blocks that need to be swaped or copied before model execution.
+        # Blocks that need to be swapped or copied before model execution.
         blocks_to_swap_in: Dict[int, int] = {}
         blocks_to_swap_out: Dict[int, int] = {}
         blocks_to_copy: Dict[int, List[int]] = {}

ChatTTS/utils/dl.py

@@ -143,15 +143,7 @@ def download_and_extract_zip(
         logger.get_logger().info(f"extracted into {folder}")
 
 
-def download_dns_yaml(url: str, folder: str, headers: Dict[str, str]):
-    logger.get_logger().info(f"downloading {url}")
-    response = requests.get(url, headers=headers, stream=True, timeout=(100, 3))
-    with open(os.path.join(folder, "dns.yaml"), "wb") as out_file:
-        out_file.write(response.content)
-        logger.get_logger().info(f"downloaded into {folder}")
-
-
-def download_all_assets(tmpdir: str, homedir: str, version="0.2.10"):
+def download_all_assets(tmpdir: str, homedir: str, version="0.2.11"):
     import subprocess
     import platform
 
@@ -175,48 +167,15 @@ def download_all_assets(tmpdir: str, homedir: str, version="0.2.10"):
     if not architecture:
         logger.get_logger().error(f"architecture {architecture} is not supported")
         exit(1)
-    try:
-        BASE_URL = "https://github.com/fumiama/RVC-Models-Downloader/releases/download/"
-        suffix = "zip" if is_win else "tar.gz"
-        RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
-        cmdfile = os.path.join(tmpdir, "rvcmd")
-        if is_win:
-            download_and_extract_zip(RVCMD_URL, tmpdir)
-            cmdfile += ".exe"
-        else:
-            download_and_extract_tar_gz(RVCMD_URL, tmpdir)
-            os.chmod(cmdfile, 0o755)
-        subprocess.run([cmdfile, "-notui", "-w", "0", "-H", homedir, "assets/chtts"])
-    except Exception:
-        BASE_URL = (
-            "https://gitea.seku.su/fumiama/RVC-Models-Downloader/releases/download/"
-        )
-        suffix = "zip" if is_win else "tar.gz"
-        RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
-        download_dns_yaml(
-            "https://gitea.seku.su/fumiama/RVC-Models-Downloader/raw/branch/main/dns.yaml",
-            tmpdir,
-            headers={
-                "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0"
-            },
-        )
-        cmdfile = os.path.join(tmpdir, "rvcmd")
-        if is_win:
-            download_and_extract_zip(RVCMD_URL, tmpdir)
-            cmdfile += ".exe"
-        else:
-            download_and_extract_tar_gz(RVCMD_URL, tmpdir)
-            os.chmod(cmdfile, 0o755)
-        subprocess.run(
-            [
-                cmdfile,
-                "-notui",
-                "-w",
-                "0",
-                "-dns",
-                os.path.join(tmpdir, "dns.yaml"),
-                "-H",
-                homedir,
-                "assets/chtts",
-            ]
-        )
+
+    BASE_URL = "https://github.com/fumiama/RVC-Models-Downloader/releases/download/"
+    suffix = "zip" if is_win else "tar.gz"
+    RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
+    cmdfile = os.path.join(tmpdir, "rvcmd")
+    if is_win:
+        download_and_extract_zip(RVCMD_URL, tmpdir)
+        cmdfile += ".exe"
+    else:
+        download_and_extract_tar_gz(RVCMD_URL, tmpdir)
+        os.chmod(cmdfile, 0o755)
+    subprocess.run([cmdfile, "-notui", "-w", "0", "-H", homedir, "assets/chtts"])

ChatTTS/utils/gpu.py

@@ -1,3 +1,5 @@
+import importlib.util
+
 import torch
 
 try:
@@ -38,11 +40,24 @@ def select_device(min_memory=2047, experimental=False):
         """
         if experimental:
             # For Apple M1/M2 chips with Metal Performance Shaders
-            logger.get_logger().warning("experimantal: found apple GPU, using MPS.")
+            logger.get_logger().warning("experimental: found apple GPU, using MPS.")
             device = torch.device("mps")
         else:
             logger.get_logger().info("found Apple GPU, but use CPU.")
             device = torch.device("cpu")
+    elif importlib.util.find_spec("torch_directml") is not None:
+        """
+        Currently DML is under developing and may output wrong result,
+        so only enable this for experimental use.
+        """
+        if experimental:
+            logger.get_logger().warning("experimental: using DML.")
+            import torch_directml
+
+            device = torch_directml.device(torch_directml.default_device())
+        else:
+            logger.get_logger().info("found DML, but use CPU.")
+            device = torch.device("cpu")
     else:
         logger.get_logger().warning("no GPU or NPU found, use CPU instead")
         device = torch.device("cpu")

README.md

@@ -21,7 +21,8 @@ A generative speech model for daily dialogue.
 > This repo contains the algorithm infrastructure and some simple examples.
 
 > [!Tip]
-> For the extended end-user products, please refer to the index repo [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) maintained by the community.
+> For the extended end-user products, please refer to the index repo [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) maintained by the community.  
+> You can find a diagram visualization of the codebase [here](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md).
 
 ChatTTS is a text-to-speech model designed specifically for dialogue scenarios such as LLM assistant.
 

docs/cn/README.md

@@ -25,7 +25,8 @@
 > 这个仓库包含算法架构和一些简单的示例。
 
 > [!Tip]
-> 由本仓库衍生出的用户端产品，请参见由社区维护的索引仓库  [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS)。
+> 由本仓库衍生出的用户端产品，请参见由社区维护的索引仓库  [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS)。  
+> 您可以在[这里](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md)查看代码库的图解。
 
 ChatTTS 是一款专门为对话场景（例如 LLM 助手）设计的文本转语音模型。
 

docs/es/README.md

@@ -17,6 +17,11 @@ Un modelo de generación de voz para la conversación diaria.
 > [!NOTE]
 > Atención, es posible que esta versión no sea la última. Por favor, consulte la versión en inglés para conocer todo el contenido.
 
+> [!Tip]
+> Para los productos finales ampliados, consulta el repositorio índice [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) mantenido por la comunidad.  
+> Puedes encontrar una visualización en forma de diagrama del código [aquí](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md).
+
+
 ## Introducción
 
 ChatTTS es un modelo de texto a voz diseñado específicamente para escenarios conversacionales como LLM assistant.

docs/fr/README.md

@@ -21,7 +21,8 @@ Un modèle de parole génératif pour le dialogue quotidien.
 > Ce dépôt contient l'infrastructure de l'algorithme et quelques exemples simples.
 
 > [!Tip]
-> Pour les produits finaux étendus pour les utilisateurs, veuillez consulter le dépôt index [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) maintenu par la communauté.
+> Pour les produits finaux étendus pour les utilisateurs, veuillez consulter le dépôt index [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) maintenu par la communauté.  
+> Vous pouvez consulter un diagramme du code [ici](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md).
 
 ChatTTS est un modèle de synthèse vocale conçu spécifiquement pour les scénarios de dialogue tels que les assistants LLM.
 

docs/jp/README.md

@@ -10,6 +10,15 @@ ChatTTSは、LLMアシスタントなどの対話シナリオ用に特別に設
 
 モデルやロードマップについての正式なお問い合わせは、**open-source@2noise.com**までご連絡ください。QQグループ：808364215に参加してディスカッションすることもできます。GitHubでの問題提起も歓迎します。
 
+## はじめに
+> [!Note]
+> このリポジトリにはアルゴリズムのインフラといくつかの簡単な例が含まれています。
+
+> [!Tip]
+> エンドユーザー向けに拡張された製品については、コミュニティによって管理されているインデックスリポジトリ [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en) を参照してください。  
+> コードベースの図解は[こちら](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md)でご覧いただけます。
+
+
 ---
 ## ハイライト
 1. **会話型TTS**: ChatTTSは対話ベースのタスクに最適化されており、自然で表現豊かな音声合成を実現します。複数の話者をサポートし、対話型の会話を容易にします。

docs/kr/README.md

@@ -25,7 +25,8 @@
 > 이 저장소에는 알고리즘 구조와 간단한 예시들이 포함되어 있습니다.
 
 > [!Tip]
-> 이 프로젝트에서 파생된 프로젝트는 커뮤니티가 유지 관리하는 커뮤니티[Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS)를 참조하시길 바랍니다.
+> 이 프로젝트에서 파생된 프로젝트는 커뮤니티가 유지 관리하는 커뮤니티[Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS)를 참조하시길 바랍니다.  
+> 코드베이스의 다이어그램 시각화는 [여기](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md)에서 확인할 수 있습니다.
 
 ChatTTS는 대화 기반 작업(예: LLM 어시스턴트)을 위해 설계된 텍스트-음성 변환(TTS) 모델입니다.
 

docs/ru/README.md

@@ -6,6 +6,15 @@
 
 [**English**](../../README.md) | [**简体中文**](../cn/README.md) | [**日本語**](../jp/README.md) | **Русский** | [**Español**](../es/README.md) | [**Français**](../fr/README.md) | [**한국어**](../kr/README.md)
 
+## Введение
+> [!Note]
+> Этот репозиторий содержит инфраструктуру алгоритма и некоторые простые примеры.
+
+> [!Tip]
+> Для полнофункциональных пользовательских продуктов обратитесь к индексному репозиторию [Awesome-ChatTTS](https://github.com/libukai/Awesome-ChatTTS/tree/en), поддерживаемому сообществом.  
+> Схематичную визуализацию кодовой базы можно найти [здесь](https://github.com/CodeBoarding/GeneratedOnBoardings/blob/main/ChatTTS/on_boarding.md).
+
+
 ChatTTS - это модель преобразования текста в речь, специально разработанная для диалоговых сценариев, таких как помощник LLM. Она поддерживает как английский, так и китайский языки. Наша модель обучена на более чем 100 000 часах английского и китайского языков. Открытая версия на **[HuggingFace](https://huggingface.co/2Noise/ChatTTS)** - это предварительно обученная модель с 40 000 часами без SFT.
 
 Для официальных запросов о модели и плане развития, пожалуйста, свяжитесь с нами по адресу **open-source@2noise.com**. Вы можете присоединиться к нашей группе QQ: 808364215 для обсуждения. Добавление вопросов на GitHub также приветствуется.

examples/api/main.py

@@ -6,7 +6,6 @@ import zipfile
 from fastapi import FastAPI
 from fastapi.responses import StreamingResponse
 
-
 if sys.platform == "darwin":
     os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
 

examples/onnx/modeling_llama.py

@@ -21,6 +21,7 @@
 PyTorch LLaMA model.
 Copied from https://github.com/sophgo/LLM-TPU/blob/main/models/Llama2/compile/files/llama-2-7b-chat-hf/modeling_llama.py
 """
+
 import math
 from typing import List, Optional, Tuple, Union
 
@@ -45,7 +46,6 @@ from transformers.utils import (
 )
 from transformers.models.llama.configuration_llama import LlamaConfig
 
-
 logger = logging.get_logger(__name__)
 
 _CONFIG_FOR_DOC = "LlamaConfig"

examples/web/funcs.py

@@ -25,6 +25,9 @@ custom_path: Optional[str] = None
 has_interrupted = False
 is_in_generate = False
 
+enable_cache = True
+experimental = False
+
 seed_min = 1
 seed_max = 4294967295
 
@@ -62,12 +65,27 @@ def on_audio_seed_change(audio_seed_input):
     return rand_spk
 
 
+def set_params(en_cache, exp):
+    global enable_cache, experimental
+
+    enable_cache = en_cache
+    experimental = exp
+
+
 def load_chat(cust_path: Optional[str], coef: Optional[str]) -> bool:
+    global enable_cache, experimental
+
     if cust_path == None:
-        ret = chat.load(coef=coef)
+        ret = chat.load(coef=coef, enable_cache=enable_cache, experimental=experimental)
     else:
         logger.info("local model path: %s", cust_path)
-        ret = chat.load("custom", custom_path=cust_path, coef=coef)
+        ret = chat.load(
+            "custom",
+            custom_path=cust_path,
+            coef=coef,
+            enable_cache=enable_cache,
+            experimental=experimental,
+        )
         global custom_path
         custom_path = cust_path
     if ret:
@@ -102,7 +120,7 @@ def reload_chat(coef: Optional[str]) -> str:
     chat.unload()
     gr.Info("Model unloaded.")
     if len(coef) != 230:
-        gr.Warning("Ingore invalid DVAE coefficient.")
+        gr.Warning("Ignore invalid DVAE coefficient.")
         coef = None
     try:
         global custom_path
@@ -111,7 +129,7 @@ def reload_chat(coef: Optional[str]) -> str:
         raise gr.Error(str(e))
     if not ret:
         raise gr.Error("Unable to load model.")
-    gr.Info("Reload succeess.")
+    gr.Info("Reload success.")
     return chat.coef
 
 

examples/web/webui.py

@@ -116,14 +116,14 @@ def main():
             spk_emb_text = gr.Textbox(
                 label="Speaker Embedding",
                 max_lines=3,
-                show_copy_button=True,
+                buttons=["copy"],
                 interactive=True,
                 scale=2,
             )
             dvae_coef_text = gr.Textbox(
                 label="DVAE Coefficient",
                 max_lines=3,
-                show_copy_button=True,
+                buttons=["copy"],
                 interactive=True,
                 scale=2,
             )
@@ -161,7 +161,7 @@ def main():
         text_output = gr.Textbox(
             label="Output Text",
             interactive=False,
-            show_copy_button=True,
+            buttons=["copy"],
         )
 
         sample_audio_input.change(
@@ -261,8 +261,14 @@ def main():
     parser.add_argument("--root_path", type=str, help="root path")
     parser.add_argument("--custom_path", type=str, help="custom model path")
     parser.add_argument("--coef", type=str, help="custom dvae coefficient")
+    parser.add_argument(
+        "--disable_cache", action="store_true", help="enable model cache"
+    )
+    parser.add_argument(
+        "--experimental", action="store_true", help="enable model cache"
+    )
     args = parser.parse_args()
-
+    set_params(not args.disable_cache, args.experimental)
     logger.info("loading ChatTTS model...")
 
     if load_chat(args.custom_path, args.coef):
@@ -279,7 +285,7 @@ def main():
         server_port=args.server_port,
         root_path=args.root_path,
         inbrowser=True,
-        show_api=False,
+        footer_links=["api", "gradio", "settings"],
     )
 
 

requirements.txt

@@ -14,3 +14,4 @@ WeTextProcessing; sys_platform == 'linux'
 nemo_text_processing; sys_platform == 'linux'
 av
 pydub
+requests

tools/audio/av.py

@@ -7,7 +7,6 @@ from av.audio.frame import AudioFrame
 from av.audio.resampler import AudioResampler
 import numpy as np
 
-
 video_format_dict: Dict[str, str] = {
     "m4a": "mp4",
 }