add cosyvoice tts code

README.md

@@ -245,6 +245,9 @@ This repository supports the `Wan2.2-S2V-14B` Speech-to-Video model and can simu
 ```sh
 python generate.py  --task s2v-14B --size 1024*704 --ckpt_dir ./Wan2.2-S2V-14B/ --offload_model True --convert_model_dtype --prompt "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard."  --image "examples/i2v_input.JPG" --audio "examples/talk.wav"
 # Without setting --num_clip, the generated video length will automatically adjust based on the input audio length
+
+# You can use CosyVoice to generate audio with --enable_tts
+python generate.py  --task s2v-14B --size 1024*704 --ckpt_dir ./Wan2.2-S2V-14B/ --offload_model True --convert_model_dtype --prompt "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard."  --image "examples/i2v_input.JPG" --enable_tts --tts_prompt_audio "examples/zero_shot_prompt.wav" --tts_prompt_text "希望你以后能够做的比我还好呦。" --tts_text "收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。"
 ```
 
 > 💡 This command can run on a GPU with at least 80GB VRAM.

generate.py

@@ -42,6 +42,12 @@ EXAMPLE_PROMPT = {
             "examples/i2v_input.JPG",
         "audio":
             "examples/talk.wav",
+        "tts_prompt_audio":
+            "examples/zero_shot_prompt.wav",
+        "tts_prompt_text":
+            "希望你以后能够做的比我还好呦。",
+        "tts_text":
+            "收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。"
     },
 }
 
@@ -56,8 +62,12 @@ def _validate_args(args):
         args.prompt = EXAMPLE_PROMPT[args.task]["prompt"]
     if args.image is None and "image" in EXAMPLE_PROMPT[args.task]:
         args.image = EXAMPLE_PROMPT[args.task]["image"]
-    if args.audio is None and "audio" in EXAMPLE_PROMPT[args.task]:
+    if args.audio is None and args.enable_tts is False and "audio" in EXAMPLE_PROMPT[args.task]:
         args.audio = EXAMPLE_PROMPT[args.task]["audio"]
+    if (args.tts_prompt_audio is None or args.tts_text is None) and args.enable_tts is True and "audio" in EXAMPLE_PROMPT[args.task]:
+        args.tts_prompt_audio = EXAMPLE_PROMPT[args.task]["tts_prompt_audio"]
+        args.tts_prompt_text = EXAMPLE_PROMPT[args.task]["tts_prompt_text"]
+        args.tts_text = EXAMPLE_PROMPT[args.task]["tts_text"]
 
     if args.task == "i2v-A14B":
         assert args.image is not None, "Please specify the image path for i2v."
@@ -217,6 +227,26 @@ def _parse_args():
         type=str,
         default=None,
         help="Path to the audio file, e.g. wav, mp3")
+    parser.add_argument(
+        "--enable_tts",
+        action="store_true",
+        default=False,
+        help="Use CosyVoice to synthesis audio")
+    parser.add_argument(
+        "--tts_prompt_audio",
+        type=str,
+        default=None,
+        help="Path to the tts prompt audio file, e.g. wav, mp3. Must be greater than 16khz, and between 5s to 15s.")
+    parser.add_argument(
+        "--tts_prompt_text",
+        type=str,
+        default=None,
+        help="Content to the tts prompt audio. If provided, must exactly match tts_prompt_audio")
+    parser.add_argument(
+        "--tts_text",
+        type=str,
+        default=None,
+        help="Text wish to synthesize")
     parser.add_argument(
         "--pose_video",
         type=str,
@@ -412,6 +442,10 @@ def generate(args):
             input_prompt=args.prompt,
             ref_image_path=args.image,
             audio_path=args.audio,
+            enable_tts=args.enable_tts,
+            tts_prompt_audio=args.tts_prompt_audio,
+            tts_prompt_text=args.tts_prompt_text,
+            tts_text=args.tts_text,
             num_repeat=args.num_clip,
             pose_video=args.pose_video,
             max_area=MAX_AREA_CONFIGS[args.size],
@@ -469,7 +503,10 @@ def generate(args):
             normalize=True,
             value_range=(-1, 1))
         if "s2v" in args.task:
-            merge_video_audio(video_path=args.save_file, audio_path=args.audio)
+            if args.enable_tts is False:
+                merge_video_audio(video_path=args.save_file, audio_path=args.audio)
+            else:
+                merge_video_audio(video_path=args.save_file, audio_path="tts.wav")
     del video
 
     torch.cuda.synchronize()

requirements.txt

@@ -2,7 +2,7 @@ torch>=2.4.0
 torchvision>=0.19.0
 opencv-python>=4.9.0.80
 diffusers>=0.31.0
-transformers>=4.49.0
+transformers>=4.49.0,<=4.51.3
 tokenizers>=0.20.3
 accelerate>=1.1.1
 tqdm
@@ -13,3 +13,23 @@ dashscope
 imageio-ffmpeg
 flash_attn
 numpy>=1.23.5,<2
+openai-whisper
+HyperPyYAML
+onnxruntime
+torchaudio
+inflect
+wetext
+omegaconf
+conformer
+hydra-core
+lightning
+rich
+gdown
+matplotlib
+wget
+pyarrow
+pyworld
+librosa
+decord
+modelscope
+GitPython

wan/speech2video.py

@@ -394,6 +394,10 @@ class WanS2V:
         input_prompt,
         ref_image_path,
         audio_path,
+        enable_tts,
+        tts_prompt_audio,
+        tts_prompt_text,
+        tts_text,
         num_repeat=1,
         pose_video=None,
         max_area=720 * 1280,
@@ -478,6 +482,8 @@ class WanS2V:
                 device=self.device)
 
         # extract audio emb
+        if enable_tts is True:
+            audio_path = self.tts(tts_prompt_audio, tts_prompt_text, tts_text)
         audio_emb, nr = self.encode_audio(audio_path, infer_frames=infer_frames)
         if num_repeat is None or num_repeat > nr:
             num_repeat = nr
@@ -671,3 +677,31 @@ class WanS2V:
             dist.barrier()
 
         return videos[0] if self.rank == 0 else None
+
+    def tts(self, tts_prompt_audio, tts_prompt_text, tts_text):
+        if not hasattr(self, 'cosyvoice'):
+            self.load_tts()
+        speech_list = []
+        from cosyvoice.utils.file_utils import load_wav
+        import torchaudio
+        prompt_speech_16k = load_wav(tts_prompt_audio, 16000)
+        if tts_prompt_text is not None:
+            for i in self.cosyvoice.inference_zero_shot(tts_text, tts_prompt_text, prompt_speech_16k):
+                speech_list.append(i['tts_speech'])
+        else:
+            for i in self.cosyvoice.inference_cross_lingual(tts_text, prompt_speech_16k):
+                speech_list.append(i['tts_speech'])
+        torchaudio.save('tts.wav', torch.concat(speech_list, dim=1), self.cosyvoice.sample_rate)
+        return 'tts.wav'
+
+    def load_tts(self):
+        if not os.path.exists('CosyVoice'):
+            from wan.utils.utils import download_cosyvoice_repo
+            download_cosyvoice_repo('CosyVoice')
+        if not os.path.exists('CosyVoice2-0.5B'):
+            from wan.utils.utils import download_cosyvoice_model
+            download_cosyvoice_model('CosyVoice2-0.5B', 'CosyVoice2-0.5B')
+        sys.path.append('CosyVoice')
+        sys.path.append('CosyVoice/third_party/Matcha-TTS')
+        from cosyvoice.cli.cosyvoice import CosyVoice2
+        self.cosyvoice = CosyVoice2('CosyVoice2-0.5B')

wan/utils/utils.py

@@ -223,3 +223,16 @@ def best_output_size(w, h, dw, dh, expected_area):
         return ow1, oh1
     else:
         return ow2, oh2
+
+
+def download_cosyvoice_repo(repo_path):
+    try:
+        import git
+    except ImportError:
+        raise ImportError('failed to import git, please run pip install GitPython')
+    repo = git.Repo.clone_from('https://github.com/FunAudioLLM/CosyVoice.git', repo_path, multi_options=['--recursive'], branch='main')
+
+
+def download_cosyvoice_model(model_name, model_path):
+    from modelscope import snapshot_download
+    snapshot_download('iic/{}'.format(model_name), local_dir=model_path)