diff --git a/README.md b/README.md index 31566db..7d9be0e 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,9 @@ This repository supports the `Wan2.2-S2V-14B` Speech-to-Video model and can simu ```sh python generate.py --task s2v-14B --size 1024*704 --ckpt_dir ./Wan2.2-S2V-14B/ --offload_model True --convert_model_dtype --prompt "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard." --image "examples/i2v_input.JPG" --audio "examples/talk.wav" # Without setting --num_clip, the generated video length will automatically adjust based on the input audio length + +# You can use CosyVoice to generate audio with --enable_tts +python generate.py --task s2v-14B --size 1024*704 --ckpt_dir ./Wan2.2-S2V-14B/ --offload_model True --convert_model_dtype --prompt "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard." --image "examples/i2v_input.JPG" --enable_tts --tts_prompt_audio "examples/zero_shot_prompt.wav" --tts_prompt_text "希望你以后能够做的比我还好呦。" --tts_text "收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。" ``` > 💡 This command can run on a GPU with at least 80GB VRAM. diff --git a/examples/zero_shot_prompt.wav b/examples/zero_shot_prompt.wav new file mode 100644 index 0000000..25fbf59 Binary files /dev/null and b/examples/zero_shot_prompt.wav differ diff --git a/generate.py b/generate.py index b95424c..9735afa 100644 --- a/generate.py +++ b/generate.py @@ -42,6 +42,12 @@ EXAMPLE_PROMPT = { "examples/i2v_input.JPG", "audio": "examples/talk.wav", + "tts_prompt_audio": + "examples/zero_shot_prompt.wav", + "tts_prompt_text": + "希望你以后能够做的比我还好呦。", + "tts_text": + "收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。" }, } @@ -56,8 +62,12 @@ def _validate_args(args): args.prompt = EXAMPLE_PROMPT[args.task]["prompt"] if args.image is None and "image" in EXAMPLE_PROMPT[args.task]: args.image = EXAMPLE_PROMPT[args.task]["image"] - if args.audio is None and "audio" in EXAMPLE_PROMPT[args.task]: + if args.audio is None and args.enable_tts is False and "audio" in EXAMPLE_PROMPT[args.task]: args.audio = EXAMPLE_PROMPT[args.task]["audio"] + if (args.tts_prompt_audio is None or args.tts_text is None) and args.enable_tts is True and "audio" in EXAMPLE_PROMPT[args.task]: + args.tts_prompt_audio = EXAMPLE_PROMPT[args.task]["tts_prompt_audio"] + args.tts_prompt_text = EXAMPLE_PROMPT[args.task]["tts_prompt_text"] + args.tts_text = EXAMPLE_PROMPT[args.task]["tts_text"] if args.task == "i2v-A14B": assert args.image is not None, "Please specify the image path for i2v." @@ -217,6 +227,26 @@ def _parse_args(): type=str, default=None, help="Path to the audio file, e.g. wav, mp3") + parser.add_argument( + "--enable_tts", + action="store_true", + default=False, + help="Use CosyVoice to synthesis audio") + parser.add_argument( + "--tts_prompt_audio", + type=str, + default=None, + help="Path to the tts prompt audio file, e.g. wav, mp3. Must be greater than 16khz, and between 5s to 15s.") + parser.add_argument( + "--tts_prompt_text", + type=str, + default=None, + help="Content to the tts prompt audio. If provided, must exactly match tts_prompt_audio") + parser.add_argument( + "--tts_text", + type=str, + default=None, + help="Text wish to synthesize") parser.add_argument( "--pose_video", type=str, @@ -412,6 +442,10 @@ def generate(args): input_prompt=args.prompt, ref_image_path=args.image, audio_path=args.audio, + enable_tts=args.enable_tts, + tts_prompt_audio=args.tts_prompt_audio, + tts_prompt_text=args.tts_prompt_text, + tts_text=args.tts_text, num_repeat=args.num_clip, pose_video=args.pose_video, max_area=MAX_AREA_CONFIGS[args.size], @@ -469,7 +503,10 @@ def generate(args): normalize=True, value_range=(-1, 1)) if "s2v" in args.task: - merge_video_audio(video_path=args.save_file, audio_path=args.audio) + if args.enable_tts is False: + merge_video_audio(video_path=args.save_file, audio_path=args.audio) + else: + merge_video_audio(video_path=args.save_file, audio_path="tts.wav") del video torch.cuda.synchronize() diff --git a/requirements.txt b/requirements.txt index 77c1e6d..f0917aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ torch>=2.4.0 torchvision>=0.19.0 opencv-python>=4.9.0.80 diffusers>=0.31.0 -transformers>=4.49.0 +transformers>=4.49.0,<=4.51.3 tokenizers>=0.20.3 accelerate>=1.1.1 tqdm @@ -13,3 +13,23 @@ dashscope imageio-ffmpeg flash_attn numpy>=1.23.5,<2 +openai-whisper +HyperPyYAML +onnxruntime +torchaudio +inflect +wetext +omegaconf +conformer +hydra-core +lightning +rich +gdown +matplotlib +wget +pyarrow +pyworld +librosa +decord +modelscope +GitPython diff --git a/wan/speech2video.py b/wan/speech2video.py index 8b79548..be9f5f1 100644 --- a/wan/speech2video.py +++ b/wan/speech2video.py @@ -394,6 +394,10 @@ class WanS2V: input_prompt, ref_image_path, audio_path, + enable_tts, + tts_prompt_audio, + tts_prompt_text, + tts_text, num_repeat=1, pose_video=None, max_area=720 * 1280, @@ -478,6 +482,8 @@ class WanS2V: device=self.device) # extract audio emb + if enable_tts is True: + audio_path = self.tts(tts_prompt_audio, tts_prompt_text, tts_text) audio_emb, nr = self.encode_audio(audio_path, infer_frames=infer_frames) if num_repeat is None or num_repeat > nr: num_repeat = nr @@ -671,3 +677,31 @@ class WanS2V: dist.barrier() return videos[0] if self.rank == 0 else None + + def tts(self, tts_prompt_audio, tts_prompt_text, tts_text): + if not hasattr(self, 'cosyvoice'): + self.load_tts() + speech_list = [] + from cosyvoice.utils.file_utils import load_wav + import torchaudio + prompt_speech_16k = load_wav(tts_prompt_audio, 16000) + if tts_prompt_text is not None: + for i in self.cosyvoice.inference_zero_shot(tts_text, tts_prompt_text, prompt_speech_16k): + speech_list.append(i['tts_speech']) + else: + for i in self.cosyvoice.inference_cross_lingual(tts_text, prompt_speech_16k): + speech_list.append(i['tts_speech']) + torchaudio.save('tts.wav', torch.concat(speech_list, dim=1), self.cosyvoice.sample_rate) + return 'tts.wav' + + def load_tts(self): + if not os.path.exists('CosyVoice'): + from wan.utils.utils import download_cosyvoice_repo + download_cosyvoice_repo('CosyVoice') + if not os.path.exists('CosyVoice2-0.5B'): + from wan.utils.utils import download_cosyvoice_model + download_cosyvoice_model('CosyVoice2-0.5B', 'CosyVoice2-0.5B') + sys.path.append('CosyVoice') + sys.path.append('CosyVoice/third_party/Matcha-TTS') + from cosyvoice.cli.cosyvoice import CosyVoice2 + self.cosyvoice = CosyVoice2('CosyVoice2-0.5B') \ No newline at end of file diff --git a/wan/utils/utils.py b/wan/utils/utils.py index 8601100..4bc4e36 100644 --- a/wan/utils/utils.py +++ b/wan/utils/utils.py @@ -223,3 +223,16 @@ def best_output_size(w, h, dw, dh, expected_area): return ow1, oh1 else: return ow2, oh2 + + +def download_cosyvoice_repo(repo_path): + try: + import git + except ImportError: + raise ImportError('failed to import git, please run pip install GitPython') + repo = git.Repo.clone_from('https://github.com/FunAudioLLM/CosyVoice.git', repo_path, multi_options=['--recursive'], branch='main') + + +def download_cosyvoice_model(model_name, model_path): + from modelscope import snapshot_download + snapshot_download('iic/{}'.format(model_name), local_dir=model_path)