From 9827a24929e62ada30bd1efa0f5ce5d4fee4e0c5 Mon Sep 17 00:00:00 2001 From: xming521 <1223398803@qq.com> Date: Tue, 29 Apr 2025 22:47:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BE=9D=E8=B5=96=E9=A1=B9?= =?UTF-8?q?=E7=89=88=E6=9C=AC=EF=BC=8C=E6=8F=90=E5=8D=87torch=E5=92=8Ctorc?= =?UTF-8?q?haudio=E8=87=B32.6.0=EF=BC=8C=E6=9B=B4=E6=96=B0openai=E8=87=B31?= =?UTF-8?q?.52.0=20=E7=9B=B8=E5=BA=94=E6=9B=B4=E6=96=B0test=5Fmodel?= =?UTF-8?q?=EF=BC=8C=E8=B0=83=E6=95=B4pytorch=E6=BA=90=E4=B8=BAcu124?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 13 +-- pyproject.toml | 22 ++-- weclone/core/inference/vllm_infer.py | 162 +++++++++++++++++++++++++++ weclone/data/qa_generator.py | 2 +- weclone/eval/test_model.py | 22 +++- weclone/utils/length_cdf.py | 14 +++ 6 files changed, 210 insertions(+), 25 deletions(-) create mode 100644 weclone/core/inference/vllm_infer.py diff --git a/README.md b/README.md index d4b0d1d..544678a 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,9 @@ ### 环境搭建 -cuda安装(已安装可跳过)：[LLaMA Factory](https://llamafactory.readthedocs.io/zh-cn/latest/getting_started/installation.html#cuda) +1.cuda安装(已安装可跳过，**要求版本12.4及以上**)：[LLaMA Factory](https://llamafactory.readthedocs.io/zh-cn/latest/getting_started/installation.html#cuda) - -建议使用 [uv](https://docs.astral.sh/uv/)，这是一个非常快速的 Python 环境管理器。安装uv后，您可以使用以下命令创建一个新的Python环境并安装依赖项，注意这不包含音频克隆功能的依赖： +2.建议使用 [uv](https://docs.astral.sh/uv/)安装依赖，这是一个非常快速的 Python 环境管理器。安装uv后，您可以使用以下命令创建一个新的Python环境并安装依赖项，注意这不包含音频克隆功能的依赖： ```bash git clone https://github.com/xming521/WeClone.git cd WeClone @@ -61,21 +60,19 @@ uv venv .venv --python=3.10 source .venv/bin/activate uv pip install --group main -e . ``` - -将配置文件模板复制一份并重命名为`settings.json`，后续配置修改在此文件进行： +3.将配置文件模板复制一份并重命名为`settings.json`，后续配置修改在此文件进行： ```bash cp settings.template.json settings.json ``` > [!NOTE] > 训练以及推理相关配置统一在文件[settings.json](settings.json) -使用以下命令测试CUDA环境是否正确配置并可被PyTorch识别，Mac不需要： +4.使用以下命令测试CUDA环境是否正确配置并可被PyTorch识别，Mac不需要： ```bash python -c "import torch; print('CUDA是否可用:', torch.cuda.is_available());" ``` -（可选）安装FlashAttention，加速训练和推理：`uv pip install flash-attn --no-build-isolation` - +5.（可选）安装FlashAttention，加速训练和推理：`uv pip install flash-attn --no-build-isolation` ### 数据准备 diff --git a/pyproject.toml b/pyproject.toml index 7f98ec8..83342be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pydantic==2.10.6", "setuptools>=78.1.0", "loguru>=0.7.3", - "torch>=2.5.1", + "torch>=2.6.0", "transformers==4.49.0", "tomli; python_version < '3.11'", ] @@ -38,10 +38,10 @@ sparktts = [ "safetensors>=0.5.2", "soundfile>=0.12.1", "soxr>=0.5.0.post1", - "torchaudio>=2.5.1", + "torchaudio>=2.6.0", "tqdm>=4.66.5", ] -main = ["llamafactory>=0.9.2", "openai==0.28.0"] +main = ["llamafactory>=0.9.2", "openai==1.76.0", "vllm==0.8.0"] dev = ["pytest", "pyright", "ruff"] [project.scripts] @@ -54,16 +54,16 @@ conflicts = [ [tool.uv.sources] torch = [ - { index = "pytorch-cu121", marker = "platform_system == 'Windows'" }, - { index = "pytorch-cu121", marker = "platform_system == 'Linux'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Windows'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Linux'" }, ] torchaudio = [ - { index = "pytorch-cu121", marker = "platform_system == 'Windows'" }, - { index = "pytorch-cu121", marker = "platform_system == 'Linux'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Windows'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Linux'" }, ] torchvision = [ - { index = "pytorch-cu121", marker = "platform_system == 'Windows'" }, - { index = "pytorch-cu121", marker = "platform_system == 'Linux'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Windows'" }, + { index = "pytorch-cu124", marker = "platform_system == 'Linux'" }, ] @@ -72,8 +72,8 @@ url = "https://pypi.tuna.tsinghua.edu.cn/simple/" default = true [[tool.uv.index]] -name = "pytorch-cu121" -url = "https://download.pytorch.org/whl/cu121" +name = "pytorch-cu124" +url = "https://download.pytorch.org/whl/cu124" explicit = true [tool.setuptools.packages.find] diff --git a/weclone/core/inference/vllm_infer.py b/weclone/core/inference/vllm_infer.py new file mode 100644 index 0000000..5526528 --- /dev/null +++ b/weclone/core/inference/vllm_infer.py @@ -0,0 +1,162 @@ +# Copyright 2025 the LlamaFactory team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Optional + +import fire +from transformers import Seq2SeqTrainingArguments + +from llamafactory.data import get_dataset, get_template_and_fix_tokenizer +from llamafactory.extras.constants import IGNORE_INDEX +from llamafactory.extras.misc import get_device_count +from llamafactory.extras.packages import is_vllm_available +from llamafactory.hparams import get_infer_args +from llamafactory.model import load_tokenizer + + +if is_vllm_available(): + from vllm import LLM, SamplingParams + from vllm.lora.request import LoRARequest + + +def vllm_infer( + model_name_or_path: str, + adapter_name_or_path: str = None, + dataset: str = "alpaca_en_demo", + dataset_dir: str = "data", + template: str = "default", + cutoff_len: int = 2048, + max_samples: Optional[int] = None, + vllm_config: str = "{}", + save_name: str = "generated_predictions.jsonl", + temperature: float = 0.95, + top_p: float = 0.7, + top_k: int = 50, + max_new_tokens: int = 1024, + repetition_penalty: float = 1.0, + skip_special_tokens: bool = True, + seed: Optional[int] = None, + pipeline_parallel_size: int = 1, + image_max_pixels: int = 768 * 768, + image_min_pixels: int = 32 * 32, +): + r"""Perform batch generation using vLLM engine, which supports tensor parallelism. + + Usage: python vllm_infer.py --model_name_or_path meta-llama/Llama-2-7b-hf --template llama --dataset alpaca_en_demo + """ + if pipeline_parallel_size > get_device_count(): + raise ValueError("Pipeline parallel size should be smaller than the number of gpus.") + + model_args, data_args, _, generating_args = get_infer_args( + dict( + model_name_or_path=model_name_or_path, + adapter_name_or_path=adapter_name_or_path, + dataset=dataset, + dataset_dir=dataset_dir, + template=template, + cutoff_len=cutoff_len, + max_samples=max_samples, + preprocessing_num_workers=16, + vllm_config=vllm_config, + temperature=temperature, + top_p=top_p, + top_k=top_k, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + ) + ) + + training_args = Seq2SeqTrainingArguments(output_dir="dummy_dir") + tokenizer_module = load_tokenizer(model_args) + tokenizer = tokenizer_module["tokenizer"] + template_obj = get_template_and_fix_tokenizer(tokenizer, data_args) + template_obj.mm_plugin.expand_mm_tokens = False # for vllm generate + dataset_module = get_dataset(template_obj, model_args, data_args, training_args, "ppo", **tokenizer_module) + + inputs, prompts, labels = [], [], [] + for sample in dataset_module["train_dataset"]: + if sample["images"]: + multi_modal_data = { + "image": template_obj.mm_plugin._regularize_images( + sample["images"], image_max_pixels=image_max_pixels, image_min_pixels=image_min_pixels + )["images"] + } + elif sample["videos"]: + multi_modal_data = { + "video": template_obj.mm_plugin._regularize_videos( + sample["videos"], image_max_pixels=image_max_pixels, image_min_pixels=image_min_pixels + )["videos"] + } + elif sample["audios"]: + audio_data = template_obj.mm_plugin._regularize_audios( + sample["audios"], + sampling_rate=16000, + ) + multi_modal_data = {"audio": zip(audio_data["audios"], audio_data["sampling_rates"])} + else: + multi_modal_data = None + + inputs.append({"prompt_token_ids": sample["input_ids"], "multi_modal_data": multi_modal_data}) + prompts.append(tokenizer.decode(sample["input_ids"], skip_special_tokens=skip_special_tokens)) + labels.append( + tokenizer.decode( + list(filter(lambda x: x != IGNORE_INDEX, sample["labels"])), skip_special_tokens=skip_special_tokens + ) + ) + + sampling_params = SamplingParams( + repetition_penalty=generating_args.repetition_penalty or 1.0, # repetition_penalty must > 0 + temperature=generating_args.temperature, + top_p=generating_args.top_p or 1.0, # top_p must > 0 + top_k=generating_args.top_k or -1, # top_k must > 0 + stop_token_ids=template_obj.get_stop_token_ids(tokenizer), + max_tokens=generating_args.max_new_tokens, + skip_special_tokens=skip_special_tokens, + seed=seed, + ) + if model_args.adapter_name_or_path is not None: + lora_request = LoRARequest("default", 1, model_args.adapter_name_or_path[0]) + else: + lora_request = None + + engine_args = { + "model": model_args.model_name_or_path, + "trust_remote_code": True, + "dtype": model_args.infer_dtype, + "max_model_len": cutoff_len + max_new_tokens, + "tensor_parallel_size": (get_device_count() // pipeline_parallel_size) or 1, + "pipeline_parallel_size": pipeline_parallel_size, + "disable_log_stats": True, + "enable_lora": model_args.adapter_name_or_path is not None, + } + if template_obj.mm_plugin.__class__.__name__ != "BasePlugin": + engine_args["limit_mm_per_prompt"] = {"image": 4, "video": 2, "audio": 2} + + if isinstance(model_args.vllm_config, dict): + engine_args.update(model_args.vllm_config) + + results = LLM(**engine_args).generate(inputs, sampling_params, lora_request=lora_request) + preds = [result.outputs[0].text for result in results] + with open(save_name, "w", encoding="utf-8") as f: + for text, pred, label in zip(prompts, preds, labels): + f.write(json.dumps({"prompt": text, "predict": pred, "label": label}, ensure_ascii=False) + "\n") + + print("*" * 70) + print(f"{len(prompts)} generated results have been saved at {save_name}.") + print("*" * 70) + + +if __name__ == "__main__": + fire.Fire(vllm_infer) \ No newline at end of file diff --git a/weclone/data/qa_generator.py b/weclone/data/qa_generator.py index ea5c9c8..d8bcb06 100644 --- a/weclone/data/qa_generator.py +++ b/weclone/data/qa_generator.py @@ -75,6 +75,7 @@ class DataProcessor: template=self.c["template"], interval=self.c["cutoff_len"], ) + logger.success(f"聊天记录处理成功，共{len(qa_res)}条，保存到./dataset/res_csv/sft/sft-my.json") def get_csv_files(self): """遍历文件夹获取所有CSV文件路径""" @@ -354,7 +355,6 @@ class DataProcessor: encoding="utf-8", ) as f: json.dump(qa_res, f, ensure_ascii=False) - logger.success(f"聊天记录处理成功，共{len(qa_res)}条，保存到 {f.name}") if __name__ == "__main__": diff --git a/weclone/eval/test_model.py b/weclone/eval/test_model.py index 514e4dc..59eeb2f 100644 --- a/weclone/eval/test_model.py +++ b/weclone/eval/test_model.py @@ -1,8 +1,10 @@ import json import openai +from openai import OpenAI # 导入 OpenAI 类 from tqdm import tqdm -from typing import List, Dict +from typing import List, Dict, cast # 导入 cast +from openai.types.chat import ChatCompletionMessageParam # 导入消息参数类型 from weclone.utils.config import load_config @@ -16,18 +18,28 @@ config = { config = type("Config", (object,), config)() -openai.api_key = """sk-test""" -openai.api_base = "http://127.0.0.1:8005/v1" +# 初始化 OpenAI 客户端 +client = OpenAI( + api_key="""sk-test""", + base_url="http://127.0.0.1:8005/v1" +) -def handler_text(content: str, history: List[Dict[str, str]], config): +def handler_text(content: str, history: list, config): messages = [{"role": "system", "content": f"{config.default_prompt}"}] for item in history: messages.append(item) messages.append({"role": "user", "content": content}) history.append({"role": "user", "content": content}) try: - response = openai.ChatCompletion.create(model=config.model, messages=messages, max_tokens=50) + # 使用新的 API 调用方式 + # 将 messages 转换为正确的类型 + typed_messages = cast(List[ChatCompletionMessageParam], messages) + response = client.chat.completions.create( + model=config.model, + messages=typed_messages, # 传递转换后的列表 + max_tokens=50 + ) except openai.APIError as e: history.pop() return "AI接口出错,请重试\n" + str(e) diff --git a/weclone/utils/length_cdf.py b/weclone/utils/length_cdf.py index 75a4020..986578a 100644 --- a/weclone/utils/length_cdf.py +++ b/weclone/utils/length_cdf.py @@ -1,3 +1,17 @@ +# Copyright 2025 the LlamaFactory team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import defaultdict from tqdm import tqdm