opensource/WeClone-new
3
0
Fork 0
mirror of https://github.com/xming521/WeClone.git synced 2026-08-28 18:07:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(tests): 添加测试模型参数配置，允许自定义测试集文件

Browse Source
This commit is contained in:
xming521
2025-06-13 15:49:02 +08:00
parent 45a579716f
commit e239893884
11 changed files with 115 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/full_pipe.jsonc
+3
View File
@@ -36,6 +36,9 @@
}
}
},
"test_model_args": {
"test_data_path": "tests/tests_data/test_model_data.json"
},
"train_sft_args": {
//微调配置
"stage": "sft",
tests/full_pipeV2.jsonc
+10
View File
@@ -33,8 +33,18 @@
"llm": {
"accept_score": 2, //可以接受的llm打分阈值,1分最差，5分最好,低于此分数的数据不会用于训练
}
},
"vision_api": {
"enable": false, // 设置为 true 来开启此功能
"api_key": "xxx",
"api_url": "https://xxx/v1", // 例如阿里云，或替换为其他兼容OpenAI的API地址
"model_name": "xxx", // 要使用的多模态模型名称,例如qwen-vl-max
"max_workers": 5 // 并行调用API的线程数，最多不要超过8
}
},
"test_model_args": {
"test_data_path": "tests/tests_data/test_model_data.json"
},
"train_sft_args": {
//微调配置
"stage": "sft",
tests/test_full_pipe.py
+14 -9
View File
@@ -43,16 +43,21 @@ def setup_make_dataset_test_data():
TESTS_DIR = os.path.dirname(__file__)
TEST_DATA_PERSON_DIR = os.path.join(TESTS_DIR, "tests_data", "test_person")
os.makedirs(DATASET_CSV_DIR, exist_ok=True)
# 先删除目录，再重新创建
if os.path.exists(DATASET_CSV_DIR):
shutil.rmtree(DATASET_CSV_DIR)
os.makedirs(DATASET_CSV_DIR)
# 创建test_person子目录
test_person_csv_dir = os.path.join(DATASET_CSV_DIR, "test_person")
os.makedirs(test_person_csv_dir)
if os.path.exists(DATASET_CSV_DIR) and os.listdir(DATASET_CSV_DIR):
if all(f.startswith('.') or f.lower() == 'readme.md' for f in os.listdir(DATASET_CSV_DIR)):
for item_name in os.listdir(TEST_DATA_PERSON_DIR):
source_item_path = os.path.join(TEST_DATA_PERSON_DIR, item_name)
if os.path.isfile(source_item_path) and item_name.lower().endswith('.csv'):
destination_item_path = os.path.join(DATASET_CSV_DIR, item_name)
shutil.copy2(source_item_path, destination_item_path)
# 复制测试数据到test_person目录
for item_name in os.listdir(TEST_DATA_PERSON_DIR):
source_item_path = os.path.join(TEST_DATA_PERSON_DIR, item_name)
if os.path.isfile(source_item_path) and item_name.lower().endswith('.csv'):
destination_item_path = os.path.join(test_person_csv_dir, item_name)
shutil.copy2(source_item_path, destination_item_path)
def run_cli_command(command: list[str], timeout: int | None = None, background: bool = False) -> Union[subprocess.CompletedProcess, subprocess.Popen]:
"""Execute a CLI command and return the result.
tests/test_full_pipeV2.py
+17 -8
View File
@@ -43,15 +43,21 @@ def setup_make_dataset_test_data():
TESTS_DIR = os.path.dirname(__file__)
TEST_DATA_PERSON_DIR = os.path.join(TESTS_DIR, "tests_data", "test_person")
os.makedirs(DATASET_CSV_DIR, exist_ok=True)
# 先删除目录，再重新创建
if os.path.exists(DATASET_CSV_DIR):
shutil.rmtree(DATASET_CSV_DIR)
os.makedirs(DATASET_CSV_DIR)
# 创建test_person子目录
test_person_csv_dir = os.path.join(DATASET_CSV_DIR, "test_person")
os.makedirs(test_person_csv_dir)
if os.path.exists(DATASET_CSV_DIR) and os.listdir(DATASET_CSV_DIR):
if all(f.startswith('.') or f.lower() == 'readme.md' for f in os.listdir(DATASET_CSV_DIR)):
for item_name in os.listdir(TEST_DATA_PERSON_DIR):
source_item_path = os.path.join(TEST_DATA_PERSON_DIR, item_name)
if os.path.isfile(source_item_path) and item_name.lower().endswith('.csv'):
destination_item_path = os.path.join(DATASET_CSV_DIR, item_name)
shutil.copy2(source_item_path, destination_item_path)
# 复制测试数据到test_person目录
for item_name in os.listdir(TEST_DATA_PERSON_DIR):
source_item_path = os.path.join(TEST_DATA_PERSON_DIR, item_name)
if os.path.isfile(source_item_path) and item_name.lower().endswith('.csv'):
destination_item_path = os.path.join(test_person_csv_dir, item_name)
shutil.copy2(source_item_path, destination_item_path)
def run_cli_command(command: list[str], timeout: int | None = None, background: bool = False) -> Union[subprocess.CompletedProcess, subprocess.Popen]:
@@ -166,3 +172,6 @@ def test_cli_test_model():
if server_process.poll() is None:
server_process.kill() # Force kill if the process hasn't terminated
test_logger.info("服务器已关闭")
if __name__ == "__main__":
setup_make_dataset_test_data()
tests/tests_data/test_model_data.json
+31
View File
@@ -0,0 +1,31 @@
{
"questions": [
[
"吃了吗？",
"吃的什么啊",
"好吃吗",
"多少钱啊",
"可以请我吃吗"
],
[
"干嘛呢？",
"等会准备干什么去"
],
[
"最近有什么新鲜事发生吗？",
"有没有什么有趣的故事可以分享？"
],
[
"周末过得怎么样？",
"做了什么好玩的？"
],
[
"今天天气怎么样？",
"你那里呢？"
],
[
"最近工作/学习顺利吗？",
"有没有遇到什么挑战？"
]
]
}
weclone/data/clean/strategies.py
+4 -4
View File
@@ -10,7 +10,7 @@ from langchain_core.prompts import PromptTemplate
from tqdm import tqdm
from weclone.core.inference.online_infer import OnlineLLM
from weclone.data.models import QaPairScore, QaPairV2
from weclone.data.models import QaPair, QaPairScore
from weclone.prompts.clean_data import CLEAN_PROMPT, ONLINE_LLM_CLEAN_PROMPT
from weclone.utils.config_models import WCMakeDatasetConfig
from weclone.utils.log import logger
@@ -23,7 +23,7 @@ class CleaningStrategy(ABC):
make_dataset_config: WCMakeDatasetConfig
@abstractmethod
def judge(self, data: List[QaPairV2]) -> None:
def judge(self, data: List[QaPair]) -> None:
"""
打分方法是抽象的，强制每个子类根据自己的方式去实现。
"""
@@ -93,7 +93,7 @@ class LLMCleaningStrategy(CleaningStrategy):
make_dataset_config: WCMakeDatasetConfig
def judge(self, data: List[QaPairV2]) -> None:
def judge(self, data: List[QaPair]) -> None:
"""
调用llm打分，并将分数直接赋值给传入的QaPair。
"""
@@ -159,7 +159,7 @@ class LLMCleaningStrategy(CleaningStrategy):
class OlineLLMCleaningStrategy(CleaningStrategy):
"""使用大模型进行数据清洗的策略"""
def judge(self, data: List[QaPairV2]) -> None:
def judge(self, data: List[QaPair]) -> None:
config = self.make_dataset_config
logger.info("开始使用在线模型对数据打分")
logger.info(f"使用模型 {config.model_name}")
weclone/data/models.py
+1 -14
View File
@@ -32,19 +32,6 @@ class QaPairFormat(Enum):
SHAREGPT = "sharegpt"
@dataclass
class QaPair:
"""原始QaPair类，保持向后兼容"""
id: int
system: str
instruction: str
output: str
history: list[list[str]]
time: Timestamp
score: int
@dataclass
class Message:
role: str
@@ -52,7 +39,7 @@ class Message:
@dataclass
class QaPairV2:
class QaPair:
"""支持sharegpt格式的QA对类"""
id: int
weclone/data/qa_generator.py
+15 -15
View File
@@ -16,7 +16,7 @@ from weclone.data.models import (
ChatMessage,
CutMessage,
Message,
QaPairV2,
QaPair,
cut_type_list,
skip_type_list,
)
@@ -35,7 +35,7 @@ class DataProcessor:
self.enable_clean = self.config.clean_dataset.enable_clean
# msg_type
self.QaPair = QaPairV2
self.QaPair = QaPair
self.include_type = self.config.include_type
if self.config.platform == PlatformType.WECHAT:
@@ -103,9 +103,9 @@ class DataProcessor:
vision_config = self.config.vision_api
if vision_config.enable and vision_config.api_key:
self.image_processor = ImageToTextProcessor(
api_url=vision_config.api_url,
api_key=vision_config.api_key,
model_name=vision_config.model_name,
api_url=vision_config.api_url, # type: ignore
api_key=vision_config.api_key, # type: ignore
model_name=vision_config.model_name, # type: ignore
)
logger.info(f"已启用图片识别功能, 模型: {self.image_processor.model_name}")
else:
@@ -113,7 +113,7 @@ class DataProcessor:
self.c = self.config
def _process_images_in_parallel(self, qa_list: List[QaPairV2]) -> List[QaPairV2]:
def _process_images_in_parallel(self, qa_list: List[QaPair]) -> List[QaPair]:
"""并行处理所有对话中的图片，并将描述替换回对话文本。"""
all_image_paths = []
media_dir = self.c.media_dir
@@ -136,7 +136,7 @@ class DataProcessor:
# 使用线程池并行调用API，executor.map 会保持结果顺序与输入一致
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
# 现在传递给 image_processor 的是完整的路径
image_descriptions = list(executor.map(self.image_processor.describe_image, all_image_paths))
image_descriptions = list(executor.map(self.image_processor.describe_image, all_image_paths)) # type: ignore
desc_iterator = iter(image_descriptions)
for qa_pair in qa_list:
@@ -167,7 +167,7 @@ class DataProcessor:
logger.error(
f"错误：目录 '{self.csv_folder}' 不存在或为空，请检查路径并确保其中包含 CSV 聊天数据文件。"
)
return
sys.exit(1)
csv_files = self.get_csv_files()
logger.info(f"共发现 {len(csv_files)} 个 CSV 文件,开始处理,请耐心等待...")
@@ -179,7 +179,7 @@ class DataProcessor:
# self.process_by_msgtype(chat_message)
logger.debug(f"处理完成: {csv_file}，共加载 {len(chat_messages)} 条消息")
qa_res = self.match_qa(message_list)
qa_res = [item for item in qa_res if isinstance(item, QaPairV2)]
qa_res = [item for item in qa_res if isinstance(item, QaPair)]
# 如果启用图片识别，则执行并行处理
if self.image_processor:
@@ -263,7 +263,7 @@ class DataProcessor:
csv_files.sort(key=extract_start)
return csv_files
def match_qa(self, messages: List[ChatMessage]) -> List[Union[QaPairV2, CutMessage]]:
def match_qa(self, messages: List[ChatMessage]) -> List[Union[QaPair, CutMessage]]:
"""
匹配问答对，直接处理历史对话
@@ -271,14 +271,14 @@ class DataProcessor:
messages: 消息列表
Returns:
List[Union[QaPairV2, CutMessage]]: 包含指令和输出的问答对列表
List[Union[QaPair, CutMessage]]: 包含指令和输出的问答对列表
"""
# 状态定义
WAITING_INSTRUCTION = "waiting_instruction" # 等待指令
WAITING_RESPONSE = "waiting_response" # 等待回复
current_state = WAITING_INSTRUCTION
qa_res: List[Union[QaPairV2, CutMessage]] = []
qa_res: List[Union[QaPair, CutMessage]] = []
last_message = None
current_instruction = None
qa_id_counter = 0
@@ -616,12 +616,12 @@ class DataProcessor:
def process_text(self, chat_message: ChatMessage):
pass
def save_result(self, qa_res: List[QaPairV2]):
def save_result(self, qa_res: List[QaPair]):
"""
Saves the list of QaPairV2 objects to a JSON file after converting them to dictionaries.
Saves the list of QaPair objects to a JSON file after converting them to dictionaries.
Args:
qa_res: A list of QaPairV2 objects.
qa_res: A list of QaPair objects.
"""
processed_qa_res = []
for idx, item in enumerate(qa_res):
weclone/eval/test_model.py
+8 -7
View File
@@ -7,17 +7,18 @@ from openai.types.chat import ChatCompletionMessageParam # 导入消息参数
from tqdm import tqdm
from weclone.utils.config import load_config
from weclone.utils.config_models import WCInferConfig
from weclone.utils.config_models import TestModelArgs, WCInferConfig
config = cast(WCInferConfig, load_config("web_demo"))
infer_config = cast(WCInferConfig, load_config("web_demo"))
test_config = cast(TestModelArgs, load_config("test_model"))
config = {
"default_prompt": config.default_system,
completion_config = {
"default_prompt": infer_config.default_system,
"model": "gpt-3.5-turbo",
"history_len": 15,
}
config = type("Config", (object,), config)()
completion_config = type("Config", (object,), completion_config)()
# 初始化 OpenAI 客户端
client = OpenAI(api_key="""sk-test""", base_url="http://127.0.0.1:8005/v1")
@@ -49,12 +50,12 @@ def handler_text(content: str, history: list, config):
def main():
test_list = json.loads(open("dataset/test_data.json", "r", encoding="utf-8").read())["questions"]
test_list = json.loads(open(test_config.test_data_path, "r", encoding="utf-8").read())["questions"]
res = []
for questions in tqdm(test_list, desc=" Testing..."):
history = []
for q in questions:
handler_text(q, history=history, config=config)
handler_text(q, history=history, config=completion_config)
res.append(history)
res_file = open("test_result-my.txt", "w")
weclone/utils/config.py
+3
View File
@@ -58,6 +58,9 @@ def create_config_by_arg_type(arg_type: str, wc_config: WcConfig) -> BaseModel:
config_dict = {**common_config, **wc_config.infer_args.model_dump()}
return WCInferConfig(**config_dict)
elif arg_type == "test_model":
return wc_config.test_model_args
elif arg_type == "train_sft":
config_dict = {**common_config, **wc_config.train_sft_args.model_dump()}
return WCTrainSftConfig(**config_dict)
weclone/utils/config_models.py
+9 -4
View File
@@ -58,8 +58,8 @@ class FinetuningType(StrEnum):
"""Finetuning type"""
LORA = "lora"
FULL = "full"
FREEZE = "freeze"
# FULL = "full"
# FREEZE = "freeze"
class CommonArgs(BaseModel):
@@ -96,7 +96,7 @@ class CleanDatasetConfig(BaseModel):
class VisionApiConfig(BaseModel):
"""Vision API specific configuration"""
enable: bool = Field(False, description="是否启用Vision API进行图像识别")
enable: bool = Field(default=False, description="是否启用Vision API进行图像识别")
api_key: Optional[str] = None
api_url: Optional[str] = None
model_name: Optional[str] = None
@@ -123,7 +123,7 @@ class MakeDatasetArgs(BaseModel):
llm_api_key: Optional[str] = Field(None, description="在线LLM的api_key")
model_name: Optional[str] = Field(None, description="在线LLM的模型名称, 建议使用参数较大的模型")
clean_batch_size: int = Field(10, description="数据清洗批次大小")
vision_api: VisionApiConfig = Field(default_factory=VisionApiConfig)
vision_api: VisionApiConfig = Field(VisionApiConfig())
class TrainSftArgs(BaseModel):
@@ -164,6 +164,10 @@ class InferArgs(BaseModel):
max_length: int = Field(..., description="最大生成长度")
class TestModelArgs(BaseModel):
test_data_path: str = Field("dataset/test_data.json", description="测试数据路径")
class WcConfig(BaseModel):
version: str = Field(..., description="配置文件版本")
common_args: CommonArgs = Field(..., description="通用参数")
@@ -171,6 +175,7 @@ class WcConfig(BaseModel):
make_dataset_args: MakeDatasetArgs = Field(..., description="数据处理参数")
train_sft_args: TrainSftArgs = Field(..., description="SFT微调参数")
infer_args: InferArgs = Field(..., description="推理参数")
test_model_args: TestModelArgs = TestModelArgs(test_data_path="dataset/test_data.json")
class WCInferConfig(CommonArgs, InferArgs):