diff --git a/dataset/res_csv/sft/dataset_info.json b/dataset/res_csv/sft/dataset_info.json index f1133df..7470a08 100644 --- a/dataset/res_csv/sft/dataset_info.json +++ b/dataset/res_csv/sft/dataset_info.json @@ -1,22 +1,5 @@ { - "wechat-sft": { - "file_name": "sft-my.json", - "columns": { - "prompt": "instruction", - "response": "output", - "system": "system" - } - }, - "wechat-sft-with-history": { - "file_name": "sft-my.json", - "columns": { - "prompt": "instruction", - "response": "output", - "system": "system", - "history": "history" - } - }, - "wechat-mllm-sft": { + "chat-sft": { "file_name": "./sft-my.json", "formatting": "sharegpt", "columns": { diff --git a/examples/mllm.template.jsonc b/examples/mllm.template.jsonc index bc5efb3..35e56a4 100644 --- a/examples/mllm.template.jsonc +++ b/examples/mllm.template.jsonc @@ -1,5 +1,5 @@ { - "version": "0.2.22", + "version": "0.2.23", "common_args": { "model_name_or_path": "./models/Qwen2.5-VL-7B-Instruct", "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir @@ -51,7 +51,7 @@ "train_sft_args": { //微调配置 "stage": "sft", - "dataset": "wechat-mllm-sft", + "dataset": "chat-sft", "dataset_dir": "./dataset/res_csv/sft", "freeze_multi_modal_projector": false, //MLLM 训练时是否冻结多模态投影器。 "use_fast_tokenizer": true, diff --git a/settings.template.jsonc b/settings.template.jsonc index 84393e7..9347e37 100644 --- a/settings.template.jsonc +++ b/settings.template.jsonc @@ -1,5 +1,5 @@ { - "version": "0.2.22", + "version": "0.2.23", "common_args": { "model_name_or_path": "./Qwen2.5-7B-Instruct", "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir @@ -44,7 +44,7 @@ "train_sft_args": { //微调配置 "stage": "sft", - "dataset": "wechat-sft", + "dataset": "chat-sft", "dataset_dir": "./dataset/res_csv/sft", "use_fast_tokenizer": true, "lora_target": "q_proj,v_proj", diff --git a/weclone/cli.py b/weclone/cli.py index 0d80e7f..64035a0 100644 --- a/weclone/cli.py +++ b/weclone/cli.py @@ -2,14 +2,16 @@ import functools import os import sys from pathlib import Path +from typing import cast import click import commentjson -from weclone.utils.config import load_config +from weclone.utils.config_models import CliArgs +from weclone.utils.configV2 import load_config from weclone.utils.log import capture_output, logger -cli_config: dict | None = None +cli_config: CliArgs | None = None try: import tomllib # type: ignore Python 3.11+ @@ -43,7 +45,7 @@ def apply_common_decorators(capture_output_enabled=False): def decorator(original_cmd_func): @functools.wraps(original_cmd_func) def new_runtime_wrapper(*args, **kwargs): - if cli_config and cli_config.get("full_log", False): + if cli_config and cli_config.full_log: return capture_output(original_cmd_func)(*args, **kwargs) else: return original_cmd_func(*args, **kwargs) @@ -61,23 +63,14 @@ def cli(): _check_project_root() _check_versions() global cli_config - cli_config = load_config(arg_type="cli_args") + cli_config = cast(CliArgs, load_config(arg_type="cli_args")) @cli.command("make-dataset", help="处理聊天记录CSV文件,生成问答对数据集。") @apply_common_decorators() def qa_generator(): """处理聊天记录CSV文件,生成问答对数据集。""" - config = load_config(arg_type="make_dataset") - - if "image" in config.get("include_type", []): - from weclone.data.qa_generatorV2 import DataProcessor - - logger.info("检测到配置包含image类型,使用qa_generatorV2") - else: - from weclone.data.qa_generator import DataProcessor - - logger.info("使用标准qa_generator") + from weclone.data.qa_generatorV2 import DataProcessor processor = DataProcessor() processor.main() diff --git a/weclone/utils/config_models.py b/weclone/utils/config_models.py index 77e64d8..ad79d9a 100644 --- a/weclone/utils/config_models.py +++ b/weclone/utils/config_models.py @@ -180,14 +180,12 @@ class WCTrainSftConfig(CommonArgs, TrainSftArgs): @model_validator(mode="after") def process_config(self): - """当包含图像模态时,自动设置为多模态数据集,同时处理adapter_name_or_path转换""" # 保存需要的值 - include_type_value = getattr(self, "include_type", []) adapter_name_value = getattr(self, "adapter_name_or_path", None) # 进行业务逻辑处理 - if DataModality.IMAGE in include_type_value: - self.dataset = "wechat-mllm-sft" + if self.dataset == "wechat-sft": + self.dataset = "chat-sft" if adapter_name_value: self.output_dir = adapter_name_value @@ -211,7 +209,6 @@ class WCMakeDatasetConfig(CommonArgs, MakeDatasetArgs): @model_validator(mode="after") def process_config(self): - """当包含图像模态时,自动设置为多模态数据集""" - if DataModality.IMAGE in self.include_type: - self.dataset = "wechat-mllm-sft" + if self.dataset == "wechat-sft": + self.dataset = "chat-sft" return self diff --git a/weclone/utils/length_cdf.py b/weclone/utils/length_cdf.py index 5032176..1c06e57 100644 --- a/weclone/utils/length_cdf.py +++ b/weclone/utils/length_cdf.py @@ -25,7 +25,7 @@ from weclone.utils.log import logger def length_cdf( model_name_or_path: str = "./Qwen2.5-7B-Instruct", - dataset: str = "wechat-sft", + dataset: str = "chat-sft", dataset_dir: str = "./dataset/res_csv/sft", media_dir: str = "./dataset/media", template: str = "qwen",