修改数据集名称为chat-sft， 纯文字模型微调数据切换到sharegpt格式。

dataset/res_csv/sft/dataset_info.json

@@ -1,22 +1,5 @@
 {
-    "wechat-sft": {
-        "file_name": "sft-my.json",
-        "columns": {
-            "prompt": "instruction",
-            "response": "output",
-            "system": "system"
-        }
-    },
-    "wechat-sft-with-history": {
-        "file_name": "sft-my.json",
-        "columns": {
-            "prompt": "instruction",
-            "response": "output",
-            "system": "system",
-            "history": "history"
-        }
-    },
-    "wechat-mllm-sft": {
+    "chat-sft": {
         "file_name": "./sft-my.json",
         "formatting": "sharegpt",
         "columns": {

examples/mllm.template.jsonc

@@ -1,5 +1,5 @@
 {
-    "version": "0.2.22",
+    "version": "0.2.23",
     "common_args": {
         "model_name_or_path": "./models/Qwen2.5-VL-7B-Instruct",
         "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir
@@ -51,7 +51,7 @@
     "train_sft_args": {
         //微调配置
         "stage": "sft",
-        "dataset": "wechat-mllm-sft",
+        "dataset": "chat-sft",
         "dataset_dir": "./dataset/res_csv/sft",
         "freeze_multi_modal_projector": false, //MLLM 训练时是否冻结多模态投影器。
         "use_fast_tokenizer": true,

settings.template.jsonc

@@ -1,5 +1,5 @@
 {
-    "version": "0.2.22",
+    "version": "0.2.23",
     "common_args": {
         "model_name_or_path": "./Qwen2.5-7B-Instruct",
         "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir
@@ -44,7 +44,7 @@
     "train_sft_args": {
         //微调配置
         "stage": "sft",
-        "dataset": "wechat-sft",
+        "dataset": "chat-sft",
         "dataset_dir": "./dataset/res_csv/sft",
         "use_fast_tokenizer": true,
         "lora_target": "q_proj,v_proj",

weclone/cli.py

@@ -2,14 +2,16 @@ import functools
 import os
 import sys
 from pathlib import Path
+from typing import cast
 
 import click
 import commentjson
 
-from weclone.utils.config import load_config
+from weclone.utils.config_models import CliArgs
+from weclone.utils.configV2 import load_config
 from weclone.utils.log import capture_output, logger
 
-cli_config: dict | None = None
+cli_config: CliArgs | None = None
 
 try:
     import tomllib  # type: ignore Python 3.11+
@@ -43,7 +45,7 @@ def apply_common_decorators(capture_output_enabled=False):
     def decorator(original_cmd_func):
         @functools.wraps(original_cmd_func)
         def new_runtime_wrapper(*args, **kwargs):
-            if cli_config and cli_config.get("full_log", False):
+            if cli_config and cli_config.full_log:
                 return capture_output(original_cmd_func)(*args, **kwargs)
             else:
                 return original_cmd_func(*args, **kwargs)
@@ -61,23 +63,14 @@ def cli():
     _check_project_root()
     _check_versions()
     global cli_config
-    cli_config = load_config(arg_type="cli_args")
+    cli_config = cast(CliArgs, load_config(arg_type="cli_args"))
 
 
 @cli.command("make-dataset", help="处理聊天记录CSV文件，生成问答对数据集。")
 @apply_common_decorators()
 def qa_generator():
     """处理聊天记录CSV文件，生成问答对数据集。"""
-    config = load_config(arg_type="make_dataset")
-
-    if "image" in config.get("include_type", []):
-        from weclone.data.qa_generatorV2 import DataProcessor
-
-        logger.info("检测到配置包含image类型，使用qa_generatorV2")
-    else:
-        from weclone.data.qa_generator import DataProcessor
-
-        logger.info("使用标准qa_generator")
+    from weclone.data.qa_generatorV2 import DataProcessor
 
     processor = DataProcessor()
     processor.main()

weclone/utils/config_models.py

@@ -180,14 +180,12 @@ class WCTrainSftConfig(CommonArgs, TrainSftArgs):
 
     @model_validator(mode="after")
     def process_config(self):
-        """当包含图像模态时，自动设置为多模态数据集，同时处理adapter_name_or_path转换"""
         # 保存需要的值
-        include_type_value = getattr(self, "include_type", [])
         adapter_name_value = getattr(self, "adapter_name_or_path", None)
 
         # 进行业务逻辑处理
-        if DataModality.IMAGE in include_type_value:
-            self.dataset = "wechat-mllm-sft"
+        if self.dataset == "wechat-sft":
+            self.dataset = "chat-sft"
         if adapter_name_value:
             self.output_dir = adapter_name_value
 
@@ -211,7 +209,6 @@ class WCMakeDatasetConfig(CommonArgs, MakeDatasetArgs):
 
     @model_validator(mode="after")
     def process_config(self):
-        """当包含图像模态时，自动设置为多模态数据集"""
-        if DataModality.IMAGE in self.include_type:
-            self.dataset = "wechat-mllm-sft"
+        if self.dataset == "wechat-sft":
+            self.dataset = "chat-sft"
         return self

weclone/utils/length_cdf.py

@@ -25,7 +25,7 @@ from weclone.utils.log import logger
 
 def length_cdf(
     model_name_or_path: str = "./Qwen2.5-7B-Instruct",
-    dataset: str = "wechat-sft",
+    dataset: str = "chat-sft",
     dataset_dir: str = "./dataset/res_csv/sft",
     media_dir: str = "./dataset/media",
     template: str = "qwen",