添加计算cutoff_len

.gitignore

@@ -151,7 +151,7 @@ data/test
 .vscode
 *-my.*
 *.csv
-test.*
+*test.*
 *users.json   
 Spark-TTS-0.5B/
 uv.lock

settings.json

@@ -31,7 +31,7 @@
         "per_device_train_batch_size": 8,
         "gradient_accumulation_steps": 4,
         "lr_scheduler_type": "cosine",
-        "cutoff_len": 512, //后续自动计算
+        "cutoff_len": 256, 
         "logging_steps": 10,
         "save_steps": 100,
         "learning_rate": 1e-4,

weclone/data/qa_generator.py

@@ -9,6 +9,7 @@ from weclone.utils.config import load_config
 from weclone.utils.log import logger
 from weclone.data.models import ChatMessage, CutMessage, skip_type_list
 from weclone.data.strategies import TimeWindowStrategy, LLMStrategy
+from weclone.utils.length_cdf import length_cdf
 
 
 class DataProcessor:
@@ -67,6 +68,13 @@ class DataProcessor:
         if self.c["prompt_with_history"]:
             qa_res = self.add_history_to_qa(qa_res)
         self.save_result(qa_res)
+        length_cdf(
+            model_name_or_path=self.c["model_name_or_path"],
+            dataset=self.c["dataset"],
+            dataset_dir=self.c["dataset_dir"],
+            template=self.c["template"],
+            interval=self.c["cutoff_len"],
+        )
 
     def get_csv_files(self):
         """遍历文件夹获取所有CSV文件路径"""

weclone/train/train_sft.py

@@ -5,7 +5,6 @@ from llamafactory.extras.misc import get_current_device
 from weclone.utils.config import load_config
 from weclone.utils.log import logger
 
-# todo 添加一个test的环境变量 放一个test的配置文件
 
 def main():
     config = load_config(arg_type="train_sft")

weclone/utils/config.py

@@ -7,8 +7,18 @@ from .tools import dict_to_argv
 
 
 def load_config(arg_type: str):
-    with open("./settings.json", "r", encoding="utf-8") as f:
-        s_config: dict = commentjson.load(f)
+    config_path = os.environ.get("WECLONE_CONFIG_PATH", "./settings.json")
+    logger.info(f"Loading configuration from: {config_path}")  # Add logging to see which file is loaded
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            s_config: dict = commentjson.load(f)
+    except FileNotFoundError:
+        logger.error(f"Configuration file not found: {config_path}")
+        sys.exit(1)  # Exit if config file is not found
+    except Exception as e:
+        logger.error(f"Error loading configuration file {config_path}: {e}")
+        sys.exit(1)
+
     if arg_type == "web_demo" or arg_type == "api_service":
         # infer_args和common_args求并集
         config = {**s_config["infer_args"], **s_config["common_args"]}
@@ -25,6 +35,9 @@ def load_config(arg_type: str):
 
     elif arg_type == "make_dataset":
         config = {**s_config["make_dataset_args"], **s_config["common_args"]}
+        config["dataset"] = s_config["train_sft_args"]["dataset"]
+        config["dataset_dir"] = s_config["train_sft_args"]["dataset_dir"]
+        config["cutoff_len"] = s_config["train_sft_args"]["cutoff_len"]
     else:
         raise ValueError("暂不支持的参数类型")
 

weclone/utils/length_cdf.py

@@ -0,0 +1,66 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+
+from tqdm import tqdm
+from weclone.utils.log import logger
+
+from llamafactory.data import get_dataset, get_template_and_fix_tokenizer
+from llamafactory.hparams import get_train_args
+from llamafactory.model import load_tokenizer
+
+
+def length_cdf(
+    model_name_or_path: str,
+    dataset: str = "alpaca_en_demo",
+    dataset_dir: str = "data",
+    template: str = "default",
+    interval: int = 1000,
+):
+    r"""Calculate the distribution of the input lengths in the dataset.
+
+    Usage: export CUDA_VISIBLE_DEVICES=0
+    python length_cdf.py --model_name_or_path path_to_model --dataset alpaca_en_demo --template default
+    """
+    model_args, data_args, training_args, _, _ = get_train_args(
+        dict(
+            stage="sft",
+            model_name_or_path=model_name_or_path,
+            dataset=dataset,
+            dataset_dir=dataset_dir,
+            template=template,
+            cutoff_len=1_000_000,
+            preprocessing_num_workers=16,
+            output_dir="dummy_dir",
+            overwrite_cache=True,
+            do_train=True,
+        )
+    )
+    tokenizer_module = load_tokenizer(model_args)
+    template = get_template_and_fix_tokenizer(tokenizer_module["tokenizer"], data_args) # type: ignore
+    trainset = get_dataset(template, model_args, data_args, training_args, "sft", **tokenizer_module)["train_dataset"] # type: ignore
+    total_num = len(trainset) # type: ignore
+    length_dict = defaultdict(int)
+    for sample in tqdm(trainset["input_ids"], desc="Collecting lengths"): # type: ignore
+        length_dict[len(sample) // interval * interval] += 1
+
+    length_tuples = list(length_dict.items())
+    length_tuples.sort()
+    count_accu, prob_accu = 0, 0
+    logger.info(" cutoff_len设置建议：")
+    for length, count in length_tuples:
+        count_accu += count
+        prob_accu += count / total_num * 100
+        logger.info(f"{count_accu:d} ({prob_accu:.2f}%) samples have length < {length + interval}.")