diff --git a/.gitignore b/.gitignore index 037a1e1..fcb3f22 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,7 @@ data/test .vscode *-my.* *.csv -test.* +*test.* *users.json Spark-TTS-0.5B/ uv.lock diff --git a/settings.json b/settings.json index fc452cf..a7462b6 100644 --- a/settings.json +++ b/settings.json @@ -31,7 +31,7 @@ "per_device_train_batch_size": 8, "gradient_accumulation_steps": 4, "lr_scheduler_type": "cosine", - "cutoff_len": 512, //后续自动计算 + "cutoff_len": 256, "logging_steps": 10, "save_steps": 100, "learning_rate": 1e-4, diff --git a/weclone/data/qa_generator.py b/weclone/data/qa_generator.py index fd8dfe1..ea5c9c8 100644 --- a/weclone/data/qa_generator.py +++ b/weclone/data/qa_generator.py @@ -9,6 +9,7 @@ from weclone.utils.config import load_config from weclone.utils.log import logger from weclone.data.models import ChatMessage, CutMessage, skip_type_list from weclone.data.strategies import TimeWindowStrategy, LLMStrategy +from weclone.utils.length_cdf import length_cdf class DataProcessor: @@ -67,6 +68,13 @@ class DataProcessor: if self.c["prompt_with_history"]: qa_res = self.add_history_to_qa(qa_res) self.save_result(qa_res) + length_cdf( + model_name_or_path=self.c["model_name_or_path"], + dataset=self.c["dataset"], + dataset_dir=self.c["dataset_dir"], + template=self.c["template"], + interval=self.c["cutoff_len"], + ) def get_csv_files(self): """遍历文件夹获取所有CSV文件路径""" diff --git a/weclone/train/train_sft.py b/weclone/train/train_sft.py index a4a308c..28ffcc7 100644 --- a/weclone/train/train_sft.py +++ b/weclone/train/train_sft.py @@ -5,7 +5,6 @@ from llamafactory.extras.misc import get_current_device from weclone.utils.config import load_config from weclone.utils.log import logger -# todo 添加一个test的环境变量 放一个test的配置文件 def main(): config = load_config(arg_type="train_sft") diff --git a/weclone/utils/config.py b/weclone/utils/config.py index 1d9b4ef..a718bab 100644 --- a/weclone/utils/config.py +++ b/weclone/utils/config.py @@ -7,8 +7,18 @@ from .tools import dict_to_argv def load_config(arg_type: str): - with open("./settings.json", "r", encoding="utf-8") as f: - s_config: dict = commentjson.load(f) + config_path = os.environ.get("WECLONE_CONFIG_PATH", "./settings.json") + logger.info(f"Loading configuration from: {config_path}") # Add logging to see which file is loaded + try: + with open(config_path, "r", encoding="utf-8") as f: + s_config: dict = commentjson.load(f) + except FileNotFoundError: + logger.error(f"Configuration file not found: {config_path}") + sys.exit(1) # Exit if config file is not found + except Exception as e: + logger.error(f"Error loading configuration file {config_path}: {e}") + sys.exit(1) + if arg_type == "web_demo" or arg_type == "api_service": # infer_args和common_args求并集 config = {**s_config["infer_args"], **s_config["common_args"]} @@ -25,6 +35,9 @@ def load_config(arg_type: str): elif arg_type == "make_dataset": config = {**s_config["make_dataset_args"], **s_config["common_args"]} + config["dataset"] = s_config["train_sft_args"]["dataset"] + config["dataset_dir"] = s_config["train_sft_args"]["dataset_dir"] + config["cutoff_len"] = s_config["train_sft_args"]["cutoff_len"] else: raise ValueError("暂不支持的参数类型") diff --git a/weclone/utils/length_cdf.py b/weclone/utils/length_cdf.py new file mode 100644 index 0000000..986578a --- /dev/null +++ b/weclone/utils/length_cdf.py @@ -0,0 +1,66 @@ +# Copyright 2025 the LlamaFactory team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict + +from tqdm import tqdm +from weclone.utils.log import logger + +from llamafactory.data import get_dataset, get_template_and_fix_tokenizer +from llamafactory.hparams import get_train_args +from llamafactory.model import load_tokenizer + + +def length_cdf( + model_name_or_path: str, + dataset: str = "alpaca_en_demo", + dataset_dir: str = "data", + template: str = "default", + interval: int = 1000, +): + r"""Calculate the distribution of the input lengths in the dataset. + + Usage: export CUDA_VISIBLE_DEVICES=0 + python length_cdf.py --model_name_or_path path_to_model --dataset alpaca_en_demo --template default + """ + model_args, data_args, training_args, _, _ = get_train_args( + dict( + stage="sft", + model_name_or_path=model_name_or_path, + dataset=dataset, + dataset_dir=dataset_dir, + template=template, + cutoff_len=1_000_000, + preprocessing_num_workers=16, + output_dir="dummy_dir", + overwrite_cache=True, + do_train=True, + ) + ) + tokenizer_module = load_tokenizer(model_args) + template = get_template_and_fix_tokenizer(tokenizer_module["tokenizer"], data_args) # type: ignore + trainset = get_dataset(template, model_args, data_args, training_args, "sft", **tokenizer_module)["train_dataset"] # type: ignore + total_num = len(trainset) # type: ignore + length_dict = defaultdict(int) + for sample in tqdm(trainset["input_ids"], desc="Collecting lengths"): # type: ignore + length_dict[len(sample) // interval * interval] += 1 + + length_tuples = list(length_dict.items()) + length_tuples.sort() + count_accu, prob_accu = 0, 0 + logger.info(" cutoff_len设置建议：") + for length, count in length_tuples: + count_accu += count + prob_accu += count / total_num * 100 + logger.info(f"{count_accu:d} ({prob_accu:.2f}%) samples have length < {length + interval}.")