mirror of
https://github.com/xming521/WeClone.git
synced 2026-08-28 18:07:28 +08:00
xming521
7e9886c020
Reduces the default LoRA rank in training configuration templates from 16 to 8. This change aims to optimize resource usage and potentially accelerate training.
100 lines
3.5 KiB
JSON
100 lines
3.5 KiB
JSON
{
|
|
"version": "0.3.01",
|
|
"common_args": {
|
|
"model_name_or_path": "./models/Qwen2.5-7B-Instruct",
|
|
"adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir
|
|
"template": "qwen",
|
|
"default_system": "请你扮演一名人类，不要说自己是人工智能",
|
|
"media_dir": "dataset/media",
|
|
"finetuning_type": "lora",
|
|
"enable_thinking": false,
|
|
"trust_remote_code": true
|
|
},
|
|
"cli_args": {
|
|
"full_log": false,
|
|
"log_level": "INFO"
|
|
},
|
|
"make_dataset_args": {
|
|
//数据处理配置
|
|
"platform": "chat", //chat,telegram
|
|
"language": "zh", // 聊天常用语言: zh(中文) 或 en(英文)
|
|
"telegram_args": {
|
|
"my_id": "user1234567890"
|
|
},
|
|
"include_type": [
|
|
"text"
|
|
],
|
|
"blocked_words": [ // 禁用词
|
|
"例如 姓名",
|
|
"例如 密码",
|
|
"//....."
|
|
],
|
|
"add_time": false,
|
|
"single_combine_strategy": "time_window", // 单人组成单句策略
|
|
"qa_match_strategy": "time_window", // 组成qa策略
|
|
"single_combine_time_window": 2, // 单人组成单句时间窗口（分钟）,
|
|
"qa_match_time_window": 5, // 组成qa时间窗口（分钟）,
|
|
"combine_msg_max_length": 2048, // 组合后消息最大长度 配合cutoff_len 使用
|
|
"messages_max_length": 2048, // messages最长字符数量 配合cutoff_len 使用
|
|
"clean_dataset": {
|
|
"enable_clean": false,
|
|
"clean_strategy": "llm",
|
|
"llm": {
|
|
"accept_score": 2, //可以接受的llm打分阈值,1分最差，5分最好,低于此分数的数据不会用于训练
|
|
}
|
|
},
|
|
"online_llm_clear": false,
|
|
"base_url": "https://xxx/v1",
|
|
"llm_api_key": "xxxxx",
|
|
"model_name": "xxx", //建议使用参数较大的模型，例如DeepSeek-V3
|
|
"clean_batch_size": 10,
|
|
"vision_api": {
|
|
"enable": false, // 设置为 true 来开启此功能
|
|
"api_key": "xxx",
|
|
"api_url": "https://xxx/v1", // 兼容OpenAI的API地址
|
|
"model_name": "xxx", // 要使用的多模态模型名称,例如qwen-vl-max
|
|
"max_workers": 5 // 并行调用API的线程数，最多不要超过8
|
|
}
|
|
},
|
|
"train_sft_args": {
|
|
//微调配置
|
|
"stage": "sft",
|
|
"dataset": "chat-sft",
|
|
"dataset_dir": "./dataset/res_csv/sft",
|
|
"use_fast_tokenizer": true,
|
|
"lora_target": "q_proj,v_proj",
|
|
"lora_rank": 8,
|
|
"lora_dropout": 0.25,
|
|
"weight_decay": 0.1,
|
|
"overwrite_cache": true,
|
|
"per_device_train_batch_size": 2,
|
|
"gradient_accumulation_steps": 16,
|
|
"lr_scheduler_type": "cosine",
|
|
"cutoff_len": 2048,
|
|
"logging_steps": 10,
|
|
"save_steps": 100,
|
|
"learning_rate": 1e-4,
|
|
"warmup_ratio": 0.1,
|
|
"num_train_epochs": 2,
|
|
"plot_loss": true,
|
|
"fp16": true,
|
|
"flash_attn": "fa2",
|
|
// "deepspeed": "ds_config.json" //多卡训练
|
|
},
|
|
"infer_args": {
|
|
"repetition_penalty": 1.2,
|
|
"temperature": 0.5,
|
|
"max_length": 256,
|
|
"top_p": 0.65
|
|
},
|
|
"vllm_args": {
|
|
"gpu_memory_utilization": 0.9,
|
|
// "data_parallel_size": 2,
|
|
// "quantization": "bitsandbytes",
|
|
// "load_format": "bitsandbytes"
|
|
},
|
|
"test_model_args": {
|
|
"test_data_path": "dataset/eval/test_data-zh.json"
|
|
}
|
|
}
|