mirror of
https://github.com/xming521/WeClone.git
synced 2026-08-28 18:07:28 +08:00
xming521
7e9886c020
Reduces the default LoRA rank in training configuration templates from 16 to 8. This change aims to optimize resource usage and potentially accelerate training.
97 lines
3.5 KiB
JSON
97 lines
3.5 KiB
JSON
{
|
|
"version": "0.2.24",
|
|
"common_args": {
|
|
"model_name_or_path": "./models/Qwen2.5-VL-7B-Instruct",
|
|
"adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir
|
|
"template": "qwen2_vl",
|
|
"default_system": "请你扮演一名人类，不要说自己是人工智能",
|
|
"finetuning_type": "lora",
|
|
"media_dir": "dataset/media",
|
|
"image_max_pixels": 409920, //720P
|
|
"enable_thinking": false,
|
|
"trust_remote_code": true
|
|
},
|
|
"cli_args": {
|
|
"full_log": false
|
|
},
|
|
"make_dataset_args": {
|
|
//数据处理配置
|
|
"platform": "chat", //chat,telegram
|
|
"include_type": [
|
|
"text",
|
|
"image"
|
|
],
|
|
"max_image_num": 2, // 单条数据最大图片数量
|
|
"blocked_words": [ // 禁用词
|
|
"例如 姓名",
|
|
"例如 密码",
|
|
"//....."
|
|
],
|
|
"single_combine_strategy": "time_window", // 单人组成单句策略
|
|
"qa_match_strategy": "time_window", // 组成qa策略
|
|
"single_combine_time_window": 2, // 单人组成单句时间窗口（分钟）,
|
|
"qa_match_time_window": 5, // 组成qa时间窗口（分钟）,
|
|
"combine_msg_max_length": 2048, // 组合后消息最大长度
|
|
"messages_max_length": 2048, // messages最长字符数量 配合cutoff_len 使用
|
|
"clean_dataset": {
|
|
"enable_clean": false,
|
|
"clean_strategy": "llm",
|
|
"llm": {
|
|
"accept_score": 2, //可以接受的llm打分阈值,1分最差，5分最好,低于此分数的数据不会用于训练
|
|
}
|
|
},
|
|
"online_llm_clear": false,
|
|
"base_url": "https://xxx/v1",
|
|
"llm_api_key": "xxxxx",
|
|
"model_name": "xxx", //建议使用参数较大的模型，例如DeepSeek-V3
|
|
"clean_batch_size": 10,
|
|
"vision_api": {
|
|
"enable": false, // 设置为 true 来开启此功能
|
|
"api_key": "xxx",
|
|
"api_url": "https://xxx/v1", // 例如阿里云，或替换为其他兼容OpenAI的API地址
|
|
"model_name": "xxx", // 要使用的多模态模型名称,例如qwen-vl-max
|
|
"max_workers": 5 // 并行调用API的线程数，最多不要超过8
|
|
}
|
|
},
|
|
"train_sft_args": {
|
|
//微调配置
|
|
"stage": "sft",
|
|
"dataset": "chat-sft",
|
|
"dataset_dir": "./dataset/res_csv/sft",
|
|
"freeze_multi_modal_projector": false, //MLLM 训练时是否冻结多模态投影器。
|
|
"use_fast_tokenizer": true,
|
|
"lora_target": "q_proj,v_proj,visual.merger.mlp.0,visual.merger.mlp.2",
|
|
"lora_rank": 8,
|
|
"lora_dropout": 0.25,
|
|
"weight_decay": 0.1,
|
|
"overwrite_cache": true,
|
|
"per_device_train_batch_size": 2,
|
|
"gradient_accumulation_steps": 16,
|
|
"lr_scheduler_type": "cosine",
|
|
"cutoff_len": 4096,
|
|
"logging_steps": 10,
|
|
"save_steps": 100,
|
|
"learning_rate": 1e-4,
|
|
"warmup_ratio": 0.1,
|
|
"num_train_epochs": 2,
|
|
"plot_loss": true,
|
|
"fp16": true,
|
|
"flash_attn": "fa2",
|
|
"preprocessing_num_workers": 16,
|
|
"dataloader_num_workers": 4
|
|
// "deepspeed": "ds_config.json" //多卡训练
|
|
},
|
|
"infer_args": {
|
|
"repetition_penalty": 1.2,
|
|
"temperature": 0.65,
|
|
"max_length": 512,
|
|
"top_p": 0.75
|
|
},
|
|
"vllm_args": {
|
|
"gpu_memory_utilization": 0.9
|
|
},
|
|
"test_model_args": {
|
|
"test_data_path": "dataset/eval/test_data-en.json"
|
|
}
|
|
}
|