mirror of
https://github.com/xming521/WeClone.git
synced 2026-08-28 18:07:28 +08:00
xming521
7e9886c020
Reduces the default LoRA rank in training configuration templates from 16 to 8. This change aims to optimize resource usage and potentially accelerate training.
102 lines
4.0 KiB
JSON
102 lines
4.0 KiB
JSON
{
|
|
"version": "0.3.0",
|
|
"common_args": {
|
|
"model_name_or_path": "./models/Qwen2.5-VL-7B-Instruct",
|
|
"adapter_name_or_path": "./model_output", // Also serves as the output_dir for train_sft_args
|
|
"template": "qwen2_vl",
|
|
"default_system": "Please act like a human and don't say you are an artificial intelligence",
|
|
"finetuning_type": "lora",
|
|
"media_dir": "dataset/media",
|
|
"image_max_pixels": 409920, //720P
|
|
"enable_thinking": false,
|
|
"trust_remote_code": true
|
|
},
|
|
"cli_args": {
|
|
"full_log": false
|
|
},
|
|
"make_dataset_args": {
|
|
// Data processing configuration
|
|
"platform": "telegram", //chat,telegram
|
|
"language": "en", // Common chat language: zh(中文), en(English)
|
|
"telegram_args": {
|
|
"my_id": "user1234567890"
|
|
},
|
|
"include_type": [
|
|
"text",
|
|
"image",
|
|
// "sticker" //Converting stickers to emojis can lead to the model outputting too many emojis.
|
|
],
|
|
"max_image_num": 2, // Maximum number of images per data entry
|
|
"blocked_words": [ // Blocked words
|
|
"e.g. Name",
|
|
"e.g. Password",
|
|
"//....."
|
|
],
|
|
"single_combine_strategy": "time_window", // Single person message combination strategy
|
|
"qa_match_strategy": "time_window", // QA combination strategy
|
|
"single_combine_time_window": 2, // Time window for single person message combination (minutes)
|
|
"qa_match_time_window": 5, // Time window for QA combination (minutes)
|
|
"combine_msg_max_length": 2048, // Maximum length of combined messages
|
|
"messages_max_length": 2048, // Maximum character count for messages, used with cutoff_len
|
|
"clean_dataset": {
|
|
"enable_clean": false,
|
|
"clean_strategy": "llm",
|
|
"llm": {
|
|
"accept_score": 2, // Acceptable LLM score threshold, 1 is worst, 5 is best, data below this score will not be used for training
|
|
}
|
|
},
|
|
"online_llm_clear": false,
|
|
"base_url": "https://xxx/v1",
|
|
"llm_api_key": "xxxxx",
|
|
"model_name": "xxx", // Recommend using models with larger parameters, e.g. DeepSeek-V3
|
|
"clean_batch_size": 10,
|
|
"vision_api": {
|
|
"enable": false, // Set to true to enable this feature
|
|
"api_key": "xxx",
|
|
"api_url": "https://xxx/v1", // e.g. Alibaba Cloud, or replace with other OpenAI-compatible API addresses
|
|
"model_name": "xxx", // Multimodal model name to use, e.g. qwen-vl-max
|
|
"max_workers": 5 // Number of parallel API call threads, maximum should not exceed 8
|
|
}
|
|
},
|
|
"train_sft_args": {
|
|
// Fine-tuning configuration
|
|
"stage": "sft",
|
|
"dataset": "chat-sft",
|
|
"dataset_dir": "./dataset/res_csv/sft",
|
|
"freeze_multi_modal_projector": false, // Whether to freeze the multimodal projector during MLLM training
|
|
"use_fast_tokenizer": true,
|
|
"lora_target": "q_proj,v_proj,visual.merger.mlp.0,visual.merger.mlp.2",
|
|
"lora_rank": 8,
|
|
"lora_dropout": 0.25,
|
|
"weight_decay": 0.1,
|
|
"overwrite_cache": true,
|
|
"per_device_train_batch_size": 2,
|
|
"gradient_accumulation_steps": 16,
|
|
"lr_scheduler_type": "cosine",
|
|
"cutoff_len": 4096,
|
|
"logging_steps": 10,
|
|
"save_steps": 100,
|
|
"learning_rate": 1e-4,
|
|
"warmup_ratio": 0.1,
|
|
"num_train_epochs": 2,
|
|
"plot_loss": true,
|
|
"fp16": true,
|
|
"flash_attn": "fa2",
|
|
"preprocessing_num_workers": 16,
|
|
"dataloader_num_workers": 4
|
|
// "deepspeed": "ds_config.json" // Multi-GPU training
|
|
},
|
|
"infer_args": {
|
|
"repetition_penalty": 1.2,
|
|
"temperature": 0.7,
|
|
"max_length": 512,
|
|
"top_p": 0.8
|
|
},
|
|
"vllm_args": {
|
|
"gpu_memory_utilization": 0.9
|
|
},
|
|
"test_model_args": {
|
|
"test_data_path": "dataset/eval/test_data-en.json"
|
|
}
|
|
}
|