diff --git a/.gitignore b/.gitignore index 5514ab9..45efd30 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,4 @@ Qwen*/ settings.jsonc settings.json dataset/blocked_words.json +dataset/wechat/* diff --git a/settings.template.jsonc b/settings.template.jsonc index 6a2b3c0..2aff44b 100644 --- a/settings.template.jsonc +++ b/settings.template.jsonc @@ -1,5 +1,5 @@ { - "version": "0.2.2", + "version": "0.2.21", "common_args": { "model_name_or_path": "./Qwen2.5-7B-Instruct", "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir @@ -14,7 +14,8 @@ "make_dataset_args": { //数据处理配置 "include_type": [ - "文本" + "text", + // "image" ], "blocked_words": [ // 禁用词 "例如 姓名", @@ -34,11 +35,11 @@ "accept_score": 2, //可以接受的llm打分阈值,1分最差,5分最好,低于此分数的数据不会用于训练 } }, - "online_llm_clear":false, - "base_url":"https://xxx/v1", - "llm_api_key":"xxxxx", - "model_name":"xxx", //建议使用参数较大的模型,例如DeepSeek-V3 - "clean_batch_size":10 + "online_llm_clear": false, + "base_url": "https://xxx/v1", + "llm_api_key": "xxxxx", + "model_name": "xxx", //建议使用参数较大的模型,例如DeepSeek-V3 + "clean_batch_size": 10 }, "train_pt_args": { //预训练微调配置 @@ -91,4 +92,4 @@ "max_length": 50, "top_p": 0.65 } -} +} \ No newline at end of file diff --git a/weclone/data/chat_parsers/wechat_parser.py b/weclone/data/chat_parsers/wechat_parser.py new file mode 100644 index 0000000..9661c88 --- /dev/null +++ b/weclone/data/chat_parsers/wechat_parser.py @@ -0,0 +1,8 @@ +class WeChatParser: + def decrypt_wechat_image(self, encrypted_path, output_path): + """解密微信加密的图片文件""" + pass + + def parse_chat_records(self, db_path): + """解析聊天记录数据库""" + pass