From f46b873f92e5f1e018c68d259dd4d56d32e7b2c5 Mon Sep 17 00:00:00 2001 From: xming521 <1223398803@qq.com> Date: Sun, 13 Apr 2025 19:14:27 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0pyproject.toml=E4=BB=A5?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=B1=BB=E5=9E=8B=E6=A3=80=E6=9F=A5=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E4=BF=AE=E6=94=B9README.md=E4=BB=A5=E6=94=B9?= =?UTF-8?q?=E5=96=84=E6=95=B0=E6=8D=AE=E9=A2=84=E5=A4=84=E7=90=86=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=EF=BC=8C=E9=87=8D=E6=9E=84settings.json=E4=BB=A5?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE=E9=9B=86=E5=8F=82=E6=95=B0?= =?UTF-8?q?=EF=BC=8C=E6=9B=B4=E6=96=B0qa=5Fgenerator.py=E4=BB=A5=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E6=96=B0=E7=9A=84=E6=B6=88=E6=81=AF=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E5=88=A0=E9=99=A4=E6=97=A7=E7=9A=84?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6test=5Fold=5Fcsv=5Fto=5Fjso?= =?UTF-8?q?n=20copy.py=E5=B9=B6=E6=9B=B4=E6=96=B0=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cursor/rules/weclone-rules.mdc | 11 + README.md | 16 +- make_dataset/qa_generator.py | 17 +- pyproject.toml | 2 +- settings.json | 12 +- tests/test_old_csv_to_json copy.py | 311 ----------------------------- tests/test_old_csv_to_json.py | 17 +- 7 files changed, 45 insertions(+), 341 deletions(-) create mode 100644 .cursor/rules/weclone-rules.mdc delete mode 100644 tests/test_old_csv_to_json copy.py diff --git a/.cursor/rules/weclone-rules.mdc b/.cursor/rules/weclone-rules.mdc new file mode 100644 index 0000000..f0d225b --- /dev/null +++ b/.cursor/rules/weclone-rules.mdc @@ -0,0 +1,11 @@ +--- +description: +globs: +alwaysApply: true +--- + +# Your rule content +- You can @ files here +- The project uses uv as the package manager and pyproject.toml as the project configuration file. + + diff --git a/README.md b/README.md index 1a776ef..93ae987 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ > [!IMPORTANT] -> WeClone仍在快速迭代期，当前效果不代表最终效果 -> 微调LLM效果很大程度取决于聊天数据的数量和质量 +> WeClone仍在快速迭代期，当前效果不代表最终效果。 +> 微调LLM效果很大程度取决于聊天数据的数量和质量。 ### 硬件要求 @@ -50,15 +50,9 @@ uv pip install --group main -e . ### 数据预处理 -项目默认去除了数据中的手机号、身份证号、邮箱、网址。还提供了一个禁用词词库[blocked_words](make_dataset/blocked_words.json)，可以自行添加需要过滤的词句（会默认去掉包括禁用词的整句）。 -执行 `./make_dataset/csv_to_json.py` 脚本对数据进行处理。 - -在同一人连续回答多句的情况下，有三种处理方式： -| 文件 | 处理方式 | -| --- | --- | -| csv_to_json.py | 用逗号连接 | -| csv_to_json-单句回答.py(已废弃) | 只选择最长的回答作为最终数据 | -| csv_to_json-单句多轮.py | 放在了提示词的'history'中 | +项目默认去除了数据中的手机号、身份证号、邮箱、网址。还提供了一个禁用词词库[blocked_words](make_dataset/blocked_words.json)，可以自行添加需要过滤的词句（会默认去掉包括禁用词的整句）。 +执行 `python ./make_dataset/qa_generator.py` 对数据进行处理，可以根据自己的聊天风格修改settings.json的`make_dataset_args`。 +目前仅支持时间窗口策略，根据`single_combine_time_window`将单人连续消息通过逗号连接合并为一句，根据`qa_match_time_window`匹配问答对。后续将增加大模型清洗数据的功能。 ### 模型下载 diff --git a/make_dataset/qa_generator.py b/make_dataset/qa_generator.py index f1a9606..638056d 100644 --- a/make_dataset/qa_generator.py +++ b/make_dataset/qa_generator.py @@ -126,6 +126,15 @@ class DataProcessor: qa_res.append( {"instruction": current_instruction, "output": msg.msg} ) + else: + if self.c["prompt_with_history"]: + qa_res.append( + CutMessage( + is_sender=msg.is_sender, + cut_type=msg.type_name, + CreateTime=msg.CreateTime, + ) + ) # 无论是否匹配，都重置状态 current_state = WAITING_INSTRUCTION current_instruction = None @@ -133,7 +142,7 @@ class DataProcessor: return qa_res - def add_history_to_qa(self, qa_res: List[Dict]): + def add_history_to_qa(self, qa_res: List[Dict]) -> List[Dict]: qa_res_with_history = [] last_res = {"instruction": "", "output": "", "history": []} @@ -187,10 +196,10 @@ class DataProcessor: ChatMessage: 合并后的消息 """ base_msg = messages[0] - combined_content = messages[0].msg.strip() + combined_content = messages[0].msg for i in messages[1:]: - content = i.msg.strip() + content = i.msg if not content: continue @@ -343,7 +352,7 @@ class DataProcessor: def save_result(self, qa_res: List[Dict]): # 保存结果 with open( - f"./data/res_csv/sft/sft-{self.c['single_combine_strategy']}-{self.c['qa_match_strategy']}-my.json", + f"./data/res_csv/sft/sft-my.json", "w", encoding="utf-8", ) as f: diff --git a/pyproject.toml b/pyproject.toml index d6bfc14..f249063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ default = true [tool.pyright] typeCheckingMode = "basic" -include = ["src"] +include = ["make_dataset"] exclude = ["**/archive","**/tests"] ignore = ["**/archive"] diff --git a/settings.json b/settings.json index bd07b35..8677bf1 100644 --- a/settings.json +++ b/settings.json @@ -44,13 +44,13 @@ }, "make_dataset_args": { // "enable_vision_model": false,//后续实现 - "include_type": [ - "文本" - ], + // "include_type": [ + // "文本" + // ], "single_combine_strategy": "time_window", // 单人组成单句策略 - "qa_match_strategy": "time_window", // 多人组成qa策略 - "single_combine_time_window": 10, // 单人组成单句时间窗口（分钟）, - "qa_match_time_window": 60, // 多人组成qa时间窗口（分钟）, + "qa_match_strategy": "time_window", // 组成qa策略 + "single_combine_time_window": 2, // 单人组成单句时间窗口（分钟）, + "qa_match_time_window": 5, // 组成qa时间窗口（分钟）, "prompt_with_history": false // 是否在prompt中包含历史对话 }, "common_args": { diff --git a/tests/test_old_csv_to_json copy.py b/tests/test_old_csv_to_json copy.py deleted file mode 100644 index c2dff02..0000000 --- a/tests/test_old_csv_to_json copy.py +++ /dev/null @@ -1,311 +0,0 @@ -import csv -import json -import os -import sys - -import pandas as pd -from collections import deque - -current_dir = os.path.dirname(os.path.abspath(__file__)) -root_dir = os.path.dirname(current_dir) -sys.path.append(root_dir) - -from make_dataset.qa_generator import DataProcessor - -csv_folder = "./data/csv" -# csv_folder = './data/test' -os.chdir(root_dir) - -print(f"当前处理目录{csv_folder}") - - -def handle_pt_csv(csvfile): - chat_df = pd.read_csv(csvfile) - # 选择type_name为文本的行、is_sender为1的行 - chat_df = chat_df[chat_df["type_name"] == "文本"] - chat_df = chat_df[chat_df["is_sender"] == 1] - # 对每一行的content进行处理 转为dict 再取'msg'字段 - chat_df["content"] = chat_df["content"].apply(lambda x: json.loads(x)["msg"]) - # 如果content 包含 手机号、身份证号、邮箱、网址则删除这行 - chat_df = chat_df[~chat_df["content"].str.contains("1\d{10}")] - chat_df = chat_df[~chat_df["content"].str.contains("\d{18}")] - chat_df = chat_df[~chat_df["content"].str.contains("\w+@\w+")] - chat_df = chat_df[~chat_df["content"].str.contains("http")] - chat_df = chat_df[~chat_df["content"].str.contains(r"\\xa0")] - chat_df = chat_df[~chat_df["content"].str.contains(r"\\u")] - - # 纯content - chat_df = chat_df["content"] - chat_df = chat_df.dropna() - - return chat_df - - -def make_pt_dataset(): - csv_res = [] - # csv文件夹里全是不同聊天对象文件夹 每个文件夹里是csv文件 先遍历不同聊天对象文件夹 再遍历聊天对象的csv文件 - for chat_obj_folder in os.listdir(csv_folder): - chat_obj_folder_path = os.path.join(csv_folder, chat_obj_folder) - for csvfile in os.listdir(chat_obj_folder_path): - if not csvfile.endswith(".csv"): - continue - csvfile_path = os.path.join(chat_obj_folder_path, csvfile) - chat_df = handle_pt_csv(csvfile_path) - csv_res.append(chat_df) - - csv_res = pd.concat(csv_res) - csv_res = csv_res.apply(lambda x: {"c": x}) # 设置数据集prompt键为c - - csv_res.to_json("./data/res_csv/pt-my.json", orient="records", force_ascii=False) - - -def handle_sft_csv(csvfile): - chat_df = pd.read_csv(csvfile) - blocked_words = json.load( - open("./make_dataset/blocked_words.json", encoding="utf-8") - )["blocked_words"] - # 选择type_name为文本的行、is_sender为1的行 - # 需要保留的type_name字段名 - type_list = [ - "文本", - "图片", - "视频", - "合并转发的聊天记录", - "语音", - "(分享)音乐", - "(分享)卡片式链接", - "(分享)笔记", - "(分享)小程序", - "(分享)收藏夹", - "(分享)小说(猜)", - "(分享)视频号名片", - "(分享)视频号视频", - "粘贴的文本", # 无法解析的分享链接 - ] - chat_df = chat_df[chat_df["type_name"].isin(values=type_list)] - - # chat_df['content'] = chat_df['content'].apply(func=lambda x: json.loads(x)['msg']) - chat_df["content"] = chat_df["msg"] - - # 如果type_name为文本 并且content 包含 手机号、身份证号、邮箱、网址则删除这行 - for i in chat_df.index: - if chat_df.loc[i, "type_name"] == "文本": - if ( - "1\d{10}" in chat_df.loc[i, "content"] - or "\d{18}" in chat_df.loc[i, "content"] - or "\w+@\w+" in chat_df.loc[i, "content"] - or "http" in chat_df.loc[i, "content"] - or r"\\xa0" in chat_df.loc[i, "content"] - or r"\\u" in chat_df.loc[i, "content"] - ): - chat_df = chat_df.drop(index=i) - continue - for blocked_word in blocked_words: - if blocked_word in chat_df.loc[i, "content"]: - chat_df = chat_df.drop(index=i) - break - else: - chat_df.loc[i, "content"] = "" - - chat_df = chat_df[["is_sender", "type_name", "content", "CreateTime"]] - chat_df = chat_df.dropna() - # 时间格式 2021-07-07 10:27:23 - # 遍历行 相同is_sender的行合并content（）遇到不同is_sender就重新开始 - # CreateTime字段保留最后的CreateTime - chat_df["CreateTime"] = pd.to_datetime(chat_df["CreateTime"]) - - # 改到这了 - - type_list.remove("文本") - skip_list = type_list - res_df = [] - last_is_sender = chat_df.iloc[0]["is_sender"] - last_content: str = chat_df.iloc[0]["content"] - last_CreateTime = chat_df.iloc[0]["CreateTime"] - # 超时处理 半天没说话就重新开始 - # 注意这里只是处理了组装成一个句子 最后封装对话、配对在make_sft_dataset - # 遇到图片 连接 直接封装成一个句子 - for i, row in chat_df.iterrows(): - if row["type_name"] in skip_list: - if last_content != "": - if last_content[-1] == "，": - last_content = last_content[:-1] + "。" - elif last_content[-1] not in ["。", "！", "？", "…", "."]: - last_content += "。" - res_df.append( - { - "is_sender": last_is_sender, - "content": last_content, - "CreateTime": last_CreateTime, - } - ) - last_CreateTime = row["CreateTime"] - last_content = "" - # cut表示被skip字段截断 - res_df.append( - { - "is_sender": row["is_sender"], - "content": "cut", - "CreateTime": row["CreateTime"], - } - ) - continue - if last_content == "": # 重新开始 - last_content = row["content"] - last_is_sender = row["is_sender"] - last_CreateTime = row["CreateTime"] - continue - if row["is_sender"] == last_is_sender: - if row["CreateTime"] - last_CreateTime > pd.Timedelta(value="10m"): - # 如果超时 前面的添加到res_df 并重新开始 - if last_content[-1] == "，": - last_content = last_content[:-1] + "。" - elif last_content[-1] not in ["。", "！", "？", "…", "."]: - last_content += "。" - res_df.append( - { - "is_sender": last_is_sender, - "content": last_content, - "CreateTime": last_CreateTime, - } - ) - last_content = row["content"] - last_CreateTime = row["CreateTime"] - continue - # 如果content的结尾没有标点符号则添加逗号，最后结尾是句号 - if last_content[-1] not in ["。", "！", "？", "…", "，"]: - last_content += "，" - last_content = last_content + row["content"] - last_CreateTime = row["CreateTime"] - else: - if last_content[-1] == "，": - last_content = last_content[:-1] + "。" - elif last_content[-1] not in ["。", "！", "？", "…", "."]: - last_content += "。" - res_df.append( - { - "is_sender": last_is_sender, - "content": last_content, - "CreateTime": last_CreateTime, - } - ) - last_is_sender = row["is_sender"] - last_content = row["content"] - last_CreateTime = row["CreateTime"] - res_df = pd.DataFrame(res_df) - return res_df - - -def make_sft_dataset(): - processor = DataProcessor() - csv_files = processor.get_csv_files() - - csv_concat = [] - csv_res = [] - - for csvfile_path in csv_files: - chat_df = handle_sft_csv(csvfile_path) - csv_concat.append(chat_df) - - # 后续代码保持不变 - csv_concat = pd.concat(csv_concat) - - # 更全面地处理cut标记 - # 1. 将连续的cut标记合并为一个 - # 2. 标记数据区块的开始和结束 - processed_rows = [] - skip_row = False - last_row_was_cut = False - - for i in range(len(csv_concat)): - if skip_row: - skip_row = False - continue - - current_row = csv_concat.iloc[i].copy() - - # 处理当前行是cut的情况 - if current_row["content"] == "cut": - # 如果上一行已经是cut，则跳过当前行 - if last_row_was_cut: - continue - - # 查找连续的cut - j = i + 1 - while j < len(csv_concat) and csv_concat.iloc[j]["content"] == "cut": - j += 1 - - # 如果有连续的cut，只保留最后一个 - if j > i + 1: - current_row = csv_concat.iloc[j - 1].copy() - skip_row = True - - last_row_was_cut = True - else: - last_row_was_cut = False - - processed_rows.append(current_row) - - # 创建新的DataFrame - csv_concat = pd.DataFrame(processed_rows) - - # csv_res里is_sender必须是01 01 01 的顺序 csv_concat里不一定是01 01 - # 相差超过1小时的时间戳分为不同的对话 - # temp_res为一个长度为2的队列 - # 将合并后的数据保存到CSV文件中 - output_dir = "./test_output" - - # 生成带时间戳的文件名 - import datetime - - now = datetime.datetime.now() - output_file = os.path.join(output_dir, f"csv_old_.csv") - - # 保存合并后的数据 - # csv_concat.to_csv(output_file, index=False, encoding="utf-8-sig") - # print(f"已将合并后的数据保存到: {output_file}") - # print(f"合并后数据总量: {len(csv_concat)} 条记录") - - temp_res = deque(maxlen=2) - # 6种情况 - # temp_res 为空 遇到 0入队 遇到1不处理 遇到cut不处理 - # temp_res 有0 遇到0清空队列再入队 遇到1相差超过1小时清空队列 没有相差一小时入队再全部出队 遇到cut清空队列 - - for i, row in csv_concat.iterrows(): - if len(temp_res) == 0: - if row["content"] == "cut": - continue - if row["is_sender"] == 0: - temp_res.append(row["content"]) - last_CreateTime = row["CreateTime"] - else: - continue - elif len(temp_res) == 1: - if row["content"] == "cut": - temp_res.clear() - last_CreateTime = row["CreateTime"] - elif row["is_sender"] == 0: - # 遇到0 清空队列再入队 - temp_res.clear() - temp_res.append(row["content"]) - last_CreateTime = row["CreateTime"] - else: - if row["CreateTime"] - last_CreateTime > pd.Timedelta("10m"): - # 相差超过1小时清空队列 - temp_res.clear() - last_CreateTime = row["CreateTime"] - else: - # 没有相差一小时入队再全部出队 - temp_res.append(row["content"]) - csv_res.append({"instruction": temp_res[0], "output": temp_res[1]}) - temp_res.clear() - last_CreateTime = row["CreateTime"] - - csv_res_df = pd.DataFrame(csv_res) - print(f"处理后数据量：{csv_res_df.shape[0]}") - csv_res_df.to_json('./data/res_csv/sft/sft-old-my.json', orient='records', force_ascii=False) - - -if __name__ == "__main__": - # make_pt_dataset() - make_sft_dataset() diff --git a/tests/test_old_csv_to_json.py b/tests/test_old_csv_to_json.py index ddb368a..e386e51 100644 --- a/tests/test_old_csv_to_json.py +++ b/tests/test_old_csv_to_json.py @@ -1,6 +1,7 @@ import csv import json import os +import re import sys import pandas as pd @@ -91,12 +92,12 @@ def handle_sft_csv(csvfile): for i in chat_df.index: if chat_df.loc[i, "type_name"] == "文本": if ( - re.search(r"1\d{10}", msg_str) - or re.search(r"\d{18}", msg_str) - or re.search(r"\w+@\w+", msg_str) - or "http" in msg_str - or r"\\xa0" in msg_str - or r"\\u" in msg_str + re.search(r"1\d{10}", chat_df.loc[i, "content"]) + or re.search(r"\d{18}", chat_df.loc[i, "content"]) + or re.search(r"\w+@\w+", chat_df.loc[i, "content"]) + or "http" in chat_df.loc[i, "content"] + or r"\\xa0" in chat_df.loc[i, "content"] + or r"\\u" in chat_df.loc[i, "content"] ): chat_df = chat_df.drop(index=i) continue @@ -156,7 +157,7 @@ def handle_sft_csv(csvfile): last_CreateTime = row["CreateTime"] continue if row["is_sender"] == last_is_sender: - if row["CreateTime"] - last_CreateTime > pd.Timedelta(value="10m"): + if row["CreateTime"] - last_CreateTime > pd.Timedelta(value="2m"): # 如果超时 前面的添加到res_df 并重新开始 if last_content[-1] == "，": last_content = last_content[:-1] @@ -290,7 +291,7 @@ def make_sft_dataset(): temp_res.append(row["content"]) last_CreateTime = row["CreateTime"] else: - if row["CreateTime"] - last_CreateTime > pd.Timedelta("1h"): + if row["CreateTime"] - last_CreateTime > pd.Timedelta("5m"): # 相差超过1小时清空队列 temp_res.clear() last_CreateTime = row["CreateTime"]