From ed9d8e320e60a48057ad6d3232efb88dd16d9ec4 Mon Sep 17 00:00:00 2001 From: xming521 <1223398803@qq.com> Date: Wed, 21 May 2025 21:57:12 +0800 Subject: [PATCH] Adjust the default values of lora_dropout and num_train_epochs, while applying a unified decorator in the CLI to support log capturing. Use an inelegant approach to print all terminal content to a file, enabled via `full_log`. --- .cursor/rules/weclone-rules.mdc | 2 +- settings.template.jsonc | 7 ++- weclone/cli.py | 40 ++++++++++++++--- weclone/data/qa_generator.py | 2 +- weclone/utils/config.py | 4 +- weclone/utils/log.py | 80 +++++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 12 deletions(-) diff --git a/.cursor/rules/weclone-rules.mdc b/.cursor/rules/weclone-rules.mdc index 1edfe21..df98ef9 100644 --- a/.cursor/rules/weclone-rules.mdc +++ b/.cursor/rules/weclone-rules.mdc @@ -12,7 +12,7 @@ alwaysApply: true # Your rule content - You can @ files here - The project uses uv as the package manager and pyproject.toml as the project configuration file. -- Unless I ask you to, code comments don't need to be excessive.You should prioritize using Chinese to comment code. +- Unless I ask you to, code comments don't need to be excessive. - Prefer using the encapsulated logger `from weclone.utils.log import logger` for printing. - When retrieving values from a parameter dictionary read from a configuration file, the `get` method should be preferred whenever possible. diff --git a/settings.template.jsonc b/settings.template.jsonc index 919f760..155bde2 100644 --- a/settings.template.jsonc +++ b/settings.template.jsonc @@ -8,6 +8,9 @@ "finetuning_type": "lora", "trust_remote_code": true }, + "cli_args": { + "full_log": false + }, "make_dataset_args": { //数据处理配置 "include_type": [ @@ -60,7 +63,7 @@ "use_fast_tokenizer": true, "lora_target": "q_proj,v_proj", "lora_rank": 4, - "lora_dropout": 0.4, + "lora_dropout": 0.3, "weight_decay": 0.1, "overwrite_cache": true, "per_device_train_batch_size": 8, @@ -71,7 +74,7 @@ "save_steps": 100, "learning_rate": 1e-4, "warmup_ratio": 0.1, - "num_train_epochs": 3, + "num_train_epochs": 2, "plot_loss": true, "fp16": true, "flash_attn": "fa2", diff --git a/weclone/cli.py b/weclone/cli.py index 138c5cb..914acd6 100644 --- a/weclone/cli.py +++ b/weclone/cli.py @@ -4,7 +4,11 @@ from pathlib import Path import os import sys import functools -from weclone.utils.log import logger + +from weclone.utils.log import logger, capture_output +from weclone.utils.config import load_config + +cli_config: dict | None = None try: import tomllib # type: ignore Python 3.11+ @@ -30,15 +34,37 @@ def clear_argv(func): return wrapper +def apply_common_decorators(capture_output_enabled=False): + """ + A unified decorator for applications + """ + + def decorator(original_cmd_func): + @functools.wraps(original_cmd_func) + def new_runtime_wrapper(*args, **kwargs): + if cli_config and cli_config.get("full_log", False): + return capture_output(original_cmd_func)(*args, **kwargs) + else: + return original_cmd_func(*args, **kwargs) + + func_with_clear_argv = clear_argv(new_runtime_wrapper) + + return functools.wraps(original_cmd_func)(func_with_clear_argv) + + return decorator + + @click.group() def cli(): """WeClone: 从聊天记录创造数字分身的一站式解决方案""" _check_project_root() _check_versions() + global cli_config + cli_config = load_config(arg_type="cli_args") @cli.command("make-dataset", help="处理聊天记录CSV文件，生成问答对数据集。") -@clear_argv +@apply_common_decorators() def qa_generator(): """处理聊天记录CSV文件，生成问答对数据集。""" from weclone.data.qa_generator import DataProcessor @@ -48,7 +74,7 @@ def qa_generator(): @cli.command("train-sft", help="使用准备好的数据集对模型进行微调。") -@clear_argv +@apply_common_decorators() def train_sft(): """使用准备好的数据集对模型进行微调。""" from weclone.train.train_sft import main as train_sft_main @@ -57,7 +83,7 @@ def train_sft(): @cli.command("webchat-demo", help="启动 Web UI 与微调后的模型进行交互测试。") # 命令名修改为 web-demo -@clear_argv +@apply_common_decorators() def web_demo(): """启动 Web UI 与微调后的模型进行交互测试。""" from weclone.eval.web_demo import main as web_demo_main @@ -66,7 +92,7 @@ def web_demo(): # TODO 添加评估功能 @cli.command("eval-model", help="使用从训练数据中划分出来的验证集评估。") -@clear_argv +@apply_common_decorators() def eval_model(): """使用从训练数据中划分出来的验证集评估。""" from weclone.eval.eval_model import main as evaluate_main @@ -75,7 +101,7 @@ def eval_model(): @cli.command("test-model", help="使用常见聊天问题测试模型。") -@clear_argv +@apply_common_decorators() def test_model(): """测试""" from weclone.eval.test_model import main as test_main @@ -84,7 +110,7 @@ def test_model(): @cli.command("server", help="启动API服务，提供模型推理接口。") -@clear_argv +@apply_common_decorators() def server(): """启动API服务，提供模型推理接口。""" from weclone.server.api_service import main as server_main diff --git a/weclone/data/qa_generator.py b/weclone/data/qa_generator.py index b99655d..98a5673 100644 --- a/weclone/data/qa_generator.py +++ b/weclone/data/qa_generator.py @@ -47,7 +47,7 @@ class DataProcessor: pass self.blocked_words = list(set(config_blocked_words + file_blocked_words)) - logger.info(f"聊天记录禁用词: {self.blocked_words}") + # logger.info(f"聊天记录禁用词: {self.blocked_words}") if self.config["single_combine_strategy"] == "time_window": self.single_combine_strategy = TimeWindowStrategy( diff --git a/weclone/utils/config.py b/weclone/utils/config.py index b6bdf8a..69e2fd4 100644 --- a/weclone/utils/config.py +++ b/weclone/utils/config.py @@ -19,7 +19,9 @@ def load_config(arg_type: str): logger.error(f"Error loading configuration file {config_path}: {e}") sys.exit(1) - if arg_type == "web_demo" or arg_type == "api_service": + if arg_type == "cli_args": + config = s_config["cli_args"] + elif arg_type == "web_demo" or arg_type == "api_service": # infer_args和common_args求并集 config = {**s_config["infer_args"], **s_config["common_args"]} elif arg_type == "train_pt": diff --git a/weclone/utils/log.py b/weclone/utils/log.py index 922fb52..ad164a8 100644 --- a/weclone/utils/log.py +++ b/weclone/utils/log.py @@ -1,5 +1,6 @@ from loguru import logger import sys +from functools import wraps logger.remove() @@ -9,3 +10,82 @@ logger.add( colorize=True, level="INFO", ) + +logger.add( + "logs/weclone.log", # 日志文件路径 + rotation="1 day", # 每天轮换一个新的日志文件 + retention="7 days", # 保留最近7天的日志文件 + compression="zip", # 压缩旧的日志文件 + level="DEBUG", # 文件日志级别 + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}", # 日志格式 + encoding="utf-8", # 文件编码 + enqueue=True, # 异步写入，避免阻塞 +) + + +def capture_output(func): + @wraps(func) + def wrapper(*args, **kwargs): + log_sink_buffer = [] + + def list_sink(message): + log_sink_buffer.append(message.record["message"]) + + sink_id = logger.add(list_sink, format="{message}", level="INFO") + + original_stdout = sys.stdout + original_stderr = sys.stderr + + class OutputTeeToGlobalLog: + def __init__(self, original_stream, log_method): + self.original_stream = original_stream + self.log_method = log_method + self.current_line_content = "" # Represents the current state of the line to be logged + + def write(self, data_chunk): + self.original_stream.write(data_chunk) # Pass through to console + + if data_chunk.endswith("\\r") and "\\n" not in data_chunk: + self.current_line_content = data_chunk[:-1] # Store without the trailing \\r + return + + full_buffer = self.current_line_content + data_chunk + lines_to_process = full_buffer.split("\\n") + + for i in range(len(lines_to_process) - 1): + line = lines_to_process[i] + final_content_of_line = line + last_cr = line.rfind("\\r") + if last_cr != -1: + final_content_of_line = line[last_cr + 1 :] + + escaped_log = final_content_of_line.replace("{", "{{").replace("}", "}}") + if final_content_of_line.strip() or line: + self.log_method(escaped_log, raw=True) + + self.current_line_content = lines_to_process[-1] + + def flush(self): + self.original_stream.flush() + if self.current_line_content: + final_content_of_line = self.current_line_content + last_cr = self.current_line_content.rfind("\\r") + if last_cr != -1: + final_content_of_line = self.current_line_content[last_cr + 1 :] + + escaped_log = final_content_of_line.replace("{", "{{").replace("}", "}}") + if final_content_of_line.strip() or self.current_line_content: + self.log_method(escaped_log, raw=True) + self.current_line_content = "" + + sys.stdout = OutputTeeToGlobalLog(original_stdout, logger.opt(raw=True).info) + sys.stderr = OutputTeeToGlobalLog(original_stderr, logger.opt(raw=True).error) + + try: + func(*args, **kwargs) + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr + logger.remove(sink_id) + + return wrapper