🎈 auto fixes by pre-commit hooks

This commit is contained in:
pre-commit-ci[bot]
2025-06-21 03:21:05 +00:00
parent 8c739d5f0b
commit 3e0f08c7cb
2 changed files with 8 additions and 9 deletions
+2 -3
View File
@@ -7,7 +7,6 @@ from llamafactory.model import load_tokenizer
from pydantic import BaseModel
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
from vllm.sampling_params import GuidedDecodingParams
from weclone.utils.config import load_config
from weclone.utils.config_models import VllmArgs
@@ -73,7 +72,7 @@ def vllm_infer(
template_obj.mm_plugin.expand_mm_tokens = False # for vllm generate
if guided_decoding_class:
json_schema = guided_decoding_class.model_json_schema()
json_schema = guided_decoding_class.model_json_schema()
sampling_params = SamplingParams(
repetition_penalty=generating_args.repetition_penalty or 1.0, # repetition_penalty must > 0
@@ -118,5 +117,5 @@ def vllm_infer(
results = LLM(**engine_args).chat(
messages_list, sampling_params, lora_request=lora_request, chat_template_kwargs=extra_body
) # type: ignore
return results
+6 -6
View File
@@ -29,17 +29,17 @@ def calculate_token_length(
template: str = "qwen3",
) -> int:
"""计算指定文本的token长度
Args:
text: 要计算token长度的文本
model_name_or_path: 模型路径
template: 模板名称
Returns:
文本的token长度
"""
logger.info(f"正在计算文本token长度: {text[:50]}...")
model_args, data_args, _, _, _ = get_train_args(
{
"stage": "sft",
@@ -50,14 +50,14 @@ def calculate_token_length(
"do_train": True,
}
)
tokenizer_module = load_tokenizer(model_args)
tokenizer = tokenizer_module["tokenizer"]
# 直接使用tokenizer编码文本
tokens = tokenizer.encode(text, add_special_tokens=False)
token_length = len(tokens)
logger.info(f"文本token长度: {token_length}")
return token_length