mirror of
https://github.com/xming521/WeClone.git
synced 2026-09-01 04:58:16 +08:00
🎈 auto fixes by pre-commit hooks
This commit is contained in:
@@ -7,7 +7,6 @@ from llamafactory.model import load_tokenizer
|
||||
from pydantic import BaseModel
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.sampling_params import GuidedDecodingParams
|
||||
|
||||
from weclone.utils.config import load_config
|
||||
from weclone.utils.config_models import VllmArgs
|
||||
@@ -73,7 +72,7 @@ def vllm_infer(
|
||||
template_obj.mm_plugin.expand_mm_tokens = False # for vllm generate
|
||||
|
||||
if guided_decoding_class:
|
||||
json_schema = guided_decoding_class.model_json_schema()
|
||||
json_schema = guided_decoding_class.model_json_schema()
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
repetition_penalty=generating_args.repetition_penalty or 1.0, # repetition_penalty must > 0
|
||||
@@ -118,5 +117,5 @@ def vllm_infer(
|
||||
results = LLM(**engine_args).chat(
|
||||
messages_list, sampling_params, lora_request=lora_request, chat_template_kwargs=extra_body
|
||||
) # type: ignore
|
||||
|
||||
|
||||
return results
|
||||
|
||||
@@ -29,17 +29,17 @@ def calculate_token_length(
|
||||
template: str = "qwen3",
|
||||
) -> int:
|
||||
"""计算指定文本的token长度
|
||||
|
||||
|
||||
Args:
|
||||
text: 要计算token长度的文本
|
||||
model_name_or_path: 模型路径
|
||||
template: 模板名称
|
||||
|
||||
|
||||
Returns:
|
||||
文本的token长度
|
||||
"""
|
||||
logger.info(f"正在计算文本token长度: {text[:50]}...")
|
||||
|
||||
|
||||
model_args, data_args, _, _, _ = get_train_args(
|
||||
{
|
||||
"stage": "sft",
|
||||
@@ -50,14 +50,14 @@ def calculate_token_length(
|
||||
"do_train": True,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
tokenizer_module = load_tokenizer(model_args)
|
||||
tokenizer = tokenizer_module["tokenizer"]
|
||||
|
||||
|
||||
# 直接使用tokenizer编码文本
|
||||
tokens = tokenizer.encode(text, add_special_tokens=False)
|
||||
token_length = len(tokens)
|
||||
|
||||
|
||||
logger.info(f"文本token长度: {token_length}")
|
||||
return token_length
|
||||
|
||||
|
||||
Reference in New Issue
Block a user