opensource/self-llm
3
0
Fork 0
mirror of https://github.com/datawhalechina/self-llm.git synced 2026-08-28 23:27:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Qwen2.5 fastapi

Browse Source
This commit is contained in:
KMnO4-zx
2025-01-05 17:59:36 +08:00
parent 27d82f88e4
commit 4ec0b08d1e
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/Qwen2.5/01-Qwen2.5-7B-Instruct FastApi 部署调用.md
+2 -2
View File
@@ -93,7 +93,7 @@ async def create_item(request: Request):
# 调用模型进行对话生成
input_ids = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True)
model_inputs = tokenizer([input_ids], return_tensors="pt").to('cuda')
model_inputs = tokenizer([input_ids], return_tensors="pt").to(model.device)
generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=512)
generated_ids = [
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
@@ -118,7 +118,7 @@ if __name__ == '__main__':
# 加载预训练的分词器和模型
model_name_or_path = '/root/autodl-tmp/qwen/Qwen2.5-7B-Instruct'
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=False)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="auto", torch_dtype=torch.bfloat16)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map=CUDA_DEVICE, torch_dtype=torch.bfloat16)
# 启动FastAPI应用
# 用6006端口可以将autodl的端口映射到本地，从而在本地使用api