mirror of
https://github.com/datawhalechina/self-llm.git
synced 2026-08-30 17:48:29 +08:00
Merge branch 'datawhalechina:master' into master
This commit is contained in:
Wang Zeyu
@@ -71,7 +71,7 @@
|
||||
|
||||
- [Qwen2-vl](https://github.com/QwenLM/Qwen2-VL)
|
||||
- [x] [Qwen2-vl-2B FastApi 部署调用](./models/Qwen2-VL/01-Qwen2-VL-2B-Instruct%20FastApi%20部署调用.md) @姜舒凡
|
||||
- [ ] [Qwen2-vl-2B WebDemo 部署]() @赵伟
|
||||
- [x] [Qwen2-vl-2B WebDemo 部署](./models/Qwen2-VL/02-Qwen2-VL-2B-Instruct%20Web%20Demo部署.md) @赵伟
|
||||
- [ ] [Qwen2-vl-2B vLLM 部署]() @荞麦
|
||||
- [ ] [Qwen2-vl-2B Lora 微调]() @李柯辰
|
||||
- [x] [Qwen2-vl-2B Lora 微调 SwanLab 可视化记录版](./models/Qwen2-VL/05-Qwen2-VL-2B-Instruct%20Lora%20微调%20SwanLab%20可视化记录版.md) @林泽毅
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from threading import Thread
|
||||
from typing import Any, Dict, Generator, List, Tuple
|
||||
|
||||
import gradio as gr
|
||||
from qwen_vl_utils import process_vision_info
|
||||
import torch
|
||||
from transformers import (
|
||||
AutoProcessor,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
Qwen2VLProcessor,
|
||||
TextIteratorStreamer,
|
||||
GenerationConfig,
|
||||
)
|
||||
from transformers.utils import is_flash_attn_2_available
|
||||
from transformers.modeling_utils import get_first_parameter_dtype
|
||||
from accelerate import init_empty_weights
|
||||
from accelerate.utils import calculate_maximum_sizes, convert_bytes
|
||||
from accelerate.commands.estimate import create_ascii_table
|
||||
|
||||
# copy from qwen_vl_utils.process_vision_info
|
||||
MIN_PIXELS = 4 * 28 * 28 # 一张图最小占4个token
|
||||
MAX_PIXELS = 16384 * 28 * 28 # 一张图最大占16384个token
|
||||
VIDEO_MIN_PIXELS = 128 * 28 * 28 # 一个视频里一帧最小占128个token
|
||||
VIDEO_MAX_PIXELS = 768 * 28 * 28 # 一个视频里一帧最大占768个token
|
||||
VIDEO_TOTAL_PIXELS = 24576 * 28 * 28 # 一个视频里所有帧总共占最多24576个token
|
||||
|
||||
# default
|
||||
DEFAULT_CKPT_PATH = "path/to/Qwen2-VL-2B-Instruct"
|
||||
VIDEO_EXTENSIONS = [
|
||||
".mp4",
|
||||
".avi",
|
||||
".mkv",
|
||||
".mov",
|
||||
".wmv",
|
||||
".flv",
|
||||
".webm",
|
||||
".mpeg",
|
||||
]
|
||||
IMAGE_EXTENSIONS = [".png", ".jpg"]
|
||||
# end default
|
||||
print("*" * 60)
|
||||
print("*Qwen2-vl 图片视频模态token限制如下:")
|
||||
print(f"*单张图片最大/最小token长度限制:{MAX_PIXELS//(28*28)}/{MIN_PIXELS//(28*28)}")
|
||||
print(
|
||||
f"*单个视频最大/最小/总token长度限制:{VIDEO_MAX_PIXELS//(28*28)}/{VIDEO_MIN_PIXELS//(28*28)}/{VIDEO_TOTAL_PIXELS//(28*28)}"
|
||||
)
|
||||
print("*" * 60, end="\n\n")
|
||||
|
||||
|
||||
# modify from https://github.com/huggingface/accelerate/blob/c0552c9012a9bae7f125e1df89cf9ee0b0d250fd/src/accelerate/commands/estimate.py#L285
|
||||
def cal_model_size(args):
|
||||
"""计算模型在各种数据类型下的存储占用
|
||||
主要计算方法是
|
||||
借助calculate_maximum_sizes函数计算所有参数数量在特定下的存储->float32,float16,int8,int4分别进行进一步乘除即可.
|
||||
convert_bytes: 将计算结果转为不超过1024的TB/GB/MB/KB等单位下的结果表示.
|
||||
"""
|
||||
model_name = Path(args.model_path).name
|
||||
model_path = Path(args.model_path).as_posix()
|
||||
# 空加载模型, 可以几乎免去对存储空间的占用, 只记录每层有几个参数, 而不实际去申请内存初始化这些参数, 在加载大模型时有很多好处, 比如这里用来计算模型存储空间的占用, 毕竟加载一次大模型还是挺费时间的~
|
||||
with init_empty_weights():
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_path, torch_dtype="auto"
|
||||
) # 这里auto时会加载bfloat16格式，占用和float16一致
|
||||
total_size, largest_layer = calculate_maximum_sizes(model)
|
||||
data = []
|
||||
|
||||
for dtype in ["float32", "float16", "int8", "int4"]:
|
||||
dtype_total_size = total_size
|
||||
dtype_largest_layer = largest_layer[0]
|
||||
if dtype == "float32":
|
||||
dtype_total_size *= 2
|
||||
dtype_largest_layer *= 2
|
||||
elif dtype == "float16":
|
||||
pass
|
||||
elif dtype == "int8":
|
||||
dtype_total_size /= 2
|
||||
dtype_largest_layer /= 2
|
||||
elif dtype == "int4":
|
||||
dtype_total_size /= 4
|
||||
dtype_largest_layer /= 4
|
||||
row = [dtype, dtype_largest_layer, dtype_total_size]
|
||||
for i, item in enumerate(row):
|
||||
if isinstance(item, (int, float)):
|
||||
row[i] = convert_bytes(item)
|
||||
elif isinstance(item, dict):
|
||||
training_usage = max(item.values())
|
||||
row[i] = (
|
||||
convert_bytes(training_usage) if training_usage != -1 else "N/A"
|
||||
)
|
||||
data.append(row)
|
||||
|
||||
headers = ["dtype", "Largest Layer", "Total Size"]
|
||||
title = f"Memory Usage for loading `{model_name}`"
|
||||
table = create_ascii_table(headers, data, title)
|
||||
print(table)
|
||||
|
||||
|
||||
def _get_args() -> Namespace:
|
||||
"""命令行参数解析为命名空间(可以看作可以用.来访问的字典)"""
|
||||
parser = ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
default=DEFAULT_CKPT_PATH,
|
||||
help="模型路径, 默认为%(default)r。",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cpu",
|
||||
action="store_true",
|
||||
help="仅CPU模式运行。(不启用则默认平均分到所有显卡上)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
default="auto",
|
||||
choices=["auto", "fp32", "fp16", "bf16"],
|
||||
help="加载特定类型的模型。(不启用则默认`auto`, 从config获取。其他类型请自己修改。)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cal-size",
|
||||
action="store_true",
|
||||
help="仅输出模型显存占用。(默认输出float32、float16、int8、int4的占用)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--flash-attn2",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="使用 `flash_attention_2` 推理。(不启用则根据环境使用eager或sdpa)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=12345, help="Demo服务器端口, 默认为`12345`。"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host", default="127.0.0.1", help="DDemo服务器地址, 默认为`127.0.0.1`。"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
class LazyModelLoader:
|
||||
"""延迟加载模型以达到快速显示页面的目的
|
||||
|
||||
延迟加载需要用到多线程,主线程执行web页面的时候,用子线程去加载模型,只需要记录好模型的引用对象即可.
|
||||
(利用延迟加载,主线程中不加载而是放到子线程中,这样而等到页面渲染好,
|
||||
用户输入完提问后,取出模型做推理时,子线程已经加载好模型.)
|
||||
"""
|
||||
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.model = None
|
||||
self.proc = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""加载模型和processor"""
|
||||
with self.lock:
|
||||
if self.model is None: # 确保模型只加载一次
|
||||
print(f"Loading model: {self.args.model_path}")
|
||||
try:
|
||||
model, proc = self._load_model_processor()
|
||||
# model不一定是存有dtype变量的nn.Module类,
|
||||
# 因此可以用这个函数来快速获取里面第一个参数的dtype。
|
||||
dtype = get_first_parameter_dtype(model)
|
||||
except Exception:
|
||||
self.lock.release()
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
exit(-1)
|
||||
self.model = model
|
||||
self.proc = proc
|
||||
print(f"Model {self.args.model_path} loaded")
|
||||
print(f"{model.device=} model.dtype={dtype}")
|
||||
|
||||
def _load_model_processor(
|
||||
self,
|
||||
) -> tuple[Qwen2VLForConditionalGeneration, Qwen2VLProcessor]:
|
||||
"""Qwen2-vl 加载模型时需要加载两个东西:
|
||||
1. 模型, 对应Qwen2VLForConditionalGeneration类
|
||||
2. processor(一个对图片和文本进行处理,转换为模型输入的预处理工具),对应AutoProcessor类
|
||||
|
||||
借助from_pretrained方法,我们可以在加载模型,预处理器时自动处理某些步骤(比如一般加载模型的流程是:初始化->从文件中加载权重并复制到初始化后的类中)而直接返回结果.
|
||||
"""
|
||||
args = self.args
|
||||
device_map = "cpu" if args.cpu else "auto"
|
||||
use_fa2 = (
|
||||
"flash_attention_2"
|
||||
if args.flash_attn2 and is_flash_attn_2_available()
|
||||
else None
|
||||
)
|
||||
dtype = (
|
||||
{
|
||||
"fp16": torch.float16,
|
||||
"fp32": torch.float32,
|
||||
"bf16": torch.bfloat16,
|
||||
"int4": "auto",
|
||||
"int8": "auto", # 不提供量化，自己改吧
|
||||
}[args.dtype]
|
||||
if args.dtype != "auto" # auto会采用config中的配置
|
||||
else args.dtype
|
||||
)
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
args.model_path,
|
||||
torch_dtype=dtype,
|
||||
# 支持: eager/flash_attention_2/sdpa
|
||||
attn_implementation=use_fa2,
|
||||
# auto: 平均分配到每个 GPU.
|
||||
device_map=device_map,
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(args.model_path)
|
||||
return model, processor
|
||||
|
||||
def get_model(self) -> Qwen2VLForConditionalGeneration:
|
||||
"""获取加载的模型，若尚未加载则触发加载"""
|
||||
if self.model is None:
|
||||
threading.Thread(target=self._load_model).start()
|
||||
return self.model
|
||||
|
||||
def get_processor(self) -> Qwen2VLProcessor:
|
||||
"""获取加载的processor"""
|
||||
if self.proc is None:
|
||||
threading.Thread(target=self._load_model).start()
|
||||
return self.proc
|
||||
|
||||
|
||||
def _transform_messages(
|
||||
messages: List[List[str | Tuple[str, ...]]],
|
||||
video_extensions=VIDEO_EXTENSIONS,
|
||||
image_extensions=IMAGE_EXTENSIONS,
|
||||
user_tag="user",
|
||||
assistant_tag="assistant",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""gradio的messages格式与qwen2的conversation不一致,需要转换
|
||||
|
||||
模型的问答是按轮次来划分的:
|
||||
第一轮: <提问>-><回答>-> 第二轮: <提问>-><回答>-> ...
|
||||
(即便是加入文件,也是放在提问里面.)
|
||||
具体来说:
|
||||
1. gradio目前有多种`对话`的处理格式, 本代码中采用的格式为:
|
||||
[
|
||||
[(<文件1>,<文件2>, ...), None], # 如果传入文件,那么没有对应回答,如果这一行是文件,那么下一行跟用户提问
|
||||
[<提问>, <回答>], # 注意,和上面的区别是提问是一个字符串,而上一行同样位置是一个存储文件的tuple.
|
||||
[("xxx1.jpg","xxx2.jpg"), None],
|
||||
["描述下这两张图片", "这张图片xxx"],
|
||||
...
|
||||
]
|
||||
2. Qwen中的格式采用:
|
||||
[
|
||||
# 这里角色可以包括: system, user, assistant, 内容则是对应角色的提问或回答.
|
||||
{"role":<角色>, "content":<内容>},
|
||||
# 针对图片和视频的传输, Qwen2-vl 在 user 的 <内容> 部分会进一步处理, 因此我们可以将这两类文件放到其 <内容> 中:
|
||||
{"role":"user", "content":"你是谁?"}, # 纯文字
|
||||
{"role":"user", "content":[{"type":"image", "image": "xxx.jpg"}, {"type":"text", "text": "这张图里有什么?"}]}, # 图片+文字
|
||||
{"role":"user", "content":[{"type":"video", "video": "xxx.mp4"}, {"type":"text", "text": "这个视频讲了什么?"}]}, # 视频+文字
|
||||
...
|
||||
]
|
||||
(值得注意的是, 对视频或图片的token限制也可以加在content里面. 可以参考下面的处理)
|
||||
3. 发现了吗,上面两种对话格式不统一,因此送入模型的预处理器前还需要做一次处理,将gradio格式转为qwen预处理支持的格式.而gradio中文件和提问是放在多个列表里的,对话轮次的切换仅通过回答是否是None来判断.
|
||||
"""
|
||||
transformed_messages = [{"role": user_tag, "content": []}]
|
||||
for message in messages:
|
||||
q = message[0]
|
||||
if isinstance(q, tuple):
|
||||
for it in q:
|
||||
if Path(it).suffix in video_extensions:
|
||||
new_item = {
|
||||
"type": "video",
|
||||
"video": it,
|
||||
"min_pixels": VIDEO_MIN_PIXELS,
|
||||
"max_pixels": VIDEO_MAX_PIXELS,
|
||||
"total_pixels": VIDEO_TOTAL_PIXELS,
|
||||
}
|
||||
elif Path(it).suffix in image_extensions:
|
||||
new_item = {
|
||||
"type": "image",
|
||||
"image": it,
|
||||
"min_pixels": MIN_PIXELS,
|
||||
"max_pixels": MAX_PIXELS,
|
||||
}
|
||||
transformed_messages[-1]["content"].append(new_item)
|
||||
elif isinstance(q, str):
|
||||
if transformed_messages[-1]["content"]:
|
||||
new_item = {"type": "text", "text": it}
|
||||
transformed_messages[-1]["content"].append(new_item)
|
||||
else:
|
||||
transformed_messages[-1]["content"] = q
|
||||
|
||||
if message[1]: # 如果回答里有值，说明当前轮对话完成，接下来做下一轮对话的处理。
|
||||
transformed_messages.extend(
|
||||
[
|
||||
{"role": assistant_tag, "content": message[1]},
|
||||
{"role": user_tag, "content": []},
|
||||
]
|
||||
)
|
||||
return transformed_messages
|
||||
|
||||
|
||||
def _gc():
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
# modify from https://github.com/gradio-app/gradio/blob/4e1f7dbcb2ea2a0cc29bb76faf5758a9f4afcd6d/demo/chatbot_examples/run.py#L1, 参考这里可以看到gradio给出的带文件传输的chatbot实现
|
||||
def print_like_dislike(x: gr.LikeData) -> None:
|
||||
print(f"{x.index=} {x.value=}{x.liked=}")
|
||||
|
||||
|
||||
def add_message(
|
||||
history: List[List[str | Tuple[str, ...]]], message: Dict
|
||||
) -> tuple[List[List[str | Tuple[str, ...]]], gr.MultimodalTextbox]:
|
||||
"""
|
||||
Params:
|
||||
history: gradio的一种对话格式, 可以参考 `_transform_messages` 的文档注释.
|
||||
message: gr.MultimodalTextbox类, 可以当作字典访问,里面有file和text,分别表示提供的文件和提问.
|
||||
"""
|
||||
for x in message["files"]:
|
||||
history.append(((x,), None))
|
||||
if message["text"] is not None:
|
||||
history.append(
|
||||
(message["text"], None)
|
||||
) # 这里填空是因为还需要把history数据转换后给模型进行回复，然后才能赋值到这里。
|
||||
return history, gr.MultimodalTextbox(value=None, interactive=False)
|
||||
|
||||
|
||||
def _pred(
|
||||
messages: List[List[str | Tuple[str, ...]]],
|
||||
temperature: float,
|
||||
topk: int,
|
||||
topp: float,
|
||||
processor: Qwen2VLProcessor,
|
||||
model: Qwen2VLForConditionalGeneration,
|
||||
):
|
||||
"""模型对话的主要逻辑, 这段代码参考了Qwen2-vl官方的 web demo的一部分流程.
|
||||
先转换出qwen2-vl需要的格式
|
||||
然后将文本和图像/视频分别送入预处理器(在此之前,图像/视频要借助官方提供的process_vision_info函数resize为28*28的倍数)
|
||||
然后送入模型进行推理,模型推理的结果作为回答."""
|
||||
messages = _transform_messages(messages)
|
||||
|
||||
# 这里首先把messages对话格式转为纯文本的特殊格式
|
||||
text = processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
# 这里对图片/视频做resize处理，主要是模型内视觉层对图片的宽高有特定限制。
|
||||
image_inputs, video_inputs = process_vision_info(messages)
|
||||
# 开始通过预处理器， 将文本和图片/视频作为输入， 处理出模型需要的数据: token_id列表 和 特定形状的一堆像素点
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to(model.device) # 结果送入模型所在的设备（CPU或某GPU卡）
|
||||
|
||||
streamer = TextIteratorStreamer(
|
||||
processor.tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True
|
||||
) # 借助TextIteratorStreamer可以提供一个流式的接口，是的模型每生成一个token就返回这个token对应的文本。
|
||||
|
||||
# 模型在生成token前有一个后处理，这里简单介绍贪心解码和采样解码：
|
||||
# 当使用贪心解码时，设置do_sample = False, 对于下一个token，模型总会选择预测的概率最大的那个。
|
||||
# 当使用采样时，字如其名，就是随机的选择。首先会对输出的下一个token的概率分布做一些简单变换（比如temperature越大，可以让概率分布越平均， topK和topP则减小待采样的词表），然后对剩余的词表进行加权的随机选择（因为加权，所以概率大的还是有大的机率被选中，但是如果temperature设置过大，反而把剩余所有词表的概率平均化了，这样大家的权重都接近1:1）
|
||||
# 因此也可以说，temperature控制模型的创造性，越大，模型采样到不同词的可能越大，模型的回答便越发散。
|
||||
_gen_kwargs = (
|
||||
dict(temperature=temperature, top_p=topp, top_k=topk)
|
||||
if temperature
|
||||
else dict(do_sample=False)
|
||||
)
|
||||
# max_new_tokens主要限制模型回答的最大token长度，当超过这个token就会停止。
|
||||
gen_config = GenerationConfig(max_new_tokens=512, **_gen_kwargs)
|
||||
|
||||
# 使用子线程启动模型的推理，结果会自动添加到streamer接口中。
|
||||
thread = Thread(
|
||||
target=model.generate,
|
||||
kwargs=dict(
|
||||
**inputs,
|
||||
generation_config=gen_config,
|
||||
streamer=streamer,
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return streamer
|
||||
|
||||
|
||||
def bot(
|
||||
history: List[List[str | Tuple[str, ...]]],
|
||||
temperature: float,
|
||||
topk: int,
|
||||
topp: float,
|
||||
) -> Generator[List[List[str | Tuple[str, ...]]], Any, None]:
|
||||
"""这里是输入提问并点击提交后触发回答的逻辑"""
|
||||
_gc() # 可以清除一下上一次回答的存储碎片
|
||||
# 然后将提问与之前轮次的对话送入_pred让模型针对这些上文进行推理
|
||||
model, proc = loader.get_model(), loader.get_processor()
|
||||
# 这里会返回一个流式的接口,通过for循环即可获取接口里新添加进去的回答,然后拼接到history里流式的返回给gradio即可.
|
||||
stream = _pred(history, temperature, topk, topp, processor=proc, model=model)
|
||||
history[-1][1] = ""
|
||||
for it in stream:
|
||||
history[-1][1] += it
|
||||
yield history
|
||||
|
||||
|
||||
def web_demo(args: Namespace):
|
||||
"""创建gradio应用程序"""
|
||||
with gr.Blocks(fill_height=True) as demo:
|
||||
with gr.Column(scale=6):
|
||||
chatbot = gr.Chatbot(
|
||||
label="Qwen2VL demo",
|
||||
elem_id="chatbot",
|
||||
bubble_full_width=False,
|
||||
scale=1,
|
||||
type="tuples",
|
||||
)
|
||||
chat_input = gr.MultimodalTextbox(
|
||||
interactive=True,
|
||||
file_count="multiple",
|
||||
placeholder="Enter message or upload file...",
|
||||
show_label=False,
|
||||
)
|
||||
with gr.Column(scale=1):
|
||||
with gr.Accordion("Gen Config", open=False):
|
||||
# 一个隐藏的选项，可以控制 Temperature、top p、top k
|
||||
temperature = gr.Slider(0.0, 1.0, step=0.01, label="Temperature")
|
||||
topk = gr.Slider(-1, 1000, step=2, label="Top K") # need?
|
||||
topp = gr.Slider(0.0, 1.0, step=0.01, label="Top P") # need?
|
||||
# 多模态的输入会先调用 add_message，然后调用 bot，最后清除输入框中的内容（因为已经显示在chatbot里了）
|
||||
chat_msg = chat_input.submit(
|
||||
add_message, [chatbot, chat_input], [chatbot, chat_input]
|
||||
)
|
||||
bot_msg = chat_msg.then(
|
||||
bot, [chatbot, temperature, topk, topp], chatbot, api_name="bot_response"
|
||||
)
|
||||
bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
|
||||
|
||||
# 这里主要是给chatbot的每个回答绑定一个用户偏好反馈的结果打印
|
||||
chatbot.like(print_like_dislike, None, None)
|
||||
demo.launch(max_threads=2, server_name=args.host, server_port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = _get_args()
|
||||
# 在这里检测flash-attn是否安装和启用
|
||||
print("flash-attn 已安装" if is_flash_attn_2_available() else "flash-attn 未安装")
|
||||
print(
|
||||
"flash-attn 已启用。"
|
||||
if args.flash_attn2 and is_flash_attn_2_available()
|
||||
else "flash-attn 未启用。"
|
||||
)
|
||||
cal_model_size(args) # 在每次启动模型时会先显示模型占用
|
||||
if args.cal_size is False:
|
||||
loader = LazyModelLoader(args)
|
||||
loader.get_model() # 在这里手动提前触发一下模型加载
|
||||
web_demo(args) # 运行web demo
|
||||
@@ -0,0 +1,10 @@
|
||||
# requirements.txt
|
||||
qwen_vl_utils==0.0.8
|
||||
transformers==4.46.2
|
||||
accelerate==1.1.1
|
||||
gradio==5.5.0
|
||||
torchvision==0.19.0
|
||||
modelscope==1.20.0
|
||||
# # 如果安装了flash-attn，则会多出这两个库
|
||||
# einops==0.8.0
|
||||
# flash-attn==2.7.0
|
||||
@@ -0,0 +1,96 @@
|
||||
# Qwen2-VL-2B-Instruct WebDemo 部署
|
||||
|
||||
|
||||
# 环境准备
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.10
|
||||
cuda 11.8
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
|
||||
# 环境安装
|
||||
|
||||
```python
|
||||
# 换源
|
||||
python -m pip install --upgrade pip
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
|
||||
# 需要安装的库
|
||||
# torchvision需要安装匹配对应torch的版本
|
||||
pip install qwen_vl_utils==0.0.8 transformers==4.46.2 accelerate==1.1.0 gradio==5.5.0 torchvision==0.18.0 av==13.1.0
|
||||
|
||||
# 如需使用魔搭(国内推荐)下载模型, 需安装这个库
|
||||
pip install modelscope==1.20.0
|
||||
|
||||
# 安装flash-attn(可选)
|
||||
# 如显卡支持flash-attn，在确认对应python、pytorch、cuda版本后, 下载对应的release版本.
|
||||
wegt https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
# 镜像加速链接:
|
||||
# wget https://github.moeyy.xyz/https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
pip install flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
```
|
||||
> 完整的pip列表(包含依赖)请参考[02-Qwen2-VL-2B-Instruct Web Demo 参考代码/requirements.txt](./02-Qwen2-VL-2B-Instruct%20Web%20Demo%20参考代码/requirements.txt)
|
||||
|
||||
# 下载模型(两种下载方法二选一即可~)
|
||||
## 1. 借助 modelscope 下载
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
新建 `model_download.py` 文件输入以下代码，并运行 `python model_download.py` 执行下载。
|
||||
|
||||
此处使用 `modelscope` 提供的 `snapshot_download` 函数进行下载，该方法对国内的用户十分友好。
|
||||
|
||||
```python
|
||||
# model_download.py
|
||||
from modelscope import snapshot_download
|
||||
model_dir = snapshot_download('Qwen/Qwen2-VL-2B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：请记得修改 `cache_dir` 为你自己的模型下载路径 ~
|
||||
|
||||
## 2. 借助 git lfs 下载
|
||||
```python
|
||||
# 进入autodl-tmp/ 或者你要保存的路径
|
||||
cd autodl-tmp/
|
||||
|
||||
# 首先安装lfs，便于通过git直接下载模型。
|
||||
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
|
||||
sudo apt-get install git-lfs
|
||||
|
||||
# 需要下载的模型
|
||||
MODEL=Qwen2-VL-2B-Instruct
|
||||
# MODEL=Qwen2-VL-7B-Instruct
|
||||
# MODEL=Qwen2-VL-72B-Instruct
|
||||
|
||||
# # huggingface 下载
|
||||
# URL="https://huggingface.co/Qwen/"
|
||||
# git clone "${URL}/${MODEL}"
|
||||
|
||||
# 魔搭下载（国内推荐）
|
||||
URL="https://www.modelscope.cn/Qwen"
|
||||
git clone "${URL}/${MODEL}.git"
|
||||
|
||||
# 返回根目录
|
||||
cd ..
|
||||
```
|
||||
|
||||
# 运行Demo
|
||||
|
||||
```python
|
||||
# 可以使用 python mm_qwen2vl.py -h 或查看代码来查看命令帮助
|
||||
# Ampere/Ada/Hopper架构显卡可以启用flash attn2加速推理，autodl要通过6006端口对外访问。(没安装flash-attn库的忽略)
|
||||
# python mm_qwen2vl.py --flash-attn2 --model-path ./autodl-tmp/Qwen2-VL-2B-Instruct --host 0.0.0.0 --port 6006
|
||||
python mm_qwen2vl.py --model-path ./autodl-tmp/Qwen2-VL-2B-Instruct --host 0.0.0.0 --port 6006
|
||||
```
|
||||
> 完整代码及详细注释请参考[mm_qwen2vl.py](./02-Qwen2-VL-2B-Instruct%20Web%20Demo%20参考代码/mm_qwen2vl.py)
|
||||
|
||||
# 测试效果
|
||||
## 图片
|
||||
|
||||
## 视频
|
||||
|
||||
|
||||
> 如果觉得2B理解能力较差, 建议用7B以上模型.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 384 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 289 KiB |
@@ -0,0 +1,297 @@
|
||||
# Qwen2.5-7B-Instruct FastApi 部署调用
|
||||
|
||||
## 环境准备
|
||||
|
||||
本文基础环境如下：
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.12
|
||||
cuda 12.1
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
> 本文默认学习者已安装好以上 Pytorch(cuda) 环境，如未安装请自行安装。
|
||||
|
||||
首先 `pip` 换源加速下载并安装依赖包
|
||||
|
||||
```shell
|
||||
# 升级pip
|
||||
python -m pip install --upgrade pip
|
||||
# 更换 pypi 源加速库的安装
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install requests==2.31.0
|
||||
pip install fastapi==0.115.1
|
||||
pip install uvicorn==0.30.6
|
||||
pip install transformers==4.46.2
|
||||
pip install huggingface-hub==0.26.2
|
||||
pip install accelerate==0.34.2
|
||||
pip install modelscope==1.20.0
|
||||
```
|
||||
|
||||
> 考虑到部分同学配置环境可能会遇到一些问题，我们在AutoDL平台准备了Qwen2.5的环境镜像，点击下方链接并直接创建Autodl示例即可。
|
||||
> ***https://www.codewithgpu.com/i/datawhalechina/self-llm/Qwen2.5-Coder-fastapi-self-llm***
|
||||
|
||||
|
||||
## 模型下载
|
||||
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
新建 `model_download.py` 文件并在其中输入以下内容，粘贴代码后请及时保存文件，如下图所示。并运行 `python model_download.py` 执行下载，模型大小为 15GB，下载模型大概需要 5 分钟。
|
||||
|
||||
```python
|
||||
import torch
|
||||
from modelscope import snapshot_download, AutoModel, AutoTokenizer
|
||||
import os
|
||||
model_dir = snapshot_download('qwen/Qwen2.5-Coder-7B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：记得修改 `cache_dir` 为你的模型下载路径哦~
|
||||
|
||||
## 代码准备
|
||||
|
||||
新建 `api.py` 文件并在其中输入以下内容，粘贴代码后请及时保存文件。以下代码有很详细的注释，大家如有不理解的地方，欢迎提出 issue 。
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI, Request
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
|
||||
import uvicorn
|
||||
import json
|
||||
import datetime
|
||||
import torch
|
||||
|
||||
# 设置设备参数
|
||||
DEVICE = "cuda" # 使用CUDA
|
||||
DEVICE_ID = "0" # CUDA设备ID，如果未设置则为空
|
||||
CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE # 组合CUDA设备信息
|
||||
|
||||
# 清理GPU内存函数
|
||||
def torch_gc():
|
||||
if torch.cuda.is_available(): # 检查是否可用CUDA
|
||||
with torch.cuda.device(CUDA_DEVICE): # 指定CUDA设备
|
||||
torch.cuda.empty_cache() # 清空CUDA缓存
|
||||
torch.cuda.ipc_collect() # 收集CUDA内存碎片
|
||||
|
||||
# 创建FastAPI应用
|
||||
app = FastAPI()
|
||||
|
||||
# 处理POST请求的端点
|
||||
@app.post("/")
|
||||
async def create_item(request: Request):
|
||||
global model, tokenizer # 声明全局变量以便在函数内部使用模型和分词器
|
||||
json_post_raw = await request.json() # 获取POST请求的JSON数据
|
||||
json_post = json.dumps(json_post_raw) # 将JSON数据转换为字符串
|
||||
json_post_list = json.loads(json_post) # 将字符串转换为Python对象
|
||||
prompt = json_post_list.get('prompt') # 获取请求中的提示
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
|
||||
# 调用模型进行对话生成
|
||||
input_ids = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True)
|
||||
model_inputs = tokenizer([input_ids], return_tensors="pt").to('cuda')
|
||||
generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=512)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
now = datetime.datetime.now() # 获取当前时间
|
||||
time = now.strftime("%Y-%m-%d %H:%M:%S") # 格式化时间为字符串
|
||||
# 构建响应JSON
|
||||
answer = {
|
||||
"response": response,
|
||||
"status": 200,
|
||||
"time": time
|
||||
}
|
||||
# 构建日志信息
|
||||
log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(response) + '"'
|
||||
print(log) # 打印日志
|
||||
torch_gc() # 执行GPU内存清理
|
||||
return answer # 返回响应
|
||||
|
||||
# 主函数入口
|
||||
if __name__ == '__main__':
|
||||
# 加载预训练的分词器和模型
|
||||
model_name_or_path = '/root/autodl-tmp/qwen/Qwen2___5-Coder-7B-Instruct'
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=False)
|
||||
model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="auto", torch_dtype=torch.bfloat16)
|
||||
|
||||
# 启动FastAPI应用
|
||||
# 用6006端口可以将autodl的端口映射到本地，从而在本地使用api
|
||||
uvicorn.run(app, host='0.0.0.0', port=6006, workers=1) # 在指定端口和主机上启动应用
|
||||
```
|
||||
|
||||
> 注意：记得修改 `model_name_or_path` 为你的模型下载路径哦~
|
||||
|
||||
## Api 部署
|
||||
|
||||
在终端输入以下命令启动api服务：
|
||||
|
||||
```shell
|
||||
cd /root/autodl-tmp
|
||||
python api.py
|
||||
# or
|
||||
python /root/autodl-tmp/api.py
|
||||
```
|
||||
|
||||
加载完毕后出现如下信息说明成功。
|
||||
|
||||
|
||||
|
||||
默认部署在 6006 端口，通过 POST 方法进行调用，可以使用 curl 调用，如下所示：
|
||||
|
||||
```shell
|
||||
curl -X POST "http://127.0.0.1:6006" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"prompt": "帮我生成一份深度学习网络训练的python代码。"}'
|
||||
```
|
||||
|
||||
得到的返回值如下所示：
|
||||
```json
|
||||
{"response":" 当然可以！以下是一个使用TensorFlow和Keras构建和训练简单卷积神经网络（CNN）的Python代码示例。这个示例使用了MNIST数据集，这是一个手写数字识别的数据集。\n\n```python\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n\n# 加载MNIST数据集\n(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n\n# 归一化像素值到0-1之间\ntrain_images, test_images = train_images / 255.0, test_images / 255.0\n\n# 将标签转换为one-hot编码\ntrain_labels = tf.keras.utils.to_categorical(train_labels)\ntest_labels = tf.keras.utils.to_categorical(test_labels)\n\n# 构建卷积神经网络模型\nmodel = models.Sequential([\n layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n layers.MaxPooling2D((2, 2)),\n layers.Conv2D(64, (3, 3), activation='relu'),\n layers.MaxPooling2D((2, 2)),\n layers.Conv2D(64, (3, 3), activation='relu'),\n layers.Flatten(),\n layers.Dense(64, activation='relu'),\n layers.Dense(10, activation='softmax')\n])\n\n# 编译模型\nmodel.compile(optimizer='adam',\n loss='categorical_crossentropy',\n metrics=['accuracy'])\n\n# 训练模型\nhistory = model.fit(train_images.reshape(-1, 28, 28, 1), train_labels,\n epochs=5,\n validation_data=(test_images.reshape(-1, 28, 28, 1), test_labels))\n\n# 评估模型\ntest_loss, test_acc = model.evaluate(test_images.reshape(-1, 28, 28, 1), test_labels, verbose=2)\nprint(f'\\nTest accuracy: {test_acc}')\n\n# 绘制训练过程中的准确率和损失\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.title","status":200,"time":"2024-11-15 13:47:30"}
|
||||
```
|
||||
|
||||
对应的python代码：
|
||||
```python
|
||||
import tensorflow as tf
|
||||
from tensorflow.keras import layers, models
|
||||
|
||||
# 加载MNIST数据集
|
||||
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
|
||||
|
||||
# 归一化像素值到0-1之间
|
||||
train_images, test_images = train_images / 255.0, test_images / 255.0
|
||||
|
||||
# 将标签转换为one-hot编码
|
||||
train_labels = tf.keras.utils.to_categorical(train_labels)
|
||||
test_labels = tf.keras.utils.to_categorical(test_labels)
|
||||
|
||||
# 构建卷积神经网络模型
|
||||
model = models.Sequential([
|
||||
layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
|
||||
layers.MaxPooling2D((2, 2)),
|
||||
layers.Conv2D(64, (3, 3), activation='relu'),
|
||||
layers.MaxPooling2D((2, 2)),
|
||||
layers.Conv2D(64, (3, 3), activation='relu'),
|
||||
layers.Flatten(),
|
||||
layers.Dense(64, activation='relu'),
|
||||
layers.Dense(10, activation='softmax')
|
||||
])
|
||||
|
||||
# 编译模型
|
||||
model.compile(optimizer='adam',
|
||||
loss='categorical_crossentropy',
|
||||
metrics=['accuracy'])
|
||||
|
||||
# 训练模型
|
||||
history = model.fit(train_images.reshape(-1, 28, 28, 1), train_labels,
|
||||
epochs=5,
|
||||
validation_data=(test_images.reshape(-1, 28, 28, 1), test_labels))
|
||||
|
||||
# 评估模型
|
||||
test_loss, test_acc = model.evaluate(test_images.reshape(-1, 28, 28, 1), test_labels, verbose=2)
|
||||
print(f'\nTest accuracy: {test_acc}')
|
||||
|
||||
# 绘制训练过程中的准确率和损失
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
plt.figure(figsize=(12, 4))
|
||||
plt.subplot(1, 2, 1)
|
||||
plt.plot(history.history['accuracy'], label='Training Accuracy')
|
||||
plt.plot(history.history['val_accuracy'], label='Validation Accuracy')
|
||||
plt.xlabel('Epochs')
|
||||
plt.ylabel('Accuracy')
|
||||
plt.legend()
|
||||
plt.title
|
||||
```
|
||||
|
||||
|
||||
|
||||
也可以使用 python 中的 requests 库进行调用，如下所示：
|
||||
|
||||
```python
|
||||
import requests
|
||||
import json
|
||||
|
||||
def get_completion(prompt):
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
data = {"prompt": prompt}
|
||||
response = requests.post(url='http://127.0.0.1:6006', headers=headers, data=json.dumps(data))
|
||||
return response.json()['response']
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(get_completion('帮我生成一份深度学习网络训练的python代码(pytorch)。'))
|
||||
```
|
||||
|
||||
得到的返回值如下所示：
|
||||
|
||||
当然可以！下面是一个使用PyTorch进行深度学习网络训练的示例代码。这个示例使用了一个简单的卷积神经网络（CNN）来分类MNIST数据集。
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
from torchvision import datasets, transforms
|
||||
|
||||
# 定义超参数
|
||||
batch_size = 64
|
||||
learning_rate = 0.001
|
||||
num_epochs = 5
|
||||
|
||||
# 数据预处理
|
||||
transform = transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.5,), (0.5,))
|
||||
])
|
||||
|
||||
# 加载MNIST数据集
|
||||
train_dataset = datasets.MNIST(root='./data', train=True, transform=transform, download=True)
|
||||
test_dataset = datasets.MNIST(root='./data', train=False, transform=transform)
|
||||
|
||||
# 创建数据加载器
|
||||
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
|
||||
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)
|
||||
|
||||
# 定义卷积神经网络模型
|
||||
class CNN(nn.Module):
|
||||
def __init__(self):
|
||||
super(CNN, self).__init__()
|
||||
self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)
|
||||
self.relu = nn.ReLU()
|
||||
self.pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
|
||||
self.fc1 = nn.Linear(32 * 14 * 14, 128)
|
||||
self.fc2 = nn.Linear(128, 10)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.conv1(x)
|
||||
out = self.relu(out)
|
||||
out = self.pool(out)
|
||||
out = out.view(out.size(0), -1)
|
||||
out = self.fc1(out)
|
||||
out = self.relu(out)
|
||||
out = self.fc2(out)
|
||||
return out
|
||||
|
||||
model = CNN()
|
||||
|
||||
# 定义损失函数和优化器
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
|
||||
|
||||
# 训练模型
|
||||
for epoch in range(num_epochs):
|
||||
model.train()
|
||||
for i, (images, labels) in enumerate(train_loader):
|
||||
# 前向传播
|
||||
outputs = model(images)
|
||||
loss = criterion(outputs, labels)
|
||||
|
||||
# 反向传播和优化
|
||||
```
|
||||
这里代码不完全是因为设置了max_new_tokens=512。
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
# Qwen2.5-7B-Instruct Langchain 接入
|
||||
|
||||
## 环境准备
|
||||
|
||||
本文基础环境如下：
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.12
|
||||
cuda 12.1
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
|
||||
> 本文默认学习者已安装好以上 Pytorch(cuda) 环境，如未安装请自行安装。
|
||||
|
||||
pip 换源加速下载并安装依赖包
|
||||
|
||||
```shell
|
||||
# 升级pip
|
||||
python -m pip install --upgrade pip
|
||||
# 更换 pypi 源加速库的安装
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install transformers==4.46.2
|
||||
pip install modelscope==1.20.0
|
||||
pip install langchain==0.3.7
|
||||
pip install accelerate==1.1.1
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 模型下载
|
||||
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
在新建 `model_download.py` 文件并在其中输入以下内容，粘贴代码后记得保存文件，如下图所示。并运行 `python model_download.py` 执行下载，模型大小为 16 GB，下载模型大概需要 12 分钟。
|
||||
|
||||
```python
|
||||
import torch
|
||||
from modelscope import snapshot_download, AutoModel, AutoTokenizer
|
||||
import os
|
||||
model_dir = snapshot_download('Qwen/Qwen2.5-Coder-7B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：记得修改 `cache_dir` 为你的模型下载路径哦~
|
||||
|
||||
## 代码准备
|
||||
|
||||
为便捷构建 `LLM` 应用，我们需要基于本地部署的 `Qwen2_5_Coder`，自定义一个 `LLM` 类，将 `Qwen2.5-Coder` 接入到 `LangChain` 框架中。完成自定义 `LLM` 类之后，可以以完全一致的方式调用 `LangChain` 的接口，而无需考虑底层模型调用的不一致。
|
||||
|
||||
基于本地部署的 `Qwen2_5_Coder` 自定义 `LLM` 类并不复杂，我们只需从 `LangChain.llms.base.LLM` 类继承一个子类，并重写构造函数与 `_call` 函数即可：
|
||||
|
||||
在当前路径新建一个 `LLM.py` 文件，并输入以下内容，粘贴代码后记得保存文件。
|
||||
|
||||
```python
|
||||
from langchain.llms.base import LLM
|
||||
from typing import Any, List, Optional
|
||||
from langchain.callbacks.manager import CallbackManagerForLLMRun
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig, LlamaTokenizerFast
|
||||
import torch
|
||||
|
||||
class Qwen2_5_Coder(LLM):
|
||||
# 基于本地 Qwen2_5-Coder 自定义 LLM 类
|
||||
tokenizer: AutoTokenizer = None
|
||||
model: AutoModelForCausalLM = None
|
||||
def __init__(self, mode_name_or_path :str):
|
||||
|
||||
super().__init__()
|
||||
print("正在从本地加载模型...")
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(mode_name_or_path, use_fast=False)
|
||||
self.model = AutoModelForCausalLM.from_pretrained(mode_name_or_path, torch_dtype=torch.bfloat16, device_map="auto")
|
||||
self.model.generation_config = GenerationConfig.from_pretrained(mode_name_or_path)
|
||||
print("完成本地模型的加载")
|
||||
|
||||
def _call(self, prompt : str, stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any):
|
||||
|
||||
messages = [{"role": "user", "content": prompt }]
|
||||
input_ids = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
model_inputs = self.tokenizer([input_ids], return_tensors="pt").to('cuda')
|
||||
generated_ids = self.model.generate(model_inputs.input_ids, attention_mask=model_inputs['attention_mask'], max_new_tokens=512)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
return response
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "Qwen2_5_Coder"
|
||||
```
|
||||
|
||||
在上述类定义中，我们分别重写了构造函数和 `_call` 函数：对于构造函数，我们在对象实例化的一开始加载本地部署的 `Qwen2_5_Coder` `模型，从而避免每一次调用都需要重新加载模型带来的时间过长；_call` 函数是 `LLM` 类的核心函数，`LangChain` 会调用该函数来调用 `LLM`，在该函数中，我们调用已实例化模型的 `generate` 方法，从而实现对模型的调用并返回调用结果。
|
||||
|
||||
在整体项目中，我们将上述代码封装为 `LLM.py`，后续将直接从该文件中引入自定义的 LLM 类。
|
||||
|
||||
## 调用
|
||||
|
||||
然后就可以像使用任何其他的langchain大模型功能一样使用了。
|
||||
|
||||
> 注意：记得修改模型路径为你的路径哦~
|
||||
|
||||
```python
|
||||
from LLM import Qwen2_5_Coder
|
||||
llm = Qwen2_5_Coder(mode_name_or_path = "autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct")
|
||||
print(llm.invoke("你是谁"))
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
|
||||
既然是Coder模型，当然要试着让它编写代码
|
||||
|
||||
```python
|
||||
text = llm.invoke("为我用python写一个简单的猜拳小游戏，三局两胜")
|
||||
print(text)
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
我们试着运行一下这段代码：
|
||||
|
||||
成功运行！
|
||||
@@ -0,0 +1,217 @@
|
||||
# Qwen2.5-Coder-7B-Instruct Lora 微调
|
||||
|
||||
本节我们简要介绍如何基于 transformers、peft 等框架，对Qwen2.5-Coder-7B-Instruct 模型进行 Lora 微调。Lora 是一种高效微调方法，深入了解其原理可参见博客：[知乎|深入浅出Lora](https://zhuanlan.zhihu.com/p/650197598)。
|
||||
|
||||
|
||||
这个教程会在同目录下给大家提供一个 [nodebook](./05-Qwen2-7B-Instruct%20Lora.ipynb) 文件，来让大家更好的学习。
|
||||
|
||||
## 模型下载
|
||||
|
||||
使用 modelscope 中的 snapshot_download 函数下载模型，第一个参数为模型名称，参数 cache_dir 为模型的下载路径。
|
||||
|
||||
在 /root/autodl-tmp 路径下新建 model_download.py 文件并在其中输入以下内容，粘贴代码后请及时保存文件，如下图所示。并运行 `python /root/autodl-tmp/model_download.py` 执行下载，模型大小为 14.18GB，下载模型大概需要 5 分钟。
|
||||
|
||||
```python
|
||||
import torch
|
||||
from modelscope import snapshot_download
|
||||
import os
|
||||
model_dir = snapshot_download('Qwen/Qwen2.5-Coder-7B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
## 环境配置
|
||||
|
||||
在完成基本环境配置和本地模型部署的情况下，你还需要安装一些第三方库，可以使用以下命令：
|
||||
|
||||
```bash
|
||||
python -m pip install --upgrade pip
|
||||
|
||||
pip install modelscope==1.20.0
|
||||
pip install transformers==4.46.2
|
||||
pip install sentencepiece==0.2.0
|
||||
pip install accelerate==1.1.1
|
||||
pip install datasets==3.1.0
|
||||
pip install peft==0.13.2
|
||||
|
||||
```
|
||||
|
||||
> 考虑到部分同学配置环境可能会遇到一些问题，我们在AutoDL平台准备了Qwen2.5-Coder-7B-Instruct的环境镜像。点击下方链接并直接创建Autodl示例即可。
|
||||
> ***https://www.codewithgpu.com/i/datawhalechina/self-llm/Qwen2.5-Coder-7B-Instruct***
|
||||
|
||||
在本节教程里，我们将微调数据集放置在根目录 [/dataset](../dataset/huanhuan.json)。
|
||||
|
||||
## 指令集构建
|
||||
|
||||
LLM 的微调一般指指令微调过程。所谓指令微调，是说我们使用的微调数据形如：
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction":"回答以下用户问题，仅输出答案。",
|
||||
"input":"1+1等于几?",
|
||||
"output":"2"
|
||||
}
|
||||
```
|
||||
|
||||
其中，`instruction` 是用户指令，告知模型其需要完成的任务；`input` 是用户输入，是完成用户指令所必须的输入内容；`output` 是模型应该给出的输出。
|
||||
|
||||
即我们的核心训练目标是让模型具有理解并遵循用户指令的能力。因此，在指令集构建时，我们应针对我们的目标任务，针对性构建任务指令集。例如，在本节我们使用由笔者合作开源的 [Chat-甄嬛](https://github.com/KMnO4-zx/huanhuan-chat) 项目作为示例，我们的目标是构建一个能够模拟甄嬛对话风格的个性化 LLM，因此我们构造的指令形如：
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "你是谁？",
|
||||
"input":"",
|
||||
"output":"家父是大理寺少卿甄远道。"
|
||||
}
|
||||
```
|
||||
|
||||
我们所构造的全部指令数据集在根目录下。
|
||||
|
||||
|
||||
## 数据格式化
|
||||
|
||||
`Lora` 训练的数据是需要经过格式化、编码之后再输入给模型进行训练的，如果是熟悉 `Pytorch` 模型训练流程的同学会知道，我们一般需要将输入文本编码为 input_ids，将输出文本编码为 `labels`，编码之后的结果都是多维的向量。我们首先定义一个预处理函数，这个函数用于对每一个样本，编码其输入、输出文本并返回一个编码后的字典：
|
||||
|
||||
```python
|
||||
def process_func(example):
|
||||
MAX_LENGTH = 384 # Llama分词器会将一个中文字切分为多个token，因此需要放开一些最大长度，保证数据的完整性
|
||||
input_ids, attention_mask, labels = [], [], []
|
||||
instruction = tokenizer(f"<|im_start|>system\n现在你要扮演皇帝身边的女人--甄嬛<|im_end|>\n<|im_start|>user\n{example['instruction'] + example['input']}<|im_end|>\n<|im_start|>assistant\n", add_special_tokens=False) # add_special_tokens 不在开头加 special_tokens
|
||||
response = tokenizer(f"{example['output']}", add_special_tokens=False)
|
||||
input_ids = instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
|
||||
attention_mask = instruction["attention_mask"] + response["attention_mask"] + [1] # 因为eos token咱们也是要关注的所以 补充为1
|
||||
labels = [-100] * len(instruction["input_ids"]) + response["input_ids"] + [tokenizer.pad_token_id]
|
||||
if len(input_ids) > MAX_LENGTH: # 做一个截断
|
||||
input_ids = input_ids[:MAX_LENGTH]
|
||||
attention_mask = attention_mask[:MAX_LENGTH]
|
||||
labels = labels[:MAX_LENGTH]
|
||||
return {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
"labels": labels
|
||||
}
|
||||
```
|
||||
|
||||
`Qwen2.5-Coder` 采用的`Prompt Template`格式如下：
|
||||
|
||||
```text
|
||||
<|im_start|>system
|
||||
You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>
|
||||
<|im_start|>user
|
||||
{user_prompt}<|im_end|>
|
||||
<|im_start|>assistant
|
||||
{assistant_response}<|im_end|>
|
||||
|
||||
```
|
||||
|
||||
## 加载tokenizer和半精度模型
|
||||
|
||||
模型以半精度形式加载，如果你的显卡比较新的话，可以用`torch.bfolat`形式加载。对于自定义的模型一定要指定`trust_remote_code`参数为`True`。
|
||||
|
||||
```python
|
||||
tokenizer = AutoTokenizer.from_pretrained('/root/autodl-tmp/qwen/Qwen2-7B-Instruct/', use_fast=False, trust_remote_code=True)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained('/root/autodl-tmp/qwen/Qwen2-7B-Instruct/', device_map="auto",torch_dtype=torch.bfloat16)
|
||||
```
|
||||
|
||||
## 定义LoraConfig
|
||||
|
||||
`LoraConfig`这个类中可以设置很多参数，但主要的参数没多少，简单讲一讲，感兴趣的同学可以直接看源码。
|
||||
|
||||
- `task_type`：模型类型
|
||||
- `target_modules`：需要训练的模型层的名字，主要就是`attention`部分的层，不同的模型对应的层的名字不同，可以传入数组，也可以字符串，也可以正则表达式。
|
||||
- `r`：`lora`的秩，具体可以看`Lora`原理
|
||||
- `lora_alpha`：`Lora alaph`，具体作用参见 `Lora` 原理
|
||||
|
||||
`Lora`的缩放是啥嘞？当然不是`r`（秩），这个缩放就是`lora_alpha/r`, 在这个`LoraConfig`中缩放就是4倍。
|
||||
|
||||
```python
|
||||
config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
|
||||
inference_mode=False, # 训练模式
|
||||
r=8, # Lora 秩
|
||||
lora_alpha=32, # Lora alaph，具体作用参见 Lora 原理
|
||||
lora_dropout=0.1# Dropout 比例
|
||||
)
|
||||
```
|
||||
|
||||
## 自定义 TrainingArguments 参数
|
||||
|
||||
`TrainingArguments`这个类的源码也介绍了每个参数的具体作用，当然大家可以来自行探索，这里就简单说几个常用的。
|
||||
|
||||
- `output_dir`：模型的输出路径
|
||||
- `per_device_train_batch_size`：顾名思义 `batch_size`
|
||||
- `gradient_accumulation_steps`: 梯度累加，如果你的显存比较小，那可以把 `batch_size` 设置小一点，梯度累加增大一些。
|
||||
- `logging_steps`：多少步，输出一次`log`
|
||||
- `num_train_epochs`：顾名思义 `epoch`
|
||||
- `gradient_checkpointing`：梯度检查，这个一旦开启，模型就必须执行`model.enable_input_require_grads()`，这个原理大家可以自行探索，这里就不细说了。
|
||||
|
||||
```python
|
||||
args = TrainingArguments(
|
||||
output_dir="./output/Qwen2_instruct_lora",
|
||||
per_device_train_batch_size=4,
|
||||
gradient_accumulation_steps=4,
|
||||
logging_steps=10,
|
||||
num_train_epochs=3,
|
||||
save_steps=100,
|
||||
learning_rate=1e-4,
|
||||
save_on_each_node=True,
|
||||
gradient_checkpointing=True
|
||||
)
|
||||
```
|
||||
|
||||
## 使用 Trainer 训练
|
||||
|
||||
```python
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=args,
|
||||
train_dataset=tokenized_id,
|
||||
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
|
||||
)
|
||||
trainer.train()
|
||||
```
|
||||
|
||||
## 加载 lora 权重推理
|
||||
|
||||
训练好了之后可以使用如下方式加载`lora`权重进行推理：
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import torch
|
||||
from peft import PeftModel
|
||||
|
||||
model_path = 'Qwen/Qwen2.5-Coder-7B-Instruct'
|
||||
lora_path = 'lora_path'
|
||||
|
||||
# 加载tokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
||||
|
||||
# 加载模型
|
||||
model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto",torch_dtype=torch.bfloat16)
|
||||
|
||||
# 加载lora权重
|
||||
model = PeftModel.from_pretrained(model, model_id=lora_path, config=config)
|
||||
|
||||
prompt = "你是谁？"
|
||||
messages = [
|
||||
{"role": "system", "content": "现在你要扮演皇帝身边的女人--甄嬛"},
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
|
||||
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to('cuda')
|
||||
|
||||
generated_ids = model.generate(
|
||||
model_inputs.input_ids,
|
||||
max_new_tokens=512
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
@@ -0,0 +1,861 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "968b3659-a0f4-4dd9-866e-936cd3b3512c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datasets import Dataset\n",
|
||||
"import pandas as pd\n",
|
||||
"from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "3e4b2293-3762-4b5c-939b-2dab7a0955ae",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 将JSON文件转换为CSV文件\n",
|
||||
"df = pd.read_json('/root/self-llm-master/dataset/huanhuan.json')\n",
|
||||
"ds = Dataset.from_pandas(df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "2400d0f1-162c-477c-a650-f5ce1a3a72d0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'instruction': ['小姐，别的秀女都在求中选，唯有咱们小姐想被撂牌子，菩萨一定记得真真儿的——',\n",
|
||||
" '这个温太医啊，也是古怪，谁不知太医不得皇命不能为皇族以外的人请脉诊病，他倒好，十天半月便往咱们府里跑。',\n",
|
||||
" '嬛妹妹，刚刚我去府上请脉，听甄伯母说你来这里进香了。'],\n",
|
||||
" 'input': ['', '', ''],\n",
|
||||
" 'output': ['嘘——都说许愿说破是不灵的。', '你们俩话太多了，我该和温太医要一剂药，好好治治你们。', '出来走走，也是散心。']}"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"ds[:3]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "56aa4253-a3b1-441d-b24f-88a78d1732d1",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Qwen2Tokenizer(name_or_path='/root/autodl-tmp/Qwen/Qwen2.5-Coder-7B-Instruct', vocab_size=151643, model_max_length=131072, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}, clean_up_tokenization_spaces=False), added_tokens_decoder={\n",
|
||||
"\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151646: AddedToken(\"<|object_ref_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151647: AddedToken(\"<|object_ref_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151648: AddedToken(\"<|box_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151649: AddedToken(\"<|box_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151650: AddedToken(\"<|quad_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151651: AddedToken(\"<|quad_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151652: AddedToken(\"<|vision_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151653: AddedToken(\"<|vision_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151654: AddedToken(\"<|vision_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151655: AddedToken(\"<|image_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151656: AddedToken(\"<|video_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
|
||||
"\t151657: AddedToken(\"<tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151658: AddedToken(\"</tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151659: AddedToken(\"<|fim_prefix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151660: AddedToken(\"<|fim_middle|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151661: AddedToken(\"<|fim_suffix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151662: AddedToken(\"<|fim_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151663: AddedToken(\"<|repo_name|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"\t151664: AddedToken(\"<|file_sep|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tokenizer = AutoTokenizer.from_pretrained('/root/autodl-tmp/Qwen/Qwen2.5-Coder-7B-Instruct', use_fast=False, trust_remote_code=True)\n",
|
||||
"tokenizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "12df8a44-3ab8-4a19-8b39-967cb76d0709",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def process_func(example):\n",
|
||||
" MAX_LENGTH = 384 # Llama分词器会将一个中文字切分为多个token，因此需要放开一些最大长度，保证数据的完整性\n",
|
||||
" input_ids, attention_mask, labels = [], [], []\n",
|
||||
" instruction = tokenizer(f\"<|im_start|>system\\n现在你要扮演皇帝身边的女人--甄嬛<|im_end|>\\n<|im_start|>user\\n{example['instruction'] + example['input']}<|im_end|>\\n<|im_start|>assistant\\n\", add_special_tokens=False) # add_special_tokens 不在开头加 special_tokens\n",
|
||||
" response = tokenizer(f\"{example['output']}\", add_special_tokens=False)\n",
|
||||
" input_ids = instruction[\"input_ids\"] + response[\"input_ids\"] + [tokenizer.pad_token_id]\n",
|
||||
" attention_mask = instruction[\"attention_mask\"] + response[\"attention_mask\"] + [1] # 因为eos token咱们也是要关注的所以 补充为1\n",
|
||||
" labels = [-100] * len(instruction[\"input_ids\"]) + response[\"input_ids\"] + [tokenizer.pad_token_id] \n",
|
||||
" if len(input_ids) > MAX_LENGTH: # 做一个截断\n",
|
||||
" input_ids = input_ids[:MAX_LENGTH]\n",
|
||||
" attention_mask = attention_mask[:MAX_LENGTH]\n",
|
||||
" labels = labels[:MAX_LENGTH]\n",
|
||||
" return {\n",
|
||||
" \"input_ids\": input_ids,\n",
|
||||
" \"attention_mask\": attention_mask,\n",
|
||||
" \"labels\": labels\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "7319b6c4-e5d0-47da-9dea-4f28f82e01dc",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3531234a30734470b291a9c9a56ebb78",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Map: 0%| | 0/3729 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Dataset({\n",
|
||||
" features: ['input_ids', 'attention_mask', 'labels'],\n",
|
||||
" num_rows: 3729\n",
|
||||
"})"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tokenized_id = ds.map(process_func, remove_columns=ds.column_names)\n",
|
||||
"tokenized_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "0c2071a6-b31b-4a5e-9b21-bdd6d9ae41c9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'<|im_start|>system\\n现在你要扮演皇帝身边的女人--甄嬛<|im_end|>\\n<|im_start|>user\\n小姐，别的秀女都在求中选，唯有咱们小姐想被撂牌子，菩萨一定记得真真儿的——<|im_end|>\\n<|im_start|>assistant\\n嘘——都说许愿说破是不灵的。<|endoftext|>'"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tokenizer.decode(tokenized_id[0]['input_ids'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "5bc95adb-6b98-4378-bf90-863c5e367491",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'你们俩话太多了，我该和温太医要一剂药，好好治治你们。<|endoftext|>'"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tokenizer.decode(list(filter(lambda x: x != -100, tokenized_id[1][\"labels\"])))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "daac681e-015b-4412-8c9f-9da8361d28de",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "7b5d6464-acc0-4a94-ad43-3bfdd23e6f4c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "fa76e5d2475b42e398189d119da8ec8c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Qwen2ForCausalLM(\n",
|
||||
" (model): Qwen2Model(\n",
|
||||
" (embed_tokens): Embedding(152064, 3584)\n",
|
||||
" (layers): ModuleList(\n",
|
||||
" (0-27): 28 x Qwen2DecoderLayer(\n",
|
||||
" (self_attn): Qwen2SdpaAttention(\n",
|
||||
" (q_proj): Linear(in_features=3584, out_features=3584, bias=True)\n",
|
||||
" (k_proj): Linear(in_features=3584, out_features=512, bias=True)\n",
|
||||
" (v_proj): Linear(in_features=3584, out_features=512, bias=True)\n",
|
||||
" (o_proj): Linear(in_features=3584, out_features=3584, bias=False)\n",
|
||||
" (rotary_emb): Qwen2RotaryEmbedding()\n",
|
||||
" )\n",
|
||||
" (mlp): Qwen2MLP(\n",
|
||||
" (gate_proj): Linear(in_features=3584, out_features=18944, bias=False)\n",
|
||||
" (up_proj): Linear(in_features=3584, out_features=18944, bias=False)\n",
|
||||
" (down_proj): Linear(in_features=18944, out_features=3584, bias=False)\n",
|
||||
" (act_fn): SiLU()\n",
|
||||
" )\n",
|
||||
" (input_layernorm): Qwen2RMSNorm((3584,), eps=1e-06)\n",
|
||||
" (post_attention_layernorm): Qwen2RMSNorm((3584,), eps=1e-06)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (norm): Qwen2RMSNorm((3584,), eps=1e-06)\n",
|
||||
" (rotary_emb): Qwen2RotaryEmbedding()\n",
|
||||
" )\n",
|
||||
" (lm_head): Linear(in_features=3584, out_features=152064, bias=False)\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = AutoModelForCausalLM.from_pretrained('/root/autodl-tmp/Qwen/Qwen2.5-Coder-7B-Instruct/', device_map=\"auto\",torch_dtype=torch.bfloat16)\n",
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "63124620-6f4f-4020-bfee-7be44aedc805",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "c4903968-c561-40d8-b450-a5a7ec231037",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"torch.bfloat16"
|
||||
]
|
||||
},
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.dtype"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5f9fbb37-ccff-42ef-aa60-6e01c10084ae",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# lora"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "45ab5181-b560-42f5-b485-991feacf8779",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path=None, revision=None, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=False, r=8, target_modules={'gate_proj', 'v_proj', 'down_proj', 'o_proj', 'up_proj', 'k_proj', 'q_proj'}, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', loftq_config={}, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False))"
|
||||
]
|
||||
},
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from peft import LoraConfig, TaskType, get_peft_model\n",
|
||||
"\n",
|
||||
"config = LoraConfig(\n",
|
||||
" task_type=TaskType.CAUSAL_LM, \n",
|
||||
" target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
|
||||
" inference_mode=False, # 训练模式\n",
|
||||
" r=8, # Lora 秩\n",
|
||||
" lora_alpha=32, # Lora alaph，具体作用参见 Lora 原理\n",
|
||||
" lora_dropout=0.1# Dropout 比例\n",
|
||||
")\n",
|
||||
"config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "73403447-e6f8-436d-a6ae-9a3e62d697dd",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path='/root/autodl-tmp/Qwen/Qwen2.5-Coder-7B-Instruct/', revision=None, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=False, r=8, target_modules={'gate_proj', 'v_proj', 'down_proj', 'o_proj', 'up_proj', 'k_proj', 'q_proj'}, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', loftq_config={}, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False))"
|
||||
]
|
||||
},
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = get_peft_model(model, config)\n",
|
||||
"config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "bdadeda7-40c7-4d13-bb10-2342f2589b59",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 20,185,088 || all params: 7,635,801,600 || trainable%: 0.2643\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f1bcf2d8-27ba-46e7-afc3-0af4564f0bcd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 配置训练参数"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "f2ebb791-793f-43c4-95a1-25b5d5e0b070",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"args = TrainingArguments(\n",
|
||||
" output_dir=\"./output/Qwen2.5-Coder-7B-Instruct\",\n",
|
||||
" per_device_train_batch_size=4,\n",
|
||||
" gradient_accumulation_steps=4,\n",
|
||||
" logging_steps=10,\n",
|
||||
" num_train_epochs=3,\n",
|
||||
" save_steps=10, # 为了快速演示，这里设置10，建议你设置成100\n",
|
||||
" learning_rate=1e-4,\n",
|
||||
" save_on_each_node=True,\n",
|
||||
" gradient_checkpointing=True\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "9daa30c2-752c-49db-888a-af48baaa434e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer = Trainer(\n",
|
||||
" model=model,\n",
|
||||
" args=args,\n",
|
||||
" train_dataset=tokenized_id,\n",
|
||||
" data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "d051dc78-00a8-4d89-aa36-7ed70b5bf71b",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"\n",
|
||||
" <div>\n",
|
||||
" \n",
|
||||
" <progress value='699' max='699' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
|
||||
" [699/699 23:35, Epoch 2/3]\n",
|
||||
" </div>\n",
|
||||
" <table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: left;\">\n",
|
||||
" <th>Step</th>\n",
|
||||
" <th>Training Loss</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>3.968300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>20</td>\n",
|
||||
" <td>3.456000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>30</td>\n",
|
||||
" <td>3.343800</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>40</td>\n",
|
||||
" <td>3.229300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>50</td>\n",
|
||||
" <td>3.239300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>60</td>\n",
|
||||
" <td>3.198400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>70</td>\n",
|
||||
" <td>3.202300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>80</td>\n",
|
||||
" <td>3.261800</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>90</td>\n",
|
||||
" <td>3.303900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>100</td>\n",
|
||||
" <td>3.208000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>110</td>\n",
|
||||
" <td>3.207700</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>120</td>\n",
|
||||
" <td>3.209000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>130</td>\n",
|
||||
" <td>3.126500</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>140</td>\n",
|
||||
" <td>3.127100</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>150</td>\n",
|
||||
" <td>3.195700</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>160</td>\n",
|
||||
" <td>3.172500</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>170</td>\n",
|
||||
" <td>3.143400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>180</td>\n",
|
||||
" <td>3.092800</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>190</td>\n",
|
||||
" <td>3.088900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>200</td>\n",
|
||||
" <td>3.083900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>210</td>\n",
|
||||
" <td>3.070200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>220</td>\n",
|
||||
" <td>3.062300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>230</td>\n",
|
||||
" <td>3.113700</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>240</td>\n",
|
||||
" <td>3.393000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>250</td>\n",
|
||||
" <td>2.783600</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>260</td>\n",
|
||||
" <td>2.682300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>270</td>\n",
|
||||
" <td>2.778900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>280</td>\n",
|
||||
" <td>2.802900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>290</td>\n",
|
||||
" <td>2.798200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>300</td>\n",
|
||||
" <td>2.679400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>310</td>\n",
|
||||
" <td>2.679900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>320</td>\n",
|
||||
" <td>2.698100</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>330</td>\n",
|
||||
" <td>2.751100</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>340</td>\n",
|
||||
" <td>2.765400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>350</td>\n",
|
||||
" <td>2.768200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>360</td>\n",
|
||||
" <td>2.715900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>370</td>\n",
|
||||
" <td>2.709400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>380</td>\n",
|
||||
" <td>2.795400</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>390</td>\n",
|
||||
" <td>2.670100</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>400</td>\n",
|
||||
" <td>2.608800</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>410</td>\n",
|
||||
" <td>2.795300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>420</td>\n",
|
||||
" <td>2.679900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>430</td>\n",
|
||||
" <td>2.775000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>440</td>\n",
|
||||
" <td>2.684300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>450</td>\n",
|
||||
" <td>2.634800</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>460</td>\n",
|
||||
" <td>2.697600</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>470</td>\n",
|
||||
" <td>2.634500</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>480</td>\n",
|
||||
" <td>2.363200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>490</td>\n",
|
||||
" <td>2.281900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>500</td>\n",
|
||||
" <td>2.294300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>510</td>\n",
|
||||
" <td>2.185200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>520</td>\n",
|
||||
" <td>2.302200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>530</td>\n",
|
||||
" <td>2.258700</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>540</td>\n",
|
||||
" <td>2.351600</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>550</td>\n",
|
||||
" <td>2.430900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>560</td>\n",
|
||||
" <td>2.360900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>570</td>\n",
|
||||
" <td>2.356000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>580</td>\n",
|
||||
" <td>2.145100</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>590</td>\n",
|
||||
" <td>2.343300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>600</td>\n",
|
||||
" <td>2.179200</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>610</td>\n",
|
||||
" <td>2.300300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>620</td>\n",
|
||||
" <td>2.259300</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>630</td>\n",
|
||||
" <td>2.329900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>640</td>\n",
|
||||
" <td>2.309000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>650</td>\n",
|
||||
" <td>2.242900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>660</td>\n",
|
||||
" <td>2.310000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>670</td>\n",
|
||||
" <td>2.378500</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>680</td>\n",
|
||||
" <td>2.170900</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <td>690</td>\n",
|
||||
" <td>2.234300</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table><p>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"TrainOutput(global_step=699, training_loss=2.7544531378793784, metrics={'train_runtime': 1418.2557, 'train_samples_per_second': 7.888, 'train_steps_per_second': 0.493, 'total_flos': 4.575963998146867e+16, 'train_loss': 2.7544531378793784, 'epoch': 2.996784565916399})"
|
||||
]
|
||||
},
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"trainer.train()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0814a560-1e42-4ced-99e6-de9648f8bfd8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 合并加载模型"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "7219bb51-66fa-4dae-bf5a-997382c220df",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b8588305bba14213ada7ca70dac68f5f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"我是甄嬛，家父是大理寺少卿甄远道。\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"import torch\n",
|
||||
"from peft import PeftModel\n",
|
||||
"\n",
|
||||
"mode_path = '/root/autodl-tmp/Qwen/Qwen2.5-Coder-7B-Instruct/'\n",
|
||||
"lora_path = '/root/output/Qwen2.5-Coder-7B-Instruct/checkpoint-690/' # 这里改称你的 lora 输出对应 checkpoint 地址\n",
|
||||
"\n",
|
||||
"# 加载tokenizer\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(mode_path, trust_remote_code=True)\n",
|
||||
"\n",
|
||||
"# 加载模型\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(mode_path, device_map=\"auto\",torch_dtype=torch.bfloat16, trust_remote_code=True).eval()\n",
|
||||
"\n",
|
||||
"# 加载lora权重\n",
|
||||
"model = PeftModel.from_pretrained(model, model_id=lora_path)\n",
|
||||
"\n",
|
||||
"prompt = \"你是谁？\"\n",
|
||||
"inputs = tokenizer.apply_chat_template([{\"role\": \"user\", \"content\": \"假设你是皇帝身边的女人--甄嬛。\"},{\"role\": \"user\", \"content\": prompt}],\n",
|
||||
" add_generation_prompt=True,\n",
|
||||
" tokenize=True,\n",
|
||||
" return_tensors=\"pt\",\n",
|
||||
" return_dict=True\n",
|
||||
" ).to('cuda')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 1}\n",
|
||||
"with torch.no_grad():\n",
|
||||
" outputs = model.generate(**inputs, **gen_kwargs)\n",
|
||||
" outputs = outputs[:, inputs['input_ids'].shape[1]:]\n",
|
||||
" print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aac6cd05-b62b-465a-8ecf-0678adc7297c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 86 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 502 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 144 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 52 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 161 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Reference in New Issue
Block a user