mirror of
https://github.com/datawhalechina/self-llm.git
synced 2026-09-19 01:36:47 +08:00
Merge branch 'master' of https://github.com/Joe-2002/self-llm
This commit is contained in:
Joe-2002
@@ -61,11 +61,19 @@
|
||||
|
||||
### 已支持模型
|
||||
|
||||
- [Qwen2.5-Coder](https://github.com/QwenLM/Qwen2.5-Coder)
|
||||
- [ ] Qwen2.5-Coder-7B-Instruct FastApi部署调用 @赵文恺
|
||||
- [ ] Qwen2.5-Coder-7B-Instruct Langchian接入 @杨晨旭
|
||||
- [ ] Qwen2.5-Coder-7B-Instruct WebDemo 部署 @王泽宇
|
||||
- [ ] Qwen2.5-Coder-7B-Instruct vLLM 部署 @王泽宇
|
||||
- [ ] Qwen2.5-Coder-7B-Instruct Lora 微调 @荞麦
|
||||
- [x] [Qwen2.5-Coder-7B-Instruct Lora 微调 SwanLab 可视化记录版](./models/Qwen2.5-Coder/05-Qwen2.5-Coder-7B-Instruct%20Lora%20微调%20SwanLab%20可视化记录版.md) @杨卓
|
||||
|
||||
- [Qwen2-vl](https://github.com/QwenLM/Qwen2-VL)
|
||||
- [ ] [Qwen2-vl-2B FastApi 部署调用]()
|
||||
- [ ] [Qwen2-vl-2B WebDemo 部署]()
|
||||
- [ ] [Qwen2-vl-2B vLLM 部署]()
|
||||
- [ ] [Qwen2-vl-2B Lora 微调]() @李柯辰
|
||||
- [ ] [Qwen2-vl-2B Lora 微调]()
|
||||
|
||||
- [Qwen2.5](https://github.com/QwenLM/Qwen2.5)
|
||||
- [x] [Qwen2.5-7B-Instruct FastApi 部署调用](./models/Qwen2.5/01-Qwen2.5-7B-Instruct%20FastApi%20部署调用.md) @娄天奥
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
from qwen_vl_utils.vision_process import process_vision_info
|
||||
from fastapi import FastAPI, Request
|
||||
import uvicorn
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Union
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
model_name_or_path = '/root/autodl-tmp/Qwen/Qwen2-VL-2B-Instruct'
|
||||
# 初始化模型和处理器（保持在全局范围内，这样只需加载一次）
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_name_or_path, torch_dtype="auto", device_map="auto"
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(model_name_or_path)
|
||||
|
||||
# 定义请求体模型
|
||||
class MessageContent(BaseModel):
|
||||
type: str
|
||||
text: str = None
|
||||
image: str = None
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
messages: List[Dict[str, Union[str, List[Dict[str, str]]]]]
|
||||
|
||||
@app.post("/generate")
|
||||
async def generate_response(chat_message: ChatMessage):
|
||||
# 直接使用请求中的 messages
|
||||
text = processor.apply_chat_template(
|
||||
chat_message.messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
|
||||
image_inputs, video_inputs = process_vision_info(chat_message.messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# 生成输出
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=1024)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False
|
||||
)
|
||||
|
||||
return {"response": output_text[0]}
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
@@ -0,0 +1,60 @@
|
||||
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
from qwen_vl_utils.vision_process import process_vision_info
|
||||
from fastapi import FastAPI, Request
|
||||
import uvicorn
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Union
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
model_name_or_path = '/root/autodl-tmp/Qwen/Qwen2-VL-2B-Instruct'
|
||||
# 初始化模型和处理器（保持在全局范围内，这样只需加载一次）
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_name_or_path, torch_dtype="auto", device_map="auto"
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(model_name_or_path)
|
||||
|
||||
# 定义请求体模型
|
||||
class MessageContent(BaseModel):
|
||||
type: str
|
||||
text: str = None
|
||||
image: str = None
|
||||
video: str = None
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
messages: List[Dict[str, Union[str, List[Dict[str, str]]]]]
|
||||
|
||||
@app.post("/generate")
|
||||
async def generate_response(chat_message: ChatMessage):
|
||||
# 直接使用请求中的 messages
|
||||
text = processor.apply_chat_template(
|
||||
chat_message.messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
|
||||
image_inputs, video_inputs = process_vision_info(chat_message.messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# 生成输出
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=1024)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False
|
||||
)
|
||||
|
||||
return {"response": output_text[0]}
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
@@ -0,0 +1,22 @@
|
||||
import requests
|
||||
|
||||
url = "http://localhost:8000/generate"
|
||||
payload = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
response = requests.post(url, json=payload)
|
||||
print(response.json())
|
||||
@@ -0,0 +1,23 @@
|
||||
import requests
|
||||
|
||||
url = "http://localhost:8000/generate"
|
||||
payload = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video",
|
||||
"video": "./space_woaudio.mp4"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this video."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = requests.post(url, json=payload)
|
||||
print(response.json())
|
||||
@@ -0,0 +1,2 @@
|
||||
from modelscope import snapshot_download
|
||||
model_dir = snapshot_download('Qwen/Qwen2-VL-2B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
from functools import lru_cache
|
||||
from io import BytesIO
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import torchvision
|
||||
from packaging import version
|
||||
from PIL import Image
|
||||
from torchvision import io, transforms
|
||||
from torchvision.transforms import InterpolationMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_FACTOR = 28
|
||||
MIN_PIXELS = 4 * 28 * 28
|
||||
MAX_PIXELS = 16384 * 28 * 28
|
||||
MAX_RATIO = 200
|
||||
|
||||
VIDEO_MIN_PIXELS = 128 * 28 * 28
|
||||
VIDEO_MAX_PIXELS = 768 * 28 * 28
|
||||
VIDEO_TOTAL_PIXELS = 24576 * 28 * 28
|
||||
FRAME_FACTOR = 2
|
||||
FPS = 2.0
|
||||
FPS_MIN_FRAMES = 4
|
||||
FPS_MAX_FRAMES = 768
|
||||
|
||||
|
||||
def round_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
||||
return round(number / factor) * factor
|
||||
|
||||
|
||||
def ceil_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
|
||||
return math.ceil(number / factor) * factor
|
||||
|
||||
|
||||
def floor_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
def smart_resize(
|
||||
height: int, width: int, factor: int = IMAGE_FACTOR, min_pixels: int = MIN_PIXELS, max_pixels: int = MAX_PIXELS
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Rescales the image so that the following conditions are met:
|
||||
|
||||
1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
|
||||
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
|
||||
|
||||
3. The aspect ratio of the image is maintained as closely as possible.
|
||||
"""
|
||||
if max(height, width) / min(height, width) > MAX_RATIO:
|
||||
raise ValueError(
|
||||
f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
|
||||
)
|
||||
h_bar = max(factor, round_by_factor(height, factor))
|
||||
w_bar = max(factor, round_by_factor(width, factor))
|
||||
if h_bar * w_bar > max_pixels:
|
||||
beta = math.sqrt((height * width) / max_pixels)
|
||||
h_bar = floor_by_factor(height / beta, factor)
|
||||
w_bar = floor_by_factor(width / beta, factor)
|
||||
elif h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (height * width))
|
||||
h_bar = ceil_by_factor(height * beta, factor)
|
||||
w_bar = ceil_by_factor(width * beta, factor)
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def fetch_image(ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR) -> Image.Image:
|
||||
if "image" in ele:
|
||||
image = ele["image"]
|
||||
else:
|
||||
image = ele["image_url"]
|
||||
image_obj = None
|
||||
if isinstance(image, Image.Image):
|
||||
image_obj = image
|
||||
elif image.startswith("http://") or image.startswith("https://"):
|
||||
image_obj = Image.open(requests.get(image, stream=True).raw)
|
||||
elif image.startswith("file://"):
|
||||
image_obj = Image.open(image[7:])
|
||||
elif image.startswith("data:image"):
|
||||
if "base64," in image:
|
||||
_, base64_data = image.split("base64,", 1)
|
||||
data = base64.b64decode(base64_data)
|
||||
image_obj = Image.open(BytesIO(data))
|
||||
else:
|
||||
image_obj = Image.open(image)
|
||||
if image_obj is None:
|
||||
raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
|
||||
image = image_obj.convert("RGB")
|
||||
## resize
|
||||
if "resized_height" in ele and "resized_width" in ele:
|
||||
resized_height, resized_width = smart_resize(
|
||||
ele["resized_height"],
|
||||
ele["resized_width"],
|
||||
factor=size_factor,
|
||||
)
|
||||
else:
|
||||
width, height = image.size
|
||||
min_pixels = ele.get("min_pixels", MIN_PIXELS)
|
||||
max_pixels = ele.get("max_pixels", MAX_PIXELS)
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=size_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
image = image.resize((resized_width, resized_height))
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def smart_nframes(
|
||||
ele: dict,
|
||||
total_frames: int,
|
||||
video_fps: int | float,
|
||||
) -> int:
|
||||
"""calculate the number of frames for video used for model inputs.
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support either `fps` or `nframes`:
|
||||
- nframes: the number of frames to extract for model inputs.
|
||||
- fps: the fps to extract frames for model inputs.
|
||||
- min_frames: the minimum number of frames of the video, only used when fps is provided.
|
||||
- max_frames: the maximum number of frames of the video, only used when fps is provided.
|
||||
total_frames (int): the original total number of frames of the video.
|
||||
video_fps (int | float): the original fps of the video.
|
||||
|
||||
Raises:
|
||||
ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
|
||||
|
||||
Returns:
|
||||
int: the number of frames for video used for model inputs.
|
||||
"""
|
||||
assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
|
||||
if "nframes" in ele:
|
||||
nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
|
||||
else:
|
||||
fps = ele.get("fps", FPS)
|
||||
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
|
||||
max_frames = floor_by_factor(ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR)
|
||||
nframes = total_frames / video_fps * fps
|
||||
nframes = min(max(nframes, min_frames), max_frames)
|
||||
nframes = round_by_factor(nframes, FRAME_FACTOR)
|
||||
if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
|
||||
raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
|
||||
return nframes
|
||||
|
||||
|
||||
def _read_video_torchvision(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using torchvision.io.read_video
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://", "https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
torch.Tensor: the video tensor with shape (T, C, H, W).
|
||||
"""
|
||||
video_path = ele["video"]
|
||||
if version.parse(torchvision.__version__) < version.parse("0.19.0"):
|
||||
if "http://" in video_path or "https://" in video_path:
|
||||
warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
|
||||
if "file://" in video_path:
|
||||
video_path = video_path[7:]
|
||||
st = time.time()
|
||||
video, audio, info = io.read_video(
|
||||
video_path,
|
||||
start_pts=ele.get("video_start", 0.0),
|
||||
end_pts=ele.get("video_end", None),
|
||||
pts_unit="sec",
|
||||
output_format="TCHW",
|
||||
)
|
||||
total_frames, video_fps = video.size(0), info["video_fps"]
|
||||
logger.info(f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long()
|
||||
video = video[idx]
|
||||
return video
|
||||
|
||||
|
||||
def is_decord_available() -> bool:
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec("decord") is not None
|
||||
|
||||
|
||||
def _read_video_decord(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using decord.VideoReader
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://", "https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
torch.Tensor: the video tensor with shape (T, C, H, W).
|
||||
"""
|
||||
import decord
|
||||
video_path = ele["video"]
|
||||
st = time.time()
|
||||
vr = decord.VideoReader(video_path)
|
||||
# TODO: support start_pts and end_pts
|
||||
if 'video_start' in ele or 'video_end' in ele:
|
||||
raise NotImplementedError("not support start_pts and end_pts in decord for now.")
|
||||
total_frames, video_fps = len(vr), vr.get_avg_fps()
|
||||
logger.info(f"decord: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
|
||||
video = vr.get_batch(idx).asnumpy()
|
||||
video = torch.tensor(video).permute(0, 3, 1, 2) # Convert to TCHW format
|
||||
return video
|
||||
|
||||
|
||||
VIDEO_READER_BACKENDS = {
|
||||
"decord": _read_video_decord,
|
||||
"torchvision": _read_video_torchvision,
|
||||
}
|
||||
|
||||
FORCE_QWENVL_VIDEO_READER = os.getenv("FORCE_QWENVL_VIDEO_READER", None)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_video_reader_backend() -> str:
|
||||
if FORCE_QWENVL_VIDEO_READER is not None:
|
||||
video_reader_backend = FORCE_QWENVL_VIDEO_READER
|
||||
elif is_decord_available():
|
||||
video_reader_backend = "decord"
|
||||
else:
|
||||
video_reader_backend = "torchvision"
|
||||
print(f"qwen-vl-utils using {video_reader_backend} to read video.", file=sys.stderr)
|
||||
return video_reader_backend
|
||||
|
||||
|
||||
def fetch_video(ele: dict, image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
|
||||
if isinstance(ele["video"], str):
|
||||
video_reader_backend = get_video_reader_backend()
|
||||
video = VIDEO_READER_BACKENDS[video_reader_backend](ele)
|
||||
nframes, _, height, width = video.shape
|
||||
|
||||
min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
|
||||
total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
|
||||
max_pixels = max(min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR), int(min_pixels * 1.05))
|
||||
max_pixels = ele.get("max_pixels", max_pixels)
|
||||
if "resized_height" in ele and "resized_width" in ele:
|
||||
resized_height, resized_width = smart_resize(
|
||||
ele["resized_height"],
|
||||
ele["resized_width"],
|
||||
factor=image_factor,
|
||||
)
|
||||
else:
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=image_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
video = transforms.functional.resize(
|
||||
video,
|
||||
[resized_height, resized_width],
|
||||
interpolation=InterpolationMode.BICUBIC,
|
||||
antialias=True,
|
||||
).float()
|
||||
return video
|
||||
else:
|
||||
assert isinstance(ele["video"], (list, tuple))
|
||||
process_info = ele.copy()
|
||||
process_info.pop("type", None)
|
||||
process_info.pop("video", None)
|
||||
images = [
|
||||
fetch_image({"image": video_element, **process_info}, size_factor=image_factor)
|
||||
for video_element in ele["video"]
|
||||
]
|
||||
nframes = ceil_by_factor(len(images), FRAME_FACTOR)
|
||||
if len(images) < nframes:
|
||||
images.extend([images[-1]] * (nframes - len(images)))
|
||||
return images
|
||||
|
||||
|
||||
def extract_vision_info(conversations: list[dict] | list[list[dict]]) -> list[dict]:
|
||||
vision_infos = []
|
||||
if isinstance(conversations[0], dict):
|
||||
conversations = [conversations]
|
||||
for conversation in conversations:
|
||||
for message in conversation:
|
||||
if isinstance(message["content"], list):
|
||||
for ele in message["content"]:
|
||||
if (
|
||||
"image" in ele
|
||||
or "image_url" in ele
|
||||
or "video" in ele
|
||||
or ele["type"] in ("image", "image_url", "video")
|
||||
):
|
||||
vision_infos.append(ele)
|
||||
return vision_infos
|
||||
|
||||
|
||||
def process_vision_info(
|
||||
conversations: list[dict] | list[list[dict]],
|
||||
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None]:
|
||||
vision_infos = extract_vision_info(conversations)
|
||||
## Read images or videos
|
||||
image_inputs = []
|
||||
video_inputs = []
|
||||
for vision_info in vision_infos:
|
||||
if "image" in vision_info or "image_url" in vision_info:
|
||||
image_inputs.append(fetch_image(vision_info))
|
||||
elif "video" in vision_info:
|
||||
video_inputs.append(fetch_video(vision_info))
|
||||
else:
|
||||
raise ValueError("image, image_url or video should in content.")
|
||||
if len(image_inputs) == 0:
|
||||
image_inputs = None
|
||||
if len(video_inputs) == 0:
|
||||
video_inputs = None
|
||||
return image_inputs, video_inputs
|
||||
@@ -0,0 +1,6 @@
|
||||
pip install modelscope==1.20.0
|
||||
pip install fastapi==0.115.4
|
||||
pip install uvicorn==0.32.0
|
||||
pip install accelerate==1.1.1
|
||||
pip install torchvision==0.19.0
|
||||
pip install av==13.1.0
|
||||
Binary file not shown.
@@ -0,0 +1,326 @@
|
||||
# Qwen2-VL-2B-Instruct FastApi 部署调用
|
||||
|
||||
|
||||
|
||||
## 环境准备
|
||||
|
||||
基础环境：
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.12
|
||||
cuda 12.1
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
> 本文默认学习者已安装好以上 PyTorch (cuda) 环境，如未安装请自行安装。
|
||||
|
||||
首先 `pip` 换源加速下载并安装依赖包
|
||||
|
||||
```shell
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install modelscope==1.20.0
|
||||
pip install fastapi==0.115.4
|
||||
pip install uvicorn==0.32.0
|
||||
pip install transformers==4.46.2
|
||||
pip install accelerate==1.1.1
|
||||
pip install torchvision==0.19.0
|
||||
pip install av==13.1.0
|
||||
```
|
||||
|
||||
> 考虑到部分同学配置环境可能会遇到一些问题，我们在AutoDL平台准备了Qwen2-VL的环境镜像，点击下方链接并直接创建Autodl示例即可。
|
||||
> ***https://www.codewithgpu.com/i/datawhalechina/self-llm/Qwen2-VL-self-llm***
|
||||
|
||||
|
||||
|
||||
## 模型下载
|
||||
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
新建 `model_download.py` 文件输入以下代码，并运行 `python model_download.py` 执行下载。
|
||||
|
||||
此处使用 `modelscope` 提供的 `snapshot_download` 函数进行下载，该方法对国内的用户十分友好。
|
||||
|
||||
```python
|
||||
# model_download.py
|
||||
from modelscope import snapshot_download
|
||||
model_dir = snapshot_download('Qwen/Qwen2-VL-2B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：请记得修改 `cache_dir` 为你自己的模型下载路径 ~
|
||||
|
||||
|
||||
|
||||
## 代码准备
|
||||
|
||||
新建 `api_image.py` 文件并在其中输入以下内容，粘贴代码后请记得及时保存文件。
|
||||
|
||||
```python
|
||||
# api_server_image.py
|
||||
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
from qwen_vl_utils.vision_process import process_vision_info
|
||||
from fastapi import FastAPI, Request
|
||||
import uvicorn
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Union
|
||||
|
||||
# 创建FastAPI应用
|
||||
app = FastAPI()
|
||||
|
||||
# 下载好的模型本地路径
|
||||
model_name_or_path = '/root/autodl-tmp/Qwen/Qwen2-VL-2B-Instruct'
|
||||
|
||||
# 初始化模型和处理器
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_name_or_path, torch_dtype="auto", device_map="auto"
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(model_name_or_path)
|
||||
|
||||
# 定义请求体模型
|
||||
class MessageContent(BaseModel):
|
||||
type: str
|
||||
text: str = None
|
||||
image: str = None
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
messages: List[Dict[str, Union[str, List[Dict[str, str]]]]]
|
||||
|
||||
# 处理POST请求的端点
|
||||
@app.post("/generate")
|
||||
async def generate_response(chat_message: ChatMessage):
|
||||
# 直接使用请求中的 messages
|
||||
text = processor.apply_chat_template(
|
||||
chat_message.messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
|
||||
# 预先写好的辅助函数，位于参考代码中
|
||||
image_inputs, video_inputs = process_vision_info(chat_message.messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# 生成输出
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=1024)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False
|
||||
)
|
||||
|
||||
return {"response": output_text[0]}
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 启动FastAPI应用，端口为8000
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
```
|
||||
|
||||
> 注意：同样记得修改 `model_name_or_path` 为你自己的模型下载路径 ~
|
||||
|
||||
|
||||
|
||||
## 图像问答 API 服务启动
|
||||
|
||||
在终端输入以下命令启动 `api` 服务：
|
||||
|
||||
```shell
|
||||
python api_server_image.py
|
||||
```
|
||||
|
||||
加载完毕后出现如下信息说明成功。
|
||||
|
||||
|
||||
|
||||
我们可以使用 `Python` 中的 `requests` 库对api服务的端口进行请求从而调用 `Qwen2-VL-2B-Instruct` 的多模态图片理解能力来生成回复，示例代码如下——
|
||||
|
||||
```python
|
||||
# fastapi_request_image.py
|
||||
import requests
|
||||
|
||||
url = "http://localhost:8000/generate"
|
||||
payload = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
response = requests.post(url, json=payload)
|
||||
print(response.json())
|
||||
```
|
||||
|
||||
```shell
|
||||
python fastapi_request_image.py
|
||||
```
|
||||
|
||||
执行后得到的返回结果如下所示：
|
||||
|
||||
```json
|
||||
{'response': "The image depicts a serene beach scene with a woman and a dog. The woman is sitting on the sand, wearing a plaid shirt and black pants, and appears to be smiling. She is holding the dog's paw in a high-five gesture. The dog, which is a large breed, is sitting on the sand with its front paws raised, possibly in response to the woman's gesture. The background shows the ocean with gentle waves, and the sky is clear with a soft light, suggesting it might be either sunrise or sunset. The overall atmosphere is peaceful and joyful."}
|
||||
```
|
||||
|
||||
|
||||
|
||||
我们再回过头来检查以下示例代码中的 `demo.jpeg` ，可以观察到模型的回复质量还是非常高的，正确且完整地叙述了图片。
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## 进阶实践
|
||||
|
||||
由于 `Qwen2-VL-2B-Instruct` 具备强大的多模态能力，其除了对图片进行问答之外，同样也支持视频形式的交互。
|
||||
|
||||
我们需要对原先的 `api_image.py` 的代码做一些修改来使我们的 `FastApi` 服务支持视频流推理。新建 `api_server_image_and_video.py` ，复制如下代码——
|
||||
|
||||
```python
|
||||
# api_server_image_and_video.py
|
||||
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
from qwen_vl_utils.vision_process import process_vision_info
|
||||
from fastapi import FastAPI, Request
|
||||
import uvicorn
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Union
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
model_name_or_path = '/root/autodl-tmp/Qwen/Qwen2-VL-2B-Instruct'
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_name_or_path, torch_dtype="auto", device_map="auto"
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(model_name_or_path)
|
||||
|
||||
# 定义请求体模型
|
||||
class MessageContent(BaseModel):
|
||||
type: str
|
||||
text: str = None
|
||||
image: str = None
|
||||
video: str = None # 添加对video的支持
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
messages: List[Dict[str, Union[str, List[Dict[str, str]]]]]
|
||||
|
||||
@app.post("/generate")
|
||||
async def generate_response(chat_message: ChatMessage):
|
||||
# 直接使用请求中的 messages
|
||||
text = processor.apply_chat_template(
|
||||
chat_message.messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
|
||||
image_inputs, video_inputs = process_vision_info(chat_message.messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# 生成输出
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=1024)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False
|
||||
)
|
||||
|
||||
return {"response": output_text[0]}
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 视频问答 API 服务启动
|
||||
|
||||
在终端输入以下命令启动 `api` 服务：
|
||||
|
||||
```shell
|
||||
python api_server_image_and_video.py
|
||||
```
|
||||
|
||||
加载完毕后出现如下信息说明成功。
|
||||
|
||||
|
||||
|
||||
同样的，我们使用 `Python` 中的 `requests` 库对服务的端口进行请求从而调用 `Qwen2-VL-2B-Instruct` 的多模态能力来生成回复，示例代码如下——
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
url = "http://localhost:8000/generate"
|
||||
payload = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video",
|
||||
"video": "./space_woaudio.mp4"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this video."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = requests.post(url, json=payload)
|
||||
print(response.json())
|
||||
```
|
||||
|
||||
上述代码中，我们在 `messages.content` 中添加了一个视频，预览如下：
|
||||
|
||||
|
||||
|
||||
```shell
|
||||
python fastapi_request_video.py
|
||||
```
|
||||
|
||||
此时得到的模型的返回结果如下：
|
||||
|
||||
```json
|
||||
{'response': "The video shows a man standing in a Mission Control Center, speaking to the camera. The center is equipped with various monitors and control panels, and there are several large screens displaying maps and data. The man appears to be giving a presentation or explaining something related to the center's operations."}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## 参考代码及其使用
|
||||
|
||||
本次教程涉及到的代码文件较多，因此额外提供了参考代码供读者参考，但依然建议初学者在理解的基础上妥善使用。
|
||||
|
||||
完成上述所有教程后的目录结构应该与下图类似，关于文件路径还请读者根据自己的实际存放情况进行修正。
|
||||
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from threading import Thread
|
||||
from typing import Any, Dict, Generator, List, Tuple
|
||||
|
||||
import gradio as gr
|
||||
from qwen_vl_utils import process_vision_info
|
||||
import torch
|
||||
from transformers import (
|
||||
AutoProcessor,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
Qwen2VLProcessor,
|
||||
TextIteratorStreamer,
|
||||
GenerationConfig,
|
||||
)
|
||||
from transformers.utils import is_flash_attn_2_available
|
||||
from transformers.modeling_utils import get_first_parameter_dtype
|
||||
from accelerate import init_empty_weights
|
||||
from accelerate.utils import calculate_maximum_sizes, convert_bytes
|
||||
from accelerate.commands.estimate import create_ascii_table
|
||||
|
||||
# copy from qwen_vl_utils.process_vision_info
|
||||
MIN_PIXELS = 4 * 28 * 28 # 一张图最小占4个token
|
||||
MAX_PIXELS = 16384 * 28 * 28 # 一张图最大占16384个token
|
||||
VIDEO_MIN_PIXELS = 128 * 28 * 28 # 一个视频里一帧最小占128个token
|
||||
VIDEO_MAX_PIXELS = 768 * 28 * 28 # 一个视频里一帧最大占768个token
|
||||
VIDEO_TOTAL_PIXELS = 24576 * 28 * 28 # 一个视频里所有帧总共占最多24576个token
|
||||
|
||||
# default
|
||||
DEFAULT_CKPT_PATH = "path/to/Qwen2-VL-2B-Instruct"
|
||||
VIDEO_EXTENSIONS = [
|
||||
".mp4",
|
||||
".avi",
|
||||
".mkv",
|
||||
".mov",
|
||||
".wmv",
|
||||
".flv",
|
||||
".webm",
|
||||
".mpeg",
|
||||
]
|
||||
IMAGE_EXTENSIONS = [".png", ".jpg"]
|
||||
# end default
|
||||
print("*" * 60)
|
||||
print("*Qwen2-vl 图片视频模态token限制如下:")
|
||||
print(f"*单张图片最大/最小token长度限制:{MAX_PIXELS//(28*28)}/{MIN_PIXELS//(28*28)}")
|
||||
print(
|
||||
f"*单个视频最大/最小/总token长度限制:{VIDEO_MAX_PIXELS//(28*28)}/{VIDEO_MIN_PIXELS//(28*28)}/{VIDEO_TOTAL_PIXELS//(28*28)}"
|
||||
)
|
||||
print("*" * 60, end="\n\n")
|
||||
|
||||
|
||||
# modify from https://github.com/huggingface/accelerate/blob/c0552c9012a9bae7f125e1df89cf9ee0b0d250fd/src/accelerate/commands/estimate.py#L285
|
||||
def cal_model_size(args):
|
||||
"""计算模型在各种数据类型下的存储占用
|
||||
主要计算方法是
|
||||
借助calculate_maximum_sizes函数计算所有参数数量在特定下的存储->float32,float16,int8,int4分别进行进一步乘除即可.
|
||||
convert_bytes: 将计算结果转为不超过1024的TB/GB/MB/KB等单位下的结果表示.
|
||||
"""
|
||||
model_name = Path(args.model_path).name
|
||||
model_path = Path(args.model_path).as_posix()
|
||||
# 空加载模型, 可以几乎免去对存储空间的占用, 只记录每层有几个参数, 而不实际去申请内存初始化这些参数, 在加载大模型时有很多好处, 比如这里用来计算模型存储空间的占用, 毕竟加载一次大模型还是挺费时间的~
|
||||
with init_empty_weights():
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_path, torch_dtype="auto"
|
||||
) # 这里auto时会加载bfloat16格式，占用和float16一致
|
||||
total_size, largest_layer = calculate_maximum_sizes(model)
|
||||
data = []
|
||||
|
||||
for dtype in ["float32", "float16", "int8", "int4"]:
|
||||
dtype_total_size = total_size
|
||||
dtype_largest_layer = largest_layer[0]
|
||||
if dtype == "float32":
|
||||
dtype_total_size *= 2
|
||||
dtype_largest_layer *= 2
|
||||
elif dtype == "float16":
|
||||
pass
|
||||
elif dtype == "int8":
|
||||
dtype_total_size /= 2
|
||||
dtype_largest_layer /= 2
|
||||
elif dtype == "int4":
|
||||
dtype_total_size /= 4
|
||||
dtype_largest_layer /= 4
|
||||
row = [dtype, dtype_largest_layer, dtype_total_size]
|
||||
for i, item in enumerate(row):
|
||||
if isinstance(item, (int, float)):
|
||||
row[i] = convert_bytes(item)
|
||||
elif isinstance(item, dict):
|
||||
training_usage = max(item.values())
|
||||
row[i] = (
|
||||
convert_bytes(training_usage) if training_usage != -1 else "N/A"
|
||||
)
|
||||
data.append(row)
|
||||
|
||||
headers = ["dtype", "Largest Layer", "Total Size"]
|
||||
title = f"Memory Usage for loading `{model_name}`"
|
||||
table = create_ascii_table(headers, data, title)
|
||||
print(table)
|
||||
|
||||
|
||||
def _get_args() -> Namespace:
|
||||
"""命令行参数解析为命名空间(可以看作可以用.来访问的字典)"""
|
||||
parser = ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
default=DEFAULT_CKPT_PATH,
|
||||
help="模型路径, 默认为%(default)r。",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cpu",
|
||||
action="store_true",
|
||||
help="仅CPU模式运行。(不启用则默认平均分到所有显卡上)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
default="auto",
|
||||
choices=["auto", "fp32", "fp16", "bf16"],
|
||||
help="加载特定类型的模型。(不启用则默认`auto`, 从config获取。其他类型请自己修改。)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cal-size",
|
||||
action="store_true",
|
||||
help="仅输出模型显存占用。(默认输出float32、float16、int8、int4的占用)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--flash-attn2",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="使用 `flash_attention_2` 推理。(不启用则根据环境使用eager或sdpa)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=12345, help="Demo服务器端口, 默认为`12345`。"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host", default="127.0.0.1", help="DDemo服务器地址, 默认为`127.0.0.1`。"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
class LazyModelLoader:
|
||||
"""延迟加载模型以达到快速显示页面的目的
|
||||
|
||||
延迟加载需要用到多线程,主线程执行web页面的时候,用子线程去加载模型,只需要记录好模型的引用对象即可.
|
||||
(利用延迟加载,主线程中不加载而是放到子线程中,这样而等到页面渲染好,
|
||||
用户输入完提问后,取出模型做推理时,子线程已经加载好模型.)
|
||||
"""
|
||||
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.model = None
|
||||
self.proc = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""加载模型和processor"""
|
||||
with self.lock:
|
||||
if self.model is None: # 确保模型只加载一次
|
||||
print(f"Loading model: {self.args.model_path}")
|
||||
try:
|
||||
model, proc = self._load_model_processor()
|
||||
# model不一定是存有dtype变量的nn.Module类,
|
||||
# 因此可以用这个函数来快速获取里面第一个参数的dtype。
|
||||
dtype = get_first_parameter_dtype(model)
|
||||
except Exception:
|
||||
self.lock.release()
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
exit(-1)
|
||||
self.model = model
|
||||
self.proc = proc
|
||||
print(f"Model {self.args.model_path} loaded")
|
||||
print(f"{model.device=} model.dtype={dtype}")
|
||||
|
||||
def _load_model_processor(
|
||||
self,
|
||||
) -> tuple[Qwen2VLForConditionalGeneration, Qwen2VLProcessor]:
|
||||
"""Qwen2-vl 加载模型时需要加载两个东西:
|
||||
1. 模型, 对应Qwen2VLForConditionalGeneration类
|
||||
2. processor(一个对图片和文本进行处理,转换为模型输入的预处理工具),对应AutoProcessor类
|
||||
|
||||
借助from_pretrained方法,我们可以在加载模型,预处理器时自动处理某些步骤(比如一般加载模型的流程是:初始化->从文件中加载权重并复制到初始化后的类中)而直接返回结果.
|
||||
"""
|
||||
args = self.args
|
||||
device_map = "cpu" if args.cpu else "auto"
|
||||
use_fa2 = (
|
||||
"flash_attention_2"
|
||||
if args.flash_attn2 and is_flash_attn_2_available()
|
||||
else None
|
||||
)
|
||||
dtype = (
|
||||
{
|
||||
"fp16": torch.float16,
|
||||
"fp32": torch.float32,
|
||||
"bf16": torch.bfloat16,
|
||||
"int4": "auto",
|
||||
"int8": "auto", # 不提供量化，自己改吧
|
||||
}[args.dtype]
|
||||
if args.dtype != "auto" # auto会采用config中的配置
|
||||
else args.dtype
|
||||
)
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
args.model_path,
|
||||
torch_dtype=dtype,
|
||||
# 支持: eager/flash_attention_2/sdpa
|
||||
attn_implementation=use_fa2,
|
||||
# auto: 平均分配到每个 GPU.
|
||||
device_map=device_map,
|
||||
)
|
||||
processor = AutoProcessor.from_pretrained(args.model_path)
|
||||
return model, processor
|
||||
|
||||
def get_model(self) -> Qwen2VLForConditionalGeneration:
|
||||
"""获取加载的模型，若尚未加载则触发加载"""
|
||||
if self.model is None:
|
||||
threading.Thread(target=self._load_model).start()
|
||||
return self.model
|
||||
|
||||
def get_processor(self) -> Qwen2VLProcessor:
|
||||
"""获取加载的processor"""
|
||||
if self.proc is None:
|
||||
threading.Thread(target=self._load_model).start()
|
||||
return self.proc
|
||||
|
||||
|
||||
def _transform_messages(
|
||||
messages: List[List[str | Tuple[str, ...]]],
|
||||
video_extensions=VIDEO_EXTENSIONS,
|
||||
image_extensions=IMAGE_EXTENSIONS,
|
||||
user_tag="user",
|
||||
assistant_tag="assistant",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""gradio的messages格式与qwen2的conversation不一致,需要转换
|
||||
|
||||
模型的问答是按轮次来划分的:
|
||||
第一轮: <提问>-><回答>-> 第二轮: <提问>-><回答>-> ...
|
||||
(即便是加入文件,也是放在提问里面.)
|
||||
具体来说:
|
||||
1. gradio目前有多种`对话`的处理格式, 本代码中采用的格式为:
|
||||
[
|
||||
[(<文件1>,<文件2>, ...), None], # 如果传入文件,那么没有对应回答,如果这一行是文件,那么下一行跟用户提问
|
||||
[<提问>, <回答>], # 注意,和上面的区别是提问是一个字符串,而上一行同样位置是一个存储文件的tuple.
|
||||
[("xxx1.jpg","xxx2.jpg"), None],
|
||||
["描述下这两张图片", "这张图片xxx"],
|
||||
...
|
||||
]
|
||||
2. Qwen中的格式采用:
|
||||
[
|
||||
# 这里角色可以包括: system, user, assistant, 内容则是对应角色的提问或回答.
|
||||
{"role":<角色>, "content":<内容>},
|
||||
# 针对图片和视频的传输, Qwen2-vl 在 user 的 <内容> 部分会进一步处理, 因此我们可以将这两类文件放到其 <内容> 中:
|
||||
{"role":"user", "content":"你是谁?"}, # 纯文字
|
||||
{"role":"user", "content":[{"type":"image", "image": "xxx.jpg"}, {"type":"text", "text": "这张图里有什么?"}]}, # 图片+文字
|
||||
{"role":"user", "content":[{"type":"video", "video": "xxx.mp4"}, {"type":"text", "text": "这个视频讲了什么?"}]}, # 视频+文字
|
||||
...
|
||||
]
|
||||
(值得注意的是, 对视频或图片的token限制也可以加在content里面. 可以参考下面的处理)
|
||||
3. 发现了吗,上面两种对话格式不统一,因此送入模型的预处理器前还需要做一次处理,将gradio格式转为qwen预处理支持的格式.而gradio中文件和提问是放在多个列表里的,对话轮次的切换仅通过回答是否是None来判断.
|
||||
"""
|
||||
transformed_messages = [{"role": user_tag, "content": []}]
|
||||
for message in messages:
|
||||
q = message[0]
|
||||
if isinstance(q, tuple):
|
||||
for it in q:
|
||||
if Path(it).suffix in video_extensions:
|
||||
new_item = {
|
||||
"type": "video",
|
||||
"video": it,
|
||||
"min_pixels": VIDEO_MIN_PIXELS,
|
||||
"max_pixels": VIDEO_MAX_PIXELS,
|
||||
"total_pixels": VIDEO_TOTAL_PIXELS,
|
||||
}
|
||||
elif Path(it).suffix in image_extensions:
|
||||
new_item = {
|
||||
"type": "image",
|
||||
"image": it,
|
||||
"min_pixels": MIN_PIXELS,
|
||||
"max_pixels": MAX_PIXELS,
|
||||
}
|
||||
transformed_messages[-1]["content"].append(new_item)
|
||||
elif isinstance(q, str):
|
||||
if transformed_messages[-1]["content"]:
|
||||
new_item = {"type": "text", "text": it}
|
||||
transformed_messages[-1]["content"].append(new_item)
|
||||
else:
|
||||
transformed_messages[-1]["content"] = q
|
||||
|
||||
if message[1]: # 如果回答里有值，说明当前轮对话完成，接下来做下一轮对话的处理。
|
||||
transformed_messages.extend(
|
||||
[
|
||||
{"role": assistant_tag, "content": message[1]},
|
||||
{"role": user_tag, "content": []},
|
||||
]
|
||||
)
|
||||
return transformed_messages
|
||||
|
||||
|
||||
def _gc():
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
# modify from https://github.com/gradio-app/gradio/blob/4e1f7dbcb2ea2a0cc29bb76faf5758a9f4afcd6d/demo/chatbot_examples/run.py#L1, 参考这里可以看到gradio给出的带文件传输的chatbot实现
|
||||
def print_like_dislike(x: gr.LikeData) -> None:
|
||||
print(f"{x.index=} {x.value=}{x.liked=}")
|
||||
|
||||
|
||||
def add_message(
|
||||
history: List[List[str | Tuple[str, ...]]], message: Dict
|
||||
) -> tuple[List[List[str | Tuple[str, ...]]], gr.MultimodalTextbox]:
|
||||
"""
|
||||
Params:
|
||||
history: gradio的一种对话格式, 可以参考 `_transform_messages` 的文档注释.
|
||||
message: gr.MultimodalTextbox类, 可以当作字典访问,里面有file和text,分别表示提供的文件和提问.
|
||||
"""
|
||||
for x in message["files"]:
|
||||
history.append(((x,), None))
|
||||
if message["text"] is not None:
|
||||
history.append(
|
||||
(message["text"], None)
|
||||
) # 这里填空是因为还需要把history数据转换后给模型进行回复，然后才能赋值到这里。
|
||||
return history, gr.MultimodalTextbox(value=None, interactive=False)
|
||||
|
||||
|
||||
def _pred(
|
||||
messages: List[List[str | Tuple[str, ...]]],
|
||||
temperature: float,
|
||||
topk: int,
|
||||
topp: float,
|
||||
processor: Qwen2VLProcessor,
|
||||
model: Qwen2VLForConditionalGeneration,
|
||||
):
|
||||
"""模型对话的主要逻辑, 这段代码参考了Qwen2-vl官方的 web demo的一部分流程.
|
||||
先转换出qwen2-vl需要的格式
|
||||
然后将文本和图像/视频分别送入预处理器(在此之前,图像/视频要借助官方提供的process_vision_info函数resize为28*28的倍数)
|
||||
然后送入模型进行推理,模型推理的结果作为回答."""
|
||||
messages = _transform_messages(messages)
|
||||
|
||||
# 这里首先把messages对话格式转为纯文本的特殊格式
|
||||
text = processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
# 这里对图片/视频做resize处理，主要是模型内视觉层对图片的宽高有特定限制。
|
||||
image_inputs, video_inputs = process_vision_info(messages)
|
||||
# 开始通过预处理器， 将文本和图片/视频作为输入， 处理出模型需要的数据: token_id列表 和 特定形状的一堆像素点
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to(model.device) # 结果送入模型所在的设备（CPU或某GPU卡）
|
||||
|
||||
streamer = TextIteratorStreamer(
|
||||
processor.tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True
|
||||
) # 借助TextIteratorStreamer可以提供一个流式的接口，是的模型每生成一个token就返回这个token对应的文本。
|
||||
|
||||
# 模型在生成token前有一个后处理，这里简单介绍贪心解码和采样解码：
|
||||
# 当使用贪心解码时，设置do_sample = False, 对于下一个token，模型总会选择预测的概率最大的那个。
|
||||
# 当使用采样时，字如其名，就是随机的选择。首先会对输出的下一个token的概率分布做一些简单变换（比如temperature越大，可以让概率分布越平均， topK和topP则减小待采样的词表），然后对剩余的词表进行加权的随机选择（因为加权，所以概率大的还是有大的机率被选中，但是如果temperature设置过大，反而把剩余所有词表的概率平均化了，这样大家的权重都接近1:1）
|
||||
# 因此也可以说，temperature控制模型的创造性，越大，模型采样到不同词的可能越大，模型的回答便越发散。
|
||||
_gen_kwargs = (
|
||||
dict(temperature=temperature, top_p=topp, top_k=topk)
|
||||
if temperature
|
||||
else dict(do_sample=False)
|
||||
)
|
||||
# max_new_tokens主要限制模型回答的最大token长度，当超过这个token就会停止。
|
||||
gen_config = GenerationConfig(max_new_tokens=512, **_gen_kwargs)
|
||||
|
||||
# 使用子线程启动模型的推理，结果会自动添加到streamer接口中。
|
||||
thread = Thread(
|
||||
target=model.generate,
|
||||
kwargs=dict(
|
||||
**inputs,
|
||||
generation_config=gen_config,
|
||||
streamer=streamer,
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return streamer
|
||||
|
||||
|
||||
def bot(
|
||||
history: List[List[str | Tuple[str, ...]]],
|
||||
temperature: float,
|
||||
topk: int,
|
||||
topp: float,
|
||||
) -> Generator[List[List[str | Tuple[str, ...]]], Any, None]:
|
||||
"""这里是输入提问并点击提交后触发回答的逻辑"""
|
||||
_gc() # 可以清除一下上一次回答的存储碎片
|
||||
# 然后将提问与之前轮次的对话送入_pred让模型针对这些上文进行推理
|
||||
model, proc = loader.get_model(), loader.get_processor()
|
||||
# 这里会返回一个流式的接口,通过for循环即可获取接口里新添加进去的回答,然后拼接到history里流式的返回给gradio即可.
|
||||
stream = _pred(history, temperature, topk, topp, processor=proc, model=model)
|
||||
history[-1][1] = ""
|
||||
for it in stream:
|
||||
history[-1][1] += it
|
||||
yield history
|
||||
|
||||
|
||||
def web_demo(args: Namespace):
|
||||
"""创建gradio应用程序"""
|
||||
with gr.Blocks(fill_height=True) as demo:
|
||||
with gr.Column(scale=6):
|
||||
chatbot = gr.Chatbot(
|
||||
label="Qwen2VL demo",
|
||||
elem_id="chatbot",
|
||||
bubble_full_width=False,
|
||||
scale=1,
|
||||
type="tuples",
|
||||
)
|
||||
chat_input = gr.MultimodalTextbox(
|
||||
interactive=True,
|
||||
file_count="multiple",
|
||||
placeholder="Enter message or upload file...",
|
||||
show_label=False,
|
||||
)
|
||||
with gr.Column(scale=1):
|
||||
with gr.Accordion("Gen Config", open=False):
|
||||
# 一个隐藏的选项，可以控制 Temperature、top p、top k
|
||||
temperature = gr.Slider(0.0, 1.0, step=0.01, label="Temperature")
|
||||
topk = gr.Slider(-1, 1000, step=2, label="Top K") # need?
|
||||
topp = gr.Slider(0.0, 1.0, step=0.01, label="Top P") # need?
|
||||
# 多模态的输入会先调用 add_message，然后调用 bot，最后清除输入框中的内容（因为已经显示在chatbot里了）
|
||||
chat_msg = chat_input.submit(
|
||||
add_message, [chatbot, chat_input], [chatbot, chat_input]
|
||||
)
|
||||
bot_msg = chat_msg.then(
|
||||
bot, [chatbot, temperature, topk, topp], chatbot, api_name="bot_response"
|
||||
)
|
||||
bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
|
||||
|
||||
# 这里主要是给chatbot的每个回答绑定一个用户偏好反馈的结果打印
|
||||
chatbot.like(print_like_dislike, None, None)
|
||||
demo.launch(max_threads=2, server_name=args.host, server_port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = _get_args()
|
||||
# 在这里检测flash-attn是否安装和启用
|
||||
print("flash-attn 已安装" if is_flash_attn_2_available() else "flash-attn 未安装")
|
||||
print(
|
||||
"flash-attn 已启用。"
|
||||
if args.flash_attn2 and is_flash_attn_2_available()
|
||||
else "flash-attn 未启用。"
|
||||
)
|
||||
cal_model_size(args) # 在每次启动模型时会先显示模型占用
|
||||
if args.cal_size is False:
|
||||
loader = LazyModelLoader(args)
|
||||
loader.get_model() # 在这里手动提前触发一下模型加载
|
||||
web_demo(args) # 运行web demo
|
||||
@@ -0,0 +1,10 @@
|
||||
# requirements.txt
|
||||
qwen_vl_utils==0.0.8
|
||||
transformers==4.46.2
|
||||
accelerate==1.1.1
|
||||
gradio==5.5.0
|
||||
torchvision==0.19.0
|
||||
modelscope==1.20.0
|
||||
# # 如果安装了flash-attn，则会多出这两个库
|
||||
# einops==0.8.0
|
||||
# flash-attn==2.7.0
|
||||
@@ -0,0 +1,96 @@
|
||||
# Qwen2-VL-2B-Instruct WebDemo 部署
|
||||
|
||||
|
||||
# 环境准备
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.10
|
||||
cuda 11.8
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
|
||||
# 环境安装
|
||||
|
||||
```python
|
||||
# 换源
|
||||
python -m pip install --upgrade pip
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
|
||||
# 需要安装的库
|
||||
# torchvision需要安装匹配对应torch的版本
|
||||
pip install qwen_vl_utils==0.0.8 transformers==4.46.2 accelerate==1.1.0 gradio==5.5.0 torchvision==0.18.0 av==13.1.0
|
||||
|
||||
# 如需使用魔搭(国内推荐)下载模型, 需安装这个库
|
||||
pip install modelscope==1.20.0
|
||||
|
||||
# 安装flash-attn(可选)
|
||||
# 如显卡支持flash-attn，在确认对应python、pytorch、cuda版本后, 下载对应的release版本.
|
||||
wegt https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
# 镜像加速链接:
|
||||
# wget https://github.moeyy.xyz/https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.0.post2/flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
pip install flash_attn-2.7.0.post2+cu12torch2.3cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
|
||||
```
|
||||
> 完整的pip列表(包含依赖)请参考[02-Qwen2-VL-2B-Instruct Web Demo 参考代码/requirements.txt](./02-Qwen2-VL-2B-Instruct%20Web%20Demo%20参考代码/requirements.txt)
|
||||
|
||||
# 下载模型(两种下载方法二选一即可~)
|
||||
## 1. 借助 modelscope 下载
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
新建 `model_download.py` 文件输入以下代码，并运行 `python model_download.py` 执行下载。
|
||||
|
||||
此处使用 `modelscope` 提供的 `snapshot_download` 函数进行下载，该方法对国内的用户十分友好。
|
||||
|
||||
```python
|
||||
# model_download.py
|
||||
from modelscope import snapshot_download
|
||||
model_dir = snapshot_download('Qwen/Qwen2-VL-2B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：请记得修改 `cache_dir` 为你自己的模型下载路径 ~
|
||||
|
||||
## 2. 借助 git lfs 下载
|
||||
```python
|
||||
# 进入autodl-tmp/ 或者你要保存的路径
|
||||
cd autodl-tmp/
|
||||
|
||||
# 首先安装lfs，便于通过git直接下载模型。
|
||||
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
|
||||
sudo apt-get install git-lfs
|
||||
|
||||
# 需要下载的模型
|
||||
MODEL=Qwen2-VL-2B-Instruct
|
||||
# MODEL=Qwen2-VL-7B-Instruct
|
||||
# MODEL=Qwen2-VL-72B-Instruct
|
||||
|
||||
# # huggingface 下载
|
||||
# URL="https://huggingface.co/Qwen/"
|
||||
# git clone "${URL}/${MODEL}"
|
||||
|
||||
# 魔搭下载（国内推荐）
|
||||
URL="https://www.modelscope.cn/Qwen"
|
||||
git clone "${URL}/${MODEL}.git"
|
||||
|
||||
# 返回根目录
|
||||
cd ..
|
||||
```
|
||||
|
||||
# 运行Demo
|
||||
|
||||
```python
|
||||
# 可以使用 python mm_qwen2vl.py -h 或查看代码来查看命令帮助
|
||||
# Ampere/Ada/Hopper架构显卡可以启用flash attn2加速推理，autodl要通过6006端口对外访问。(没安装flash-attn库的忽略)
|
||||
# python mm_qwen2vl.py --flash-attn2 --model-path ./autodl-tmp/Qwen2-VL-2B-Instruct --host 0.0.0.0 --port 6006
|
||||
python mm_qwen2vl.py --model-path ./autodl-tmp/Qwen2-VL-2B-Instruct --host 0.0.0.0 --port 6006
|
||||
```
|
||||
> 完整代码及详细注释请参考[mm_qwen2vl.py](./02-Qwen2-VL-2B-Instruct%20Web%20Demo%20参考代码/mm_qwen2vl.py)
|
||||
|
||||
# 测试效果
|
||||
## 图片
|
||||
|
||||
## 视频
|
||||
|
||||
|
||||
> 如果觉得2B理解能力较差, 建议用7B以上模型.
|
||||
@@ -0,0 +1,587 @@
|
||||
# Qwen2-VL-2B-Instruct Lora 微调 SwanLab可视化记录版
|
||||
|
||||
本节我们简要介绍基于 transformers、peft 等框架，使用 Qwen2-VL-2B-Instruct 模型在**COCO2014图像描述** 上进行Lora微调训练，同时使用 [SwanLab](https://github.com/swanhubx/swanlab) 监控训练过程与评估模型效果。
|
||||
|
||||
Lora 是一种高效微调方法，深入了解其原理可参见博客：[知乎|深入浅出 Lora](https://zhuanlan.zhihu.com/p/650197598)。
|
||||
|
||||
训练过程：<a href="https://swanlab.cn/@ZeyiLin/Qwen2-VL-finetune/runs/53vm3y7sp5h5fzlmlc5up/chart" target="_blank">Qwen2-VL-finetune
|
||||
</a>
|
||||
|
||||
## 目录
|
||||
|
||||
- [SwanLab简介](#-SwanLab简介)
|
||||
- [环境配置](#-环境配置)
|
||||
- [准备数据集](#-准备数据集)
|
||||
- [模型下载与加载](#-模型下载与加载)
|
||||
- [集成SwanLab](#-集成SwanLab)
|
||||
- [开始微调（完整代码）](#-开始微调)
|
||||
- [训练结果演示](#-训练结果演示)
|
||||
- [推理LoRA微调后的模型](#-推理LoRA微调后的模型)
|
||||
- [补充](#补充)
|
||||
|
||||
## 👋 SwanLab简介
|
||||
|
||||
|
||||
|
||||
[SwanLab](https://github.com/swanhubx/swanlab) 是一个开源的模型训练记录工具，常被称为"中国版 Weights&Biases + Tensorboard"。SwanLab面向AI研究者，提供了训练可视化、自动日志记录、超参数记录、实验对比、多人协同等功能。在SwanLab上，研究者能基于直观的可视化图表发现训练问题，对比多个实验找到研究灵感，并通过在线链接的分享与基于组织的多人协同训练，打破团队沟通的壁垒。
|
||||
|
||||
**为什么要记录训练？**
|
||||
|
||||
相较于软件开发，模型训练更像一个实验科学。一个品质优秀的模型背后，往往是成千上万次实验。研究者需要不断尝试、记录、对比，积累经验，才能找到最佳的模型结构、超参数与数据配比。在这之中，如何高效进行记录与对比，对于研究效率的提升至关重要。
|
||||
|
||||
**可视化的价值在哪里？**
|
||||
|
||||
机器学习模型训练往往伴随着大量的超参数、指标、日志等数据，很多关键信息往往存在于实验的中间而非结尾，如果不对连续的指标通过图表进行可视化，往往会错失发现问题的最佳时机，甚至错过关键信息。同时不进行可视化，也难以对比多个实验之间的差异。
|
||||
可视化也为AI研究者提供了良好的交流基础，研究者们可以基于图表进行沟通、分析与优化，而非以往看着枯燥的终端打印。这打破了团队沟通的壁垒，提高了整体的研发效率。
|
||||
|
||||
|
||||
## 🌍 环境配置
|
||||
|
||||
环境配置分为三步：
|
||||
|
||||
1. 确保你的电脑上至少有一张英伟达显卡，并已安装好了CUDA环境。
|
||||
|
||||
2. 安装Python（版本>=3.8）以及能够调用CUDA加速的PyTorch。
|
||||
|
||||
3. 安装Qwen2-VL微调相关的第三方库，可以使用以下命令：
|
||||
|
||||
```bash
|
||||
python -m pip install --upgrade pip
|
||||
# 更换 pypi 源加速库的安装
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install modelscope==1.18.0
|
||||
pip install transformers==4.46.2
|
||||
pip install sentencepiece==0.2.0
|
||||
pip install accelerate==1.1.1
|
||||
pip install datasets==2.18.0
|
||||
pip install peft==0.13.2
|
||||
pip install swanlab==0.3.25
|
||||
pip install qwen-vl-utils==0.0.8
|
||||
```
|
||||
|
||||
## 📚 准备数据集
|
||||
|
||||
本节使用的是 [coco_2014_caption](https://modelscope.cn/datasets/modelscope/coco_2014_caption/summary) 数据集（中的500张图），该数据集主要用于多模态（Image-to-Text）任务。
|
||||
|
||||
> 数据集介绍：COCO 2014 Caption数据集是Microsoft Common Objects in Context (COCO)数据集的一部分，主要用于图像描述任务。该数据集包含了大约40万张图像，每张图像都有至少1个人工生成的英文描述语句。这些描述语句旨在帮助计算机理解图像内容，并为图像自动生成描述提供训练数据。
|
||||
|
||||
|
||||
|
||||
在本节的任务中，我们主要使用其中的前500张图像，并对它进行处理和格式调整，目标是组合成如下格式的json文件：
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "identity_1",
|
||||
"conversations": [
|
||||
{
|
||||
"from": "user",
|
||||
"value": "COCO Yes: <|vision_start|>图像文件路径<|vision_end|>"
|
||||
},
|
||||
{
|
||||
"from": "assistant",
|
||||
"value": "A snow skier assessing the mountain before starting to sky"
|
||||
}
|
||||
]
|
||||
},
|
||||
...
|
||||
]
|
||||
```
|
||||
|
||||
其中，"from"是角色（user代表人类，assistant代表模型），"value"是聊天的内容，其中`<|vision_start|>`和`<|vision_end|>`是Qwen2-VL模型识别图像的标记，中间可以放图像的文件路径，也可以是URL。
|
||||
|
||||
**数据集下载与处理方式**
|
||||
|
||||
1. **我们需要做四件事情：**
|
||||
- 通过Modelscope下载coco_2014_caption数据集
|
||||
- 加载数据集，将图像保存到本地
|
||||
- 将图像路径和描述文本转换为一个csv文件
|
||||
- 将csv文件转换为json文件
|
||||
|
||||
2. **使用下面的代码完成从数据下载到生成csv的过程：**
|
||||
|
||||
data2csv.py：
|
||||
|
||||
```python
|
||||
# 导入所需的库
|
||||
from modelscope.msdatasets import MsDataset
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
MAX_DATA_NUMBER = 500
|
||||
|
||||
# 检查目录是否已存在
|
||||
if not os.path.exists('coco_2014_caption'):
|
||||
# 从modelscope下载COCO 2014图像描述数据集
|
||||
ds = MsDataset.load('modelscope/coco_2014_caption', subset_name='coco_2014_caption', split='train')
|
||||
print(len(ds))
|
||||
# 设置处理的图片数量上限
|
||||
total = min(MAX_DATA_NUMBER, len(ds))
|
||||
|
||||
# 创建保存图片的目录
|
||||
os.makedirs('coco_2014_caption', exist_ok=True)
|
||||
|
||||
# 初始化存储图片路径和描述的列表
|
||||
image_paths = []
|
||||
captions = []
|
||||
|
||||
for i in range(total):
|
||||
# 获取每个样本的信息
|
||||
item = ds[i]
|
||||
image_id = item['image_id']
|
||||
caption = item['caption']
|
||||
image = item['image']
|
||||
|
||||
# 保存图片并记录路径
|
||||
image_path = os.path.abspath(f'coco_2014_caption/{image_id}.jpg')
|
||||
image.save(image_path)
|
||||
|
||||
# 将路径和描述添加到列表中
|
||||
image_paths.append(image_path)
|
||||
captions.append(caption)
|
||||
|
||||
# 每处理50张图片打印一次进度
|
||||
if (i + 1) % 50 == 0:
|
||||
print(f'Processing {i+1}/{total} images ({(i+1)/total*100:.1f}%)')
|
||||
|
||||
# 将图片路径和描述保存为CSV文件
|
||||
df = pd.DataFrame({
|
||||
'image_path': image_paths,
|
||||
'caption': captions
|
||||
})
|
||||
|
||||
# 将数据保存为CSV文件
|
||||
df.to_csv('./coco-2024-dataset.csv', index=False)
|
||||
|
||||
print(f'数据处理完成，共处理了{total}张图片')
|
||||
|
||||
else:
|
||||
print('coco_2014_caption目录已存在,跳过数据处理步骤')
|
||||
```
|
||||
|
||||
|
||||
**3. 在同一目录下，用以下代码，将csv文件转换为json文件：**
|
||||
|
||||
csv2json.py：
|
||||
|
||||
```python
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
# 载入CSV文件
|
||||
df = pd.read_csv('./coco-2024-dataset.csv')
|
||||
conversations = []
|
||||
|
||||
# 添加对话数据
|
||||
for i in range(len(df)):
|
||||
conversations.append({
|
||||
"id": f"identity_{i+1}",
|
||||
"conversations": [
|
||||
{
|
||||
"from": "user",
|
||||
"value": f"COCO Yes: <|vision_start|>{df.iloc[i]['image_path']}<|vision_end|>"
|
||||
},
|
||||
{
|
||||
"from": "assistant",
|
||||
"value": df.iloc[i]['caption']
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
# 保存为Json
|
||||
with open('data_vl.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(conversations, f, ensure_ascii=False, indent=2)
|
||||
```
|
||||
|
||||
此时目录下会多出两个文件：
|
||||
- coco-2024-dataset.csv
|
||||
- data_vl.json
|
||||
|
||||
至此，我们完成了数据集的准备。
|
||||
|
||||
|
||||
## 🤖 模型下载与加载
|
||||
|
||||
这里我们使用modelscope下载Qwen2-VL-2B-Instruct模型，然后把它加载到Transformers中进行训练：
|
||||
|
||||
```python
|
||||
from modelscope import snapshot_download, AutoTokenizer
|
||||
from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq, Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
import torch
|
||||
|
||||
# 在modelscope上下载Qwen2-VL模型到本地目录下
|
||||
model_dir = snapshot_download("Qwen/Qwen2-VL-2B-Instruct", cache_dir="./", revision="master")
|
||||
|
||||
# 使用Transformers加载模型权重
|
||||
tokenizer = AutoTokenizer.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct/", use_fast=False, trust_remote_code=True)
|
||||
# 特别的，Qwen2-VL-2B-Instruct模型需要使用Qwen2VLForConditionalGeneration来加载
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct/", device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True,)
|
||||
model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法
|
||||
```
|
||||
|
||||
模型大小为 4.5GB，下载模型大概需要 5 分钟。
|
||||
|
||||
## 🐦 集成SwanLab
|
||||
|
||||
SwanLab与Transformers已经做好了集成，用法是在Trainer的`callbacks`参数中添加`SwanLabCallback`实例，就可以自动记录超参数和训练指标，简化代码如下：
|
||||
|
||||
```python
|
||||
from swanlab.integration.transformers import SwanLabCallback
|
||||
from transformers import Trainer
|
||||
|
||||
swanlab_callback = SwanLabCallback()
|
||||
|
||||
trainer = Trainer(
|
||||
...
|
||||
callbacks=[swanlab_callback],
|
||||
)
|
||||
```
|
||||
|
||||
首次使用SwanLab，需要先在[官网](https://swanlab.cn)注册一个账号，然后在用户设置页面复制你的API Key，然后在训练开始提示登录时粘贴即可，后续无需再次登录：
|
||||
|
||||
|
||||
|
||||
更多用法可参考[快速开始](https://docs.swanlab.cn/zh/guide_cloud/general/quick-start.html)、[Transformers集成](https://docs.swanlab.cn/zh/guide_cloud/integration/integration-huggingface-transformers.html)。
|
||||
|
||||
|
||||
## 🚀 开始微调
|
||||
|
||||
查看可视化训练过程：<a href="https://swanlab.cn/@ZeyiLin/Qwen2-VL-finetune/runs/53vm3y7sp5h5fzlmlc5up/chart" target="_blank">Qwen2-VL-finetune</a>
|
||||
|
||||
|
||||
**本节代码做了以下几件事：**
|
||||
1. 下载并加载Qwen2-VL-2B-Instruct模型
|
||||
2. 加载数据集，取前496条数据参与训练，4条数据进行主观评测
|
||||
3. 配置Lora，参数为r=64, lora_alpha=16, lora_dropout=0.05
|
||||
4. 使用SwanLab记录训练过程，包括超参数、指标和最终的模型输出结果
|
||||
5. 训练2个epoch
|
||||
|
||||
开始执行代码时的目录结构应该是：
|
||||
```
|
||||
|———— train.py
|
||||
|———— coco_2014_caption
|
||||
|———— coco-2024-dataset.csv
|
||||
|———— data_vl.json
|
||||
|———— data2csv.py
|
||||
|———— csv2json.py
|
||||
```
|
||||
|
||||
|
||||
**完整代码如下**
|
||||
|
||||
train.py：
|
||||
|
||||
```python
|
||||
import torch
|
||||
from datasets import Dataset
|
||||
from modelscope import snapshot_download, AutoTokenizer
|
||||
from swanlab.integration.transformers import SwanLabCallback
|
||||
from qwen_vl_utils import process_vision_info
|
||||
from peft import LoraConfig, TaskType, get_peft_model, PeftModel
|
||||
from transformers import (
|
||||
TrainingArguments,
|
||||
Trainer,
|
||||
DataCollatorForSeq2Seq,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
AutoProcessor,
|
||||
)
|
||||
import swanlab
|
||||
import json
|
||||
|
||||
|
||||
def process_func(example):
|
||||
"""
|
||||
将数据集进行预处理
|
||||
"""
|
||||
MAX_LENGTH = 8192
|
||||
input_ids, attention_mask, labels = [], [], []
|
||||
conversation = example["conversations"]
|
||||
input_content = conversation[0]["value"]
|
||||
output_content = conversation[1]["value"]
|
||||
|
||||
instruction = tokenizer(
|
||||
f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{input_content}<|im_end|>\n<|im_start|>assistant\n",
|
||||
add_special_tokens=False,
|
||||
)
|
||||
response = tokenizer(f"{output_content}", add_special_tokens=False)
|
||||
input_ids = (
|
||||
instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
|
||||
)
|
||||
attention_mask = instruction["attention_mask"] + response["attention_mask"] + [1]
|
||||
labels = (
|
||||
[-100] * len(instruction["input_ids"])
|
||||
+ response["input_ids"]
|
||||
+ [tokenizer.pad_token_id]
|
||||
)
|
||||
|
||||
if len(input_ids) > MAX_LENGTH: # 做一个截断
|
||||
input_ids = input_ids[:MAX_LENGTH]
|
||||
attention_mask = attention_mask[:MAX_LENGTH]
|
||||
labels = labels[:MAX_LENGTH]
|
||||
|
||||
return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
|
||||
|
||||
|
||||
def predict(messages, model):
|
||||
# 准备推理
|
||||
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
image_inputs, video_inputs = process_vision_info(messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# 生成输出
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=128)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
||||
)
|
||||
|
||||
return output_text[0]
|
||||
|
||||
|
||||
# 在modelscope上下载Qwen2-VL模型到本地目录下
|
||||
model_dir = snapshot_download("Qwen/Qwen2-VL-2B-Instruct", cache_dir="./", revision="master")
|
||||
|
||||
# 使用Transformers加载模型权重
|
||||
tokenizer = AutoTokenizer.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct/", use_fast=False, trust_remote_code=True)
|
||||
processor = AutoProcessor.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct")
|
||||
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct/", device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True,)
|
||||
model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法
|
||||
|
||||
# 处理数据集：读取json文件
|
||||
# 拆分成训练集和测试集，保存为data_vl_train.json和data_vl_test.json
|
||||
train_json_path = "data_vl.json"
|
||||
with open(train_json_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
train_data = data[:-4]
|
||||
test_data = data[-4:]
|
||||
|
||||
with open("data_vl_train.json", "w") as f:
|
||||
json.dump(train_data, f)
|
||||
|
||||
with open("data_vl_test.json", "w") as f:
|
||||
json.dump(test_data, f)
|
||||
|
||||
train_ds = Dataset.from_json("data_vl_train.json")
|
||||
train_dataset = train_ds.map(process_func)
|
||||
|
||||
# 配置LoRA
|
||||
config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
|
||||
inference_mode=False, # 训练模式
|
||||
r=64, # Lora 秩
|
||||
lora_alpha=16, # Lora alaph，具体作用参见 Lora 原理
|
||||
lora_dropout=0.05, # Dropout 比例
|
||||
bias="none",
|
||||
)
|
||||
|
||||
# 获取LoRA模型
|
||||
peft_model = get_peft_model(model, config)
|
||||
|
||||
# 配置训练参数
|
||||
args = TrainingArguments(
|
||||
output_dir="./output/Qwen2-VL-2B",
|
||||
per_device_train_batch_size=4,
|
||||
gradient_accumulation_steps=4,
|
||||
logging_steps=10,
|
||||
num_train_epochs=2,
|
||||
save_steps=100,
|
||||
learning_rate=1e-4,
|
||||
save_on_each_node=True,
|
||||
gradient_checkpointing=True,
|
||||
report_to="none",
|
||||
)
|
||||
|
||||
# 设置SwanLab回调
|
||||
swanlab_callback = SwanLabCallback(
|
||||
project="Qwen2-VL-finetune",
|
||||
experiment_name="qwen2-vl-coco2014",
|
||||
config={
|
||||
"model": "https://modelscope.cn/models/Qwen/Qwen2-VL-2B-Instruct",
|
||||
"dataset": "https://modelscope.cn/datasets/modelscope/coco_2014_caption/quickstart",
|
||||
"github": "https://github.com/datawhalechina/self-llm",
|
||||
"prompt": "COCO Yes: ",
|
||||
"train_data_number": len(train_data),
|
||||
"lora_rank": 64,
|
||||
"lora_alpha": 16,
|
||||
"lora_dropout": 0.1,
|
||||
},
|
||||
)
|
||||
|
||||
# 配置Trainer
|
||||
trainer = Trainer(
|
||||
model=peft_model,
|
||||
args=args,
|
||||
train_dataset=train_dataset,
|
||||
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
|
||||
callbacks=[swanlab_callback],
|
||||
)
|
||||
|
||||
# 开启模型训练
|
||||
trainer.train()
|
||||
|
||||
# ====================测试模式===================
|
||||
# 配置测试参数
|
||||
val_config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
|
||||
inference_mode=True, # 训练模式
|
||||
r=64, # Lora 秩
|
||||
lora_alpha=16, # Lora alaph，具体作用参见 Lora 原理
|
||||
lora_dropout=0.05, # Dropout 比例
|
||||
bias="none",
|
||||
)
|
||||
|
||||
# 获取测试模型
|
||||
val_peft_model = PeftModel.from_pretrained(model, model_id="./output/Qwen2-VL-2B/checkpoint-62", config=val_config)
|
||||
|
||||
# 读取测试数据
|
||||
with open("data_vl_test.json", "r") as f:
|
||||
test_dataset = json.load(f)
|
||||
|
||||
test_image_list = []
|
||||
for item in test_dataset:
|
||||
input_image_prompt = item["conversations"][0]["value"]
|
||||
# 去掉前后的<|vision_start|>和<|vision_end|>
|
||||
origin_image_path = input_image_prompt.split("<|vision_start|>")[1].split("<|vision_end|>")[0]
|
||||
|
||||
messages = [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"image": origin_image_path
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "COCO Yes:"
|
||||
}
|
||||
]}]
|
||||
|
||||
response = predict(messages, val_peft_model)
|
||||
messages.append({"role": "assistant", "content": f"{response}"})
|
||||
print(messages[-1])
|
||||
|
||||
test_image_list.append(swanlab.Image(origin_image_path, caption=response))
|
||||
|
||||
swanlab.log({"Prediction": test_image_list})
|
||||
|
||||
# 在Jupyter Notebook中运行时要停止SwanLab记录，需要调用swanlab.finish()
|
||||
swanlab.finish()
|
||||
```
|
||||
|
||||
看到下面的进度条即代表训练开始：
|
||||
|
||||
|
||||
|
||||
|
||||
## 💻 训练结果演示
|
||||
|
||||
|
||||
|
||||
从SwanLab图表中我们可以看到，lr的下降策略是线性下降，loss随epoch呈现下降趋势，而grad_norm则在上升。这种形态往往反映了模型有过拟合的风险，训练不要超过2个epoch。
|
||||
|
||||
在`Prediction`图表中记录着模型最终的输出结果，可以看到模型在回答的风格上是用的COCO数据集的简短英文风格进行的描述：
|
||||
|
||||
|
||||
|
||||
而同样的图像，没有被微调的模型输出结果如下：
|
||||
|
||||
```
|
||||
1-没有微调：The image depicts a cozy living room with a rocking chair in the center, a bookshelf filled with books, and a table with a vase and a few other items. The walls are decorated with wallpaper, and there are curtains on the windows. The room appears to be well-lit, with sunlight streaming in from the windows.
|
||||
1-微调后：A living room with a rocking chair, a bookshelf, and a table with a vase and a bowl.
|
||||
|
||||
2-没有微调：It looks like a family gathering or a party in a living room. There are several people sitting around a dining table, eating pizza. The room has a cozy and warm atmosphere.
|
||||
2-微调后：A group of people sitting around a dining table eating pizza.
|
||||
```
|
||||
|
||||
可以明显看到微调后风格的变化。
|
||||
|
||||
|
||||
## 🧐 推理LoRA微调后的模型
|
||||
|
||||
加载lora微调后的模型，并进行推理：
|
||||
|
||||
```python
|
||||
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
|
||||
from qwen_vl_utils import process_vision_info
|
||||
from peft import PeftModel, LoraConfig, TaskType
|
||||
|
||||
config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
|
||||
inference_mode=True,
|
||||
r=64, # Lora 秩
|
||||
lora_alpha=16, # Lora alaph，具体作用参见 Lora 原理
|
||||
lora_dropout=0.05, # Dropout 比例
|
||||
bias="none",
|
||||
)
|
||||
|
||||
# default: Load the model on the available device(s)
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
"./Qwen/Qwen2-VL-2B-Instruct", torch_dtype="auto", device_map="auto"
|
||||
)
|
||||
model = PeftModel.from_pretrained(model, model_id="./output/Qwen2-VL-2B/checkpoint-62", config=config)
|
||||
processor = AutoProcessor.from_pretrained("./Qwen/Qwen2-VL-2B-Instruct")
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"image": "测试图像路径",
|
||||
},
|
||||
{"type": "text", "text": "COCO Yes:"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# Preparation for inference
|
||||
text = processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
image_inputs, video_inputs = process_vision_info(messages)
|
||||
inputs = processor(
|
||||
text=[text],
|
||||
images=image_inputs,
|
||||
videos=video_inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to("cuda")
|
||||
|
||||
# Inference: Generation of the output
|
||||
generated_ids = model.generate(**inputs, max_new_tokens=128)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = processor.batch_decode(
|
||||
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
||||
)
|
||||
print(output_text)
|
||||
```
|
||||
|
||||
|
||||
## 补充
|
||||
|
||||
### 详细硬件配置和参数说明
|
||||
|
||||
使用4张A100 40GB显卡，batch size为4，gradient accumulation steps为4，训练2个epoch的用时为1分钟57秒。
|
||||
|
||||
|
||||
|
||||
### 注意
|
||||
|
||||
- 在微调脚本中，`val_peft_model`加载的是一共固定的checkpoint文件，如果你添加了数据或超参数，请根据实际情况修改checkpoint文件路径。
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.6 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 384 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 289 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 645 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 842 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 233 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 316 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 349 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 205 KiB |
@@ -0,0 +1,129 @@
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
# Qwen2.5-7B-Instruct Langchain 接入
|
||||
|
||||
## 环境准备
|
||||
|
||||
本文基础环境如下：
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.12
|
||||
cuda 12.1
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
|
||||
> 本文默认学习者已安装好以上 Pytorch(cuda) 环境，如未安装请自行安装。
|
||||
|
||||
pip 换源加速下载并安装依赖包
|
||||
|
||||
```shell
|
||||
# 升级pip
|
||||
python -m pip install --upgrade pip
|
||||
# 更换 pypi 源加速库的安装
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install transformers==4.46.2
|
||||
pip install modelscope==1.20.0
|
||||
pip install langchain==0.3.7
|
||||
pip install accelerate==1.1.1
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 模型下载
|
||||
|
||||
使用 `modelscope` 中的 `snapshot_download` 函数下载模型，第一个参数为模型名称，参数 `cache_dir` 为模型的下载路径。
|
||||
|
||||
在新建 `model_download.py` 文件并在其中输入以下内容，粘贴代码后记得保存文件，如下图所示。并运行 `python model_download.py` 执行下载，模型大小为 16 GB，下载模型大概需要 12 分钟。
|
||||
|
||||
```python
|
||||
import torch
|
||||
from modelscope import snapshot_download, AutoModel, AutoTokenizer
|
||||
import os
|
||||
model_dir = snapshot_download('Qwen/Qwen2.5-Coder-7B-Instruct', cache_dir='/root/autodl-tmp', revision='master')
|
||||
```
|
||||
|
||||
> 注意：记得修改 `cache_dir` 为你的模型下载路径哦~
|
||||
|
||||
## 代码准备
|
||||
|
||||
为便捷构建 `LLM` 应用，我们需要基于本地部署的 `Qwen2_5_Coder`，自定义一个 `LLM` 类，将 `Qwen2.5-Coder` 接入到 `LangChain` 框架中。完成自定义 `LLM` 类之后，可以以完全一致的方式调用 `LangChain` 的接口，而无需考虑底层模型调用的不一致。
|
||||
|
||||
基于本地部署的 `Qwen2_5_Coder` 自定义 `LLM` 类并不复杂，我们只需从 `LangChain.llms.base.LLM` 类继承一个子类，并重写构造函数与 `_call` 函数即可：
|
||||
|
||||
在当前路径新建一个 `LLM.py` 文件，并输入以下内容，粘贴代码后记得保存文件。
|
||||
|
||||
```python
|
||||
from langchain.llms.base import LLM
|
||||
from typing import Any, List, Optional
|
||||
from langchain.callbacks.manager import CallbackManagerForLLMRun
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig, LlamaTokenizerFast
|
||||
import torch
|
||||
|
||||
class Qwen2_5_Coder(LLM):
|
||||
# 基于本地 Qwen2_5-Coder 自定义 LLM 类
|
||||
tokenizer: AutoTokenizer = None
|
||||
model: AutoModelForCausalLM = None
|
||||
def __init__(self, mode_name_or_path :str):
|
||||
|
||||
super().__init__()
|
||||
print("正在从本地加载模型...")
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(mode_name_or_path, use_fast=False)
|
||||
self.model = AutoModelForCausalLM.from_pretrained(mode_name_or_path, torch_dtype=torch.bfloat16, device_map="auto")
|
||||
self.model.generation_config = GenerationConfig.from_pretrained(mode_name_or_path)
|
||||
print("完成本地模型的加载")
|
||||
|
||||
def _call(self, prompt : str, stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any):
|
||||
|
||||
messages = [{"role": "user", "content": prompt }]
|
||||
input_ids = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
model_inputs = self.tokenizer([input_ids], return_tensors="pt").to('cuda')
|
||||
generated_ids = self.model.generate(model_inputs.input_ids, attention_mask=model_inputs['attention_mask'], max_new_tokens=512)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
return response
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "Qwen2_5_Coder"
|
||||
```
|
||||
|
||||
在上述类定义中，我们分别重写了构造函数和 `_call` 函数：对于构造函数，我们在对象实例化的一开始加载本地部署的 `Qwen2_5_Coder` `模型，从而避免每一次调用都需要重新加载模型带来的时间过长；_call` 函数是 `LLM` 类的核心函数，`LangChain` 会调用该函数来调用 `LLM`，在该函数中，我们调用已实例化模型的 `generate` 方法，从而实现对模型的调用并返回调用结果。
|
||||
|
||||
在整体项目中，我们将上述代码封装为 `LLM.py`，后续将直接从该文件中引入自定义的 LLM 类。
|
||||
|
||||
## 调用
|
||||
|
||||
然后就可以像使用任何其他的langchain大模型功能一样使用了。
|
||||
|
||||
> 注意：记得修改模型路径为你的路径哦~
|
||||
|
||||
```python
|
||||
from LLM import Qwen2_5_Coder
|
||||
llm = Qwen2_5_Coder(mode_name_or_path = "autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct")
|
||||
print(llm.invoke("你是谁"))
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
|
||||
既然是Coder模型，当然要试着让它编写代码
|
||||
|
||||
```python
|
||||
text = llm.invoke("为我用python写一个简单的猜拳小游戏，三局两胜")
|
||||
print(text)
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
我们试着运行一下这段代码：
|
||||
|
||||
成功运行！
|
||||
@@ -0,0 +1,356 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 数据集下载后转成格式调整 jsonl to json\n",
|
||||
"\n",
|
||||
"转换后的命令格式如下：\n",
|
||||
"\n",
|
||||
"```json\n",
|
||||
"{\n",
|
||||
" \"instruction\": \"你是一个法律专家，请根据用户的问题给出专业的回答\",\n",
|
||||
" \"input\": \"诈骗罪量刑标准是什么?\",\n",
|
||||
" \"output\": \"诈骗罪指的是以非法占有为目的，使用欺骗方法，骗取数额较大的公私财物的行为...\"\n",
|
||||
"}\n",
|
||||
"```\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 与 jsonl 数据集放在同一目录下 可以新建一个 python 文件\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"# 定义固定的instruction\n",
|
||||
"INSTRUCTION = \"你是一个法律专家，请根据用户的问题给出专业的回答\"\n",
|
||||
"\n",
|
||||
"def process_jsonl(input_file, output_file):\n",
|
||||
" with open(input_file, 'r', encoding='utf-8') as infile, open(output_file, 'w', encoding='utf-8') as outfile:\n",
|
||||
" for line in infile:\n",
|
||||
" # 读取每一行并解析JSON\n",
|
||||
" data = json.loads(line)\n",
|
||||
" \n",
|
||||
" # 创建新的字典，包含instruction, input和output\n",
|
||||
" new_data = {\n",
|
||||
" \"instruction\": INSTRUCTION,\n",
|
||||
" \"input\": data[\"input\"],\n",
|
||||
" \"output\": data[\"output\"]\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" # 将新的字典写入输出文件\n",
|
||||
" json.dump(new_data, outfile, ensure_ascii=False)\n",
|
||||
" outfile.write('\\n')\n",
|
||||
"\n",
|
||||
"# 使用示例\n",
|
||||
"input_file = \"DISC-Law-SFT-Pair-QA-released.jsonl\"\n",
|
||||
"output_file = \"DISC-Law-SFT-Pair-QA-released-new.jsonl\"\n",
|
||||
"\n",
|
||||
"process_jsonl(input_file, output_file)\n",
|
||||
"print(f\"处理完成。输出文件：{output_file}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 包的导入"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import pandas as pd\n",
|
||||
"import torch\n",
|
||||
"from datasets import Dataset\n",
|
||||
"from modelscope import snapshot_download, AutoTokenizer\n",
|
||||
"from swanlab.integration.transformers import SwanLabCallback\n",
|
||||
"from peft import LoraConfig, TaskType, get_peft_model\n",
|
||||
"from transformers import (\n",
|
||||
" AutoModelForCausalLM,\n",
|
||||
" TrainingArguments,\n",
|
||||
" Trainer,\n",
|
||||
" DataCollatorForSeq2Seq,\n",
|
||||
")\n",
|
||||
"import swanlab"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 数据集预处理"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def process_func(example):\n",
|
||||
" \"\"\"\n",
|
||||
" 将数据集进行预处理\n",
|
||||
" \"\"\"\n",
|
||||
" MAX_LENGTH = 384\n",
|
||||
" input_ids, attention_mask, labels = [], [], []\n",
|
||||
" instruction = tokenizer(\n",
|
||||
" f\"<|im_start|>system\\n{example['instruction']}<|im_end|>\\n<|im_start|>user\\n{example['input']}<|im_end|>\\n<|im_start|>assistant\\n\",\n",
|
||||
" add_special_tokens=False,\n",
|
||||
" )\n",
|
||||
" response = tokenizer(f\"{example['output']}\", add_special_tokens=False)\n",
|
||||
" input_ids = (\n",
|
||||
" instruction[\"input_ids\"] + response[\"input_ids\"] + [tokenizer.pad_token_id]\n",
|
||||
" )\n",
|
||||
" attention_mask = instruction[\"attention_mask\"] + response[\"attention_mask\"] + [1]\n",
|
||||
" labels = (\n",
|
||||
" [-100] * len(instruction[\"input_ids\"])\n",
|
||||
" + response[\"input_ids\"]\n",
|
||||
" + [tokenizer.pad_token_id]\n",
|
||||
" )\n",
|
||||
" if len(input_ids) > MAX_LENGTH: # 做一个截断\n",
|
||||
" input_ids = input_ids[:MAX_LENGTH]\n",
|
||||
" attention_mask = attention_mask[:MAX_LENGTH]\n",
|
||||
" labels = labels[:MAX_LENGTH]\n",
|
||||
" \n",
|
||||
" return {\"input_ids\": input_ids, \"attention_mask\": attention_mask, \"labels\": labels}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_jsonl_path = \"DISC-Law-SFT-Pair-QA-released-new.jsonl\"\n",
|
||||
"train_df = pd.read_json(train_jsonl_path, lines=True)[5:5000]\n",
|
||||
"train_ds = Dataset.from_pandas(train_df)\n",
|
||||
"train_dataset = train_ds.map(process_func, remove_columns=train_ds.column_names)\n",
|
||||
"test_df = pd.read_json(train_jsonl_path, lines=True)[:5]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 模型下载，并加载到 Transformers 模型中\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 在modelscope上下载Qwen模型到本地目录下\n",
|
||||
"model_dir = snapshot_download(\"Qwen/Qwen2.5-Coder-7B-Instruct\", cache_dir=\"/root/autodl-tmp\", revision=\"master\")\n",
|
||||
"\n",
|
||||
"# Transformers加载模型权重\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(\"/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/\", use_fast=False, trust_remote_code=True)\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(\"/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/\", device_map=\"auto\", torch_dtype=torch.bfloat16)\n",
|
||||
"model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 模型预测函数"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict(messages, model, tokenizer):\n",
|
||||
" device = \"cuda\"\n",
|
||||
" text = tokenizer.apply_chat_template(\n",
|
||||
" messages, tokenize=False, add_generation_prompt=True\n",
|
||||
" )\n",
|
||||
" model_inputs = tokenizer([text], return_tensors=\"pt\").to(device)\n",
|
||||
"\n",
|
||||
" generated_ids = model.generate(model_inputs.input_ids, max_new_tokens=512)\n",
|
||||
" generated_ids = [\n",
|
||||
" output_ids[len(input_ids) :]\n",
|
||||
" for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
|
||||
"\n",
|
||||
" return response\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## peft model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"config = LoraConfig(\n",
|
||||
" task_type=TaskType.CAUSAL_LM,\n",
|
||||
" target_modules=[\n",
|
||||
" \"q_proj\",\n",
|
||||
" \"k_proj\",\n",
|
||||
" \"v_proj\",\n",
|
||||
" \"o_proj\",\n",
|
||||
" \"gate_proj\",\n",
|
||||
" \"up_proj\",\n",
|
||||
" \"down_proj\",\n",
|
||||
" ],\n",
|
||||
" inference_mode=False, # 训练模式\n",
|
||||
" r=64, # Lora 秩\n",
|
||||
" lora_alpha=16, # Lora alaph，具体作用参见 Lora 原理\n",
|
||||
" lora_dropout=0.1, # Dropout 比例\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"peft_model = get_peft_model(model, config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 设定参数并指定 Callback"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"args = TrainingArguments(\n",
|
||||
" output_dir=\"./output/Qwen2.5-Coder-7b\",\n",
|
||||
" per_device_train_batch_size=2,\n",
|
||||
" gradient_accumulation_steps=8,\n",
|
||||
" logging_steps=10,\n",
|
||||
" num_train_epochs=1,\n",
|
||||
" save_steps=100,\n",
|
||||
" learning_rate=1e-4,\n",
|
||||
" save_on_each_node=True,\n",
|
||||
" gradient_checkpointing=True,\n",
|
||||
" report_to=\"none\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"class HuanhuanSwanLabCallback(SwanLabCallback): \n",
|
||||
" def on_train_begin(self, args, state, control, model=None, **kwargs):\n",
|
||||
" if not self._initialized:\n",
|
||||
" self.setup(args, state, model, **kwargs)\n",
|
||||
" \n",
|
||||
" print(\"训练开始\")\n",
|
||||
" print(\"未开始微调，先取3条主观评测：\")\n",
|
||||
" test_text_list = []\n",
|
||||
" for index, row in test_df[:3].iterrows():\n",
|
||||
" instruction = row[\"instruction\"]\n",
|
||||
" input_value = row[\"input\"]\n",
|
||||
"\n",
|
||||
" messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": f\"{instruction}\"},\n",
|
||||
" {\"role\": \"user\", \"content\": f\"{input_value}\"},\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" response = predict(messages, peft_model, tokenizer)\n",
|
||||
" messages.append({\"role\": \"assistant\", \"content\": f\"{response}\"})\n",
|
||||
" \n",
|
||||
" result_text = f\"【Q】{messages[1]['content']}\\n【LLM】{messages[2]['content']}\\n\"\n",
|
||||
" print(result_text)\n",
|
||||
" \n",
|
||||
" test_text_list.append(swanlab.Text(result_text, caption=response))\n",
|
||||
"\n",
|
||||
" swanlab.log({\"Prediction\": test_text_list}, step=0)\n",
|
||||
" \n",
|
||||
" def on_epoch_end(self, args, state, control, **kwargs):\n",
|
||||
" # ===================测试阶段======================\n",
|
||||
" test_text_list = []\n",
|
||||
" for index, row in test_df.iterrows():\n",
|
||||
" instruction = row[\"instruction\"]\n",
|
||||
" input_value = row[\"input\"]\n",
|
||||
" ground_truth = row[\"output\"]\n",
|
||||
"\n",
|
||||
" messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": f\"{instruction}\"},\n",
|
||||
" {\"role\": \"user\", \"content\": f\"{input_value}\"},\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" response = predict(messages, peft_model, tokenizer)\n",
|
||||
" messages.append({\"role\": \"assistant\", \"content\": f\"{response}\"})\n",
|
||||
" \n",
|
||||
" if index == 0:\n",
|
||||
" print(\"epoch\", round(state.epoch), \"主观评测：\")\n",
|
||||
" \n",
|
||||
" result_text = f\"【Q】{messages[1]['content']}\\n【LLM】{messages[2]['content']}\\n【GT】 {ground_truth}\"\n",
|
||||
" print(result_text)\n",
|
||||
" \n",
|
||||
" test_text_list.append(swanlab.Text(result_text, caption=response))\n",
|
||||
"\n",
|
||||
" swanlab.log({\"Prediction\": test_text_list}, step=round(state.epoch))\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"swanlab_callback = HuanhuanSwanLabCallback(\n",
|
||||
" project=\"Qwen2.5-Coder-LoRA-Law\",\n",
|
||||
" experiment_name=\"7b\",\n",
|
||||
" config={\n",
|
||||
" \"model\": \"https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct\",\n",
|
||||
" \"dataset\": \"https://huggingface.co/datasets/ShengbinYue/DISC-Law-SFT\",\n",
|
||||
" \"github\": \"https://github.com/datawhalechina/self-llm\",\n",
|
||||
" \"system_prompt\": \"你是一个法律专家，请根据用户的问题给出专业的回答\",\n",
|
||||
" \"lora_rank\": 64,\n",
|
||||
" \"lora_alpha\": 16,\n",
|
||||
" \"lora_dropout\": 0.1,\n",
|
||||
" },\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 开始训练"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer = Trainer(\n",
|
||||
" model=peft_model,\n",
|
||||
" args=args,\n",
|
||||
" train_dataset=train_dataset,\n",
|
||||
" data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n",
|
||||
" callbacks=[swanlab_callback],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"trainer.train()\n",
|
||||
"\n",
|
||||
"swanlab.finish()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
# Qwen2.5-Coder-7B-Instruct Lora 微调 SwanLab 可视化记录版
|
||||
|
||||
本节我们简要介绍如何基于 transformers、peft 等框架，对Qwen2.5-Coder-7B-Instruct 模型进行Lora微调。使用的数据集是**中文法律问答数据集 DISC-Law-SFT**，同时使用 [SwanLab](https://github.com/swanhubx/swanlab) 监控训练过程与评估模型效果。
|
||||
|
||||
Lora 是一种高效微调方法，深入了解其原理可参见博客：[知乎|深入浅出 Lora](https://zhuanlan.zhihu.com/p/650197598)。
|
||||
|
||||
训练过程：<a href="https://swanlab.cn/@Harrison/Qwen2.5-Coder-LoRA-Law/overview" target="_blank">Qwen2.5-Coder-LoRA-Law</a>
|
||||
|
||||
|
||||
同时，这个教程会在同目录下给大家提供一个 [notebook](./05-Qwen2.5-Coder-7B-Instruct%20Lora%20微调%20SwanLab%20可视化记录版.ipynb)文件，方便大家快速上手。
|
||||
|
||||
## 目录
|
||||
|
||||
- [SwanLab简介](#swanlab简介)
|
||||
- [环境配置](#环境配置)
|
||||
- [准备数据集](#准备数据集)
|
||||
- [模型下载与加载](#模型下载与加载)
|
||||
- [集成SwanLab](#集成-swanlab)
|
||||
- [开始微调（完整代码）](#开始微调)
|
||||
- [训练结果展示](#训练结果展示)
|
||||
|
||||
## SwanLab简介
|
||||
|
||||
|
||||
|
||||
[SwanLab](https://github.com/swanhubx/swanlab) 是一个开源的模型训练记录工具，面向AI研究者，提供了训练可视化、自动日志记录、超参数记录、实验对比、多人协同等功能。在SwanLab上，研究者能基于直观的可视化图表发现训练问题，对比多个实验找到研究灵感，并通过在线链接的分享与基于组织的多人协同训练，打破团队沟通的壁垒。
|
||||
|
||||
**为什么要记录训练**
|
||||
|
||||
相较于软件开发，模型训练更像一个实验科学。一个品质优秀的模型背后，往往是成千上万次实验。研究者需要不断尝试、记录、对比，积累经验，才能找到最佳的模型结构、超参数与数据配比。在这之中，如何高效进行记录与对比，对于研究效率的提升至关重要。
|
||||
|
||||
## 环境配置
|
||||
|
||||
本文的基础环境如下：
|
||||
|
||||
```
|
||||
----------------
|
||||
ubuntu 22.04
|
||||
python 3.12
|
||||
cuda 12.1
|
||||
pytorch 2.3.0
|
||||
----------------
|
||||
```
|
||||
|
||||
> **注意**：本文默认学习者已安装好以上环境
|
||||
|
||||
然后我们开始环境配置
|
||||
|
||||
```bash
|
||||
python -m pip install --upgrade pip
|
||||
# 更换 pypi 源加速库的安装
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
pip install modelscope==1.20.0
|
||||
pip install transformers==4.46.2
|
||||
pip install accelerate==1.1.1
|
||||
pip install peft==0.13.2
|
||||
pip install datasets==3.1.0
|
||||
pip install swanlab==0.3.25
|
||||
```
|
||||
|
||||
## 准备数据集
|
||||
本节使用的是 [DISC-Law-SFT](https://huggingface.co/datasets/ShengbinYue/DISC-Law-SFT) 数据集，该数据集主要用于中文法律大语言模型的微调。
|
||||
|
||||
> HF README：在中国，法律智能系统需要结合各种能力，包括法律文本理解和生成。为了实现这一目标，我们构建了一个高质量的监督微调数据集，名为DISC-Law-SFT，涵盖了不同的法律场景，如法律信息提取、法律判决预测、法律文件摘要和法律问题回答。DISC-Law-SFT包括两个子集，DISC-Law-SFT-Pair和DISC-Law-SFT-Triplet。前者旨在向LLM引入法律推理能力，而后者有助于增强模型利用外部法律知识的能力。
|
||||
|
||||
|
||||
|
||||
在本节的任务中，我们主要使用其中的 [DISC-Law-SFT-Pair-QA-released](https://huggingface.co/datasets/ShengbinYue/DISC-Law-SFT/blob/main/DISC-Law-SFT-Pair-QA-released.jsonl) 子集，并对它进行格式调整，组合成如下格式的json文件：
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "你是一个法律专家，请根据用户的问题给出专业的回答",
|
||||
"input": "诈骗罪量刑标准是什么?",
|
||||
"output": "诈骗罪指的是以非法占有为目的，使用欺骗方法，骗取数额较大的公私财物的行为..."
|
||||
}
|
||||
```
|
||||
|
||||
其中，`instruction` 是用户指令，告知模型其需要完成的任务；`input` 是用户输入，即模拟真实场景下用户的问题；`output` 是模型应该给出的输出，即模型的回答。
|
||||
|
||||
**数据集下载与处理方式**
|
||||
1. 在HuggingFace上直接下载[DISC-Law-SFT-Pair-QA-released.jsonl](https://huggingface.co/datasets/ShengbinYue/DISC-Law-SFT/blob/main/DISC-Law-SFT-Pair-QA-released.jsonl)，放到项目根目录下
|
||||
2. 在同一目录下，用以下代码处理数据集，得到新数据集文件：
|
||||
```python
|
||||
import json
|
||||
|
||||
# 定义固定的instruction
|
||||
INSTRUCTION = "你是一个法律专家，请根据用户的问题给出专业的回答"
|
||||
|
||||
def process_jsonl(input_file, output_file):
|
||||
with open(input_file, 'r', encoding='utf-8') as infile, open(output_file, 'w', encoding='utf-8') as outfile:
|
||||
for line in infile:
|
||||
# 读取每一行并解析JSON
|
||||
data = json.loads(line)
|
||||
|
||||
# 创建新的字典，包含instruction, input和output
|
||||
new_data = {
|
||||
"instruction": INSTRUCTION,
|
||||
"input": data["input"],
|
||||
"output": data["output"]
|
||||
}
|
||||
|
||||
# 将新的字典写入输出文件
|
||||
json.dump(new_data, outfile, ensure_ascii=False)
|
||||
outfile.write('\n')
|
||||
|
||||
# 使用示例
|
||||
input_file = "DISC-Law-SFT-Pair-QA-released.jsonl"
|
||||
output_file = "DISC-Law-SFT-Pair-QA-released-new.jsonl"
|
||||
|
||||
process_jsonl(input_file, output_file)
|
||||
print(f"处理完成。输出文件：{output_file}")
|
||||
```
|
||||
至此，我们完成了数据集的准备
|
||||
|
||||
|
||||
## 模型下载与加载
|
||||
这里我们使用`modelscope`中的`snapshot_download`下载模型，然后加载到 Transformers 中进行训练：
|
||||
|
||||
```python
|
||||
from modelscope import snapshot_download, AutoTokenizer
|
||||
from transformers import AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForSeq2Seq
|
||||
import torch
|
||||
|
||||
# 在modelscope上下载Qwen模型到本地目录下
|
||||
model_dir = snapshot_download("Qwen/Qwen2.5-Coder-7B-Instruct", cache_dir="/root/autodl-tmp", revision="master")
|
||||
|
||||
# Transformers加载模型权重
|
||||
tokenizer = AutoTokenizer.from_pretrained("/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/", use_fast=False, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained("/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/", device_map="auto", torch_dtype=torch.bfloat16)
|
||||
model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法
|
||||
```
|
||||
|
||||
模型大小为 15GB，下载模型我大概花了 15 分钟。
|
||||
|
||||
|
||||
## 集成 SwanLab
|
||||
SwanLab与Transformers已经做好了集成，用法是在Trainer的`callbacks`参数中添加`SwanLabCallback`实例，就可以自动记录超参数和训练指标，简化代码如下：
|
||||
|
||||
```python
|
||||
from swanlab.integration.transformers import SwanLabCallback
|
||||
from transformers import Trainer
|
||||
|
||||
swanlab_callback = SwanLabCallback()
|
||||
|
||||
trainer = Trainer(
|
||||
...
|
||||
callbacks=[swanlab_callback],
|
||||
)
|
||||
|
||||
```
|
||||
首次使用SwanLab，需要先在[官网](https://swanlab.cn)注册一个账号，然后在用户设置页面复制你的API Key，然后在训练开始提示登录时粘贴即可，后续无需再次登录：
|
||||
|
||||
|
||||
|
||||
更多用法可参考[快速开始](https://docs.swanlab.cn/zh/guide_cloud/general/quick-start.html)、[Transformers集成](https://docs.swanlab.cn/zh/guide_cloud/integration/integration-huggingface-transformers.html)。
|
||||
|
||||
## 开始微调
|
||||
|
||||
查看可视化训练过程：<a href="https://swanlab.cn/@Harrison/Qwen2.5-Coder-LoRA-Law/overview" target="_blank">Qwen2.5-Coder-LoRA-Law</a>
|
||||
|
||||
**本节代码做了以下几件事：**
|
||||
1. 下载并加载Qwen2.5-7B-Coder-Instruct模型
|
||||
2. 加载数据集，取前5000条数据参与训练，5条数据进行主观评测
|
||||
3. 配置Lora，参数为r=64, lora_alpha=16, lora_dropout=0.1
|
||||
4. 使用SwanLab记录训练过程，包括超参数、指标和每个epoch的模型输出结果
|
||||
5. 训练1个epoch
|
||||
|
||||
**完整代码如下**
|
||||
|
||||
```python
|
||||
import json
|
||||
import pandas as pd
|
||||
import torch
|
||||
from datasets import Dataset
|
||||
from modelscope import snapshot_download, AutoTokenizer
|
||||
from swanlab.integration.transformers import SwanLabCallback
|
||||
from peft import LoraConfig, TaskType, get_peft_model
|
||||
from transformers import (
|
||||
AutoModelForCausalLM,
|
||||
TrainingArguments,
|
||||
Trainer,
|
||||
DataCollatorForSeq2Seq,
|
||||
)
|
||||
import swanlab
|
||||
|
||||
|
||||
def process_func(example):
|
||||
"""
|
||||
将数据集进行预处理
|
||||
"""
|
||||
MAX_LENGTH = 384
|
||||
input_ids, attention_mask, labels = [], [], []
|
||||
instruction = tokenizer(
|
||||
f"<|im_start|>system\n{example['instruction']}<|im_end|>\n<|im_start|>user\n{example['input']}<|im_end|>\n<|im_start|>assistant\n",
|
||||
add_special_tokens=False,
|
||||
)
|
||||
response = tokenizer(f"{example['output']}", add_special_tokens=False)
|
||||
input_ids = (
|
||||
instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
|
||||
)
|
||||
attention_mask = instruction["attention_mask"] + response["attention_mask"] + [1]
|
||||
labels = (
|
||||
[-100] * len(instruction["input_ids"])
|
||||
+ response["input_ids"]
|
||||
+ [tokenizer.pad_token_id]
|
||||
)
|
||||
if len(input_ids) > MAX_LENGTH: # 做一个截断
|
||||
input_ids = input_ids[:MAX_LENGTH]
|
||||
attention_mask = attention_mask[:MAX_LENGTH]
|
||||
labels = labels[:MAX_LENGTH]
|
||||
|
||||
return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
|
||||
|
||||
|
||||
def predict(messages, model, tokenizer):
|
||||
device = "cuda"
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(device)
|
||||
|
||||
generated_ids = model.generate(model_inputs.input_ids, max_new_tokens=512)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids) :]
|
||||
for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
|
||||
return response
|
||||
|
||||
|
||||
# 在modelscope上下载Qwen模型到本地目录下
|
||||
model_dir = snapshot_download("Qwen/Qwen2.5-Coder-7B-Instruct", cache_dir="/root/autodl-tmp", revision="master")
|
||||
|
||||
# Transformers加载模型权重
|
||||
tokenizer = AutoTokenizer.from_pretrained("/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/", use_fast=False, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained("/root/autodl-tmp/Qwen/Qwen2___5-Coder-7B-Instruct/", device_map="auto", torch_dtype=torch.bfloat16)
|
||||
model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法
|
||||
|
||||
# 处理数据集
|
||||
train_jsonl_path = "DISC-Law-SFT-Pair-QA-released-new.jsonl"
|
||||
train_df = pd.read_json(train_jsonl_path, lines=True)[5:5000]
|
||||
train_ds = Dataset.from_pandas(train_df)
|
||||
train_dataset = train_ds.map(process_func, remove_columns=train_ds.column_names)
|
||||
test_df = pd.read_json(train_jsonl_path, lines=True)[:5]
|
||||
|
||||
config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
target_modules=[
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
"down_proj",
|
||||
],
|
||||
inference_mode=False, # 训练模式
|
||||
r=64, # Lora 秩
|
||||
lora_alpha=16, # Lora alaph，具体作用参见 Lora 原理
|
||||
lora_dropout=0.1, # Dropout 比例
|
||||
)
|
||||
|
||||
peft_model = get_peft_model(model, config)
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir="./output/Qwen2.5-Coder-7b",
|
||||
per_device_train_batch_size=2,
|
||||
gradient_accumulation_steps=8,
|
||||
logging_steps=10,
|
||||
num_train_epochs=1,
|
||||
save_steps=100,
|
||||
learning_rate=1e-4,
|
||||
save_on_each_node=True,
|
||||
gradient_checkpointing=True,
|
||||
report_to="none",
|
||||
)
|
||||
|
||||
class HuanhuanSwanLabCallback(SwanLabCallback):
|
||||
def on_train_begin(self, args, state, control, model=None, **kwargs):
|
||||
if not self._initialized:
|
||||
self.setup(args, state, model, **kwargs)
|
||||
|
||||
print("训练开始")
|
||||
print("未开始微调，先取3条主观评测：")
|
||||
test_text_list = []
|
||||
for index, row in test_df[:3].iterrows():
|
||||
instruction = row["instruction"]
|
||||
input_value = row["input"]
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": f"{instruction}"},
|
||||
{"role": "user", "content": f"{input_value}"},
|
||||
]
|
||||
|
||||
response = predict(messages, peft_model, tokenizer)
|
||||
messages.append({"role": "assistant", "content": f"{response}"})
|
||||
|
||||
result_text = f"【Q】{messages[1]['content']}\n【LLM】{messages[2]['content']}\n"
|
||||
print(result_text)
|
||||
|
||||
test_text_list.append(swanlab.Text(result_text, caption=response))
|
||||
|
||||
swanlab.log({"Prediction": test_text_list}, step=0)
|
||||
|
||||
def on_epoch_end(self, args, state, control, **kwargs):
|
||||
# ===================测试阶段======================
|
||||
test_text_list = []
|
||||
for index, row in test_df.iterrows():
|
||||
instruction = row["instruction"]
|
||||
input_value = row["input"]
|
||||
ground_truth = row["output"]
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": f"{instruction}"},
|
||||
{"role": "user", "content": f"{input_value}"},
|
||||
]
|
||||
|
||||
response = predict(messages, peft_model, tokenizer)
|
||||
messages.append({"role": "assistant", "content": f"{response}"})
|
||||
|
||||
if index == 0:
|
||||
print("epoch", round(state.epoch), "主观评测：")
|
||||
|
||||
result_text = f"【Q】{messages[1]['content']}\n【LLM】{messages[2]['content']}\n【GT】 {ground_truth}"
|
||||
print(result_text)
|
||||
|
||||
test_text_list.append(swanlab.Text(result_text, caption=response))
|
||||
|
||||
swanlab.log({"Prediction": test_text_list}, step=round(state.epoch))
|
||||
|
||||
|
||||
swanlab_callback = HuanhuanSwanLabCallback(
|
||||
project="Qwen2.5-Coder-LoRA-Law",
|
||||
experiment_name="7b",
|
||||
config={
|
||||
"model": "https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct",
|
||||
"dataset": "https://huggingface.co/datasets/ShengbinYue/DISC-Law-SFT",
|
||||
"github": "https://github.com/datawhalechina/self-llm",
|
||||
"system_prompt": "你是一个法律专家，请根据用户的问题给出专业的回答",
|
||||
"lora_rank": 64,
|
||||
"lora_alpha": 16,
|
||||
"lora_dropout": 0.1,
|
||||
},
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=peft_model,
|
||||
args=args,
|
||||
train_dataset=train_dataset,
|
||||
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
|
||||
callbacks=[swanlab_callback],
|
||||
)
|
||||
|
||||
trainer.train()
|
||||
|
||||
# 在Jupyter Notebook中运行时要停止SwanLab记录，需要调用swanlab.finish()
|
||||
swanlab.finish()
|
||||
```
|
||||
|
||||
看到下面的进度条并成功登录 SwanLab 即代表训练开始：
|
||||
|
||||
|
||||
|
||||
|
||||
## 训练结果展示
|
||||
在我们设置的参数下，大约花了十七分钟就训练好了，训练结束后，在终端中可以看到如下的界面。
|
||||
|
||||
|
||||
|
||||
打开SwanLab，可以查看训练过程中记录的参数和指标：
|
||||
|
||||
|
||||
|
||||
更多训练细节请参考：<a href="https://swanlab.cn/@Harrison/Qwen2.5-Coder-LoRA-Law/overview" target="_blank">Qwen2.5-Coder-LoRA-Law</a>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 52 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 161 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 645 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 399 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 233 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 180 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 94 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 276 KiB |
@@ -0,0 +1 @@
|
||||
此处为Qwen2.5-Coder教程
|
||||
@@ -239,7 +239,7 @@ model = AutoModelForCausalLM.from_pretrained("./qwen/Qwen2___5-7B-Instruct/", de
|
||||
model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法
|
||||
|
||||
# 处理数据集
|
||||
train_jsonl_path = "DISC-Law-SFT-Pair-New.jsonl"
|
||||
train_jsonl_path = "DISC-Law-SFT-Pair-QA-released-new.jsonl"
|
||||
train_df = pd.read_json(train_jsonl_path, lines=True)[5:5000]
|
||||
train_ds = Dataset.from_pandas(train_df)
|
||||
train_dataset = train_ds.map(process_func, remove_columns=train_ds.column_names)
|
||||
|
||||
Reference in New Issue
Block a user