From 0ad1f3f6e2fad91dd666c3684ec9bdb368d8a666 Mon Sep 17 00:00:00 2001 From: Yang Yu <35400026+YangYu-NUAA@users.noreply.github.com> Date: Sun, 8 Feb 2026 17:45:01 +0800 Subject: [PATCH] Add files via upload --- models/GLM-4.7-Flash/GLM-4.7-Flash.ipynb | 1512 ++++++++++++++++++++++ 1 file changed, 1512 insertions(+) create mode 100644 models/GLM-4.7-Flash/GLM-4.7-Flash.ipynb diff --git a/models/GLM-4.7-Flash/GLM-4.7-Flash.ipynb b/models/GLM-4.7-Flash/GLM-4.7-Flash.ipynb new file mode 100644 index 0000000..949c987 --- /dev/null +++ b/models/GLM-4.7-Flash/GLM-4.7-Flash.ipynb @@ -0,0 +1,1512 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "360e86a9-3b26-4b32-ba2f-6b5c2ddbb786", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import Glm4MoeLiteForCausalLM, AutoTokenizer, DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig\n", + "from datasets import Dataset\n", + "import pandas as pd\n", + "from peft import LoraConfig, TaskType, get_peft_model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "13dea7c0-81ff-4c91-aaa2-4500e805cc18", + "metadata": {}, + "outputs": [], + "source": [ + "from modelscope import snapshot_download\n", + "\n", + "model_dir = snapshot_download('ZhipuAI/GLM-4.7-Flash', cache_dir='/root/autodl-fs/ZhipuAI/GLM-4.7-Flash', revision='master')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "feedcd26-f5c6-43be-bd44-839f8898c31f", + "metadata": {}, + "outputs": [], + "source": [ + "def process_func(example):\n", + " MAX_LENGTH = 1024 # 设置最大序列长度为1024个token\n", + " input_ids, attention_mask, labels = [], [], [] # 初始化返回值\n", + " # 适配chat_template\n", + " instruction = tokenizer(\n", + " f\"[gMASK]<|system|>\\n现在你要扮演皇帝身边的女人--甄嬛\" \n", + " f\"<|user|>\\n{example['instruction'] + example['input']}\" \n", + " f\"<|assistant|>\\n\\n\", \n", + " add_special_tokens=False \n", + " )\n", + " response = tokenizer(f\"{example['output']}\", add_special_tokens=False)\n", + " # 将instructio部分和response部分的input_ids拼接，并在末尾添加eos token作为标记结束的token\n", + " input_ids = instruction[\"input_ids\"] + response[\"input_ids\"]\n", + " # 注意力掩码，表示模型需要关注的位置\n", + " attention_mask = instruction[\"attention_mask\"] + response[\"attention_mask\"]\n", + " # 对于instruction，使用-100表示这些位置不计算loss（即模型不需要预测这部分）\n", + " labels = [-100] * len(instruction[\"input_ids\"]) + response[\"input_ids\"]\n", + " if len(input_ids) > MAX_LENGTH: # 超出最大序列长度截断\n", + " input_ids = input_ids[:MAX_LENGTH]\n", + " attention_mask = attention_mask[:MAX_LENGTH]\n", + " labels = labels[:MAX_LENGTH]\n", + " return {\n", + " \"input_ids\": input_ids,\n", + " \"attention_mask\": attention_mask,\n", + " \"labels\": labels\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "449740b6-cc17-43e5-a8bf-49c71b0f71a5", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_json('./dataset/huanhuan.json')\n", + "ds = Dataset.from_pandas(df)\n", + "\n", + "model_path = '/root/autodl-fs/ZhipuAI/GLM-4.7-Flash'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1e404383-f756-403f-ab6c-63b314b3de14", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "154820 154820\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2ad7e2de8ae44462b251e81f9ee2e43a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/751 [00:00<|system|>You are a helpful assistant.<|user|>你好<|assistant|>你好，我是一个AI助手<|user|>不错～<|assistant|>\n" + ] + } + ], + "source": [ + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"你好\"},\n", + " {\"role\": \"assistant\", \"content\": \"你好，我是一个AI助手\"},\n", + " {\"role\": \"user\", \"content\": \"不错～\"},\n", + "]\n", + "\n", + "text = tokenizer.apply_chat_template(\n", + "\tmessages,\n", + "\tadd_generation_prompt=True,\n", + "\ttokenize=False,\n", + ")\n", + "\n", + "print(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "72ed79ed-bb7e-4ce1-888e-c7b7ca699933", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import Glm4MoeLiteModel, Glm4MoeLiteConfig\n", + "#from transformers import AutoModelForCausalLM, AutoTokenizer, DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig\n", + "from transformers import Glm4MoeLiteForCausalLM, AutoTokenizer, DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig\n", + "from datasets import Dataset\n", + "import pandas as pd\n", + "from peft import LoraConfig, TaskType, get_peft_model " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "64eb1308-6ed8-4d2c-87d7-cf7b7fe5d1b5", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bda409422d4e43e399fe79b40479cd23", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/751 [00:00\n", + "47\n" + ] + } + ], + "source": [ + "model = Glm4MoeLiteForCausalLM.from_pretrained(\n", + " pretrained_model_name_or_path=model_path,\n", + " torch_dtype=torch.bfloat16,\n", + " device_map=\"auto\",\n", + " trust_remote_code=True,\n", + " \n", + ")\n", + "\n", + "print(type(model))\n", + "print(len(model.model.layers))\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3cc6692e-2dc4-4f98-89e9-571cb29f4b9b", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4006397e705f4061acb7e447ce6e74fb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/3729 [00:00<|system|>\n", + "现在你要扮演皇帝身边的女人--甄嬛<|user|>\n", + "小姐，别的秀女都在求中选，唯有咱们小姐想被撂牌子，菩萨一定记得真真儿的——<|assistant|>\n", + "\n", + "嘘——都说许愿说破是不灵的。\n", + "你们俩话太多了，我该和温太医要一剂药，好好治治你们。\n", + "\n", + "model\n", + "model.embed_tokens\n", + "model.layers\n", + "model.layers.0\n", + "model.layers.0.self_attn\n", + "model.layers.0.self_attn.q_a_proj\n", + "model.layers.0.self_attn.q_a_layernorm\n", + "model.layers.0.self_attn.q_b_proj\n", + "model.layers.0.self_attn.kv_a_proj_with_mqa\n", + "model.layers.0.self_attn.kv_a_layernorm\n", + "model.layers.0.self_attn.kv_b_proj\n", + "model.layers.0.self_attn.o_proj\n", + "model.layers.0.mlp\n", + "model.layers.0.mlp.gate_proj\n", + "model.layers.0.mlp.up_proj\n", + "model.layers.0.mlp.down_proj\n", + "model.layers.0.mlp.act_fn\n", + "model.layers.0.input_layernorm\n", + "model.layers.0.post_attention_layernorm\n", + "model.layers.1\n", + "model.layers.1.self_attn\n", + "model.layers.1.self_attn.q_a_proj\n", + "model.layers.1.self_attn.q_a_layernorm\n", + "model.layers.1.self_attn.q_b_proj\n", + "model.layers.1.self_attn.kv_a_proj_with_mqa\n", + "model.layers.1.self_attn.kv_a_layernorm\n", + "model.layers.1.self_attn.kv_b_proj\n", + "model.layers.1.self_attn.o_proj\n", + "model.layers.1.mlp\n", + "model.layers.1.mlp.experts\n", + "model.layers.1.mlp.experts.act_fn\n", + "model.layers.1.mlp.gate\n", + "model.layers.1.mlp.shared_experts\n", + "model.layers.1.mlp.shared_experts.gate_proj\n", + "model.layers.1.mlp.shared_experts.up_proj\n", + "model.layers.1.mlp.shared_experts.down_proj\n", + "model.layers.1.mlp.shared_experts.act_fn\n", + "model.layers.1.input_layernorm\n", + "model.layers.1.post_attention_layernorm\n", + "model.layers.2\n", + "model.layers.2.self_attn\n", + "model.layers.2.self_attn.q_a_proj\n", + "model.layers.2.self_attn.q_a_layernorm\n", + "model.layers.2.self_attn.q_b_proj\n", + "model.layers.2.self_attn.kv_a_proj_with_mqa\n", + "model.layers.2.self_attn.kv_a_layernorm\n", + "model.layers.2.self_attn.kv_b_proj\n", + "model.layers.2.self_attn.o_proj\n", + "model.layers.2.mlp\n", + "model.layers.2.mlp.experts\n", + "model.layers.2.mlp.experts.act_fn\n", + "model.layers.2.mlp.gate\n", + "model.layers.2.mlp.shared_experts\n", + "model.layers.2.mlp.shared_experts.gate_proj\n", + "model.layers.2.mlp.shared_experts.up_proj\n", + "model.layers.2.mlp.shared_experts.down_proj\n", + "model.layers.2.mlp.shared_experts.act_fn\n", + "model.layers.2.input_layernorm\n", + "model.layers.2.post_attention_layernorm\n", + "model.layers.3\n", + "model.layers.3.self_attn\n", + "model.layers.3.self_attn.q_a_proj\n", + "model.layers.3.self_attn.q_a_layernorm\n", + "model.layers.3.self_attn.q_b_proj\n", + "model.layers.3.self_attn.kv_a_proj_with_mqa\n", + "model.layers.3.self_attn.kv_a_layernorm\n", + "model.layers.3.self_attn.kv_b_proj\n", + "model.layers.3.self_attn.o_proj\n", + "model.layers.3.mlp\n", + "model.layers.3.mlp.experts\n", + "model.layers.3.mlp.experts.act_fn\n", + "model.layers.3.mlp.gate\n", + "model.layers.3.mlp.shared_experts\n", + "model.layers.3.mlp.shared_experts.gate_proj\n", + "model.layers.3.mlp.shared_experts.up_proj\n", + "model.layers.3.mlp.shared_experts.down_proj\n", + "model.layers.3.mlp.shared_experts.act_fn\n", + "model.layers.3.input_layernorm\n", + "model.layers.3.post_attention_layernorm\n", + "model.layers.4\n", + "model.layers.4.self_attn\n", + "model.layers.4.self_attn.q_a_proj\n", + "model.layers.4.self_attn.q_a_layernorm\n", + "model.layers.4.self_attn.q_b_proj\n", + "model.layers.4.self_attn.kv_a_proj_with_mqa\n", + "model.layers.4.self_attn.kv_a_layernorm\n", + "model.layers.4.self_attn.kv_b_proj\n", + "model.layers.4.self_attn.o_proj\n", + "model.layers.4.mlp\n", + "model.layers.4.mlp.experts\n", + "model.layers.4.mlp.experts.act_fn\n", + "model.layers.4.mlp.gate\n", + "model.layers.4.mlp.shared_experts\n", + "model.layers.4.mlp.shared_experts.gate_proj\n", + "model.layers.4.mlp.shared_experts.up_proj\n", + "model.layers.4.mlp.shared_experts.down_proj\n", + "model.layers.4.mlp.shared_experts.act_fn\n", + "model.layers.4.input_layernorm\n", + "model.layers.4.post_attention_layernorm\n", + "model.layers.5\n", + "model.layers.5.self_attn\n", + "model.layers.5.self_attn.q_a_proj\n", + "model.layers.5.self_attn.q_a_layernorm\n", + "model.layers.5.self_attn.q_b_proj\n", + "model.layers.5.self_attn.kv_a_proj_with_mqa\n", + "model.layers.5.self_attn.kv_a_layernorm\n", + "model.layers.5.self_attn.kv_b_proj\n", + "model.layers.5.self_attn.o_proj\n", + "model.layers.5.mlp\n", + "model.layers.5.mlp.experts\n", + "model.layers.5.mlp.experts.act_fn\n", + "model.layers.5.mlp.gate\n", + "model.layers.5.mlp.shared_experts\n", + "model.layers.5.mlp.shared_experts.gate_proj\n", + "model.layers.5.mlp.shared_experts.up_proj\n", + "model.layers.5.mlp.shared_experts.down_proj\n", + "model.layers.5.mlp.shared_experts.act_fn\n", + "model.layers.5.input_layernorm\n", + "model.layers.5.post_attention_layernorm\n", + "model.layers.6\n", + "model.layers.6.self_attn\n", + "model.layers.6.self_attn.q_a_proj\n", + "model.layers.6.self_attn.q_a_layernorm\n", + "model.layers.6.self_attn.q_b_proj\n", + "model.layers.6.self_attn.kv_a_proj_with_mqa\n", + "model.layers.6.self_attn.kv_a_layernorm\n", + "model.layers.6.self_attn.kv_b_proj\n", + "model.layers.6.self_attn.o_proj\n", + "model.layers.6.mlp\n", + "model.layers.6.mlp.experts\n", + "model.layers.6.mlp.experts.act_fn\n", + "model.layers.6.mlp.gate\n", + "model.layers.6.mlp.shared_experts\n", + "model.layers.6.mlp.shared_experts.gate_proj\n", + "model.layers.6.mlp.shared_experts.up_proj\n", + "model.layers.6.mlp.shared_experts.down_proj\n", + "model.layers.6.mlp.shared_experts.act_fn\n", + "model.layers.6.input_layernorm\n", + "model.layers.6.post_attention_layernorm\n", + "model.layers.7\n", + "model.layers.7.self_attn\n", + "model.layers.7.self_attn.q_a_proj\n", + "model.layers.7.self_attn.q_a_layernorm\n", + "model.layers.7.self_attn.q_b_proj\n", + "model.layers.7.self_attn.kv_a_proj_with_mqa\n", + "model.layers.7.self_attn.kv_a_layernorm\n", + "model.layers.7.self_attn.kv_b_proj\n", + "model.layers.7.self_attn.o_proj\n", + "model.layers.7.mlp\n", + "model.layers.7.mlp.experts\n", + "model.layers.7.mlp.experts.act_fn\n", + "model.layers.7.mlp.gate\n", + "model.layers.7.mlp.shared_experts\n", + "model.layers.7.mlp.shared_experts.gate_proj\n", + "model.layers.7.mlp.shared_experts.up_proj\n", + "model.layers.7.mlp.shared_experts.down_proj\n", + "model.layers.7.mlp.shared_experts.act_fn\n", + "model.layers.7.input_layernorm\n", + "model.layers.7.post_attention_layernorm\n", + "model.layers.8\n", + "model.layers.8.self_attn\n", + "model.layers.8.self_attn.q_a_proj\n", + "model.layers.8.self_attn.q_a_layernorm\n", + "model.layers.8.self_attn.q_b_proj\n", + "model.layers.8.self_attn.kv_a_proj_with_mqa\n", + "model.layers.8.self_attn.kv_a_layernorm\n", + "model.layers.8.self_attn.kv_b_proj\n", + "model.layers.8.self_attn.o_proj\n", + "model.layers.8.mlp\n", + "model.layers.8.mlp.experts\n", + "model.layers.8.mlp.experts.act_fn\n", + "model.layers.8.mlp.gate\n", + "model.layers.8.mlp.shared_experts\n", + "model.layers.8.mlp.shared_experts.gate_proj\n", + "model.layers.8.mlp.shared_experts.up_proj\n", + "model.layers.8.mlp.shared_experts.down_proj\n", + "model.layers.8.mlp.shared_experts.act_fn\n", + "model.layers.8.input_layernorm\n", + "model.layers.8.post_attention_layernorm\n", + "model.layers.9\n", + "model.layers.9.self_attn\n", + "model.layers.9.self_attn.q_a_proj\n", + "model.layers.9.self_attn.q_a_layernorm\n", + "model.layers.9.self_attn.q_b_proj\n", + "model.layers.9.self_attn.kv_a_proj_with_mqa\n", + "model.layers.9.self_attn.kv_a_layernorm\n", + "model.layers.9.self_attn.kv_b_proj\n", + "model.layers.9.self_attn.o_proj\n", + "model.layers.9.mlp\n", + "model.layers.9.mlp.experts\n", + "model.layers.9.mlp.experts.act_fn\n", + "model.layers.9.mlp.gate\n", + "model.layers.9.mlp.shared_experts\n", + "model.layers.9.mlp.shared_experts.gate_proj\n", + "model.layers.9.mlp.shared_experts.up_proj\n", + "model.layers.9.mlp.shared_experts.down_proj\n", + "model.layers.9.mlp.shared_experts.act_fn\n", + "model.layers.9.input_layernorm\n", + "model.layers.9.post_attention_layernorm\n", + "model.layers.10\n", + "model.layers.10.self_attn\n", + "model.layers.10.self_attn.q_a_proj\n", + "model.layers.10.self_attn.q_a_layernorm\n", + "model.layers.10.self_attn.q_b_proj\n", + "model.layers.10.self_attn.kv_a_proj_with_mqa\n", + "model.layers.10.self_attn.kv_a_layernorm\n", + "model.layers.10.self_attn.kv_b_proj\n", + "model.layers.10.self_attn.o_proj\n", + "model.layers.10.mlp\n", + "model.layers.10.mlp.experts\n", + "model.layers.10.mlp.experts.act_fn\n", + "model.layers.10.mlp.gate\n", + "model.layers.10.mlp.shared_experts\n", + "model.layers.10.mlp.shared_experts.gate_proj\n", + "model.layers.10.mlp.shared_experts.up_proj\n", + "model.layers.10.mlp.shared_experts.down_proj\n", + "model.layers.10.mlp.shared_experts.act_fn\n", + "model.layers.10.input_layernorm\n", + "model.layers.10.post_attention_layernorm\n", + "model.layers.11\n", + "model.layers.11.self_attn\n", + "model.layers.11.self_attn.q_a_proj\n", + "model.layers.11.self_attn.q_a_layernorm\n", + "model.layers.11.self_attn.q_b_proj\n", + "model.layers.11.self_attn.kv_a_proj_with_mqa\n", + "model.layers.11.self_attn.kv_a_layernorm\n", + "model.layers.11.self_attn.kv_b_proj\n", + "model.layers.11.self_attn.o_proj\n", + "model.layers.11.mlp\n", + "model.layers.11.mlp.experts\n", + "model.layers.11.mlp.experts.act_fn\n", + "model.layers.11.mlp.gate\n", + "model.layers.11.mlp.shared_experts\n", + "model.layers.11.mlp.shared_experts.gate_proj\n", + "model.layers.11.mlp.shared_experts.up_proj\n", + "model.layers.11.mlp.shared_experts.down_proj\n", + "model.layers.11.mlp.shared_experts.act_fn\n", + "model.layers.11.input_layernorm\n", + "model.layers.11.post_attention_layernorm\n", + "model.layers.12\n", + "model.layers.12.self_attn\n", + "model.layers.12.self_attn.q_a_proj\n", + "model.layers.12.self_attn.q_a_layernorm\n", + "model.layers.12.self_attn.q_b_proj\n", + "model.layers.12.self_attn.kv_a_proj_with_mqa\n", + "model.layers.12.self_attn.kv_a_layernorm\n", + "model.layers.12.self_attn.kv_b_proj\n", + "model.layers.12.self_attn.o_proj\n", + "model.layers.12.mlp\n", + "model.layers.12.mlp.experts\n", + "model.layers.12.mlp.experts.act_fn\n", + "model.layers.12.mlp.gate\n", + "model.layers.12.mlp.shared_experts\n", + "model.layers.12.mlp.shared_experts.gate_proj\n", + "model.layers.12.mlp.shared_experts.up_proj\n", + "model.layers.12.mlp.shared_experts.down_proj\n", + "model.layers.12.mlp.shared_experts.act_fn\n", + "model.layers.12.input_layernorm\n", + "model.layers.12.post_attention_layernorm\n", + "model.layers.13\n", + "model.layers.13.self_attn\n", + "model.layers.13.self_attn.q_a_proj\n", + "model.layers.13.self_attn.q_a_layernorm\n", + "model.layers.13.self_attn.q_b_proj\n", + "model.layers.13.self_attn.kv_a_proj_with_mqa\n", + "model.layers.13.self_attn.kv_a_layernorm\n", + "model.layers.13.self_attn.kv_b_proj\n", + "model.layers.13.self_attn.o_proj\n", + "model.layers.13.mlp\n", + "model.layers.13.mlp.experts\n", + "model.layers.13.mlp.experts.act_fn\n", + "model.layers.13.mlp.gate\n", + "model.layers.13.mlp.shared_experts\n", + "model.layers.13.mlp.shared_experts.gate_proj\n", + "model.layers.13.mlp.shared_experts.up_proj\n", + "model.layers.13.mlp.shared_experts.down_proj\n", + "model.layers.13.mlp.shared_experts.act_fn\n", + "model.layers.13.input_layernorm\n", + "model.layers.13.post_attention_layernorm\n", + "model.layers.14\n", + "model.layers.14.self_attn\n", + "model.layers.14.self_attn.q_a_proj\n", + "model.layers.14.self_attn.q_a_layernorm\n", + "model.layers.14.self_attn.q_b_proj\n", + "model.layers.14.self_attn.kv_a_proj_with_mqa\n", + "model.layers.14.self_attn.kv_a_layernorm\n", + "model.layers.14.self_attn.kv_b_proj\n", + "model.layers.14.self_attn.o_proj\n", + "model.layers.14.mlp\n", + "model.layers.14.mlp.experts\n", + "model.layers.14.mlp.experts.act_fn\n", + "model.layers.14.mlp.gate\n", + "model.layers.14.mlp.shared_experts\n", + "model.layers.14.mlp.shared_experts.gate_proj\n", + "model.layers.14.mlp.shared_experts.up_proj\n", + "model.layers.14.mlp.shared_experts.down_proj\n", + "model.layers.14.mlp.shared_experts.act_fn\n", + "model.layers.14.input_layernorm\n", + "model.layers.14.post_attention_layernorm\n", + "model.layers.15\n", + "model.layers.15.self_attn\n", + "model.layers.15.self_attn.q_a_proj\n", + "model.layers.15.self_attn.q_a_layernorm\n", + "model.layers.15.self_attn.q_b_proj\n", + "model.layers.15.self_attn.kv_a_proj_with_mqa\n", + "model.layers.15.self_attn.kv_a_layernorm\n", + "model.layers.15.self_attn.kv_b_proj\n", + "model.layers.15.self_attn.o_proj\n", + "model.layers.15.mlp\n", + "model.layers.15.mlp.experts\n", + "model.layers.15.mlp.experts.act_fn\n", + "model.layers.15.mlp.gate\n", + "model.layers.15.mlp.shared_experts\n", + "model.layers.15.mlp.shared_experts.gate_proj\n", + "model.layers.15.mlp.shared_experts.up_proj\n", + "model.layers.15.mlp.shared_experts.down_proj\n", + "model.layers.15.mlp.shared_experts.act_fn\n", + "model.layers.15.input_layernorm\n", + "model.layers.15.post_attention_layernorm\n", + "model.layers.16\n", + "model.layers.16.self_attn\n", + "model.layers.16.self_attn.q_a_proj\n", + "model.layers.16.self_attn.q_a_layernorm\n", + "model.layers.16.self_attn.q_b_proj\n", + "model.layers.16.self_attn.kv_a_proj_with_mqa\n", + "model.layers.16.self_attn.kv_a_layernorm\n", + "model.layers.16.self_attn.kv_b_proj\n", + "model.layers.16.self_attn.o_proj\n", + "model.layers.16.mlp\n", + "model.layers.16.mlp.experts\n", + "model.layers.16.mlp.experts.act_fn\n", + "model.layers.16.mlp.gate\n", + "model.layers.16.mlp.shared_experts\n", + "model.layers.16.mlp.shared_experts.gate_proj\n", + "model.layers.16.mlp.shared_experts.up_proj\n", + "model.layers.16.mlp.shared_experts.down_proj\n", + "model.layers.16.mlp.shared_experts.act_fn\n", + "model.layers.16.input_layernorm\n", + "model.layers.16.post_attention_layernorm\n", + "model.layers.17\n", + "model.layers.17.self_attn\n", + "model.layers.17.self_attn.q_a_proj\n", + "model.layers.17.self_attn.q_a_layernorm\n", + "model.layers.17.self_attn.q_b_proj\n", + "model.layers.17.self_attn.kv_a_proj_with_mqa\n", + "model.layers.17.self_attn.kv_a_layernorm\n", + "model.layers.17.self_attn.kv_b_proj\n", + "model.layers.17.self_attn.o_proj\n", + "model.layers.17.mlp\n", + "model.layers.17.mlp.experts\n", + "model.layers.17.mlp.experts.act_fn\n", + "model.layers.17.mlp.gate\n", + "model.layers.17.mlp.shared_experts\n", + "model.layers.17.mlp.shared_experts.gate_proj\n", + "model.layers.17.mlp.shared_experts.up_proj\n", + "model.layers.17.mlp.shared_experts.down_proj\n", + "model.layers.17.mlp.shared_experts.act_fn\n", + "model.layers.17.input_layernorm\n", + "model.layers.17.post_attention_layernorm\n", + "model.layers.18\n", + "model.layers.18.self_attn\n", + "model.layers.18.self_attn.q_a_proj\n", + "model.layers.18.self_attn.q_a_layernorm\n", + "model.layers.18.self_attn.q_b_proj\n", + "model.layers.18.self_attn.kv_a_proj_with_mqa\n", + "model.layers.18.self_attn.kv_a_layernorm\n", + "model.layers.18.self_attn.kv_b_proj\n", + "model.layers.18.self_attn.o_proj\n", + "model.layers.18.mlp\n", + "model.layers.18.mlp.experts\n", + "model.layers.18.mlp.experts.act_fn\n", + "model.layers.18.mlp.gate\n", + "model.layers.18.mlp.shared_experts\n", + "model.layers.18.mlp.shared_experts.gate_proj\n", + "model.layers.18.mlp.shared_experts.up_proj\n", + "model.layers.18.mlp.shared_experts.down_proj\n", + "model.layers.18.mlp.shared_experts.act_fn\n", + "model.layers.18.input_layernorm\n", + "model.layers.18.post_attention_layernorm\n", + "model.layers.19\n", + "model.layers.19.self_attn\n", + "model.layers.19.self_attn.q_a_proj\n", + "model.layers.19.self_attn.q_a_layernorm\n", + "model.layers.19.self_attn.q_b_proj\n", + "model.layers.19.self_attn.kv_a_proj_with_mqa\n", + "model.layers.19.self_attn.kv_a_layernorm\n", + "model.layers.19.self_attn.kv_b_proj\n", + "model.layers.19.self_attn.o_proj\n", + "model.layers.19.mlp\n", + "model.layers.19.mlp.experts\n", + "model.layers.19.mlp.experts.act_fn\n", + "model.layers.19.mlp.gate\n", + "model.layers.19.mlp.shared_experts\n", + "model.layers.19.mlp.shared_experts.gate_proj\n", + "model.layers.19.mlp.shared_experts.up_proj\n", + "model.layers.19.mlp.shared_experts.down_proj\n", + "model.layers.19.mlp.shared_experts.act_fn\n", + "model.layers.19.input_layernorm\n", + "model.layers.19.post_attention_layernorm\n", + "model.layers.20\n", + "model.layers.20.self_attn\n", + "model.layers.20.self_attn.q_a_proj\n", + "model.layers.20.self_attn.q_a_layernorm\n", + "model.layers.20.self_attn.q_b_proj\n", + "model.layers.20.self_attn.kv_a_proj_with_mqa\n", + "model.layers.20.self_attn.kv_a_layernorm\n", + "model.layers.20.self_attn.kv_b_proj\n", + "model.layers.20.self_attn.o_proj\n", + "model.layers.20.mlp\n", + "model.layers.20.mlp.experts\n", + "model.layers.20.mlp.experts.act_fn\n", + "model.layers.20.mlp.gate\n", + "model.layers.20.mlp.shared_experts\n", + "model.layers.20.mlp.shared_experts.gate_proj\n", + "model.layers.20.mlp.shared_experts.up_proj\n", + "model.layers.20.mlp.shared_experts.down_proj\n", + "model.layers.20.mlp.shared_experts.act_fn\n", + "model.layers.20.input_layernorm\n", + "model.layers.20.post_attention_layernorm\n", + "model.layers.21\n", + "model.layers.21.self_attn\n", + "model.layers.21.self_attn.q_a_proj\n", + "model.layers.21.self_attn.q_a_layernorm\n", + "model.layers.21.self_attn.q_b_proj\n", + "model.layers.21.self_attn.kv_a_proj_with_mqa\n", + "model.layers.21.self_attn.kv_a_layernorm\n", + "model.layers.21.self_attn.kv_b_proj\n", + "model.layers.21.self_attn.o_proj\n", + "model.layers.21.mlp\n", + "model.layers.21.mlp.experts\n", + "model.layers.21.mlp.experts.act_fn\n", + "model.layers.21.mlp.gate\n", + "model.layers.21.mlp.shared_experts\n", + "model.layers.21.mlp.shared_experts.gate_proj\n", + "model.layers.21.mlp.shared_experts.up_proj\n", + "model.layers.21.mlp.shared_experts.down_proj\n", + "model.layers.21.mlp.shared_experts.act_fn\n", + "model.layers.21.input_layernorm\n", + "model.layers.21.post_attention_layernorm\n", + "model.layers.22\n", + "model.layers.22.self_attn\n", + "model.layers.22.self_attn.q_a_proj\n", + "model.layers.22.self_attn.q_a_layernorm\n", + "model.layers.22.self_attn.q_b_proj\n", + "model.layers.22.self_attn.kv_a_proj_with_mqa\n", + "model.layers.22.self_attn.kv_a_layernorm\n", + "model.layers.22.self_attn.kv_b_proj\n", + "model.layers.22.self_attn.o_proj\n", + "model.layers.22.mlp\n", + "model.layers.22.mlp.experts\n", + "model.layers.22.mlp.experts.act_fn\n", + "model.layers.22.mlp.gate\n", + "model.layers.22.mlp.shared_experts\n", + "model.layers.22.mlp.shared_experts.gate_proj\n", + "model.layers.22.mlp.shared_experts.up_proj\n", + "model.layers.22.mlp.shared_experts.down_proj\n", + "model.layers.22.mlp.shared_experts.act_fn\n", + "model.layers.22.input_layernorm\n", + "model.layers.22.post_attention_layernorm\n", + "model.layers.23\n", + "model.layers.23.self_attn\n", + "model.layers.23.self_attn.q_a_proj\n", + "model.layers.23.self_attn.q_a_layernorm\n", + "model.layers.23.self_attn.q_b_proj\n", + "model.layers.23.self_attn.kv_a_proj_with_mqa\n", + "model.layers.23.self_attn.kv_a_layernorm\n", + "model.layers.23.self_attn.kv_b_proj\n", + "model.layers.23.self_attn.o_proj\n", + "model.layers.23.mlp\n", + "model.layers.23.mlp.experts\n", + "model.layers.23.mlp.experts.act_fn\n", + "model.layers.23.mlp.gate\n", + "model.layers.23.mlp.shared_experts\n", + "model.layers.23.mlp.shared_experts.gate_proj\n", + "model.layers.23.mlp.shared_experts.up_proj\n", + "model.layers.23.mlp.shared_experts.down_proj\n", + "model.layers.23.mlp.shared_experts.act_fn\n", + "model.layers.23.input_layernorm\n", + "model.layers.23.post_attention_layernorm\n", + "model.layers.24\n", + "model.layers.24.self_attn\n", + "model.layers.24.self_attn.q_a_proj\n", + "model.layers.24.self_attn.q_a_layernorm\n", + "model.layers.24.self_attn.q_b_proj\n", + "model.layers.24.self_attn.kv_a_proj_with_mqa\n", + "model.layers.24.self_attn.kv_a_layernorm\n", + "model.layers.24.self_attn.kv_b_proj\n", + "model.layers.24.self_attn.o_proj\n", + "model.layers.24.mlp\n", + "model.layers.24.mlp.experts\n", + "model.layers.24.mlp.experts.act_fn\n", + "model.layers.24.mlp.gate\n", + "model.layers.24.mlp.shared_experts\n", + "model.layers.24.mlp.shared_experts.gate_proj\n", + "model.layers.24.mlp.shared_experts.up_proj\n", + "model.layers.24.mlp.shared_experts.down_proj\n", + "model.layers.24.mlp.shared_experts.act_fn\n", + "model.layers.24.input_layernorm\n", + "model.layers.24.post_attention_layernorm\n", + "model.layers.25\n", + "model.layers.25.self_attn\n", + "model.layers.25.self_attn.q_a_proj\n", + "model.layers.25.self_attn.q_a_layernorm\n", + "model.layers.25.self_attn.q_b_proj\n", + "model.layers.25.self_attn.kv_a_proj_with_mqa\n", + "model.layers.25.self_attn.kv_a_layernorm\n", + "model.layers.25.self_attn.kv_b_proj\n", + "model.layers.25.self_attn.o_proj\n", + "model.layers.25.mlp\n", + "model.layers.25.mlp.experts\n", + "model.layers.25.mlp.experts.act_fn\n", + "model.layers.25.mlp.gate\n", + "model.layers.25.mlp.shared_experts\n", + "model.layers.25.mlp.shared_experts.gate_proj\n", + "model.layers.25.mlp.shared_experts.up_proj\n", + "model.layers.25.mlp.shared_experts.down_proj\n", + "model.layers.25.mlp.shared_experts.act_fn\n", + "model.layers.25.input_layernorm\n", + "model.layers.25.post_attention_layernorm\n", + "model.layers.26\n", + "model.layers.26.self_attn\n", + "model.layers.26.self_attn.q_a_proj\n", + "model.layers.26.self_attn.q_a_layernorm\n", + "model.layers.26.self_attn.q_b_proj\n", + "model.layers.26.self_attn.kv_a_proj_with_mqa\n", + "model.layers.26.self_attn.kv_a_layernorm\n", + "model.layers.26.self_attn.kv_b_proj\n", + "model.layers.26.self_attn.o_proj\n", + "model.layers.26.mlp\n", + "model.layers.26.mlp.experts\n", + "model.layers.26.mlp.experts.act_fn\n", + "model.layers.26.mlp.gate\n", + "model.layers.26.mlp.shared_experts\n", + "model.layers.26.mlp.shared_experts.gate_proj\n", + "model.layers.26.mlp.shared_experts.up_proj\n", + "model.layers.26.mlp.shared_experts.down_proj\n", + "model.layers.26.mlp.shared_experts.act_fn\n", + "model.layers.26.input_layernorm\n", + "model.layers.26.post_attention_layernorm\n", + "model.layers.27\n", + "model.layers.27.self_attn\n", + "model.layers.27.self_attn.q_a_proj\n", + "model.layers.27.self_attn.q_a_layernorm\n", + "model.layers.27.self_attn.q_b_proj\n", + "model.layers.27.self_attn.kv_a_proj_with_mqa\n", + "model.layers.27.self_attn.kv_a_layernorm\n", + "model.layers.27.self_attn.kv_b_proj\n", + "model.layers.27.self_attn.o_proj\n", + "model.layers.27.mlp\n", + "model.layers.27.mlp.experts\n", + "model.layers.27.mlp.experts.act_fn\n", + "model.layers.27.mlp.gate\n", + "model.layers.27.mlp.shared_experts\n", + "model.layers.27.mlp.shared_experts.gate_proj\n", + "model.layers.27.mlp.shared_experts.up_proj\n", + "model.layers.27.mlp.shared_experts.down_proj\n", + "model.layers.27.mlp.shared_experts.act_fn\n", + "model.layers.27.input_layernorm\n", + "model.layers.27.post_attention_layernorm\n", + "model.layers.28\n", + "model.layers.28.self_attn\n", + "model.layers.28.self_attn.q_a_proj\n", + "model.layers.28.self_attn.q_a_layernorm\n", + "model.layers.28.self_attn.q_b_proj\n", + "model.layers.28.self_attn.kv_a_proj_with_mqa\n", + "model.layers.28.self_attn.kv_a_layernorm\n", + "model.layers.28.self_attn.kv_b_proj\n", + "model.layers.28.self_attn.o_proj\n", + "model.layers.28.mlp\n", + "model.layers.28.mlp.experts\n", + "model.layers.28.mlp.experts.act_fn\n", + "model.layers.28.mlp.gate\n", + "model.layers.28.mlp.shared_experts\n", + "model.layers.28.mlp.shared_experts.gate_proj\n", + "model.layers.28.mlp.shared_experts.up_proj\n", + "model.layers.28.mlp.shared_experts.down_proj\n", + "model.layers.28.mlp.shared_experts.act_fn\n", + "model.layers.28.input_layernorm\n", + "model.layers.28.post_attention_layernorm\n", + "model.layers.29\n", + "model.layers.29.self_attn\n", + "model.layers.29.self_attn.q_a_proj\n", + "model.layers.29.self_attn.q_a_layernorm\n", + "model.layers.29.self_attn.q_b_proj\n", + "model.layers.29.self_attn.kv_a_proj_with_mqa\n", + "model.layers.29.self_attn.kv_a_layernorm\n", + "model.layers.29.self_attn.kv_b_proj\n", + "model.layers.29.self_attn.o_proj\n", + "model.layers.29.mlp\n", + "model.layers.29.mlp.experts\n", + "model.layers.29.mlp.experts.act_fn\n", + "model.layers.29.mlp.gate\n", + "model.layers.29.mlp.shared_experts\n", + "model.layers.29.mlp.shared_experts.gate_proj\n", + "model.layers.29.mlp.shared_experts.up_proj\n", + "model.layers.29.mlp.shared_experts.down_proj\n", + "model.layers.29.mlp.shared_experts.act_fn\n", + "model.layers.29.input_layernorm\n", + "model.layers.29.post_attention_layernorm\n", + "model.layers.30\n", + "model.layers.30.self_attn\n", + "model.layers.30.self_attn.q_a_proj\n", + "model.layers.30.self_attn.q_a_layernorm\n", + "model.layers.30.self_attn.q_b_proj\n", + "model.layers.30.self_attn.kv_a_proj_with_mqa\n", + "model.layers.30.self_attn.kv_a_layernorm\n", + "model.layers.30.self_attn.kv_b_proj\n", + "model.layers.30.self_attn.o_proj\n", + "model.layers.30.mlp\n", + "model.layers.30.mlp.experts\n", + "model.layers.30.mlp.experts.act_fn\n", + "model.layers.30.mlp.gate\n", + "model.layers.30.mlp.shared_experts\n", + "model.layers.30.mlp.shared_experts.gate_proj\n", + "model.layers.30.mlp.shared_experts.up_proj\n", + "model.layers.30.mlp.shared_experts.down_proj\n", + "model.layers.30.mlp.shared_experts.act_fn\n", + "model.layers.30.input_layernorm\n", + "model.layers.30.post_attention_layernorm\n", + "model.layers.31\n", + "model.layers.31.self_attn\n", + "model.layers.31.self_attn.q_a_proj\n", + "model.layers.31.self_attn.q_a_layernorm\n", + "model.layers.31.self_attn.q_b_proj\n", + "model.layers.31.self_attn.kv_a_proj_with_mqa\n", + "model.layers.31.self_attn.kv_a_layernorm\n", + "model.layers.31.self_attn.kv_b_proj\n", + "model.layers.31.self_attn.o_proj\n", + "model.layers.31.mlp\n", + "model.layers.31.mlp.experts\n", + "model.layers.31.mlp.experts.act_fn\n", + "model.layers.31.mlp.gate\n", + "model.layers.31.mlp.shared_experts\n", + "model.layers.31.mlp.shared_experts.gate_proj\n", + "model.layers.31.mlp.shared_experts.up_proj\n", + "model.layers.31.mlp.shared_experts.down_proj\n", + "model.layers.31.mlp.shared_experts.act_fn\n", + "model.layers.31.input_layernorm\n", + "model.layers.31.post_attention_layernorm\n", + "model.layers.32\n", + "model.layers.32.self_attn\n", + "model.layers.32.self_attn.q_a_proj\n", + "model.layers.32.self_attn.q_a_layernorm\n", + "model.layers.32.self_attn.q_b_proj\n", + "model.layers.32.self_attn.kv_a_proj_with_mqa\n", + "model.layers.32.self_attn.kv_a_layernorm\n", + "model.layers.32.self_attn.kv_b_proj\n", + "model.layers.32.self_attn.o_proj\n", + "model.layers.32.mlp\n", + "model.layers.32.mlp.experts\n", + "model.layers.32.mlp.experts.act_fn\n", + "model.layers.32.mlp.gate\n", + "model.layers.32.mlp.shared_experts\n", + "model.layers.32.mlp.shared_experts.gate_proj\n", + "model.layers.32.mlp.shared_experts.up_proj\n", + "model.layers.32.mlp.shared_experts.down_proj\n", + "model.layers.32.mlp.shared_experts.act_fn\n", + "model.layers.32.input_layernorm\n", + "model.layers.32.post_attention_layernorm\n", + "model.layers.33\n", + "model.layers.33.self_attn\n", + "model.layers.33.self_attn.q_a_proj\n", + "model.layers.33.self_attn.q_a_layernorm\n", + "model.layers.33.self_attn.q_b_proj\n", + "model.layers.33.self_attn.kv_a_proj_with_mqa\n", + "model.layers.33.self_attn.kv_a_layernorm\n", + "model.layers.33.self_attn.kv_b_proj\n", + "model.layers.33.self_attn.o_proj\n", + "model.layers.33.mlp\n", + "model.layers.33.mlp.experts\n", + "model.layers.33.mlp.experts.act_fn\n", + "model.layers.33.mlp.gate\n", + "model.layers.33.mlp.shared_experts\n", + "model.layers.33.mlp.shared_experts.gate_proj\n", + "model.layers.33.mlp.shared_experts.up_proj\n", + "model.layers.33.mlp.shared_experts.down_proj\n", + "model.layers.33.mlp.shared_experts.act_fn\n", + "model.layers.33.input_layernorm\n", + "model.layers.33.post_attention_layernorm\n", + "model.layers.34\n", + "model.layers.34.self_attn\n", + "model.layers.34.self_attn.q_a_proj\n", + "model.layers.34.self_attn.q_a_layernorm\n", + "model.layers.34.self_attn.q_b_proj\n", + "model.layers.34.self_attn.kv_a_proj_with_mqa\n", + "model.layers.34.self_attn.kv_a_layernorm\n", + "model.layers.34.self_attn.kv_b_proj\n", + "model.layers.34.self_attn.o_proj\n", + "model.layers.34.mlp\n", + "model.layers.34.mlp.experts\n", + "model.layers.34.mlp.experts.act_fn\n", + "model.layers.34.mlp.gate\n", + "model.layers.34.mlp.shared_experts\n", + "model.layers.34.mlp.shared_experts.gate_proj\n", + "model.layers.34.mlp.shared_experts.up_proj\n", + "model.layers.34.mlp.shared_experts.down_proj\n", + "model.layers.34.mlp.shared_experts.act_fn\n", + "model.layers.34.input_layernorm\n", + "model.layers.34.post_attention_layernorm\n", + "model.layers.35\n", + "model.layers.35.self_attn\n", + "model.layers.35.self_attn.q_a_proj\n", + "model.layers.35.self_attn.q_a_layernorm\n", + "model.layers.35.self_attn.q_b_proj\n", + "model.layers.35.self_attn.kv_a_proj_with_mqa\n", + "model.layers.35.self_attn.kv_a_layernorm\n", + "model.layers.35.self_attn.kv_b_proj\n", + "model.layers.35.self_attn.o_proj\n", + "model.layers.35.mlp\n", + "model.layers.35.mlp.experts\n", + "model.layers.35.mlp.experts.act_fn\n", + "model.layers.35.mlp.gate\n", + "model.layers.35.mlp.shared_experts\n", + "model.layers.35.mlp.shared_experts.gate_proj\n", + "model.layers.35.mlp.shared_experts.up_proj\n", + "model.layers.35.mlp.shared_experts.down_proj\n", + "model.layers.35.mlp.shared_experts.act_fn\n", + "model.layers.35.input_layernorm\n", + "model.layers.35.post_attention_layernorm\n", + "model.layers.36\n", + "model.layers.36.self_attn\n", + "model.layers.36.self_attn.q_a_proj\n", + "model.layers.36.self_attn.q_a_layernorm\n", + "model.layers.36.self_attn.q_b_proj\n", + "model.layers.36.self_attn.kv_a_proj_with_mqa\n", + "model.layers.36.self_attn.kv_a_layernorm\n", + "model.layers.36.self_attn.kv_b_proj\n", + "model.layers.36.self_attn.o_proj\n", + "model.layers.36.mlp\n", + "model.layers.36.mlp.experts\n", + "model.layers.36.mlp.experts.act_fn\n", + "model.layers.36.mlp.gate\n", + "model.layers.36.mlp.shared_experts\n", + "model.layers.36.mlp.shared_experts.gate_proj\n", + "model.layers.36.mlp.shared_experts.up_proj\n", + "model.layers.36.mlp.shared_experts.down_proj\n", + "model.layers.36.mlp.shared_experts.act_fn\n", + "model.layers.36.input_layernorm\n", + "model.layers.36.post_attention_layernorm\n", + "model.layers.37\n", + "model.layers.37.self_attn\n", + "model.layers.37.self_attn.q_a_proj\n", + "model.layers.37.self_attn.q_a_layernorm\n", + "model.layers.37.self_attn.q_b_proj\n", + "model.layers.37.self_attn.kv_a_proj_with_mqa\n", + "model.layers.37.self_attn.kv_a_layernorm\n", + "model.layers.37.self_attn.kv_b_proj\n", + "model.layers.37.self_attn.o_proj\n", + "model.layers.37.mlp\n", + "model.layers.37.mlp.experts\n", + "model.layers.37.mlp.experts.act_fn\n", + "model.layers.37.mlp.gate\n", + "model.layers.37.mlp.shared_experts\n", + "model.layers.37.mlp.shared_experts.gate_proj\n", + "model.layers.37.mlp.shared_experts.up_proj\n", + "model.layers.37.mlp.shared_experts.down_proj\n", + "model.layers.37.mlp.shared_experts.act_fn\n", + "model.layers.37.input_layernorm\n", + "model.layers.37.post_attention_layernorm\n", + "model.layers.38\n", + "model.layers.38.self_attn\n", + "model.layers.38.self_attn.q_a_proj\n", + "model.layers.38.self_attn.q_a_layernorm\n", + "model.layers.38.self_attn.q_b_proj\n", + "model.layers.38.self_attn.kv_a_proj_with_mqa\n", + "model.layers.38.self_attn.kv_a_layernorm\n", + "model.layers.38.self_attn.kv_b_proj\n", + "model.layers.38.self_attn.o_proj\n", + "model.layers.38.mlp\n", + "model.layers.38.mlp.experts\n", + "model.layers.38.mlp.experts.act_fn\n", + "model.layers.38.mlp.gate\n", + "model.layers.38.mlp.shared_experts\n", + "model.layers.38.mlp.shared_experts.gate_proj\n", + "model.layers.38.mlp.shared_experts.up_proj\n", + "model.layers.38.mlp.shared_experts.down_proj\n", + "model.layers.38.mlp.shared_experts.act_fn\n", + "model.layers.38.input_layernorm\n", + "model.layers.38.post_attention_layernorm\n", + "model.layers.39\n", + "model.layers.39.self_attn\n", + "model.layers.39.self_attn.q_a_proj\n", + "model.layers.39.self_attn.q_a_layernorm\n", + "model.layers.39.self_attn.q_b_proj\n", + "model.layers.39.self_attn.kv_a_proj_with_mqa\n", + "model.layers.39.self_attn.kv_a_layernorm\n", + "model.layers.39.self_attn.kv_b_proj\n", + "model.layers.39.self_attn.o_proj\n", + "model.layers.39.mlp\n", + "model.layers.39.mlp.experts\n", + "model.layers.39.mlp.experts.act_fn\n", + "model.layers.39.mlp.gate\n", + "model.layers.39.mlp.shared_experts\n", + "model.layers.39.mlp.shared_experts.gate_proj\n", + "model.layers.39.mlp.shared_experts.up_proj\n", + "model.layers.39.mlp.shared_experts.down_proj\n", + "model.layers.39.mlp.shared_experts.act_fn\n", + "model.layers.39.input_layernorm\n", + "model.layers.39.post_attention_layernorm\n", + "model.layers.40\n", + "model.layers.40.self_attn\n", + "model.layers.40.self_attn.q_a_proj\n", + "model.layers.40.self_attn.q_a_layernorm\n", + "model.layers.40.self_attn.q_b_proj\n", + "model.layers.40.self_attn.kv_a_proj_with_mqa\n", + "model.layers.40.self_attn.kv_a_layernorm\n", + "model.layers.40.self_attn.kv_b_proj\n", + "model.layers.40.self_attn.o_proj\n", + "model.layers.40.mlp\n", + "model.layers.40.mlp.experts\n", + "model.layers.40.mlp.experts.act_fn\n", + "model.layers.40.mlp.gate\n", + "model.layers.40.mlp.shared_experts\n", + "model.layers.40.mlp.shared_experts.gate_proj\n", + "model.layers.40.mlp.shared_experts.up_proj\n", + "model.layers.40.mlp.shared_experts.down_proj\n", + "model.layers.40.mlp.shared_experts.act_fn\n", + "model.layers.40.input_layernorm\n", + "model.layers.40.post_attention_layernorm\n", + "model.layers.41\n", + "model.layers.41.self_attn\n", + "model.layers.41.self_attn.q_a_proj\n", + "model.layers.41.self_attn.q_a_layernorm\n", + "model.layers.41.self_attn.q_b_proj\n", + "model.layers.41.self_attn.kv_a_proj_with_mqa\n", + "model.layers.41.self_attn.kv_a_layernorm\n", + "model.layers.41.self_attn.kv_b_proj\n", + "model.layers.41.self_attn.o_proj\n", + "model.layers.41.mlp\n", + "model.layers.41.mlp.experts\n", + "model.layers.41.mlp.experts.act_fn\n", + "model.layers.41.mlp.gate\n", + "model.layers.41.mlp.shared_experts\n", + "model.layers.41.mlp.shared_experts.gate_proj\n", + "model.layers.41.mlp.shared_experts.up_proj\n", + "model.layers.41.mlp.shared_experts.down_proj\n", + "model.layers.41.mlp.shared_experts.act_fn\n", + "model.layers.41.input_layernorm\n", + "model.layers.41.post_attention_layernorm\n", + "model.layers.42\n", + "model.layers.42.self_attn\n", + "model.layers.42.self_attn.q_a_proj\n", + "model.layers.42.self_attn.q_a_layernorm\n", + "model.layers.42.self_attn.q_b_proj\n", + "model.layers.42.self_attn.kv_a_proj_with_mqa\n", + "model.layers.42.self_attn.kv_a_layernorm\n", + "model.layers.42.self_attn.kv_b_proj\n", + "model.layers.42.self_attn.o_proj\n", + "model.layers.42.mlp\n", + "model.layers.42.mlp.experts\n", + "model.layers.42.mlp.experts.act_fn\n", + "model.layers.42.mlp.gate\n", + "model.layers.42.mlp.shared_experts\n", + "model.layers.42.mlp.shared_experts.gate_proj\n", + "model.layers.42.mlp.shared_experts.up_proj\n", + "model.layers.42.mlp.shared_experts.down_proj\n", + "model.layers.42.mlp.shared_experts.act_fn\n", + "model.layers.42.input_layernorm\n", + "model.layers.42.post_attention_layernorm\n", + "model.layers.43\n", + "model.layers.43.self_attn\n", + "model.layers.43.self_attn.q_a_proj\n", + "model.layers.43.self_attn.q_a_layernorm\n", + "model.layers.43.self_attn.q_b_proj\n", + "model.layers.43.self_attn.kv_a_proj_with_mqa\n", + "model.layers.43.self_attn.kv_a_layernorm\n", + "model.layers.43.self_attn.kv_b_proj\n", + "model.layers.43.self_attn.o_proj\n", + "model.layers.43.mlp\n", + "model.layers.43.mlp.experts\n", + "model.layers.43.mlp.experts.act_fn\n", + "model.layers.43.mlp.gate\n", + "model.layers.43.mlp.shared_experts\n", + "model.layers.43.mlp.shared_experts.gate_proj\n", + "model.layers.43.mlp.shared_experts.up_proj\n", + "model.layers.43.mlp.shared_experts.down_proj\n", + "model.layers.43.mlp.shared_experts.act_fn\n", + "model.layers.43.input_layernorm\n", + "model.layers.43.post_attention_layernorm\n", + "model.layers.44\n", + "model.layers.44.self_attn\n", + "model.layers.44.self_attn.q_a_proj\n", + "model.layers.44.self_attn.q_a_layernorm\n", + "model.layers.44.self_attn.q_b_proj\n", + "model.layers.44.self_attn.kv_a_proj_with_mqa\n", + "model.layers.44.self_attn.kv_a_layernorm\n", + "model.layers.44.self_attn.kv_b_proj\n", + "model.layers.44.self_attn.o_proj\n", + "model.layers.44.mlp\n", + "model.layers.44.mlp.experts\n", + "model.layers.44.mlp.experts.act_fn\n", + "model.layers.44.mlp.gate\n", + "model.layers.44.mlp.shared_experts\n", + "model.layers.44.mlp.shared_experts.gate_proj\n", + "model.layers.44.mlp.shared_experts.up_proj\n", + "model.layers.44.mlp.shared_experts.down_proj\n", + "model.layers.44.mlp.shared_experts.act_fn\n", + "model.layers.44.input_layernorm\n", + "model.layers.44.post_attention_layernorm\n", + "model.layers.45\n", + "model.layers.45.self_attn\n", + "model.layers.45.self_attn.q_a_proj\n", + "model.layers.45.self_attn.q_a_layernorm\n", + "model.layers.45.self_attn.q_b_proj\n", + "model.layers.45.self_attn.kv_a_proj_with_mqa\n", + "model.layers.45.self_attn.kv_a_layernorm\n", + "model.layers.45.self_attn.kv_b_proj\n", + "model.layers.45.self_attn.o_proj\n", + "model.layers.45.mlp\n", + "model.layers.45.mlp.experts\n", + "model.layers.45.mlp.experts.act_fn\n", + "model.layers.45.mlp.gate\n", + "model.layers.45.mlp.shared_experts\n", + "model.layers.45.mlp.shared_experts.gate_proj\n", + "model.layers.45.mlp.shared_experts.up_proj\n", + "model.layers.45.mlp.shared_experts.down_proj\n", + "model.layers.45.mlp.shared_experts.act_fn\n", + "model.layers.45.input_layernorm\n", + "model.layers.45.post_attention_layernorm\n", + "model.layers.46\n", + "model.layers.46.self_attn\n", + "model.layers.46.self_attn.q_a_proj\n", + "model.layers.46.self_attn.q_a_layernorm\n", + "model.layers.46.self_attn.q_b_proj\n", + "model.layers.46.self_attn.kv_a_proj_with_mqa\n", + "model.layers.46.self_attn.kv_a_layernorm\n", + "model.layers.46.self_attn.kv_b_proj\n", + "model.layers.46.self_attn.o_proj\n", + "model.layers.46.mlp\n", + "model.layers.46.mlp.experts\n", + "model.layers.46.mlp.experts.act_fn\n", + "model.layers.46.mlp.gate\n", + "model.layers.46.mlp.shared_experts\n", + "model.layers.46.mlp.shared_experts.gate_proj\n", + "model.layers.46.mlp.shared_experts.up_proj\n", + "model.layers.46.mlp.shared_experts.down_proj\n", + "model.layers.46.mlp.shared_experts.act_fn\n", + "model.layers.46.input_layernorm\n", + "model.layers.46.post_attention_layernorm\n", + "model.norm\n", + "model.rotary_emb\n", + "lm_head\n" + ] + } + ], + "source": [ + "model.config.use_cache = False\n", + "\n", + "model.enable_input_require_grads()\n", + "#print(model)\n", + "\n", + "\n", + "tokenized_id = ds.map(process_func, remove_columns=ds.column_names)\n", + "\n", + "print(tokenizer.decode(tokenized_id[0]['input_ids']))\n", + "\n", + "print(tokenizer.decode(list(filter(lambda x: x != -100, tokenized_id[1][\"labels\"]))))\n", + "\n", + "\n", + "for name, module in model.named_modules():\n", + " print(name)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "484a3882-6dec-4b87-8519-3d8c39b6098f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trainable params: 14,767,616 || all params: 29,958,158,592 || trainable%: 0.0493\n", + "None\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [30/30 03:08, Epoch 1/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
103.339206
203.111820
303.088505

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=30, training_loss=3.179843457539876, metrics={'train_runtime': 197.2214, 'train_samples_per_second': 18.908, 'train_steps_per_second': 0.152, 'total_flos': 9.046072556851046e+16, 'train_loss': 3.179843457539876, 'epoch': 1.0})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config = LoraConfig(\n", + " task_type=TaskType.CAUSAL_LM,\n", + " target_modules=[\"q_a_proj\", \"q_b_proj\", \"kv_a_proj_with_mqa\", \"kv_b_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " inference_mode=False, # 训练模式\n", + " r=8, # Lora 秩\n", + " lora_alpha=32, # Lora alpha\n", + " lora_dropout=0.1 # Dropout 比例\n", + ")\n", + "\n", + "model = get_peft_model(model, config)\n", + "\n", + "print(model.print_trainable_parameters())\n", + "\n", + "args = TrainingArguments(\n", + " output_dir=\"./output/GLM-4.7-Flash\", # 注意修改\n", + " per_device_train_batch_size=32,\n", + " gradient_accumulation_steps=4,\n", + " logging_steps=10,\n", + " num_train_epochs=1,\n", + " save_steps=100,\n", + " learning_rate=1e-4,\n", + " save_on_each_node=True,\n", + " report_to=\"none\",\n", + ")\n", + "\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=args,\n", + " train_dataset=tokenized_id,\n", + " data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n", + ")\n", + "\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0e5ceb6b-d441-4640-a6bc-e6bc4fa9632c", + "metadata": {}, + "outputs": [], + "source": [ + "model_path = '/root/autodl-fs/ZhipuAI/GLM-4.7-Flash'\n", + "lora_path = './output/GLM-4.7-Flash/checkpoint-30'" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a6898745-1146-415d-8798-43731d07d47e", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "65a2af217265442e816e35cb90e824e3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/751 [00:00我是甄嬛。皇上，您怎么来了？怎么不让人通报一声？您怎么不穿外衣？怎么不穿外衣？怎么不\n" + ] + } + ], + "source": [ + "from peft import PeftModel\n", + "# 加载tokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)\n", + "\n", + "# 加载模型\n", + "model = Glm4MoeLiteForCausalLM.from_pretrained(\n", + " pretrained_model_name_or_path=model_path,\n", + " dtype=torch.bfloat16,\n", + " device_map=\"auto\",\n", + " trust_remote_code=True,\n", + ").eval()\n", + "\n", + "#model = AutoModelForCausalLM.from_pretrained(model_path, device_map=\"auto\",torch_dtype=torch.bfloat16, trust_remote_code=True).eval()\n", + "\n", + "# 加载lora权重\n", + "model = PeftModel.from_pretrained(model, model_id=lora_path)\n", + "\n", + "messages=[\n", + " { 'role': 'system', 'content': \"假设你是皇帝身边的女人--甄嬛。\"},\n", + " { 'role': 'user', 'content': \"你是谁？\"}\n", + "]\n", + "\n", + "inputs = tokenizer.apply_chat_template(\n", + "\tmessages,\n", + "\tadd_generation_prompt=True,\n", + "\ttokenize=True,\n", + "\treturn_dict=True,\n", + "\treturn_tensors=\"pt\",\n", + ").to(model.device)\n", + "\n", + "outputs = model.generate(**inputs, max_new_tokens=64)\n", + "\n", + "\n", + "print(tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9c34b8d-a4f2-4ef8-8a37-562fd654ead6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}