From 01b094679358e08be2d6169eb35fcd721dfaa150 Mon Sep 17 00:00:00 2001 From: KMnO4-zx <1021385881@qq.com> Date: Thu, 16 Nov 2023 21:50:43 +0800 Subject: [PATCH] completed interlm --- InternLM/02-Lagent+InternLM-Chat-7B-V1.1.md | 275 ++++++++++++++++++++ InternLM/03-浦语灵笔图文理解&创作.md | 68 +++++ InternLM/images/image-10.png | Bin 0 -> 317777 bytes InternLM/images/image-7.png | Bin 0 -> 66137 bytes InternLM/images/image-8.png | Bin 0 -> 98197 bytes InternLM/images/image-9.png | Bin 0 -> 610173 bytes README.md | 8 +- 7 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 InternLM/02-Lagent+InternLM-Chat-7B-V1.1.md create mode 100644 InternLM/03-浦语灵笔图文理解&创作.md create mode 100644 InternLM/images/image-10.png create mode 100644 InternLM/images/image-7.png create mode 100644 InternLM/images/image-8.png create mode 100644 InternLM/images/image-9.png diff --git a/InternLM/02-Lagent+InternLM-Chat-7B-V1.1.md b/InternLM/02-Lagent+InternLM-Chat-7B-V1.1.md new file mode 100644 index 0000000..c729dd0 --- /dev/null +++ b/InternLM/02-Lagent+InternLM-Chat-7B-V1.1.md @@ -0,0 +1,275 @@ +# Lagent+InternLM-Chat-7B-V1.1 + +## 环境准备 + +选择和第一个 `InternLM` 一样的 `AutoDL` 镜像环境,运行以下命令安装依赖,如果上一个 `InternLM-Chat-7B` 已经配置好环境不需要重复安装. + +```shell +# 更换 pypi 源加速库的安装 +pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + +pip install modelscope +pip install transformers +pip install streamlit==1.24.0 +pip install sentencepiece +pip install accelerate +``` + +## 模型下载 + +在 `/root/autodl-tmp` 路径下新建 `download.py` 文件并在其中输入以下内容,并运行 `python /root/autodl-tmp/download.py`执行下载,模型大小为 14 GB,下载模型大概需要 10~20 分钟 + +```python +import torch +from modelscope import snapshot_download, AutoModel, AutoTokenizer +import os +model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm-chat-7b-v1_1', cache_dir='/root/autodl-tmp', revision='master') +``` + +## Lagent 安装 + +首先我们运行以下命令打开学术加速 `clone` `lagent`仓库,并通过 `pip install -e .`源码安装 `Lagent` + +```shell +source /etc/network_turbo +cd /root/autodl-tmp +git clone https://github.com/InternLM/lagent.git +git checkout 511b03889010c4811b1701abb153e02b8e94fb5e # 尽量保证和教程commit版本一致 +cd lagent +pip install -e . # 源码安装 +unset http_proxy && unset https_proxy +``` + +## 修改代码 + +由于代码修改的地方比较多,大家直接将`/root/autodl-tmp/lagent/examples/react_web_demo.py` 内容替换为以下代码 + +```python +import copy +import os + +import streamlit as st +from streamlit.logger import get_logger + +from lagent.actions import ActionExecutor, GoogleSearch, PythonInterpreter +from lagent.agents.react import ReAct +from lagent.llms import GPTAPI +from lagent.llms.huggingface import HFTransformerCasualLM + + +class SessionState: + + def init_state(self): + """Initialize session state variables.""" + st.session_state['assistant'] = [] + st.session_state['user'] = [] + + #action_list = [PythonInterpreter(), GoogleSearch()] + action_list = [PythonInterpreter()] + st.session_state['plugin_map'] = { + action.name: action + for action in action_list + } + st.session_state['model_map'] = {} + st.session_state['model_selected'] = None + st.session_state['plugin_actions'] = set() + + def clear_state(self): + """Clear the existing session state.""" + st.session_state['assistant'] = [] + st.session_state['user'] = [] + st.session_state['model_selected'] = None + if 'chatbot' in st.session_state: + st.session_state['chatbot']._session_history = [] + + +class StreamlitUI: + + def __init__(self, session_state: SessionState): + self.init_streamlit() + self.session_state = session_state + + def init_streamlit(self): + """Initialize Streamlit's UI settings.""" + st.set_page_config( + layout='wide', + page_title='lagent-web', + page_icon='./docs/imgs/lagent_icon.png') + # st.header(':robot_face: :blue[Lagent] Web Demo ', divider='rainbow') + st.sidebar.title('模型控制') + + def setup_sidebar(self): + """Setup the sidebar for model and plugin selection.""" + model_name = st.sidebar.selectbox( + '模型选择:', options=['gpt-3.5-turbo','internlm']) + if model_name != st.session_state['model_selected']: + model = self.init_model(model_name) + self.session_state.clear_state() + st.session_state['model_selected'] = model_name + if 'chatbot' in st.session_state: + del st.session_state['chatbot'] + else: + model = st.session_state['model_map'][model_name] + + plugin_name = st.sidebar.multiselect( + '插件选择', + options=list(st.session_state['plugin_map'].keys()), + default=[list(st.session_state['plugin_map'].keys())[0]], + ) + + plugin_action = [ + st.session_state['plugin_map'][name] for name in plugin_name + ] + if 'chatbot' in st.session_state: + st.session_state['chatbot']._action_executor = ActionExecutor( + actions=plugin_action) + if st.sidebar.button('清空对话', key='clear'): + self.session_state.clear_state() + uploaded_file = st.sidebar.file_uploader( + '上传文件', type=['png', 'jpg', 'jpeg', 'mp4', 'mp3', 'wav']) + return model_name, model, plugin_action, uploaded_file + + def init_model(self, option): + """Initialize the model based on the selected option.""" + if option not in st.session_state['model_map']: + if option.startswith('gpt'): + st.session_state['model_map'][option] = GPTAPI( + model_type=option) + else: + st.session_state['model_map'][option] = HFTransformerCasualLM( + '/root/autodl-tmp/Shanghai_AI_Laboratory/internlm-chat-7b-v1_1') + return st.session_state['model_map'][option] + + def initialize_chatbot(self, model, plugin_action): + """Initialize the chatbot with the given model and plugin actions.""" + return ReAct( + llm=model, action_executor=ActionExecutor(actions=plugin_action)) + + def render_user(self, prompt: str): + with st.chat_message('user'): + st.markdown(prompt) + + def render_assistant(self, agent_return): + with st.chat_message('assistant'): + for action in agent_return.actions: + if (action): + self.render_action(action) + st.markdown(agent_return.response) + + def render_action(self, action): + with st.expander(action.type, expanded=True): + st.markdown( + "
插 件:" # noqa E501 + + action.type + '
', + unsafe_allow_html=True) + st.markdown( + "思考步骤:" # noqa E501 + + action.thought + '
', + unsafe_allow_html=True) + if (isinstance(action.args, dict) and 'text' in action.args): + st.markdown( + "执行内容:
", # noqa E501 + unsafe_allow_html=True) + st.markdown(action.args['text']) + self.render_action_results(action) + + def render_action_results(self, action): + """Render the results of action, including text, images, videos, and + audios.""" + if (isinstance(action.result, dict)): + st.markdown( + "执行结果:
", # noqa E501 + unsafe_allow_html=True) + if 'text' in action.result: + st.markdown( + "" + action.result['text'] + + '
', + unsafe_allow_html=True) + if 'image' in action.result: + image_path = action.result['image'] + image_data = open(image_path, 'rb').read() + st.image(image_data, caption='Generated Image') + if 'video' in action.result: + video_data = action.result['video'] + video_data = open(video_data, 'rb').read() + st.video(video_data) + if 'audio' in action.result: + audio_data = action.result['audio'] + audio_data = open(audio_data, 'rb').read() + st.audio(audio_data) + + +def main(): + logger = get_logger(__name__) + # Initialize Streamlit UI and setup sidebar + if 'ui' not in st.session_state: + session_state = SessionState() + session_state.init_state() + st.session_state['ui'] = StreamlitUI(session_state) + + else: + st.set_page_config( + layout='wide', + page_title='lagent-web', + page_icon='./docs/imgs/lagent_icon.png') + # st.header(':robot_face: :blue[Lagent] Web Demo ', divider='rainbow') + model_name, model, plugin_action, uploaded_file = st.session_state[ + 'ui'].setup_sidebar() + + # Initialize chatbot if it is not already initialized + # or if the model has changed + if 'chatbot' not in st.session_state or model != st.session_state[ + 'chatbot']._llm: + st.session_state['chatbot'] = st.session_state[ + 'ui'].initialize_chatbot(model, plugin_action) + + for prompt, agent_return in zip(st.session_state['user'], + st.session_state['assistant']): + st.session_state['ui'].render_user(prompt) + st.session_state['ui'].render_assistant(agent_return) + # User input form at the bottom (this part will be at the bottom) + # with st.form(key='my_form', clear_on_submit=True): + + if user_input := st.chat_input(''): + st.session_state['ui'].render_user(user_input) + st.session_state['user'].append(user_input) + # Add file uploader to sidebar + if uploaded_file: + file_bytes = uploaded_file.read() + file_type = uploaded_file.type + if 'image' in file_type: + st.image(file_bytes, caption='Uploaded Image') + elif 'video' in file_type: + st.video(file_bytes, caption='Uploaded Video') + elif 'audio' in file_type: + st.audio(file_bytes, caption='Uploaded Audio') + # Save the file to a temporary location and get the path + file_path = os.path.join(root_dir, uploaded_file.name) + with open(file_path, 'wb') as tmpfile: + tmpfile.write(file_bytes) + st.write(f'File saved at: {file_path}') + user_input = '我上传了一个图像,路径为: {file_path}. {user_input}'.format( + file_path=file_path, user_input=user_input) + agent_return = st.session_state['chatbot'].chat(user_input) + st.session_state['assistant'].append(copy.deepcopy(agent_return)) + logger.info(agent_return.inner_steps) + st.session_state['ui'].render_assistant(agent_return) + + +if __name__ == '__main__': + root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + root_dir = os.path.join(root_dir, 'tmp_dir') + os.makedirs(root_dir, exist_ok=True) + main() +``` + +## Demo 运行 + +```shell +streamlit run /root/autodl-tmp/lagent/examples/react_web_demo.py --server.address 127.0.0.1 --server.port 6006 +``` + +用同样的方法将端口映射到本地,运行成功后,我们在 `Web` 页面选择 `InternLM` 模型,等待模型加载完毕后,输入数学问题 已知 `2x+3=10`,求`x` ,此时 `InternLM-Chat-7B-V1.1` 模型理解题意生成解此题的 `Python`代码,`Lagent` 调度送入 `Python` 代码解释器求出该问题的解。 + + + diff --git a/InternLM/03-浦语灵笔图文理解&创作.md b/InternLM/03-浦语灵笔图文理解&创作.md new file mode 100644 index 0000000..158753d --- /dev/null +++ b/InternLM/03-浦语灵笔图文理解&创作.md @@ -0,0 +1,68 @@ +# 浦语灵笔图文理解&创作 + +## 环境准备 + +首先在 `AutoDL` 上租一台显卡驱动支持 `11.7` 以上的双卡 `3090` 机器. +在选择镜像是选择 `Miniconda` --> `conda3` --> `3.8(ubuntu20.04)`--> `11.6` + + + +打开 `jupyter lab` 中的终端,首先运行以下命令安装 `PyTorch 2.0.1` + +```shell +pip install torch==2.0.1+cu117 torchvision==0.15.2+cu117 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu117 +``` + +接下来运行以下命令,安装 `transformers`、`gradio` 等依赖包。请严格安装以下版本安装! + +```shell +pip install transformers==4.33.1 timm==0.4.12 sentencepiece==0.1.99 gradio==3.44.4 markdown2==2.4.10 xlsxwriter==3.1.2 einops accelerate +``` +## 模型下载 + +安装`modelscope`,下载模型的老朋友了 + +```shell +pip install modelscope +``` + +在 `/root/autodl-tmp` 路径下新建 `download.py` 文件并在其中输入以下内容,并运行 `python /root/autodl-tmp/download.py`执行下载 + +```python +import torch +from modelscope import snapshot_download, AutoModel, AutoTokenizer +import os +model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm-xcomposer-7b', cache_dir='/root/autodl-tmp', revision='master') +``` + +## 代码准备 + +在 `/root/autodl-tmp` `git clone InternLM-XComposer` 仓库的代码 + +```shell +source /etc/network_turbo +cd /root/autodl-tmp +git clone https://github.com/InternLM/InternLM-XComposer.git +git checkout 3e8c79051a1356b9c388a6447867355c0634932d # 最好保证和教程的commit版本一致 +unset http_proxy && unset https_proxy +``` + +## Demo 运行 + +在终端运行以下代码: + +```shell +cd /root/autodl-tmp/InternLM-XComposer +python examples/web_demo.py \ + --folder /root/autodl-tmp/Shanghai_AI_Laboratory/internlm-xcomposer-7b \ + --num_gpus 2 \ + --port 6006 +``` + +将autodl的端口映射到本地之后,用浏览器代开`http://localhost:6006/`,我们以`又见敦煌`为提示词,体验图文创作的功能,如下图所示: + + + +接下来,我们可以体验以下图片理解的能力,如下所示~ + + \ No newline at end of file diff --git a/InternLM/images/image-10.png b/InternLM/images/image-10.png new file mode 100644 index 0000000000000000000000000000000000000000..6f9e28c1ab37771f4cb2b0513d10865b7dde3a4b GIT binary patch literal 317777 zcmcG$g;$i{_dO0s2`Gr9lprDkQi3R{gp^2^beELULrMt$EekbaXDBR}e1w%)YD>Qn#4}+S{$eI~4qo8Xhh#_#doKavjY=
zzv)kgL%#gwUne*jR4_7(UR|7HcQwk0^U7wf%W6f`mDm%Y#TTkLj@CZft}Q9~9AzVq
zT^q NIPkGpOGj@*f36F19m2OZ2kUrk&V7+l+hlbL>Y*HWGloqM+DDHL`T1qV
zlILTmDp=&ceiLX}S#d*7#Q_r3bJJ@Ui5VTe%}-x#+MLPJ9WIl&H=mLMKV+H4`{o)0
zhpCqrjzNjT%>LV2l603{V;VgAJjA>Wr@96&E%UYy8$@;Z=AxpQAcVi0jbxIOe@JPn
zLropE9L)K)-%^ZwDQLjz0du?zm)4IPDr9E@p5ntHV1O) Ju
zXspH!fa*n@dr6eq1Hg7+p`gd6301BdF6>?t6PlbH@`>Z-h&cSb0Y4UMKYnkc*g5d~
z2QDwiJ8v&&BB&y>aX(y>3*@*->B1^P0JijHm!%cllQj!^(loomU&^@9{Q#g4%J~V@
z)$Vd|%Lag1;9J7DCGT{a{aTyl;8f=QG|t%)|6@5VX--U~{{-U%H1u)AmdQC8rb{^o
zzU~ah)`|Kp{VIaxPR~Yzi*{2@p;=jd7Rx^z0K6ZH+a4;iUOm}E!WgV{eLZ6EXOfX!
zHh?KUOGg2WCcLyTdI5`-HPndPrP<#j#e7x{SQHUM7_12M*;c-Bwg8+JqW`eBr!wY9
z8b$>w4vbplA92rTw=TRE!Xn4{r$Gg?^5+WvcsG^Lz%zsAuad6;nD;4oHj#0L&}jlL
z8EAn3t0D#;o0J8s pHg
zQUqSLC^S`?ioMxN*@A2|y_HFd1%6sp2E#{1hd3!KLF(QmS?Gfx{I^=#A
z%Ujiz#eY@lu|_mHpb&8burLT&HpZsv%h1PxAsoG{vsZJIXtM-{xrVv%na*S)iW~WHv1A{+q>fk
zTL`Lk0?vrg4M(IN%V2XHk2Ni*@;8X)XrXQSvKXejirU<@(XSp0_CLcvzpiSSTz+rw
zDlwBqz7&^=b4D)UTVezSWJQu%6sws^cgg!xR>wwwKEJl_8#*a|b3vlA(>x^6uAUH4
z=bFNMkCS7_7mIjT617lX!#0ccKCC+HGICZtD
j_W#TJb(sS{A4ud<^oW9LnloL*$m0CzA_lBBqeKJw6TZSoLCHGdU
zcpAU2btfTlR*%lKh!CM45C90JWLY~N6ttQtRYIWd@#|Fto}GD75|`Ln$zf7$$}tYl
zxjTAPLI>wEkOdRJ*|TAzX&=L#CTYGPZ@qd&;(x(+BqDaz3NCeid}uJAeV1*2wLJ>3
zVX;;1y@2yt$=;&2Cp8j_zCPNPIV)LWKAUh68
1!@beE32*%YS ;dR`lpwGUmNSAeOKJWYRpWcd6!2u
V?{-&0Q$}+mn7LeVt9I>+
z!$8!Zrn9>cU0fhX=CrQbu`%bi%iwTwWO^PDk0YU(XV<&fEz#RSr|Gb4<0m}${=*vw
zdmoIxQm@v0Ybnzo!Y?8=wZ=&r4-NSTb964PrP5k^R}N(88;{fT)|RRzLrUPGw_b}0
zaHWz)x9EotgPz*=AKMJMggy}zycUX0O#A@8Bu)rt_9|*42y6k_sY9x-`(HdGCk|bH
zC4cveuRY|Yu`{