opensource/ai-edu
3
0
Fork 0
mirror of https://github.com/microsoft/ai-edu.git synced 2026-08-29 02:27:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update 20.3-GRU基本原理.md (#557)

Browse Source
This commit is contained in:
DaGang
2020-09-19 18:24:37 +08:00
committed by GitHub
parent 29f6438e81
commit 4fc7dd78dd
1 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
A-基础教程/A2-神经网络基本原理简明教程/Step9 - RNN/20.3-GRU基本原理.md
+13 -13
View File
@@ -40,8 +40,8 @@ GRU单元的前向计算公式如下：
3. 候选隐藏状态
$$
\tilde{h}_t = \tanh((r_t \circ h_{t-1}) \cdot W_h + x_t \cdot U_h)
$$
\tilde h_t = \tanh((r_t \circ h_{t-1}) \cdot W_h + x_t \cdot U_h)
\tag{3}
$$
@@ -79,7 +79,7 @@ z_{rt} = h_{t-1} \cdot W_r + x_t \cdot U_r
$$
$$
z_{\tilde{h}_t} = (r_t \circ h_{t-1}) \cdot W_h + x_t \cdot U_h
z_{\tilde h_t} = (r_t \circ h_{t-1}) \cdot W_h + x_t \cdot U_h
\tag{7}
$$
@@ -88,24 +88,24 @@ $$
$$
\begin{aligned}
\delta_{z_{zt}} &= \frac{\partial{loss}}{\partial{h_t}} \cdot \frac{\partial{h_t}}{\partial{z_t}} \cdot \frac{\partial{z_t}}{\partial{z_{z_t}}} \\
&= \delta_t \cdot (-diag[h_{t-1}] + diag[\tilde{h}_t]) \cdot diag[z_t \circ (1-z_t)] \\
&= \delta_t \circ (\tilde{h}_t - h_{t-1}) \circ z_t \circ (1-z_t)
&= \delta_t \cdot (-diag[h_{t-1}] + diag[\tilde h_t]) \cdot diag[z_t \circ (1-z_t)] \\
&= \delta_t \circ (\tilde h_t - h_{t-1}) \circ z_t \circ (1-z_t)
\end{aligned}
\tag{8}
$$
$$
\begin{aligned}
\delta_{z_{\tilde{h}t}} &= \frac{\partial{loss}}{\partial{h_t}} \cdot \frac{\partial{h_t}}{\partial{\tilde{h}_t}} \cdot \frac{\partial{\tilde{h}_t}}{\partial{z_{\tilde{h}_t}}} \\
&= \delta_t \cdot diag[z_t] \cdot diag[1-(\tilde{h}_t)^2] \\
&= \delta_t \circ z_t \circ (1-(\tilde{h}_t)^2)
\delta_{z_{\tilde{h}t}} &= \frac{\partial{loss}}{\partial{h_t}} \cdot \frac{\partial{h_t}}{\partial{\tilde h_t}} \cdot \frac{\partial{\tilde h_t}}{\partial{z_{\tilde h_t}}} \\
&= \delta_t \cdot diag[z_t] \cdot diag[1-(\tilde h_t)^2] \\
&= \delta_t \circ z_t \circ (1-(\tilde h_t)^2)
\end{aligned}
\tag{9}
$$
$$
\begin{aligned}
\delta_{z_{rt}} &= \frac{\partial{loss}}{\partial{\tilde{h}_t}} \cdot \frac{\partial{\tilde{h}_t}}{\partial{z_{\tilde{h}_t}}} \cdot \frac{\partial{z_{\tilde{h}_t}}}{\partial{r_t}} \cdot \frac{\partial{r_t}}{\partial{z_{r_t}}} \\
\delta_{z_{rt}} &= \frac{\partial{loss}}{\partial{\tilde h_t}} \cdot \frac{\partial{\tilde h_t}}{\partial{z_{\tilde h_t}}} \cdot \frac{\partial{z_{\tilde h_t}}}{\partial{r_t}} \cdot \frac{\partial{r_t}}{\partial{z_{r_t}}} \\
&= \delta_{z_{\tilde{h}t}} \cdot W_h^T \cdot diag[h_{t-1}] \cdot diag[r_t \circ (1-r_t)] \\
&= \delta_{z_{\tilde{h}t}} \cdot W_h^T \circ h_{t-1} \circ r_t \circ (1-r_t)
\end{aligned}
@@ -116,14 +116,14 @@ $$
$$
\begin{aligned}
d_{W_{h,t}} = \frac{\partial{loss}}{\partial{z_{\tilde{h}_t}}} \cdot \frac{\partial{z_{\tilde{h}_t}}}{\partial{W_h}} = (r_t \circ h_{t-1})^T \cdot \delta_{z_{\tilde{h}t}}
d_{W_{h,t}} = \frac{\partial{loss}}{\partial{z_{\tilde h_t}}} \cdot \frac{\partial{z_{\tilde h_t}}}{\partial{W_h}} = (r_t \circ h_{t-1})^T \cdot \delta_{z_{\tilde{h}t}}
\end{aligned}
\tag{11}
$$
$$
\begin{aligned}
d_{U_{h,t}} = \frac{\partial{loss}}{\partial{z_{\tilde{h}_t}}} \cdot \frac{\partial{z_{\tilde{h}_t}}}{\partial{U_h}} = x_t^T \cdot \delta_{z_{\tilde{h}t}}
d_{U_{h,t}} = \frac{\partial{loss}}{\partial{z_{\tilde h_t}}} \cdot \frac{\partial{z_{\tilde h_t}}}{\partial{U_h}} = x_t^T \cdot \delta_{z_{\tilde{h}t}}
\end{aligned}
\tag{12}
$$
@@ -194,7 +194,7 @@ $$
$$
\begin{aligned}
\delta_{h_{t-1}} = \frac{\partial{loss}}{\partial{h_{t-1}}} &= \frac{\partial{loss}}{\partial{h_t}} \cdot \frac{\partial{h_t}}{\partial{h_{t-1}}} + \frac{\partial{loss}}{\partial{z_{\tilde{h}_t}}} \cdot \frac{\partial{z_{\tilde{h}_t}}}{\partial{h_{t-1}}} \\
\delta_{h_{t-1}} = \frac{\partial{loss}}{\partial{h_{t-1}}} &= \frac{\partial{loss}}{\partial{h_t}} \cdot \frac{\partial{h_t}}{\partial{h_{t-1}}} + \frac{\partial{loss}}{\partial{z_{\tilde h_t}}} \cdot \frac{\partial{z_{\tilde h_t}}}{\partial{h_{t-1}}} \\
&+ \frac{\partial{loss}}{\partial{z_{rt}}} \cdot \frac{\partial{z_{rt}}}{\partial{h_{t-1}}} + \frac{\partial{loss}}{\partial{z_{zt}}} \cdot \frac{\partial{z_{zt}}}{\partial{h_{t-1}}} \\
&= \delta_{t} \circ (1-z_t) + \delta_{z_{\tilde{h}t}} \cdot W_h^T \circ r_t \\
&+ \delta_{z_{rt}} \cdot W_r^T + \delta_{z_{zt}} \cdot W_z^T
@@ -206,7 +206,7 @@ $$
$$
\begin{aligned}
\delta_{x_t} &= \frac{\partial{loss}}{\partial{x_t}} = \frac{\partial{loss}}{\partial{z_{\tilde{h}_t}}} \cdot \frac{\partial{z_{\tilde{h}_t}}}{\partial{x_t}} \\
\delta_{x_t} &= \frac{\partial{loss}}{\partial{x_t}} = \frac{\partial{loss}}{\partial{z_{\tilde h_t}}} \cdot \frac{\partial{z_{\tilde h_t}}}{\partial{x_t}} \\
&+ \frac{\partial{loss}}{\partial{z_{r_t}}} \cdot \frac{\partial{z_{r_t}}}{\partial{x_t}} + \frac{\partial{loss}}{\partial{z_{z_t}}} \cdot \frac{\partial{z_{z_t}}}{\partial{x_t}} \\
&= \delta_{z_{\tilde{h}t}} \cdot U_h^T + \delta_{z_{rt}} \cdot U_r^T + \delta_{z_{zt}} \cdot U_z^T
\end{aligned}