mirror of
https://github.com/microsoft/ai-edu.git
synced 2026-08-30 17:04:17 +08:00
xiaowuhu
@@ -3,6 +3,10 @@ Copyright © Microsoft Corporation. All rights reserved.
|
||||
|
||||
在这一章，我们将简要介绍一下激活函数，因为在下一章中将要使用激活函数构造神经网络。
|
||||
|
||||
- [挤压（饱和）型激活函数](06.1-挤压型激活函数.md)
|
||||
- [半线性（非饱和）激活函数](06.2-半线性激活函数.md)
|
||||
|
||||
|
||||
# 激活函数
|
||||
|
||||
看神经网络中的一个神经元，为了简化，假设该神经元接受三个输入，分别为$x_1, x_2, x_3$,那么$z=\sum\limits_{i}w_ix_i+b_i$,
|
||||
@@ -25,12 +29,6 @@ Copyright © Microsoft Corporation. All rights reserved.
|
||||
|
||||
用到神经网络中的概念，用‘1’来代表一个神经元被激活，‘0’代表一个神经元未被激活。
|
||||
|
||||
这个函数有什么不好的地方呢？主要的一点就是，他的梯度（导数）恒为零（个别点除外)。
|
||||
这个函数有什么不好的地方呢？主要的一点就是，他的梯度（导数）恒为零（个别点除外)。反向传播公式中，梯度传递用到了链式法则，如果在这样一个连乘的式子其中有一项是零，这样的梯度就会恒为零，是没有办法进行反向传播的。
|
||||
|
||||
想想我们说过的反向传播公式？梯度传递用到了链式法则，如果在这样一个连乘的式子其中有一项是零，结果会怎么样呢？这样的梯度就会恒为零，这个样子的函数是没有办法进行反向传播的。于是，数学家们发明了如下激活函数。
|
||||
|
||||
- [挤压型激活函数](06.1-挤压型激活函数.md)
|
||||
|
||||
- [半线性激活函数](06.2-半线性激活函数.md)
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ Copyright © Microsoft Corporation. All rights reserved.
|
||||
|
||||
# 挤压型激活函数 Squashing Function
|
||||
|
||||
挤压型激活函数中，用的最多的是Sigmoid函数，Sigmoid意为S型。
|
||||
又可以叫饱和型激活函数。挤压型激活函数中，用的最多的是Sigmoid函数，Sigmoid意为S型。
|
||||
|
||||
## 对数几率函数 Logistic Function
|
||||
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
Copyright © Microsoft Corporation. All rights reserved.
|
||||
适用于[License](https://github.com/Microsoft/ai-edu/blob/master/LICENSE.md)版权许可
|
||||
|
||||
|
||||
# 半线性激活函数
|
||||
|
||||
又可以叫非饱和型激活函数。
|
||||
|
||||
## ReLU函数
|
||||
Rectified Linear Unit，修正线性单元，线性整流函数，斜坡函数。
|
||||
|
||||
公式：
|
||||
|
||||
$$a(z) = max(0,z) = \begin{cases} z & z \geq 0 \\ 0 & z < 0 \end{cases}$$
|
||||
$$a(z) = max(0,z) = \begin{Bmatrix}
|
||||
z & (z \geq 0) \\
|
||||
0 & (z < 0)
|
||||
\end{Bmatrix}$$
|
||||
|
||||
导数:
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import numpy as np
|
||||
|
||||
class CSigmoid(object):
|
||||
def forward(self, z):
|
||||
a = 1.0 / (1.0 + np.exp(-z))
|
||||
return a
|
||||
|
||||
def backward(self, z, a, delta):
|
||||
da = np.multiply(a, 1-a)
|
||||
dz = np.multiply(delta, da)
|
||||
return da, dz
|
||||
|
||||
class CSoftmax(object):
|
||||
def forward(self, Z):
|
||||
shift_z = Z - np.max(Z, axis=0)
|
||||
exp_z = np.exp(shift_z)
|
||||
A = exp_z / np.sum(exp_z, axis=0)
|
||||
return A
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import numpy as np
|
||||
import struct
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
'''
|
||||
train_image_file = 'train-images-01'
|
||||
train_label_file = 'train-labels-01'
|
||||
test_image_file = 'test-images-01'
|
||||
test_label_file = 'test-labels-01'
|
||||
'''
|
||||
train_image_file = 'train-images-10'
|
||||
train_label_file = 'train-labels-10'
|
||||
test_image_file = 'test-images-10'
|
||||
test_label_file = 'test-labels-10'
|
||||
|
||||
|
||||
# output array: 768 x num_images
|
||||
def ReadImageFile(image_file_name):
|
||||
f = open(image_file_name, "rb")
|
||||
a = f.read(4)
|
||||
b = f.read(4)
|
||||
num_images = int.from_bytes(b, byteorder='big')
|
||||
c = f.read(4)
|
||||
num_rows = int.from_bytes(c, byteorder='big')
|
||||
d = f.read(4)
|
||||
num_cols = int.from_bytes(d, byteorder='big')
|
||||
|
||||
image_size = num_rows * num_cols # 784
|
||||
fmt = '>' + str(image_size) + 'B'
|
||||
image_data = np.empty((image_size, num_images)) # 784 x M
|
||||
for i in range(num_images):
|
||||
bin_data = f.read(image_size)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)
|
||||
array_data = np.array(unpacked_data)
|
||||
array_data2 = array_data.reshape((image_size, 1))
|
||||
image_data[:,i] = array_data
|
||||
f.close()
|
||||
return image_data
|
||||
|
||||
def ReadLabelFile(lable_file_name, num_output):
|
||||
f = open(lable_file_name, "rb")
|
||||
f.read(4)
|
||||
a = f.read(4)
|
||||
num_labels = int.from_bytes(a, byteorder='big')
|
||||
|
||||
fmt = '>B'
|
||||
label_data = np.zeros((num_output, num_labels)) # 10 x M
|
||||
for i in range(num_labels):
|
||||
bin_data = f.read(1)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)[0]
|
||||
label_data[unpacked_data,i] = 1
|
||||
f.close()
|
||||
return label_data
|
||||
|
||||
def NormalizeData(X):
|
||||
X_NEW = np.zeros(X.shape)
|
||||
x_max = np.max(X)
|
||||
x_min = np.min(X)
|
||||
X_NEW = (X - x_min)/(x_max-x_min)
|
||||
return X_NEW
|
||||
|
||||
def Sigmoid(x):
|
||||
s=1/(1+np.exp(-x))
|
||||
return s
|
||||
|
||||
def Softmax(Z):
|
||||
shift_z = Z - np.max(Z)
|
||||
exp_z = np.exp(shift_z)
|
||||
s = np.sum(exp_z, axis=0)
|
||||
A = exp_z / s
|
||||
return A
|
||||
|
||||
|
||||
# cross entropy: -Y*lnA
|
||||
def CalculateLoss(dict_Param, X, Y, count, forward):
|
||||
A2, dict_Cache = forward(X, dict_Param)
|
||||
p = Y * np.log(A2)
|
||||
Loss = -np.sum(p) / count
|
||||
return Loss
|
||||
|
||||
def Test(num_output, dict_Param, num_input, forward):
|
||||
raw_data = ReadImageFile(test_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(test_label_file, num_output)
|
||||
num_images = X.shape[1]
|
||||
correct = 0
|
||||
for image_idx in range(num_images):
|
||||
x = X[:,image_idx].reshape(num_input, 1)
|
||||
y = Y[:,image_idx].reshape(num_output, 1)
|
||||
A2, dict_Cache = forward(x, dict_Param)
|
||||
if np.argmax(A2) == np.argmax(y):
|
||||
correct += 1
|
||||
return correct, num_images
|
||||
|
||||
def Train(X, Y, learning_rate, max_epoch, num_images, num_input, num_output, dict_param, forward, backward, update):
|
||||
loss_history = list()
|
||||
|
||||
print("Training...")
|
||||
for iteration in range(max_epoch):
|
||||
for item in range(num_images):
|
||||
x = X[:,item].reshape(num_input,1)
|
||||
y = Y[:,item].reshape(num_output,1)
|
||||
A2, dict_Cache = forward(x, dict_param)
|
||||
dict_Grads = backward(dict_param, dict_Cache, x, y)
|
||||
dict_param = update(dict_param, dict_Grads, learning_rate)
|
||||
if item % 1000 == 0:
|
||||
Loss = CalculateLoss(dict_param, X, Y, num_images, forward)
|
||||
print(item, Loss)
|
||||
loss_history = np.append(loss_history, Loss)
|
||||
print(iteration)
|
||||
|
||||
print("Testing...")
|
||||
correct, count = Test(num_output, dict_param, num_input, forward)
|
||||
print(str.format("rate={0} / {1} = {2}", correct, count, correct/count))
|
||||
|
||||
plt.plot(loss_history, "r")
|
||||
plt.xlabel("Iteration(x1000)")
|
||||
plt.ylabel("Loss")
|
||||
plt.show()
|
||||
|
||||
def LoadData(num_output):
|
||||
raw_data = ReadImageFile(train_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(train_label_file, num_output)
|
||||
return X, Y
|
||||
@@ -5,6 +5,8 @@ import numpy as np
|
||||
import struct
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from Level0_Base import *
|
||||
|
||||
'''
|
||||
train_image_file = 'train-images-01'
|
||||
train_label_file = 'train-labels-01'
|
||||
@@ -17,77 +19,7 @@ test_image_file = 'test-images-10'
|
||||
test_label_file = 'test-labels-10'
|
||||
|
||||
|
||||
# output array: 768 x num_images
|
||||
def ReadImageFile(image_file_name):
|
||||
f = open(image_file_name, "rb")
|
||||
a = f.read(4)
|
||||
b = f.read(4)
|
||||
num_images = int.from_bytes(b, byteorder='big')
|
||||
c = f.read(4)
|
||||
num_rows = int.from_bytes(c, byteorder='big')
|
||||
d = f.read(4)
|
||||
num_cols = int.from_bytes(d, byteorder='big')
|
||||
|
||||
image_size = num_rows * num_cols # 784
|
||||
fmt = '>' + str(image_size) + 'B'
|
||||
image_data = np.empty((image_size, num_images)) # 784 x M
|
||||
for i in range(num_images):
|
||||
bin_data = f.read(image_size)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)
|
||||
array_data = np.array(unpacked_data)
|
||||
array_data2 = array_data.reshape((image_size, 1))
|
||||
image_data[:,i] = array_data
|
||||
f.close()
|
||||
return image_data
|
||||
|
||||
def ReadLabelFile(lable_file_name, num_output):
|
||||
f = open(lable_file_name, "rb")
|
||||
f.read(4)
|
||||
a = f.read(4)
|
||||
num_labels = int.from_bytes(a, byteorder='big')
|
||||
|
||||
fmt = '>B'
|
||||
label_data = np.zeros((num_output, num_labels)) # 10 x M
|
||||
for i in range(num_labels):
|
||||
bin_data = f.read(1)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)[0]
|
||||
label_data[unpacked_data,i] = 1
|
||||
f.close()
|
||||
return label_data
|
||||
|
||||
def NormalizeByRow(X):
|
||||
X_NEW = np.zeros(X.shape)
|
||||
# get number of features
|
||||
n = X.shape[0]
|
||||
for i in range(n):
|
||||
x_row = X[i,:]
|
||||
x_max = np.max(x_row)
|
||||
x_min = np.min(x_row)
|
||||
if x_max != x_min:
|
||||
x_new = (x_row - x_min)/(x_max-x_min)
|
||||
X_NEW[i,:] = x_new
|
||||
return X_NEW
|
||||
|
||||
def NormalizeData(X):
|
||||
X_NEW = np.zeros(X.shape)
|
||||
x_max = np.max(X)
|
||||
x_min = np.min(X)
|
||||
X_NEW = (X - x_min)/(x_max-x_min)
|
||||
return X_NEW
|
||||
|
||||
def Sigmoid(x):
|
||||
s=1/(1+np.exp(-x))
|
||||
return s
|
||||
|
||||
def Softmax(Z):
|
||||
shift_z = Z - np.max(Z)
|
||||
exp_z = np.exp(shift_z)
|
||||
#s = np.sum(exp_z)
|
||||
s = np.sum(exp_z, axis=0)
|
||||
A = exp_z / s
|
||||
return A
|
||||
|
||||
def ForwardCalculation(X, dict_Param):
|
||||
def Forward(X, dict_Param):
|
||||
W1 = dict_Param["W1"]
|
||||
B1 = dict_Param["B1"]
|
||||
W2 = dict_Param["W2"]
|
||||
@@ -95,7 +27,6 @@ def ForwardCalculation(X, dict_Param):
|
||||
|
||||
Z1 = np.dot(W1,X)+B1
|
||||
A1 = Sigmoid(Z1)
|
||||
#A1 = np.tanh(Z1)
|
||||
|
||||
Z2=np.dot(W2,A1)+B2
|
||||
A2=Softmax(Z2)
|
||||
@@ -106,7 +37,7 @@ def ForwardCalculation(X, dict_Param):
|
||||
"A2": A2}
|
||||
return A2, dict_Cache
|
||||
|
||||
def BackPropagation(dict_Param,cache,X,Y):
|
||||
def Backward(dict_Param,cache,X,Y):
|
||||
W1=dict_Param["W1"]
|
||||
W2=dict_Param["W2"]
|
||||
A1 = cache["A1"]
|
||||
@@ -119,7 +50,6 @@ def BackPropagation(dict_Param,cache,X,Y):
|
||||
|
||||
dLoss_A1 = np.dot(W2.T, dZ2)
|
||||
dA1_Z1 = A1 * (1 - A1) # sigmoid
|
||||
#dA1_Z1 = 1-np.power(A1,2) # tanh
|
||||
dZ1 = dLoss_A1 * dA1_Z1
|
||||
|
||||
dW1 = np.dot(dZ1, X.T)
|
||||
@@ -128,7 +58,7 @@ def BackPropagation(dict_Param,cache,X,Y):
|
||||
dict_Grads = {"dW1": dW1, "dB1": dB1, "dW2": dW2, "dB2": dB2}
|
||||
return dict_Grads
|
||||
|
||||
def UpdateParam(dict_Param, dict_Grads, learning_rate):
|
||||
def Update(dict_Param, dict_Grads, learning_rate):
|
||||
W1 = dict_Param["W1"]
|
||||
B1 = dict_Param["B1"]
|
||||
W2 = dict_Param["W2"]
|
||||
@@ -147,12 +77,6 @@ def UpdateParam(dict_Param, dict_Grads, learning_rate):
|
||||
dict_Param = {"W1": W1, "B1": B1, "W2": W2, "B2": B2}
|
||||
return dict_Param
|
||||
|
||||
# cross entropy: -Y*lnA
|
||||
def CalculateLoss(dict_Param, X, Y, count):
|
||||
A2, dict_Cache = ForwardCalculation(X, dict_Param)
|
||||
p = Y * np.log(A2)
|
||||
Loss = -np.sum(p) / count
|
||||
return Loss
|
||||
|
||||
def InitialParameters(num_input, num_hidden, num_output, flag):
|
||||
if flag == 0:
|
||||
@@ -173,61 +97,16 @@ def InitialParameters(num_input, num_hidden, num_output, flag):
|
||||
dict_Param = {"W1": W1, "B1": B1, "W2": W2, "B2": B2}
|
||||
return dict_Param
|
||||
|
||||
def Test(num_output, dict_Param, num_input):
|
||||
raw_data = ReadImageFile(test_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(test_label_file, num_output)
|
||||
num_images = X.shape[1]
|
||||
correct = 0
|
||||
for image_idx in range(num_images):
|
||||
x = X[:,image_idx].reshape(num_input, 1)
|
||||
y = Y[:,image_idx].reshape(num_output, 1)
|
||||
A2, dict_Cache = ForwardCalculation(x, dict_Param)
|
||||
if np.argmax(A2) == np.argmax(y):
|
||||
correct += 1
|
||||
return correct, num_images
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
print("Loading...")
|
||||
learning_rate = 0.1
|
||||
num_hidden = 32
|
||||
num_output = 10
|
||||
learning_rate = 0.05
|
||||
n_hidden = 32
|
||||
n_output = 10
|
||||
X,Y = LoadData(n_output)
|
||||
n_images = X.shape[1]
|
||||
n_input = X.shape[0]
|
||||
m_epoch = 1
|
||||
dict_Param = InitialParameters(n_input, n_hidden, n_output, 2)
|
||||
Train(X, Y, learning_rate, m_epoch, n_images, n_input, n_output, dict_Param, Forward, Backward, Update)
|
||||
|
||||
raw_data = ReadImageFile(train_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(train_label_file, num_output)
|
||||
|
||||
num_images = X.shape[1]
|
||||
num_input = X.shape[0]
|
||||
max_iteration = 1
|
||||
|
||||
dict_Param = InitialParameters(num_input, num_hidden, num_output, 2)
|
||||
w = dict_Param["W1"]
|
||||
print(np.var(w))
|
||||
|
||||
loss_history = list()
|
||||
|
||||
print("Training...")
|
||||
for iteration in range(max_iteration):
|
||||
for item in range(num_images):
|
||||
x = X[:,item].reshape(num_input,1)
|
||||
y = Y[:,item].reshape(num_output,1)
|
||||
A2, dict_Cache = ForwardCalculation(x, dict_Param)
|
||||
dict_Grads = BackPropagation(dict_Param, dict_Cache, x, y)
|
||||
dict_Param = UpdateParam(dict_Param, dict_Grads, learning_rate)
|
||||
if item % 1000 == 0:
|
||||
Loss = CalculateLoss(dict_Param, X, Y, num_images)
|
||||
print(item, Loss)
|
||||
loss_history = np.append(loss_history, Loss)
|
||||
print(iteration)
|
||||
|
||||
print("Testing...")
|
||||
correct, count = Test(num_output, dict_Param, num_input)
|
||||
print(str.format("rate={0} / {1} = {2}", correct, count, correct/count))
|
||||
|
||||
plt.plot(loss_history, "r")
|
||||
plt.title("Xavier Initilization")
|
||||
plt.xlabel("Iteration(x1000)")
|
||||
plt.ylabel("Loss")
|
||||
plt.show()
|
||||
|
||||
@@ -5,93 +5,14 @@ import numpy as np
|
||||
import struct
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
'''
|
||||
train_image_file = 'train-images-01'
|
||||
train_label_file = 'train-labels-01'
|
||||
test_image_file = 'test-images-01'
|
||||
test_label_file = 'test-labels-01'
|
||||
'''
|
||||
train_image_file = 'train-images-10'
|
||||
train_label_file = 'train-labels-10'
|
||||
test_image_file = 'test-images-10'
|
||||
test_label_file = 'test-labels-10'
|
||||
from Level0_TwoLayerNet import *
|
||||
|
||||
|
||||
# output array: 768 x num_images
|
||||
def ReadImageFile(image_file_name):
|
||||
f = open(image_file_name, "rb")
|
||||
a = f.read(4)
|
||||
b = f.read(4)
|
||||
num_images = int.from_bytes(b, byteorder='big')
|
||||
c = f.read(4)
|
||||
num_rows = int.from_bytes(c, byteorder='big')
|
||||
d = f.read(4)
|
||||
num_cols = int.from_bytes(d, byteorder='big')
|
||||
|
||||
image_size = num_rows * num_cols # 784
|
||||
fmt = '>' + str(image_size) + 'B'
|
||||
image_data = np.empty((image_size, num_images)) # 784 x M
|
||||
for i in range(num_images):
|
||||
bin_data = f.read(image_size)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)
|
||||
array_data = np.array(unpacked_data)
|
||||
array_data2 = array_data.reshape((image_size, 1))
|
||||
image_data[:,i] = array_data
|
||||
f.close()
|
||||
return image_data
|
||||
|
||||
def ReadLabelFile(lable_file_name, num_output):
|
||||
f = open(lable_file_name, "rb")
|
||||
f.read(4)
|
||||
a = f.read(4)
|
||||
num_labels = int.from_bytes(a, byteorder='big')
|
||||
|
||||
fmt = '>B'
|
||||
label_data = np.zeros((num_output, num_labels)) # 10 x M
|
||||
for i in range(num_labels):
|
||||
bin_data = f.read(1)
|
||||
unpacked_data = struct.unpack(fmt, bin_data)[0]
|
||||
label_data[unpacked_data,i] = 1
|
||||
f.close()
|
||||
return label_data
|
||||
|
||||
def NormalizeByRow(X):
|
||||
X_NEW = np.zeros(X.shape)
|
||||
# get number of features
|
||||
n = X.shape[0]
|
||||
for i in range(n):
|
||||
x_row = X[i,:]
|
||||
x_max = np.max(x_row)
|
||||
x_min = np.min(x_row)
|
||||
if x_max != x_min:
|
||||
x_new = (x_row - x_min)/(x_max-x_min)
|
||||
X_NEW[i,:] = x_new
|
||||
return X_NEW
|
||||
|
||||
def NormalizeData(X):
|
||||
X_NEW = np.zeros(X.shape)
|
||||
x_max = np.max(X)
|
||||
x_min = np.min(X)
|
||||
X_NEW = (X - x_min)/(x_max-x_min)
|
||||
return X_NEW
|
||||
|
||||
def Sigmoid(z):
|
||||
a=1/(1+np.exp(-z))
|
||||
return a
|
||||
|
||||
def Tanh(z):
|
||||
a = 2.0 / (1.0 + np.exp(-2*z)) - 1.0
|
||||
return a
|
||||
|
||||
|
||||
def Softmax(Z):
|
||||
shift_z = Z - np.max(Z)
|
||||
exp_z = np.exp(shift_z)
|
||||
s = np.sum(exp_z, axis=0)
|
||||
A = exp_z / s
|
||||
return A
|
||||
|
||||
def ForwardCalculation(X, dict_Param):
|
||||
def forward(X, dict_Param):
|
||||
W1 = dict_Param["W1"]
|
||||
B1 = dict_Param["B1"]
|
||||
W2 = dict_Param["W2"]
|
||||
@@ -111,7 +32,7 @@ def ForwardCalculation(X, dict_Param):
|
||||
dict_Cache = {"Z1": Z1, "A1": A1, "Z2": Z2, "A2": A2, "Z3": Z3, "A3": A3}
|
||||
return A3, dict_Cache
|
||||
|
||||
def BackPropagation(dict_Param,cache,X,Y):
|
||||
def backward(dict_Param,cache,X,Y):
|
||||
W1=dict_Param["W1"]
|
||||
W2=dict_Param["W2"]
|
||||
W3=dict_Param["W3"]
|
||||
@@ -139,7 +60,7 @@ def BackPropagation(dict_Param,cache,X,Y):
|
||||
dict_Grads = {"dW1": dW1, "dB1": dB1, "dW2": dW2, "dB2": dB2, "dW3": dW3, "dB3": dB3}
|
||||
return dict_Grads
|
||||
|
||||
def UpdateParam(dict_Param, dict_Grads, learning_rate):
|
||||
def update(dict_Param, dict_Grads, learning_rate):
|
||||
W1 = dict_Param["W1"]
|
||||
B1 = dict_Param["B1"]
|
||||
W2 = dict_Param["W2"]
|
||||
@@ -164,12 +85,6 @@ def UpdateParam(dict_Param, dict_Grads, learning_rate):
|
||||
dict_Param = {"W1": W1, "B1": B1, "W2": W2, "B2": B2, "W3": W3, "B3": B3}
|
||||
return dict_Param
|
||||
|
||||
# cross entropy: -Y*lnA
|
||||
def CalculateLoss(dict_Param, X, Y, count):
|
||||
A3, dict_Cache = ForwardCalculation(X, dict_Param)
|
||||
p = Y * np.log(A3)
|
||||
Loss = -np.sum(p) / count
|
||||
return Loss
|
||||
|
||||
def InitialParameters(num_input, num_hidden1, num_hidden2, num_output, flag):
|
||||
if flag == 0:
|
||||
@@ -199,67 +114,17 @@ def InitialParameters(num_input, num_hidden1, num_hidden2, num_output, flag):
|
||||
dict_Param = {"W1": W1, "B1": B1, "W2": W2, "B2": B2, "W3": W3, "B3": B3}
|
||||
return dict_Param
|
||||
|
||||
def Test(num_output, dict_Param, num_input):
|
||||
raw_data = ReadImageFile(test_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(test_label_file, num_output)
|
||||
num_images = X.shape[1]
|
||||
correct = 0
|
||||
for image_idx in range(num_images):
|
||||
x = X[:,image_idx].reshape(num_input, 1)
|
||||
y = Y[:,image_idx].reshape(num_output, 1)
|
||||
A2, dict_Cache = ForwardCalculation(x, dict_Param)
|
||||
if np.argmax(A2) == np.argmax(y):
|
||||
correct += 1
|
||||
return correct, num_images
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
print("Loading...")
|
||||
learning_rate = 0.05
|
||||
num_hidden1 = 64
|
||||
num_hidden2 = 16
|
||||
num_output = 10
|
||||
n_hidden1 = 64
|
||||
n_hidden2 = 16
|
||||
n_output = 10
|
||||
X,Y = LoadData(n_output)
|
||||
n_images = X.shape[1]
|
||||
n_input = X.shape[0]
|
||||
m_epoch = 1
|
||||
dict_Param = InitialParameters(n_input, n_hidden1, n_hidden2, n_output, 2)
|
||||
Train(X, Y, learning_rate, m_epoch, n_images, n_input, n_output, dict_Param, forward, backward, update)
|
||||
|
||||
raw_data = ReadImageFile(train_image_file)
|
||||
X = NormalizeData(raw_data)
|
||||
Y = ReadLabelFile(train_label_file, num_output)
|
||||
|
||||
num_images = X.shape[1]
|
||||
num_input = X.shape[0]
|
||||
max_iteration = 10
|
||||
|
||||
dict_Param = InitialParameters(num_input, num_hidden1, num_hidden2, num_output, 2)
|
||||
|
||||
loss_history = list()
|
||||
eps = 1e-1
|
||||
print("Training...")
|
||||
for iteration in range(max_iteration):
|
||||
for item in range(num_images):
|
||||
x = X[:,item].reshape(num_input,1)
|
||||
y = Y[:,item].reshape(num_output,1)
|
||||
A3, dict_Cache = ForwardCalculation(x, dict_Param)
|
||||
dict_Grads = BackPropagation(dict_Param, dict_Cache, x, y)
|
||||
dict_Param = UpdateParam(dict_Param, dict_Grads, learning_rate)
|
||||
if item % 1000 == 0:
|
||||
Loss = CalculateLoss(dict_Param, X, Y, num_images)
|
||||
print(item, Loss)
|
||||
loss_history = np.append(loss_history, Loss)
|
||||
if Loss < eps:
|
||||
break
|
||||
#end if
|
||||
#end if
|
||||
#end for
|
||||
if Loss < eps:
|
||||
break
|
||||
print(iteration)
|
||||
|
||||
print("Testing...")
|
||||
correct, count = Test(num_output, dict_Param, num_input)
|
||||
print(str.format("rate={0} / {1} = {2}", correct, count, correct/count))
|
||||
|
||||
plt.plot(loss_history, "r")
|
||||
plt.title("Xavier Initilization")
|
||||
plt.xlabel("Iteration(x1000)")
|
||||
plt.ylabel("Loss")
|
||||
plt.show()
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import numpy as np
|
||||
|
||||
class CActivator(object):
|
||||
# z = 本层的wx+b计算值矩阵
|
||||
def forward(self, z):
|
||||
pass
|
||||
|
||||
# z = 本层的wx+b计算值矩阵
|
||||
# a = 本层的激活函数输出值矩阵
|
||||
# delta = 上（后）层反传回来的梯度值矩阵
|
||||
def backward(self, z, a, delta):
|
||||
pass
|
||||
|
||||
|
||||
# 直传函数，相当于无激活
|
||||
class Identity(CActivator):
|
||||
def forward(self, z):
|
||||
return z
|
||||
|
||||
def backward(self, z, a, delta):
|
||||
return delta, a
|
||||
|
||||
|
||||
class Sigmoid(CActivator):
|
||||
def forward(self, z):
|
||||
a = 1.0 / (1.0 + np.exp(-Z))
|
||||
return a
|
||||
|
||||
def backward(self, z, a, delta):
|
||||
da = np.multiply(a, 1-a)
|
||||
dz = np.multiply(delta, da)
|
||||
return dz, da
|
||||
|
||||
|
||||
class Tanh(CActivator):
|
||||
def forward(self, z):
|
||||
a = 2.0 / (1.0 + np.exp(-2*z)) - 1.0
|
||||
return a
|
||||
|
||||
def backward(self, z, a, delta):
|
||||
da = 1 - np.multiply(a, a)
|
||||
dz = np.multiply(delta, da)
|
||||
return dz, da
|
||||
|
||||
|
||||
class Relu(CActivator):
|
||||
def forward(self, z):
|
||||
a = np.maximum(z, 0)
|
||||
return a
|
||||
|
||||
# 注意relu函数判断是否大于1的根据是正向的wx+b=z的值，而不是a值
|
||||
def backward(self, z, a, delta):
|
||||
da = np.zeros(z.shape)
|
||||
da[z>0] = 1
|
||||
dz = da * delta
|
||||
return dz, mem
|
||||
|
||||
|
||||
class Softmax(CActivator):
|
||||
def forward(self, z):
|
||||
shift_z = z - np.max(z, axis=0)
|
||||
exp_z = np.exp(shift_z)
|
||||
a = exp_z / np.sum(exp_z, axis=0)
|
||||
return a
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import numpy as np
|
||||
|
||||
from Layer import *
|
||||
from Activators import *
|
||||
|
||||
class FcLayer(CLayer):
|
||||
def __init__(self, input_size, output_size, activator):
|
||||
self.input_size = input_size
|
||||
self.output_size = output_size
|
||||
self.weights = init_array((output_size, input_size), "norm")
|
||||
self.bias = np.zeros((output_size, 1))
|
||||
self.activator = activator
|
||||
#self.x = np.zeros() # input from lower layer
|
||||
#self.z = np.zeros() # weights multiply for current layer
|
||||
#self.a = np.zeros() # outpu to upper layer
|
||||
|
||||
def forward(self, input):
|
||||
self.input_shape = input.shape
|
||||
if input.ndim == 3: # come from pooling layer
|
||||
self.x = input.reshape(input.size, 1)
|
||||
else:
|
||||
self.x = input
|
||||
self.z = np.dot(self.weights, self.x) + self.bias
|
||||
self.a = self.activator.forward(self.z)
|
||||
return self.a
|
||||
|
||||
# 把激活函数算做是当前层，上一层的误差传入后，先经过激活函数的导数，而得到本层的针对z值的误差
|
||||
def backward(self, delta_in, flag):
|
||||
if flag == LayerIndexFlags.LastLayer or flag == LayerIndexFlags.SingleLayer:
|
||||
dZ = delta_in
|
||||
else:
|
||||
#dZ = delta_in * self.activator.backward(self.a)
|
||||
dZ = self.activator.backward(self.a, delta_in)
|
||||
|
||||
delta_out = np.dot(self.weights.T, dZ)
|
||||
self.dW = np.dot(dZ, self.x.T)
|
||||
self.dB = np.sum(dZ, axis=1, keepdims=True)
|
||||
|
||||
if len(self.input_shape) > 2:
|
||||
return delta_out.reshape(self.input_shape)
|
||||
else:
|
||||
return delta_out
|
||||
|
||||
def update(self, learning_rate):
|
||||
self.weights = self.weights - learning_rate * self.dW
|
||||
self.bias = self.bias - learning_rate * self.dB
|
||||
|
||||
def save_parameters(self, name):
|
||||
np.save(name+"_w", self.weights)
|
||||
np.save(name+"_b", self.bias)
|
||||
|
||||
def load_parameters(self, name):
|
||||
self.weights = np.load(name+"_w.npy")
|
||||
self.bias = np.load(name+"_b.npy")
|
||||
@@ -0,0 +1,24 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
from enum import Enum
|
||||
|
||||
class LayerTypes(Enum):
|
||||
FC = 0 # full connection
|
||||
CONV = 1 # convalution
|
||||
POOL = 2 # pooling
|
||||
|
||||
|
||||
class CLayer(object):
|
||||
def __init__(self, layer_type):
|
||||
self.layer_type = layer_type
|
||||
|
||||
def update(self, lr):
|
||||
return
|
||||
|
||||
class LayerIndexFlags(Enum):
|
||||
SingleLayer = 0
|
||||
FirstLayer = 1
|
||||
LastLayer = -1
|
||||
MiddleLayer = 2
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
|
||||
import numpy as np
|
||||
from enum import Enum
|
||||
|
||||
from Layer import *
|
||||
from FCLayer import *
|
||||
|
||||
class CNet(object):
|
||||
def __init__(self, optimizer_type, loss_function):
|
||||
self.optimizer_type = optimizer_type
|
||||
self.loss_function = loss_function
|
||||
self.layer_list = []
|
||||
self.layer_name = []
|
||||
self.output = np.zeros((1,1))
|
||||
self.layer_count = 0
|
||||
|
||||
def add_layer(self, layer, name=""):
|
||||
self.layer_list.append(layer)
|
||||
self.layer_name.append(name)
|
||||
self.layer_count += 1
|
||||
|
||||
def forward(self, input_array):
|
||||
input = input_array
|
||||
for i in range(self.layer_count):
|
||||
layer = self.layer_list[i]
|
||||
output = layer.forward(input)
|
||||
input = output
|
||||
|
||||
self.output = output
|
||||
return self.output
|
||||
|
||||
def backward(self, X, Y):
|
||||
delta_in = self.output - Y
|
||||
for i in range(self.layer_count-1,-1,-1):
|
||||
layer = self.layer_list[i]
|
||||
flag = self.get_layer_index(i)
|
||||
delta_out = layer.backward(delta_in, flag)
|
||||
# move back to previous layer
|
||||
delta_in = delta_out
|
||||
|
||||
def update(self, learning_rate):
|
||||
for i in range(self.layer_count-1,-1,-1):
|
||||
layer = self.layer_list[i]
|
||||
layer.update(learning_rate)
|
||||
|
||||
def get_layer_index(self, idx):
|
||||
if self.layer_count == 1:
|
||||
return LayerIndexFlags.SingleLayer
|
||||
else:
|
||||
if idx == self.layer_count - 1:
|
||||
return LayerIndexFlags.LastLayer
|
||||
elif idx == 0:
|
||||
return LayerIndexFlags.FirstLayer
|
||||
else:
|
||||
return LayerIndexFlags.MiddleLayer
|
||||
|
||||
def train(self, X, Y, max_iteration, learning_rate):
|
||||
if X.ndim == 2:
|
||||
num_feature = X.shape[0]
|
||||
num_example = X.shape[1]
|
||||
elif X.ndim == 4:
|
||||
num_example = X.shape[0]
|
||||
num_feature = X.shape[1] * X.shape[2] * X.shape[3]
|
||||
|
||||
num_output = Y.shape[0]
|
||||
|
||||
# num_example = 2000
|
||||
|
||||
for i in range(max_iteration):
|
||||
print(i)
|
||||
for j in range(num_example):
|
||||
if j%10==0:
|
||||
print(i, ":", j)
|
||||
if X.ndim == 2:
|
||||
x = X[:,j].reshape(num_feature, 1)
|
||||
elif X.ndim == 4:
|
||||
x = X[j] # x.ndim == 3
|
||||
y = Y[:,j].reshape(num_output, 1)
|
||||
self.forward(x)
|
||||
self.backward(x, y)
|
||||
self.update(learning_rate)
|
||||
|
||||
def inference(self, X):
|
||||
self.forward(X)
|
||||
return self.output
|
||||
|
||||
def save_parameters(self):
|
||||
for i in range(self.layer_count):
|
||||
layer = self.layer_list[i]
|
||||
name = self.layer_name[i]
|
||||
layer.save_parameters(name)
|
||||
|
||||
def load_parameters(self):
|
||||
for i in range(self.layer_count):
|
||||
layer = self.layer_list[i]
|
||||
name = self.layer_name[i]
|
||||
layer.load_parameters(name)
|
||||
@@ -21,17 +21,30 @@
|
||||
<EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="Activations.py" />
|
||||
<Compile Include="DataReader.py" />
|
||||
<Compile Include="GDOptimizer.py" />
|
||||
<Compile Include="Level0_Base.py" />
|
||||
<Compile Include="Level2\Activators.py" />
|
||||
<Compile Include="Level2\DataReader.py" />
|
||||
<Compile Include="Level2\FCLayer.py">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Level2\GDOptimizer.py" />
|
||||
<Compile Include="Level1_ThreeLayerNet.py" />
|
||||
<Compile Include="Level0_TwoLayerNet.py" />
|
||||
<Compile Include="LossFunction.py" />
|
||||
<Compile Include="Level2\Layer.py">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Level2\LossFunction.py" />
|
||||
<Compile Include="Level2\Net.py">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="MnistData.py">
|
||||
<SubType>Code</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Parameters.py" />
|
||||
<Compile Include="WeightsBias.py" />
|
||||
<Compile Include="Level2\Parameters.py" />
|
||||
<Compile Include="Level2\WeightsBias.py" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="Level2\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
|
||||
<!-- Uncomment the CoreCompile target to enable the Build command in
|
||||
|
||||
Reference in New Issue
Block a user