Xiaowuhu/190730 (#317)

* Update Level1_Mnist.py

* pp

* kk

* kk

* hh

* hh

* nn

* jj

.gitignore

@@ -331,3 +331,5 @@ ASALocalRun/
 
 media/mpp
 media/d434528764edf0871c2991bf2da70b96/
+B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/handwriting.png
+B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/handwriting.png

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch11-NonLinearMultipleClassification/Level1_BankClassifier.py

@@ -31,8 +31,12 @@ if __name__ == '__main__':
 
     hp = HyperParameters_2_0(n_input, n_hidden, n_output, eta, max_epoch, batch_size, eps, NetType.MultipleClassifier, InitialMethod.Xavier)
     net = NeuralNet_2_2(hp, "Bank_233")
-    net.train(dataReader, 100, True)
-    net.ShowTrainingHistory()
+
+    net.LoadResult()
+
+
+    #net.train(dataReader, 100, True)
+    #net.ShowTrainingHistory()
 
     fig = plt.figure(figsize=(6,6))
     DrawThreeCategoryPoints(dataReader.XTrain[:,0], dataReader.XTrain[:,1], dataReader.YTrain, hp.toString())

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch11-NonLinearMultipleClassification/ch11.pyproj

@@ -4,7 +4,7 @@
     <SchemaVersion>2.0</SchemaVersion>
     <ProjectGuid>3de29cfe-16a6-493d-97b3-e617074af796</ProjectGuid>
     <ProjectHome>.</ProjectHome>
-    <StartupFile>Level2_HiddenUnits.py</StartupFile>
+    <StartupFile>Level2_BankHow.py</StartupFile>
     <SearchPath>
     </SearchPath>
     <WorkingDirectory>.</WorkingDirectory>

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/ActivatorFunction_2_0.py

@@ -38,7 +38,6 @@ class Sigmoid(CActivator):
         dz = np.multiply(delta, da)
         return dz, da
 
-
 class Tanh(CActivator):
     def forward(self, z):
         a = 2.0 / (1.0 + np.exp(-2*z)) - 1.0

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/LossFunction_1_1.py

@@ -54,7 +54,7 @@ class LossFunction_1_1(object):
     def CE3(self, A, Y, count):
         p1 = np.log(A)
         p2 =  np.multiply(Y, p1)
-        LOSS = np.sum(-p2) 
+        LOSS = np.sum(-p2, keepdims=False) 
         loss = LOSS / count
         return loss
     # end def

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/MnistImageDataReader.py

@@ -37,7 +37,7 @@ class MnistImageDataReader(DataReader_2_0):
         self.YTrainRaw = self.YTrainRaw[0:count]
 
         self.num_example = self.XTrainRaw.shape[0]
-        self.num_category = len(np.unique(self.YTrainRaw))
+        self.num_category = (np.unique(self.YTrainRaw)).shape[0]
         self.num_test = self.XTestRaw.shape[0]
         self.num_train = self.num_example
         if self.mode == "vector":
@@ -51,7 +51,7 @@ class MnistImageDataReader(DataReader_2_0):
         self.XTestRaw = self.__ReadImageFile(self.test_image_file)
         self.YTestRaw = self.__ReadLabelFile(self.test_label_file)
         self.num_example = self.XTrainRaw.shape[0]
-        self.num_category = len(np.unique(self.YTrainRaw))
+        self.num_category = (np.unique(self.YTrainRaw)).shape[0]
         self.num_test = self.XTestRaw.shape[0]
         self.num_train = self.num_example
         if self.mode == "vector":
@@ -100,11 +100,11 @@ class MnistImageDataReader(DataReader_2_0):
         return label_data
 
     def NormalizeX(self):
-        self.XTrain = self.__NormalizeData(self.XTrainRaw).astype(np.float32)
-        self.XTest = self.__NormalizeData(self.XTestRaw).astype(np.float32)
+        self.XTrain = self.__NormalizeData(self.XTrainRaw)
+        self.XTest = self.__NormalizeData(self.XTestRaw)
 
     def __NormalizeData(self, XRawData):
-        X_NEW = np.zeros(XRawData.shape).astype(np.float32)
+        X_NEW = np.zeros(XRawData.shape)
         x_max = np.max(XRawData)
         x_min = np.min(XRawData)
         X_NEW = (XRawData - x_min)/(x_max-x_min)

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/NeuralNet_3_0.py

@@ -25,7 +25,7 @@ class NeuralNet_3_0(object):
     def __init__(self, hp, model_name):
         self.hp = hp
         self.model_name = model_name
-        self.subfolder = os.getcwd() + "\\" + self.__create_subfolder()
+        self.subfolder = os.getcwd() + "/" + self.__create_subfolder()
         print(self.subfolder)
 
         self.wb1 = WeightsBias_1_0(self.hp.num_input, self.hp.num_hidden1, self.hp.init_method, self.hp.eta)
@@ -38,7 +38,7 @@ class NeuralNet_3_0(object):
     def __create_subfolder(self):
         if self.model_name != None:
             path = self.model_name.strip()
-            path = path.rstrip("\\")
+            path = path.rstrip("/")
             isExists = os.path.exists(path)
             if not isExists:
                 os.makedirs(path)
@@ -49,10 +49,12 @@ class NeuralNet_3_0(object):
         self.Z1 = np.dot(batch_x, self.wb1.W) + self.wb1.B
         # 公式2
         self.A1 = Sigmoid().forward(self.Z1)
+        #self.A1 = 1.0 / (1.0 + np.exp(-self.Z1))
         # 公式3
         self.Z2 = np.dot(self.A1, self.wb2.W) + self.wb2.B
         # 公式4
         self.A2 = Tanh().forward(self.Z2)
+        #self.A2 = 2.0 / (1.0 + np.exp(-2*self.Z2)) - 1.0
         # 公式5
         self.Z3 = np.dot(self.A2, self.wb3.W) + self.wb3.B
         # 公式6
@@ -74,12 +76,14 @@ class NeuralNet_3_0(object):
         # 公式8
         self.wb3.dW = np.dot(self.A2.T, dZ3)/m
         # 公式9
-        self.wb3.dB = np.sum(dZ3, axis=0, keepdims=True)/m 
+        self.wb3.dB = np.sum(dZ3, axis=0, keepdims=True)/m
 
         # 第二层的梯度输入 公式10
         dA2 = np.dot(dZ3, self.wb3.W.T)
         # 公式11
         dZ2,_ = Tanh().backward(None, self.A2, dA2)
+        #da = 1 - np.multiply(self.A2, self.A2)
+        #dZ2 = np.multiply(dA2, da)
         # 公式12
         self.wb2.dW = np.dot(self.A1.T, dZ2)/m
         # 公式13
@@ -89,6 +93,9 @@ class NeuralNet_3_0(object):
         dA1 = np.dot(dZ2, self.wb2.W.T) 
         # 第一层的dZ 公式10
         dZ1,_ = Sigmoid().backward(None, self.A1, dA1)
+        #da = np.multiply(self.A1, 1-self.A1)
+        #dZ1 = np.multiply(dA1, da)
+
         # 第一层的权重和偏移 公式11
         self.wb1.dW = np.dot(batch_x.T, dZ1)/m
         self.wb1.dB = np.sum(dZ1, axis=0, keepdims=True)/m
@@ -186,13 +193,13 @@ class NeuralNet_3_0(object):
         elif self.hp.net_type == NetType.BinaryClassifier:
             b = np.round(a)
             r = (b == y)
-            correct = r.sum()
+            correct = np.sum(r)
             return correct/m
         elif self.hp.net_type == NetType.MultipleClassifier:
             ra = np.argmax(a, axis=1)
             ry = np.argmax(y, axis=1)
             r = (ra == ry)
-            correct = r.sum()
+            correct = np.sum(r)
             return correct/m
 
     def SaveResult(self):

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/TrainingHistory_2_3.py

@@ -7,7 +7,6 @@ what's new?
 - add epoch/iteration as parameter in showLossHistory
 """
 
-import numpy as np
 import matplotlib.pyplot as plt
 import pickle
 

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/HelperClass2/WeightsBias_1_0.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 import numpy as np
+import math
 from pathlib import Path
 
 from HelperClass2.EnumDef_2_0 import *
@@ -26,39 +27,40 @@ class WeightsBias_1_0(object):
         self.dW = np.zeros(self.W.shape)
         self.dB = np.zeros(self.B.shape)
 
-    def __CreateNew(self):
+    def __CreateNew(self, file_name):
         self.W, self.B = WeightsBias_1_0.InitialParameters(self.num_input, self.num_output, self.init_method)
-        self.__SaveInitialValue()
+        self.__SaveInitialValue(file_name)
         
     def __LoadExistingParameters(self):
-        file_name = str.format("{0}\\{1}.npz", self.folder, self.initial_value_filename)
+        file_name = str.format("{0}/{1}.npz", self.folder, self.initial_value_filename)
         w_file = Path(file_name)
         if w_file.exists():
-            self.__LoadInitialValue()
+            self.__LoadInitialValue(file_name)
         else:
-            self.__CreateNew()
+            self.__CreateNew(file_name)
         # end if
 
     def Update(self):
         self.W = self.W - self.eta * self.dW
         self.B = self.B - self.eta * self.dB
 
-    def __SaveInitialValue(self):
-        file_name = str.format("{0}\\{1}.npz", self.folder, self.initial_value_filename)
+    def __SaveInitialValue(self, file_name):
+        file_name = str.format("{0}/{1}.npz", self.folder, self.initial_value_filename)
         np.savez(file_name, weights=self.W, bias=self.B)
 
-    def __LoadInitialValue(self):
-        file_name = str.format("{0}\\{1}.npz", self.folder, self.initial_value_filename)
+    def __LoadInitialValue(self, file_name):
         data = np.load(file_name)
         self.W = data["weights"]
         self.B = data["bias"]
 
     def SaveResultValue(self, folder, name):
-        file_name = str.format("{0}\\{1}.npz", folder, name)
+        file_name = str.format("{0}/{1}.npz", folder, name)
         np.savez(file_name, weights=self.W, bias=self.B)
+        print(file_name)
 
     def LoadResultValue(self, folder, name):
-        file_name = str.format("{0}\\{1}.npz", folder, name)
+        file_name = str.format("{0}/{1}.npz", folder, name)
+        print(file_name)
         data = np.load(file_name)
         self.W = data["weights"]
         self.B = data["bias"]
@@ -70,14 +72,13 @@ class WeightsBias_1_0(object):
             W = np.zeros((num_input, num_output))
         elif method == InitialMethod.Normal:
             # normalize
-            W = np.random.normal(size=(num_input, num_output))
+            W = np.random.normal((num_input, num_output))
         elif method == InitialMethod.MSRA:
-            W = np.random.normal(0, np.sqrt(2/num_output), size=(num_input, num_output))
+            W = np.random.normal(0, math.sqrt(2/num_output), (num_input, num_output))
         elif method == InitialMethod.Xavier:
             # xavier
-            W = np.random.uniform(-np.sqrt(6/(num_output+num_input)),
-                                  np.sqrt(6/(num_output+num_input)),
-                                  size=(num_input, num_output))
+            t = math.sqrt(6/(num_output+num_input))
+            W = np.random.uniform(-t, t, (num_input, num_output))
         # end if
         B = np.zeros((1, num_output))
         return W, B

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/Level1_Mnist.py

@@ -1,11 +1,11 @@
 # Copyright (c) Microsoft. All rights reserved.
 # Licensed under the MIT license. See LICENSE file in the project root for full license information.
+import time
 
 from HelperClass2.MnistImageDataReader import *
 from HelperClass2.NeuralNet_3_0 import *
 
 if __name__ == '__main__':
-
     dataReader = MnistImageDataReader(mode="vector")
     dataReader.ReadData()
     dataReader.NormalizeX()

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/Level2_Inference.py

@@ -24,7 +24,7 @@ def ReadImage(img_file_name):
 def Inference(img_array):
     output = net.inference(img_array)
     n = np.argmax(output)
-    print("------recognize result is: {0} -----", n)
+    print("------recognize result is: -----", n)
 
 def on_key_press(event):
     img_file_name = "handwriting.png"

B-教学案例与实践/B6-神经网络基本原理简明教程/SourceCode/ch12-MultipleLayerNetwork/ch12.pyproj

@@ -4,7 +4,7 @@
     <SchemaVersion>2.0</SchemaVersion>
     <ProjectGuid>a7eebfef-e8b3-46a1-92cf-a0d970de4c9a</ProjectGuid>
     <ProjectHome>.</ProjectHome>
-    <StartupFile>Level4_LR_BS.py</StartupFile>
+    <StartupFile>Level2_Inference.py</StartupFile>
     <SearchPath>
     </SearchPath>
     <WorkingDirectory>.</WorkingDirectory>