计算速度优化

• 前面的计算都是针对输入一个样本，然后更新一次权重。这里将代码改成矩阵运算，每次批量计算mini_batch对权重的更改。下面把这章节的代码和该系列文章二的代码运算速度对比，结果如下：

参数：
net.SGD(training_data, 10, 10, 0.5, test_data, False)  # 全样本
二            :0:01:19.567001
三（下）      ：0:00:42.725754

• 针对前面提到过的采用softmax作为输出层函数，和似然函数作为损失函数结(输入样本x输出a真实值为y， y对应真实值位置k与则这cost：- LOGe（a[k]）， a理解为x被分为y每类对应的概率； sum（y）=1，这是softmax函数导致的。当预测越接近真实值，a[k]越接近1， 即 - LOGe（a[k]）越接近0)。这里给出一些学习softmax函数的链接ufldl.stanford.edu、csdn
  代码如下

# encoding: utf-8

"""
@version: python3.5.2
@author: kaenlee  @contact: lichaolfm@163.com
@software: PyCharm Community Edition
@time: 2017/8/16 11:09
purpose:
"""

# 输出层采用softmax
# 似然函数作为损失函数
# minibatch训练采用矩阵乘法曾快计算
# dropout 应对过度拟合

import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import random
from functools import reduce
import operator
import datetime as dt
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mp

mp.style.use('ggplot')




# 各个层仍然会用到s函数
def Sigmod(z):
    return 1 / (1 + np.exp(-z))


def SigmodPrime(z):
    """对S函数求导"""
    return Sigmod(z) * (1 - Sigmod(z))


class CrossEntropyLossFunc:
    @staticmethod
    def loss(A, Y):
        """
        计算cost
        :param A: N X 10 ，N：样本的数量
        :param Y: N X 10
        """
        # 对应的输出index
        index = np.argmax(Y, axis=1)
        CS = [-np.log(A[row, col]) for row, col in zip(range(len(index)), index)]
        return np.sum(np.nan_to_num(CS)) / len(index)  # 似然损失函数计算方法

    @staticmethod
    def delta(A, Y):
        # L的误差向量即偏倒(C-b)
        return A - Y    # 每行对应一个样本L层delta向量




class NetWorks:
    # 定义一个神经网络，也就是定义每一层的权重以及偏置
    def __init__(self, size, lossFunc):
        """
        给出每层的节点数量，包含输出输出层
        :param size: list
        """
        self.size = size
        self.Layers = len(size)
        self.initializeWeightBias()
        self.lossFunc = lossFunc

    def initializeWeightBias(self):
        # 普通的初始化权重方法， 后面会给出更好的
        self.bias = [np.random.randn(num) for num in self.size[1:]]  # 输入层没有bias
        # 每层的权重取决于row取决于该层的节点数量，从来取决于前面一层的输出即节点数
        self.weight = [np.random.randn(row, col) for row, col in zip(self.size[1:], self.size[:-1])]

    def Feedward(self, X):
        """
        :param X:输入向量矩阵 , array
        :return:
        """
        for b, w in zip(self.bias, self.weight):
            Z = X.dot(w.T) + b    # 带全输入信号 N X ?
            X = Sigmod(Z)         # 输出信号, 每行代表一个样本 N X ?

        # 最后一层输出需要除以输出的和
        total = np.sum(X, axis=1)
        total.shape = -1, 1
        return X / total     # N X 10

    def SGD(self, training_data, epochs, minibatch_size, eta, test_data=None, isplot=False):
        """
        随机梯度下降法
        :param training_data:输入模型训练数据@[input, output] # 输入的数据格式变化
        :param epochs: 迭代的期数@ int
        :param minibatch_size: 每次计算梯度向量的取样数量
        :param eta: 学习速率
        :param p: 每次dropout的神经元百分比
        :param test_data: 训练数据
        :return:
        """
        trainX = training_data[0]
        trainY = training_data[1]
        if test_data:
            testX = test_data[0]
            testY = test_data[1]
            n_test = len(testY)
        n = len(trainY)
        accuracy_train = []
        accuracy_test = []
        cost_train = []
        cost_test = []
        for e in range(epochs):
            # 每个迭代器抽样前先打乱数据的顺序
            indices = np.arange(n)
            random.shuffle(indices)
            trainX = trainX[indices]
            trainY = trainY[indices]
            batchXs = [trainX[k:(k + minibatch_size)] for k in range(0, n, minibatch_size)]
            batchYs = [trainY[k:(k + minibatch_size)] for k in range(0, n, minibatch_size)]

            for batchX, batchY in zip(batchXs, batchYs):
                self.Update_miniBatchs(batchX, batchY, eta)

            if test_data:
                totall_predRight = self.Evalueate(test_data)
                print('Epoch {0}: {1}/{2}'.format(e, totall_predRight, n_test))

                if isplot:
                    accuracy_test.append(totall_predRight / n_test)
                    cost_test.append(self.lossFunc.loss(self.Feedward(testX), testY))

            if isplot:
                accuracy_train.append(self.Evalueate(training_data) / n)
                # 计算训练数据的cost 即loss
                cost_train.append(self.lossFunc.loss(self.lossFunc.loss(trainX), trainY))

        if isplot:
            plt.figure()
            plt.plot(np.arange(1, epochs + 1), accuracy_train, label='train')
            plt.plot(np.arange(1, epochs + 1), accuracy_test, label='test')
            axis = plt.gca()
            axis_01 = plt.twinx(axis)
            axis_01.plot(np.arange(1, epochs + 1), cost_train, label='cost')
            plt.xlabel('epoch')
            plt.legend()
            plt.savefig('dropout.png')
            plt.close()

    def Update_miniBatchs(self, batchX, batchY, eta):
        """
        对mini_batch采用梯度下降法,对网络的权重进行更新
        :param mini_batch:
        :param eta:
        :return:
        """
        # 批量计算每个样本对权重改变
        Cprime_bs, Cprime_ws = self.BackProd(batchX, batchY)
        self.bias = [bias - eta * change for bias, change in zip(self.bias, Cprime_bs)]
        self.weight = [weight - eta * change for weight, change in zip(self.weight, Cprime_ws)]

    def BackProd(self, batchX, batchY):
        """
        :param batchX: N X 748
        :param batchY: N X 10
        """
        n = len(batchY)                   # 样本的数量
        # 每层都会有n个z, a
        zs_n = []                         # 每层的加权输入向量， 第一层没有（输入层）n X ?(取决于每层的神经元个数) X layers -1
        activations_n = [batchX]          # 每层的输出信号，第一层为xmat本身        n X ? X layers
        # 计算2...L的权重和偏置(n组)
        for b, w in zip(self.bias, self.weight):
            z_n = activations_n[-1].dot(w.T) + b
            zs_n.append(z_n)                        # 从第二层开始保存带权输入，size-1个
            activations_n.append(Sigmod(z_n))       # 输出信号a

        # 计算输出层L每个节点的delta
        delta_L = self.lossFunc.delta(activations_n[-1], batchY)  # n X 10
        Cprime_bs = [delta_L]                                     # 输出成L的c对b偏倒等于delta_L
        Cprime_ws = [[np.array(np.mat(delta_L[i]).T * np.mat(activations_n[-2][i])) for i in
                      range(n)]]  # c对w的骗到等于前一层的输出信号装置乘当前层的误差
        # 计算所有的层的误差
        temp = delta_L
        for i in range(1, self.Layers - 1):
            # 仅仅需要计算到第二层（且最后一层已知），当前层的delta即b可以用下一层的w、delta表示和当前z表示
            # 从倒数第二层开始求解
            x1 = temp.dot(self.weight[-i])  # 下一层的权重的装置乘下一层的delta
            x2 = SigmodPrime(zs_n[-i - 1])  # 当前层的带权输入
            delta_now = x1 * x2
            Cprime_bs.append(delta_now)
            Cprime_ws.append([np.array(np.mat(delta_now[j]).T * np.mat(activations_n[-i - 2][j])) for j in range(n)])
            temp = delta_now

        # 把每个样本的求解权重进行加总并取平均
        Cprime_bs = [np.sum(bn, axis=0) / n for bn in Cprime_bs]
        Cprime_ws = [reduce(operator.add, wn) / n for wn in Cprime_ws]
        # print([len(b) for b in Cprime_bs])
        # print([w.shape for w in Cprime_ws])
        # 改变输出的顺序
        Cprime_bs.reverse()
        Cprime_ws.reverse()
        return (Cprime_bs, Cprime_ws)

    def Evalueate(self, test_data):
        """
        评估模型
        :param test_data:
        :return:返回预测正确的数量@int
        """
        # 最大数字位置相对应记为正确
        testX = test_data[0]
        testY = test_data[1]
        n_test = len(testY)
        res_pred = np.argmax(self.Feedward(testX), axis=1) == np.argmax(testY, axis=1)
        return sum(res_pred)


if __name__ == '__main__':
    mnist = input_data.read_data_sets(r'D:\PycharmProjects\HandWritingRecognition\TF\data', one_hot=True)
    training_data = [mnist.train.images, mnist.train.labels]
    test_data = [mnist.test.images, mnist.test.labels]
    net = NetWorks([784, 20, 10], CrossEntropyLossFunc)
    X = test_data[0][:3]
    Y = test_data[1][:3]
    # print(net.Feedward(X))
    # print(net.BackProd(X, Y))
    start = dt.datetime.now()
    net.SGD(training_data, 10, 10, 0.5, test_data, isplot=False)
    print(dt.datetime.now() - start)

DropOut

文（三）是针对解决过度拟合的问题，回归主题。这里补充上（三）上的dropout代码

1.等比例随机删除隐藏层的p比例节点，备份一份权重偏置数据
2.剩下的节点按自己原有权重，进行一次更新
3.将更新的权重，覆盖备份数据中对应位置的权重
4.预测取权重（1-p）比例进行预测，预测后将权重还原
5.回到步骤1

# encoding: utf-8

"""
@version: python3.5.2
@author: kaenlee  @contact: lichaolfm@163.com
@software: PyCharm Community Edition
@time: 2017/8/16 11:09
purpose:
"""

# 输出层采用softmax
# 似然函数作为损失函数
# minibatch训练采用矩阵乘法曾快计算
# dropout 应对过度拟合

import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import random
from functools import reduce
import operator
import datetime as dt
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mp

mp.style.use('ggplot')


# 各个层仍然会用到s函数
def Sigmod(z):
    return 1 / (1 + np.exp(-z))


def SigmodPrime(z):
    """对S函数求导"""
    return Sigmod(z) * (1 - Sigmod(z))


class CrossEntropyLossFunc:
    @staticmethod
    def loss(A, Y):
        """
        计算cost
        :param A: N X 10 ，N：样本的数量
        :param Y: N X 10
        """
        # 对应的输出index
        index = np.argmax(Y, axis=1)
        CS = [-np.log(A[row, col]) for row, col in zip(range(len(index)), index)]
        return np.sum(np.nan_to_num(CS)) / len(index)  # 似然损失函数计算方法

    @staticmethod
    def delta(A, Y):
        # L的误差向量即偏倒(C-b)
        return A - Y  # 每行对应一个样本L层delta向量


class NetWorks:
    # 定义一个神经网络，也就是定义每一层的权重以及偏置
    def __init__(self, size, lossFunc):
        """
        给出每层的节点数量，包含输出输出层
        :param size: list
        """
        self.size = size
        self.Layers = len(size)
        self.initializeWeightBias()
        self.lossFunc = lossFunc

    def initializeWeightBias(self):
        # 普通的初始化权重方法， 后面会给出更好的
        self.bias = [np.random.randn(num) for num in self.size[1:]]  # 输入层没有bias
        # 每层的权重取决于row取决于该层的节点数量，从来取决于前面一层的输出即节点数
        self.weight = [np.random.randn(row, col) for row, col in zip(self.size[1:], self.size[:-1])]

    def Feedward(self, X, p, ISpredtest=True):
        """
        :param X:输入向量矩阵 , array
        :return:
        """
        if ISpredtest:
            # 这个主要用来预测函数， 权重要乘以1-p
            weight = self.weight.copy()
            bias = self.bias.copy()
            self.bias = [(1 - p) * b for b in bias]
            self.weight = [(1 - p) * w for w in weight]

        for b, w in zip(self.bias, self.weight):
            Z = X.dot(w.T) + b  # 带全输入信号 N X ?
            X = Sigmod(Z)  # 输出信号, 每行代表一个样本 N X ?
        if ISpredtest:
            # 每个迭代器期都会预测， 预测后需要将权重返还
            self.weight = weight
            self.bias = bias
        # 最后一层输出需要除以输出的和
        total = np.sum(X, axis=1)
        total.shape = -1, 1
        return X / total  # N X 10

    def DropOut(self, p):
        # 给出隐藏层隐藏层删除的节点
        # print(p)
        weight = self.weight.copy()     # 被这个copy坑死了
        # print('that', weight[-1].shape)
        bias = self.bias
        n = len(weight)
        updateW = []
        updateB = []
        size = self.size[1:]  # 输入层没有权重
        save = []
        for i in range(0, n - 1):  # 保留全部输出
            # 删除隐藏层的部分节点
            saveIndex = []      # 无放回的抽样
            sample_num = int(size[i] * (1 - p))
            while len(saveIndex) != sample_num:
                index = np.random.randint(size[i])
                if index not in saveIndex:
                    saveIndex.append(index)

            # print(size[i], saveIndex)
            saveIndex = sorted(saveIndex)
            save.append(saveIndex)
            updateW.append(self.weight[i][saveIndex])
            updateB.append(self.bias[i][saveIndex])
            # 当删除当前层节点个数，后面一层的每个节点w权重个数也相应减少
            self.weight[i + 1] = self.weight[i + 1][:, saveIndex]
            # print(weight[i])
            # print((bias[i]))
            # print(updateB)
            # print(updateW)
        updateW.append(self.weight[-1])   # 保留输出层全部权重
        updateB.append(self.bias[-1])
        save.append(np.arange(size[-1]))
        self.weight = updateW
        self.bias = updateB
        # print('here', weight[-1].shape)
        return weight, bias, save

    def SGD(self, training_data, epochs, minibatch_size, eta, p, test_data=None, isplot=False):
        """
        随机梯度下降法
        :param training_data:输入模型训练数据@[input, output] # 输入的数据格式变化
        :param epochs: 迭代的期数@ int
        :param minibatch_size: 每次计算梯度向量的取样数量
        :param eta: 学习速率
        :param p: 每次dropout的神经元百分比
        :param test_data: 训练数据
        :return:
        """
        trainX = training_data[0]
        trainY = training_data[1]
        if test_data:
            testX = test_data[0]
            testY = test_data[1]
            n_test = len(testY)
        n = len(trainY)
        accuracy_train = []
        accuracy_test = []
        cost_train = []
        cost_test = []
        for e in range(epochs):
            # 每个迭代器抽样前先打乱数据的顺序
            indices = np.arange(n)
            random.shuffle(indices)
            trainX = trainX[indices]
            trainY = trainY[indices]
            batchXs = [trainX[k:(k + minibatch_size)] for k in range(0, n, minibatch_size)]
            batchYs = [trainY[k:(k + minibatch_size)] for k in range(0, n, minibatch_size)]

            for batchX, batchY in zip(batchXs, batchYs):
                weightBackup, biasBackup, save = self.DropOut(p)
                # print(self.bias)
                # print(self.weight)
                self.Update_miniBatchs(batchX, batchY, eta)
                # 更新完后的权重和加入的权重相结合
                for i in range(self.Layers - 1):
                    # print('i', i)
                    biasBackup[i][save[i]] = self.bias[i]
                    if i == 0:
                        # L2的层仅仅减少节点个数并没有改变每个节点权重个数，因为输出层没有变
                        weightBackup[i][save[i]] = self.weight[i]
                    else:
                        row = save[i]
                        col = save[i - 1]
                        # print(row, col)
                        # print(type(weightBackup[i]))
                        # print(weightBackup[i].shape)
                        weightBackup[i][row, :][:, col] = self.weight[i]


                self.weight = weightBackup
                self.bias = biasBackup

            if test_data:
                totall_predRight = self.Evalueate(test_data, p)
                print('Epoch {0}: {1}/{2}'.format(e, totall_predRight, n_test))

                if isplot:
                    # ???计算test data 的cost需要  * 1-p ？？？
                    accuracy_test.append(totall_predRight / n_test)
                    cost_test.append(self.lossFunc.loss(self.Feedward(testX, p), testY))

            if isplot:
                accuracy_train.append(self.Evalueate(training_data, p, False) / n)
                # 计算训练数据的cost 即loss
                cost_train.append(self.lossFunc.loss(self.Feedward(trainX, p, False), trainY))

        if isplot:
            plt.figure()
            plt.plot(np.arange(1, epochs + 1), accuracy_train, label='train')
            plt.plot(np.arange(1, epochs + 1), accuracy_test, label='test')
            axis = plt.gca()
            axis_01 = plt.twinx(axis)
            axis_01.plot(np.arange(1, epochs + 1), cost_train, label='cost')
            plt.xlabel('epoch')
            plt.legend()
            plt.savefig('dropout.png')
            plt.close()

    def Update_miniBatchs(self, batchX, batchY, eta):
        """
        对mini_batch采用梯度下降法,对网络的权重进行更新
        :param mini_batch:
        :param eta:
        :return:
        """
        # 批量计算每个样本对权重改变
        Cprime_bs, Cprime_ws = self.BackProd(batchX, batchY)
        self.bias = [bias - eta * change for bias, change in zip(self.bias, Cprime_bs)]
        self.weight = [weight - eta * change for weight, change in zip(self.weight, Cprime_ws)]

    def BackProd(self, batchX, batchY):
        """
        :param batchX: N X 748
        :param batchY: N X 10
        """
        n = len(batchY)  # 样本的数量
        # 每层都会有n个z, a
        zs_n = []  # 每层的加权输入向量， 第一层没有（输入层）n X ?(取决于每层的神经元个数) X layers -1
        activations_n = [batchX]  # 每层的输出信号，第一层为xmat本身        n X ? X layers
        # 计算2...L的权重和偏置(n组)
        # print(self.bias)
        # print(self.weight)
        for b, w in zip(self.bias, self.weight):
            # print(w.shape)
            z_n = activations_n[-1].dot(w.T) + b
            zs_n.append(z_n)  # 从第二层开始保存带权输入，size-1个
            activations_n.append(Sigmod(z_n))  # 输出信号a

        # 计算输出层L每个节点的delta
        delta_L = self.lossFunc.delta(activations_n[-1], batchY)  # n X 10
        Cprime_bs = [delta_L]  # 输出成L的c对b偏倒等于delta_L
        Cprime_ws = [[np.array(np.mat(delta_L[i]).T * np.mat(activations_n[-2][i])) for i in
                      range(n)]]  # c对w的骗到等于前一层的输出信号装置乘当前层的误差
        # 计算所有的层的误差
        temp = delta_L
        for i in range(1, self.Layers - 1):
            # 仅仅需要计算到第二层（且最后一层已知），当前层的delta即b可以用下一层的w、delta表示和当前z表示
            # 从倒数第二层开始求解
            x1 = temp.dot(self.weight[-i])  # 下一层的权重的装置乘下一层的delta
            x2 = SigmodPrime(zs_n[-i - 1])  # 当前层的带权输入
            delta_now = x1 * x2
            Cprime_bs.append(delta_now)
            Cprime_ws.append([np.array(np.mat(delta_now[j]).T * np.mat(activations_n[-i - 2][j])) for j in range(n)])
            temp = delta_now

        # 把每个样本的求解权重进行加总并取平均
        Cprime_bs = [np.sum(bn, axis=0) / n for bn in Cprime_bs]
        Cprime_ws = [reduce(operator.add, wn) / n for wn in Cprime_ws]
        # print([len(b) for b in Cprime_bs])
        # print([w.shape for w in Cprime_ws])
        # 改变输出的顺序
        Cprime_bs.reverse()
        Cprime_ws.reverse()
        return (Cprime_bs, Cprime_ws)

    def Evalueate(self, test_data, p, IStest=True):
        """
        评估模型
        :param test_data:
        :return:返回预测正确的数量@int
        """
        # 最大数字位置相对应记为正确
        testX = test_data[0]
        testY = test_data[1]
        n_test = len(testY)
        res_pred = np.argmax(self.Feedward(testX, p, IStest), axis=1) == np.argmax(testY, axis=1)
        return sum(res_pred)


if __name__ == '__main__':
    mnist = input_data.read_data_sets(r'D:\PycharmProjects\HandWritingRecognition\TF\data', one_hot=True)
    training_data = [mnist.train.images[:2000], mnist.train.labels[:2000]]
    test_data = [mnist.test.images[:1000], mnist.test.labels[:1000]]
    net = NetWorks([784, 100, 10], CrossEntropyLossFunc)
    X = test_data[0][:3]
    Y = test_data[1][:3]
    # print(net.Feedward(X))
    # print(net.BackProd(X, Y))
    start = dt.datetime.now()
    net.SGD(training_data, 100, 10, 3, 0.5, test_data, isplot=True)
    print(dt.datetime.now() - start)

结果分析：虽然train和test的accuracy几乎都是同时饱和，但是cost缺还在下降，无法解释。（难道pred train data的时候权重也需要乘以1-p！！！！有待考证）

dropout.png