激活层激活函数定义

import numpy as np


def sigmoid_forward(z):
    """
    sigmoid激活前向过程
    :param z:
    :return:
    """
    return 1 / (1 + np.exp(-z))


def sigmoid_backward(next_dz, z):
    """
    sigmoid激活反向过程
    :param next_dz:
    :param z:
    :return:
    """
    return sigmoid_forward(z) * (1 - sigmoid_forward(z)) * next_dz


def tanh_forward(z):
    """
    tanh激活前向过程
    :param z:
    :return:
    """
    return np.tanh(z)


def tanh_backward(next_dz, z):
    """
    tanh激活反向过程
    :param next_dz:
    :param z:
    :return:
    """
    return next_dz(1 - np.square(np.tanh(z)))


def relu_forward(z):
    """
    relu前向传播
    :param z: 待激活层
    :return: 激活后的结果
    """
    return np.maximum(0, z)


def relu_backward(next_dz, z):
    """
    relu反向传播
    :param next_dz: 激活后的梯度
    :param z: 激活前的值
    :return:
    """
    dz = np.where(np.greater(z, 0), next_dz, 0)
    return dz


def lrelu_forward(z, alpha=0.1):
    """
    leaky relu前向传播
    :param z: 待激活层
    :param alpha: 常量因子
    :return: 激活后的结果
    """
    return np.where(np.greater(z, 0), z, alpha * z)


def lrelu_backward(next_dz, z, alpha=0.1):
    """
    leaky relu反向传播
    :param next_dz: 激活后的梯度
    :param z: 激活前的值
    :param alpha: 常量因子
    :return:
    """
    dz = np.where(np.greater(z, 0), next_dz, alpha * next_dz)
    return dz


def prelu_forward(z, alpha):
    """
    PReLu 前向传播
    :param z: 输入
    :param alpha:需要学习的参数
    :return:
    """
    return np.where(np.greater(z, 0), z, alpha * z)


def prelu_backwark(next_dz, z, alpha):
    """
    PReLu 后向传播
    :param next_dz: 输出层的梯度
    :param z: 输入
    :param alpha:需要学习的参数
    :return:
    """
    dz = np.where(np.greater(z, 0), next_dz, alpha * next_dz)
    dalpha = np.where(np.greater(z, 0), 0, z * next_dz)
    return dalpha, dz


def elu_forward(z, alpha=0.1):
    """
    elu前向传播
    :param z: 输入
    :param alpha: 常量因子
    :return:
    """
    return np.where(np.greater(z, 0), z, alpha * (np.exp(z) - 1))


def elu_backward(next_dz, z, alpha=0.1):
    """
    elu反向传播
    :param next_dz: 输出层梯度
    :param z: 输入
    :param alpha: 常量因子
    :return:
    """
    return np.where(np.greater(z, 0), next_dz, alpha * next_dz * np.exp(z))

封装激活层

  
import numpy as np

# Collection of activation functions
# Reference: https://en.wikipedia.org/wiki/Activation_function

class Sigmoid():
    def __call__(self, x):
        return 1 / (1 + np.exp(-x))

    def gradient(self, x):
        return self.__call__(x) * (1 - self.__call__(x))

class Softmax():
    def __call__(self, x):
        e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
        return e_x / np.sum(e_x, axis=-1, keepdims=True)

    def gradient(self, x):
        p = self.__call__(x)
        return p * (1 - p)

class TanH():
    def __call__(self, x):
        return 2 / (1 + np.exp(-2*x)) - 1

    def gradient(self, x):
        return 1 - np.power(self.__call__(x), 2)

class ReLU():
    def __call__(self, x):
        return np.where(x >= 0, x, 0)

    def gradient(self, x):
        return np.where(x >= 0, 1, 0)

class LeakyReLU():
    def __init__(self, alpha=0.2):
        self.alpha = alpha

    def __call__(self, x):
        return np.where(x >= 0, x, self.alpha * x)

    def gradient(self, x):
        return np.where(x >= 0, 1, self.alpha)

class ELU():
    def __init__(self, alpha=0.1):
        self.alpha = alpha 

    def __call__(self, x):
        return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))

    def gradient(self, x):
        return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)

class SELU():
    # Reference : https://arxiv.org/abs/1706.02515,
    # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
    def __init__(self):
        self.alpha = 1.6732632423543772848170429916717
        self.scale = 1.0507009873554804934193349852946 

    def __call__(self, x):
        return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))

    def gradient(self, x):
        return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))

class SoftPlus():
    def __call__(self, x):
        return np.log(1 + np.exp(x))

    def gradient(self, x):
        return 1 / (1 + np.exp(-x))

activation_functions = {
    'relu': ReLU,
    'sigmoid': Sigmoid,
    'selu': SELU,
    'elu': ELU,
    'softmax': Softmax,
    'leaky_relu': LeakyReLU,
    'tanh': TanH,
    'softplus': SoftPlus
}

class Activation(Layer):
    """A layer that applies an activation operation to the input.
    Parameters:
    -----------
    name: string
        The name of the activation function that will be used.
    """

    def __init__(self, name):
        self.activation_name = name
        self.activation_func = activation_functions[name]()
        self.trainable = True

    def layer_name(self):
        return "Activation (%s)" % (self.activation_func.__class__.__name__)

    def forward_pass(self, X, training=True):
        self.layer_input = X
        return self.activation_func(X)

    def backward_pass(self, accum_grad):
        return accum_grad * self.activation_func.gradient(self.layer_input)

    def output_shape(self):
        return self.input_shape