导读

sigmoid
softmax
neural network training

sigmoid (logistic regression)

image.png
image.png
image.png

import numpy as np
from numpy.linalg import cholesky
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

sampleNo = 100
mu = np.array([[2, 3]])
Sigma = np.array([[1, 0.5], [1.5, 3]])
R = cholesky(Sigma)
s = np.dot(np.random.randn(sampleNo, 2), R) + mu

mu2 = np.array([[7, 6]])
t = np.dot(np.random.randn(sampleNo, 2), R) + mu2

plt.plot(s[:,0],s[:,1],'+')
plt.plot(t[:,0],t[:,1],'*')
plt.show()

image.png

#构造数据
x = np.concatenate((s, t)).T
y1 = np.zeros(100).reshape(1,100)
y2 = np.ones(100).reshape(1,100)
y = np.concatenate((y1, y2), axis=1)
#初始化参数
w = np.random.randn(2) / 2
b = np.random.randn(1)

def logistic(z):
    return 1 / (1 + np.exp(-z))

def loss(a, y):
    return -(y * np.log(a) + (1-y) * np.log(1-a))

#gradient descent训练参数
iter = 1
m = 200
alpha = 0.1
while iter < 5000:
    z = np.dot(w.T, x) + b
    a = logistic(z)
    #l = np.sum(loss(a, y)) / m
    #print(iter, l)
    dz = a - y
    dw = np.sum(np.dot(x, dz.T), axis=1) / m
    db = np.sum(dz) / m
    w = w - alpha * dw
    b = b - alpha * db
    iter += 1

#显示结果
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.scatter(x[0], x[1], y)
x1_tmp = x2_tmp = np.linspace(-5, 10, 100)
x1_tmp, x2_tmp = np.meshgrid(x1_tmp, x2_tmp)
x_tmp = np.concatenate((x1_tmp.reshape(1, 10000), x2_tmp.reshape(1, 10000)))
z_tmp = np.dot(w.T, x_tmp) + b
a_tmp = logistic(z_tmp)
y_prected = a_tmp.reshape(100, 100)
ax.plot_surface(x1_tmp, x2_tmp, y_prected)
ax.view_init(elev=15,azim=-5)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.show()

image.png

softmax

image.png
image.png

import numpy as np
from numpy.linalg import cholesky
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

sampleNo = 100
mu = np.array([[2, 3]])
Sigma = np.array([[1, 0.5], [1.5, 3]])
R = cholesky(Sigma)
r = np.dot(np.random.randn(sampleNo, 2), R) + mu

mu2 = np.array([[6, 6]])
s = np.dot(np.random.randn(sampleNo, 2), R) + mu2

mu3 = np.array([[1, 6]])
t = np.dot(np.random.randn(sampleNo, 2), R) + mu3

mu4 = np.array([[9, 4]])
v = np.dot(np.random.randn(sampleNo, 2), R) + mu4

plt.plot(r[:,0],r[:,1],'+')
plt.plot(s[:,0],s[:,1],'*')
plt.plot(t[:,0],t[:,1],'.')
plt.plot(v[:,0],v[:,1],'o')
plt.show()

image.png

#构造数据
x = np.concatenate((r,s,t,v))
x = x.T
y_tmp1 = np.array([1,0,0,0]).reshape(4,1)
y_tmp2 = np.array([0,1,0,0]).reshape(4,1)
y_tmp3 = np.array([0,0,1,0]).reshape(4,1)
y_tmp4 = np.array([0,0,0,1]).reshape(4,1)
ones = np.ones(100).reshape(1, 100)
y1 = np.dot(y_tmp1, ones)
y2 = np.dot(y_tmp2, ones)
y3 = np.dot(y_tmp3, ones)
y4 = np.dot(y_tmp4, ones)
y = np.concatenate((y1, y2, y3, y4), axis=1)
#初始化参数
w = np.random.randn(8) / 8
w = w.reshape(2,4)
b = np.random.randn(4) / 4
b = b.reshape(1,4)

def softmax(z):
    z_exp = np.exp(z)
    z_sum = np.sum(z_exp, axis=0)
    return z_exp / z_sum

#gradient descent
m = 400
alpha = 0.1
iter = 1
while iter < 5000:
    z = np.dot(w.T, x) + b.T
    a = softmax(z)
    l = -np.sum(y * np.log(a)) / m
    print(l)
    dz = a - y
    dw = np.dot(x, dz.T) / m
    db = np.sum(dz.T, axis=0) / m
    w = w - alpha * dw
    b = b - alpha * db
    iter += 1

def predict(x):
    z = np.dot(w.T, x) + b.T
    a = softmax(z)
    return np.argmax(a, axis=0)

#显示结果
x1 = x2 = np.linspace(-2, 15, 100)
x1, x2 = np.meshgrid(x1, x2)
x1 = x1.reshape(1, 10000)
x2 = x2.reshape(1, 10000)
x_tmp = np.concatenate((x1, x2))
y_tmp = predict(x_tmp)
x_0 = x_tmp[:, y_tmp==0]
plt.scatter(x_0[0], x_0[1], alpha=0.1)
x_1 = x_tmp[:, y_tmp==1]
plt.scatter(x_1[0], x_1[1], alpha=0.1)
x_2 = x_tmp[:, y_tmp==2]
plt.scatter(x_2[0], x_2[1], alpha=0.1)
x_3 = x_tmp[:, y_tmp==3]
plt.scatter(x_3[0], x_3[1], alpha=0.1)

plt.plot(r[:,0],r[:,1],'+', alpha=1)
plt.plot(s[:,0],s[:,1],'*', alpha=1)
plt.plot(t[:,0],t[:,1],'.', alpha=1)
plt.plot(v[:,0],v[:,1],'o', alpha=1)
plt.show()

image.png

neural network

假设一个三层网络，输入3个节点，第一层4个节点，第二层2个节点，第三层（输出）1个节点，假设所有层的activator都是sigmoid

image.png
image.png

#构造数据
x = np.random.randn(18).reshape(3, 6)
y1 = np.ones(3).reshape(1,3)
y2 = np.zeros(3).reshape(1,3)
y = np.concatenate((y1, y2), axis=1)
#初始化参数
w1 = np.random.randn(12) / 12
w1 = w1.reshape(3,4)
b1 = np.random.randn(4).reshape(1, 4)
w2 = np.random.randn(8) / 8
w2 = w2.reshape(4,2)
b2 = np.random.randn(2).reshape(1, 2)
w3 = np.random.randn(2) / 2
w3 = w3.reshape(2,1)
b3 = np.random.randn(1).reshape(1, 1)

#gradient descent
iter = 1
m = 6
alpha = 0.1
while iter < 100:
    z1 = np.dot(w1.T, x) + b1.T
    a1 = logistic(z1)
    
    z2 = np.dot(w2.T, a1) + b2.T
    a2 = logistic(z2)
    
    z3 = np.dot(w3.T, a2) + b3.T
    a3 = logistic(z3)
        
    dz3 = a3 - y
    dw3 = np.dot(a2, dz3.T) / m
    db3 = np.sum(dz3.T, axis=0) / m
    
    dz2 = np.dot(w3, dz3) * z2 * (1 - z2)
    dw2 = np.dot(a1, dz2.T) / m
    db2 = np.sum(dz2.T, axis=0) / m
    
    dz1 = np.dot(w2, dz2) * z1 * (1 - z1)
    dw1 = np.dot(x, dz1.T) / m
    db1 = np.sum(dz1.T, axis=0) / m
    
    w3 = w3 - alpha * dw3
    b3 = b3 - alpha * db3
    w2 = w2 - alpha * dw2
    b2 = b2 - alpha * db2
    w1 = w1 - alpha * dw1
    b1 = b1 - alpha * db1
    
    iter += 1

常用的activatior及其导数：

，值域为[0, 1]，常用于二类分类

，sigmoid的优化版，值域为[-1, 1]，常用于隐藏层，使均值为0

，保留输入值，为隐藏层的默认activator