Tensorflow实现AlexNet

---

AlexNet把CNN的基本原理应用到了深度神经网络中，同时应用了许多新的技术：

1. 将ReLU作为CNN的激活函数，成功解决了Sigmoid在网络较深时的梯度弥散问题
2. 训练时使用Dropout随机忽略一部分神经，以避免模型过拟合。

过拟合是机器学习中一个常见的问题。Hinton教授团队提出了一个简单
有效的方法，Dropout，将神经网络某一层的输出节点数据随机丢弃一
部分，实质上等于创造出了很多新的随机样本，通过增大样本、减少特
征数量来防止过拟合，可以理解为每次丢弃节点数据是对特征的一种采样。

3. 在CNN中使用重叠的最大池化。避免平均池化的模糊化效果，同时让步长比池化核的尺寸小，这样池化层的输出之间会有重叠和覆盖，提升了特征的丰富性。
4. 使用LRN层，对局部神经元的活动创建竞争机制，使得其中响应较大的值变得相对更大，并抑制其它反馈较小的神经元，增加了模型的泛化能力。
   5.数据增强。主要就是对原始图像机型截取、翻转等。使用了数据增强后可以大大减轻过拟合，提升泛化能力。
5. 对图像的RGB数据进行PCA处理，并对主成份分析做一个标准差为0.1的高斯扰动，增加一些噪声。

---

输入的图像是224x224x3的图像，以下是每个处理层的尺寸大小：
conv1 [32, 56, 56, 64]
pool1 [32, 27, 27, 64]
conv2 [32, 27, 27, 192]
pool2 [32, 13, 13, 192]
conv3 [32, 13, 13, 384]
conv4 [32, 13, 13, 256]
conv5 [32, 13, 13, 256]
pool5 [32, 6, 6, 256]
fcl1 [32, 4096]
fcl2 [32, 4096]
fcl3 [32, 1000]

---

下面是使用Tensorflow的实现：

# _*- coding:utf-8 _*_

import math
import time
import tensorflow as tf
from datetime import datetime

batch_size = 32
num_batches = 100

# 查看每一层网络结构
def print_activations(t):
    print(t.op.name, '  ', t.get_shape().as_list())

def inference(images):
    parameters = []

    # 第一个卷积层
    with tf.name_scope('conv1') as scope:
        kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 64], dtype=tf.float32, stddev=1e-1),
                             name='weights')
        conv = tf.nn.conv2d(images, kernel, [1, 4, 4, 1], padding='SAME')
        biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
                             trainable=True, name='biases')
        bias = tf.nn.bias_add(conv, biases)
        conv1 = tf.nn.relu(bias, name=scope)
        parameters += [kernel, biases]
        print_activations(conv1)

    lrn1 = tf.nn.lrn(conv1, 4, bias=1.0, alpha=0.001/9, beta=0.75, name='lrn1')
    pool1 = tf.nn.max_pool(lrn1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool1')
    print_activations(pool1)

    # 第二个卷积层
    with tf.name_scope('conv2') as scope:
        kernel = tf.Variable(tf.truncated_normal([5, 5, 64, 192], dtype=tf.float32, stddev=1e-1),
                             name='weights')
        conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding='SAME')
        biases = tf.Variable(tf.constant(0.0, shape=[192], dtype=tf.float32),
                             trainable=True, name='biases')
        bias = tf.nn.bias_add(conv, biases)
        conv2 = tf.nn.relu(bias, name=scope)
        parameters += [kernel, biases]
        print_activations(conv2)

    lrn2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001/9, beta=0.75, name='lrn2')
    pool2 = tf.nn.max_pool(lrn2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool2')
    print_activations(pool2)

    # 第三个卷积层
    with tf.name_scope('conv3') as scope:
        kernel = tf.Variable(tf.random_normal([3, 3, 192, 384], dtype=tf.float32, stddev=1e-1),
                             name='weights')
        conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding='SAME')
        biases = tf.Variable(tf.constant(0.0, shape=[384], dtype=tf.float32),
                             trainable=True, name='biases')
        bias = tf.nn.bias_add(conv, biases)
        conv3 = tf.nn.relu(bias, name=scope)
        parameters += [kernel, biases]
        print_activations(conv3)

    # 第四个卷积层
    with tf.name_scope('conv4') as scope:
        kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256], dtype=tf.float32, stddev=1e-1),
                             name='weights')
        conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding='SAME')
        biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                             trainable=True, name='biases')
        bias = tf.nn.bias_add(conv, biases)
        conv4 = tf.nn.relu(bias, name=scope)
        parameters += [kernel, biases]
        print_activations(conv4)

    # 第五个卷积层
    with tf.name_scope('conv5') as scope:
        kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=1e-1),
                             name='weights')
        conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding='SAME')
        biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                             trainable=True, name='biases')
        bias = tf.nn.bias_add(conv, biases)
        conv5 = tf.nn.relu(bias, name=scope)
        parameters += [kernel, biases]
        print_activations(conv5)

    pool5 = tf.nn.max_pool(conv5, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool5')
    print_activations(pool5)

    # 第一个全连接层
    with tf.name_scope('fcl1') as scope:
        weight = tf.Variable(tf.truncated_normal([6 * 6 * 256, 4096], stddev=0.1), name='weights')
        biases = tf.Variable(tf.constant(0.0, shape=[4096], dtype=tf.float32), trainable=True, name='biases')
        h_pool5_flat = tf.reshape(pool5, [-1, 6 * 6 * 256])
        fcl1 = tf.nn.relu(tf.matmul(h_pool5_flat, weight) + biases, name=scope)
        drop1 = tf.nn.dropout(fcl1, 0.7)
        parameters += [weight, biases]
        print_activations(fcl1)

    # 第二个全连接层
    with tf.name_scope('fcl2') as scope:
        weight = tf.Variable(tf.truncated_normal([4096, 4096], stddev=0.1), name='weights')
        biases = tf.Variable(tf.constant(0.0, shape=[4096], dtype=tf.float32), trainable=True, name='biases')
        fcl2 = tf.nn.relu(tf.matmul(drop1, weight) + biases, name=scope)
        drop2 = tf.nn.dropout(fcl2, 0.7)
        parameters += [weight, biases]
        print_activations(fcl2)

    # 第三个全连接层
    with tf.name_scope('fcl3') as scope:
        weight = tf.Variable(tf.truncated_normal([4096, 1000], stddev=0.1), name='weights')
        biases = tf.Variable(tf.constant(0.0, shape=[1000], dtype=tf.float32), trainable=True, name='biases')
        fcl3 = tf.nn.relu(tf.matmul(drop2, weight) + biases, name=scope)
        parameters += [weight, biases]
        print_activations(fcl3)

    return fcl3, parameters

---

参考自《tensorflow实战》