简介

使用TensorFlow实现快速风格迁移（fast neural style transfer）， 参考论文[1]

原理

根据内容图片和风格图片优化输入图片，使得内容损失和风格损失尽可能小，快速图像风格迁移网络结构如下所示：

image

其中风格图片时固定的，而内容是可变的，即将任意输入图片转换为指定风格的风格图片。

• 转换网络：参数要训练，将内容图片转换成迁移图片
• 损失网络：计算迁移图片和风格图片之间的风格损失，以及迁移图片和原始图片之间的内容损失。
  经过训练后，转换网络所生成的迁移图片，在内容上和输入的内容图片相似，在风格上和指定的风格图片相似。

代码

1、使用vgg19作为损失网络：

def conv_(inputs, w, b):
    return tf.nn.conv2d(inputs, w, [1, 1, 1, 1], "SAME") + b

def max_pooling(inputs):
    return tf.nn.max_pool(inputs, [1, 2, 2, 1], [1, 2, 2, 1], "SAME")

def vggnet(inputs, vgg_path='/home/luodan/project/fast_neural_style/Conditional-Instance-Norm-for-n-Style-Transfer/vgg_para/'):
    inputs = tf.reverse(inputs, [-1]) - np.array([103.939, 116.779, 123.68])
    para = np.load(vgg_path+"vgg16.npy", encoding="latin1").item()
    F = {}
    inputs = relu(conv_(inputs, para["conv1_1"][0], para["conv1_1"][1]))
    inputs = relu(conv_(inputs, para["conv1_2"][0], para["conv1_2"][1]))
    F["conv1_2"] = inputs
    inputs = max_pooling(inputs)
    inputs = relu(conv_(inputs, para["conv2_1"][0], para["conv2_1"][1]))
    inputs = relu(conv_(inputs, para["conv2_2"][0], para["conv2_2"][1]))
    F["conv2_2"] = inputs
    inputs = max_pooling(inputs)
    inputs = relu(conv_(inputs, para["conv3_1"][0], para["conv3_1"][1]))
    inputs = relu(conv_(inputs, para["conv3_2"][0], para["conv3_2"][1]))
    inputs = relu(conv_(inputs, para["conv3_3"][0], para["conv3_3"][1]))
    F["conv3_3"] = inputs
    inputs = max_pooling(inputs)
    inputs = relu(conv_(inputs, para["conv4_1"][0], para["conv4_1"][1]))
    inputs = relu(conv_(inputs, para["conv4_2"][0], para["conv4_2"][1]))
    inputs = relu(conv_(inputs, para["conv4_3"][0], para["conv4_3"][1]))
    F["conv4_3"] = inputs
    return F

2、风格网络

def transfer(image):
    conv1 = _conv_layer(image, 32, 9, 1)
    conv2 = _conv_layer(conv1, 64, 3, 2)
    conv3 = _conv_layer(conv2, 128, 3, 2)
    resid1 = _residual_block(conv3, 3)
    resid2 = _residual_block(resid1, 3)
    resid3 = _residual_block(resid2, 3)
    resid4 = _residual_block(resid3, 3)
    resid5 = _residual_block(resid4, 3)
#     conv_t1 = _conv_tranpose_layer(resid5, 64, 3, 2)
#     conv_t2 = _conv_tranpose_layer(conv_t1, 32, 3, 2)
    conv_up1 = upsampling(resid5, 64, 3)
    conv_up2 = upsampling(conv_up1, 32, 3)
    conv_up3 = _conv_layer(conv_up2, 3, 9, 1, relu=False)
    preds = tf.nn.sigmoid(conv_up3) * 255.
    return preds
def _conv_layer(net, num_filters, filter_size, strides, relu=True):
    weights_init = _conv_init_vars(net, num_filters, filter_size)
    strides_shape = [1, strides, strides, 1]
    net = tf.nn.conv2d(net, weights_init, strides_shape, padding='SAME')
    net = _instance_norm(net)
    if relu:
        net = tf.nn.relu(net)
    return net
def upsampling(net, num_filters, filter_size):
    net = tf.image.resize_nearest_neighbor(net, [tf.shape(net)[1] * 2, tf.shape(net)[2] * 2])
    weights_init = _conv_init_vars(net, num_filters, filter_size)
    net = tf.nn.conv2d(net, weights_init, [1, 1, 1, 1], padding='SAME')
    return _instance_norm(net)

def _residual_block(net, filter_size=3):
    tmp = _conv_layer(net, 128, filter_size, 1)
    return net + _conv_layer(tmp, 128, filter_size, 1, relu=False)

def _instance_norm(net, train=True):
    batch, rows, cols, channels = [i.value for i in net.get_shape()]
    var_shape = [channels]
    mu, sigma_sq = tf.nn.moments(net, [1,2], keep_dims=True)
    shift = tf.Variable(tf.zeros(var_shape))
    scale = tf.Variable(tf.ones(var_shape))
    epsilon = 1e-3
    normalized = (net-mu)/(sigma_sq + epsilon)**(.5)
    return scale * normalized + shift

def _conv_init_vars(net, out_channels, filter_size, transpose=False):
    _, rows, cols, in_channels = [i.value for i in net.get_shape()]
    if not transpose:
        weights_shape = [filter_size, filter_size, in_channels, out_channels]
    else:
        weights_shape = [filter_size, filter_size, out_channels, in_channels]
    weights_init = tf.Variable(tf.truncated_normal(weights_shape, stddev=0.02, seed=1), dtype=tf.float32)
    return weights_init

参考文献

[1]Perceptual Losses for Real-Time Style Transfer
and Super-Resolution