import os
import tensorflow as tf
tf.executing_eagerly()
from sklearn.utils import shuffle
import numpy as np
import matplotlib.pyplot as plt
AUTOTUNE = tf.data.experimental.AUTOTUNE

print(tf.__version__)

2.2.0

预处理

def load_sample(sample_dir,shuffleflag = True):
    '''递归读取文件。只支持一级。返回文件名、数值标签、数值对应的标签名'''
    print ('loading sample  dataset..')
    lfilenames = []
    labelsnames = []
    for (dirpath, dirnames, filenames) in os.walk(sample_dir):#递归遍历文件夹
        for filename in filenames:                            #遍历所有文件名
            #print(dirnames)
            filename_path = os.sep.join([dirpath, filename])
            lfilenames.append(filename_path)               #添加文件名
            labelsnames.append( dirpath.split('\\')[-1] )#添加文件名对应的标签

    lab= list(sorted(set(labelsnames)))  #生成标签名称列表
    labdict=dict( zip( lab  ,list(range(len(lab)))  )) #生成字典

    labels = [labdict[i] for i in labelsnames]
    if shuffleflag == True:
        return shuffle(np.asarray( lfilenames),np.asarray( labels)),np.asarray(lab)
    else:
        return (np.asarray( lfilenames),np.asarray( labels)),np.asarray(lab)

def _distorted_image(image,size,ch=1,shuffleflag = False,cropflag  = False,
                     brightnessflag=False,contrastflag=False):    #定义函数，实现变化图片
    distorted_image =tf.image.random_flip_left_right(image)

    if cropflag == True:                                                #随机裁剪
        s = tf.random.uniform((1,2),int(size[0]*0.8),size[0],tf.int32)
        distorted_image = tf.image.random_crop(distorted_image, [s[0][0],s[0][0],ch])

    distorted_image = tf.image.random_flip_up_down(distorted_image)#上下随机翻转
    if brightnessflag == True:#随机变化亮度
        distorted_image = tf.image.random_brightness(distorted_image,max_delta=10)
    if contrastflag == True:   #随机变化对比度
        distorted_image = tf.image.random_contrast(distorted_image,lower=0.2, upper=1.8)
    if shuffleflag==True:
        distorted_image = tf.random.shuffle(distorted_image)#沿着第0维乱序
    return distorted_image

from skimage import transform
def _random_rotated30(image, label): #定义函数实现图片随机旋转操作

    def _rotated(image):                #封装好的skimage模块，来进行图片旋转30度
        shift_y, shift_x = np.array(image.shape[:2],np.float32) / 2.
        tf_rotate = transform.SimilarityTransform(rotation=np.deg2rad(30))
        tf_shift = transform.SimilarityTransform(translation=[-shift_x, -shift_y])
        tf_shift_inv = transform.SimilarityTransform(translation=[shift_x, shift_y])
        image_rotated = transform.warp(image, (tf_shift + (tf_rotate + tf_shift_inv)).inverse)
        return image_rotated

    def _rotatedwrap():
        image_rotated = tf.py_func( _rotated,[image],[tf.float64])   #调用第三方函数
        return tf.cast(image_rotated,tf.float32)[0]

    a = tf.random.uniform([1],0,2,tf.int32)#实现随机功能
    image_decoded = tf.cond(pred=tf.equal(tf.constant(0),a[0]),true_fn=lambda: image,false_fn=_rotatedwrap)

    return image_decoded, label

def dataset(directory,size,batchsize,random_rotated=False,shuffleflag = True):#定义函数，创建数据集
    """ parse  dataset."""
    (filenames,labels),_ =load_sample(directory,shuffleflag=False) #载入文件名称与标签
#     print("labels len",len(labels))
#     p=0
#     f=0
#     for l in labels:
#         if l==1:
#             p+=1
#         elif l==0:
#             f+=1
#         else:
#             pass
#             #print(l)
#     print("positive label",p)
#     print("native label",f)
    #print(filenames,labels)
    def _parseone(filename, label):                         #解析一个图片文件
        """ Reading and handle  image"""
        image_string = tf.io.read_file(filename)         #读取整个文件
        image_decoded = tf.image.decode_image(image_string)
        image_decoded.set_shape([None, None, None])    # 必须有这句，不然下面会转化失败
        image_decoded = _distorted_image(image_decoded,size)#对图片做扭曲变化
        image_decoded = tf.image.resize(image_decoded, size)  #变化尺寸
        image_decoded = _norm_image(image_decoded,size)#归一化
        image_decoded = tf.cast(image_decoded, dtype=tf.float32)
        label = tf.cast(  tf.reshape(label, [1,]) ,tf.int32  )#将label 转为张量
        return image_decoded, label

    dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))#生成Dataset对象


    if shuffleflag == True:#乱序
        dataset = dataset.shuffle(10000)

    dataset = dataset.map(_parseone)   #有图片内容的数据集

    if random_rotated == True:#旋转
        dataset = dataset.map(_random_rotated30)

    dataset = dataset.batch(batchsize) #批次划分数据集
    dataset = dataset.prefetch(1)

    return dataset
def _norm_image(image,size,ch=1,flattenflag = False):    #定义函数，实现归一化，并且拍平
    image_decoded = image/127.5-1#image/255.0
    if flattenflag==True:
        image_decoded = tf.reshape(image_decoded, [size[0]*size[1]*ch])
    return image_decoded

from tensorflow.keras.applications.resnet50 import ResNet50

size = [224,224]
batchsize = 10

sample_dir=r"./apple2orange/train"
testsample_dir = r"./apple2orange/test"

traindataset = dataset(sample_dir,size,batchsize)#训练集
#testdataset = dataset(testsample_dir,size,batchsize,shuffleflag = False)#测试集

loading sample  dataset..

ds=traindataset

ds

<PrefetchDataset shapes: ((None, 224, 224, None), (None, 1)), types: (tf.float32, tf.int32)>

image_batch, label_batch = next(iter(ds))

label_batch

<tf.Tensor: shape=(10, 1), dtype=int32, numpy=
array([[0],
       [1],
       [0],
       [1],
       [1],
       [0],
       [1],
       [0],
       [0],
       [1]], dtype=int32)>

构建模型

#构建模型
def create_model():
    img_size = (224, 224, 3)
    inputs = tf.keras.Input(shape=img_size)
    #inputs =preprocess_input(inputs)
    #conv_base = ResNet50(weights='resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',input_tensor=inputs,input_shape = img_size
    #                 ,include_top=False, backend=tf.keras.backend, layers=tf.keras.layers, models=tf.keras.models, utils=tf.keras.utils)#创建ResNet网络

    conv_base = ResNet50(weights='resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',input_tensor=inputs,input_shape = img_size
                     ,include_top=False)#创建ResNet网络

    model = tf.keras.models.Sequential()
    model.add(conv_base)
    model.add(tf.keras.layers.Flatten())
    model.add(tf.keras.layers.Dense(256, activation='relu'))
    model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
    conv_base.trainable = False
    model.summary()
    model.compile(loss='binary_crossentropy',
                  optimizer=tf.keras.optimizers.RMSprop(lr=2e-5),
                  metrics=['acc'])
    return model

保存模型checkpoint

checkpoint_path = "training_1/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# 创建一个保存模型权重的回调
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 verbose=1)

可视化

tbCallBack = tf.keras.callbacks.TensorBoard(log_dir='./logs',  # log 目录
                 histogram_freq=0,  # 按照何等频率（epoch）来计算直方图，0为不计算
#                  batch_size=32,     # 用多大量的数据计算直方图
                 write_graph=True,  # 是否存储网络结构图
                 write_grads=True, # 是否可视化梯度直方图
                 write_images=True,# 是否可视化参数
                 embeddings_freq=0, 
                 embeddings_layer_names=None, 
                 embeddings_metadata=None)

WARNING:tensorflow:`write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.

构建并训练

model=create_model()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
resnet50 (Model)             (None, 7, 7, 2048)        23587712  
_________________________________________________________________
flatten (Flatten)            (None, 100352)            0         
_________________________________________________________________
dense (Dense)                (None, 256)               25690368  
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 257       
=================================================================
Total params: 49,278,337
Trainable params: 25,690,625
Non-trainable params: 23,587,712
_________________________________________________________________

model.fit(ds, 
          epochs=50,callbacks=[cp_callback,tbCallBack])

Epoch 1/50
202/202 [==============================] - ETA: 0s - loss: 0.7283 - acc: 0.5839
Epoch 00001: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 254ms/step - loss: 0.7283 - acc: 0.5839
Epoch 2/50
202/202 [==============================] - ETA: 0s - loss: 0.6098 - acc: 0.6778
Epoch 00002: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.6098 - acc: 0.6778
Epoch 3/50
202/202 [==============================] - ETA: 0s - loss: 0.5876 - acc: 0.7001
Epoch 00003: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 255ms/step - loss: 0.5876 - acc: 0.7001
Epoch 4/50
202/202 [==============================] - ETA: 0s - loss: 0.5464 - acc: 0.7224
Epoch 00004: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.5464 - acc: 0.7224
Epoch 5/50
202/202 [==============================] - ETA: 0s - loss: 0.5202 - acc: 0.7299
Epoch 00005: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.5202 - acc: 0.7299
Epoch 6/50
202/202 [==============================] - ETA: 0s - loss: 0.5141 - acc: 0.7537
Epoch 00006: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.5141 - acc: 0.7537
Epoch 7/50
202/202 [==============================] - ETA: 0s - loss: 0.4856 - acc: 0.7642
Epoch 00007: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 254ms/step - loss: 0.4856 - acc: 0.7642
Epoch 8/50
202/202 [==============================] - ETA: 0s - loss: 0.4822 - acc: 0.7696
Epoch 00008: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.4822 - acc: 0.7696
Epoch 9/50
202/202 [==============================] - ETA: 0s - loss: 0.4578 - acc: 0.7810
Epoch 00009: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.4578 - acc: 0.7810
Epoch 10/50
202/202 [==============================] - ETA: 0s - loss: 0.4549 - acc: 0.7776
Epoch 00010: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.4549 - acc: 0.7776
Epoch 11/50
202/202 [==============================] - ETA: 0s - loss: 0.4397 - acc: 0.7989
Epoch 00011: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.4397 - acc: 0.7989
Epoch 12/50
202/202 [==============================] - ETA: 0s - loss: 0.4168 - acc: 0.7999
Epoch 00012: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.4168 - acc: 0.7999
Epoch 13/50
202/202 [==============================] - ETA: 0s - loss: 0.4257 - acc: 0.8034
Epoch 00013: saving model to training_1/cp.ckpt
202/202 [==============================] - 50s 249ms/step - loss: 0.4257 - acc: 0.8034
Epoch 14/50
202/202 [==============================] - ETA: 0s - loss: 0.3998 - acc: 0.8203
Epoch 00014: saving model to training_1/cp.ckpt
202/202 [==============================] - 50s 250ms/step - loss: 0.3998 - acc: 0.8203
Epoch 15/50
202/202 [==============================] - ETA: 0s - loss: 0.3978 - acc: 0.8188
Epoch 00015: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.3978 - acc: 0.8188
Epoch 16/50
202/202 [==============================] - ETA: 0s - loss: 0.3801 - acc: 0.8322
Epoch 00016: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.3801 - acc: 0.8322
Epoch 17/50
202/202 [==============================] - ETA: 0s - loss: 0.3734 - acc: 0.8272
Epoch 00017: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.3734 - acc: 0.8272
Epoch 18/50
202/202 [==============================] - ETA: 0s - loss: 0.3756 - acc: 0.8267
Epoch 00018: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.3756 - acc: 0.8267
Epoch 19/50
202/202 [==============================] - ETA: 0s - loss: 0.3671 - acc: 0.8411
Epoch 00019: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.3671 - acc: 0.8411
Epoch 20/50
202/202 [==============================] - ETA: 0s - loss: 0.3677 - acc: 0.8391
Epoch 00020: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 255ms/step - loss: 0.3677 - acc: 0.8391
Epoch 21/50
202/202 [==============================] - ETA: 0s - loss: 0.3422 - acc: 0.8481
Epoch 00021: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3422 - acc: 0.8481
Epoch 22/50
202/202 [==============================] - ETA: 0s - loss: 0.3449 - acc: 0.8421
Epoch 00022: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3449 - acc: 0.8421
Epoch 23/50
202/202 [==============================] - ETA: 0s - loss: 0.3285 - acc: 0.8500
Epoch 00023: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3285 - acc: 0.8500
Epoch 24/50
202/202 [==============================] - ETA: 0s - loss: 0.3275 - acc: 0.8615
Epoch 00024: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3275 - acc: 0.8615
Epoch 25/50
202/202 [==============================] - ETA: 0s - loss: 0.3347 - acc: 0.8500
Epoch 00025: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.3347 - acc: 0.8500
Epoch 26/50
202/202 [==============================] - ETA: 0s - loss: 0.3269 - acc: 0.8550
Epoch 00026: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3269 - acc: 0.8550
Epoch 27/50
202/202 [==============================] - ETA: 0s - loss: 0.3192 - acc: 0.8669
Epoch 00027: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.3192 - acc: 0.8669
Epoch 28/50
202/202 [==============================] - ETA: 0s - loss: 0.2982 - acc: 0.8694
Epoch 00028: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2982 - acc: 0.8694
Epoch 29/50
202/202 [==============================] - ETA: 0s - loss: 0.2920 - acc: 0.8739
Epoch 00029: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 256ms/step - loss: 0.2920 - acc: 0.8739
Epoch 30/50
202/202 [==============================] - ETA: 0s - loss: 0.3030 - acc: 0.8689
Epoch 00030: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 255ms/step - loss: 0.3030 - acc: 0.8689
Epoch 31/50
202/202 [==============================] - ETA: 0s - loss: 0.2910 - acc: 0.8739
Epoch 00031: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 254ms/step - loss: 0.2910 - acc: 0.8739
Epoch 32/50
202/202 [==============================] - ETA: 0s - loss: 0.2879 - acc: 0.8744
Epoch 00032: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 254ms/step - loss: 0.2879 - acc: 0.8744
Epoch 33/50
202/202 [==============================] - ETA: 0s - loss: 0.2665 - acc: 0.8868
Epoch 00033: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2665 - acc: 0.8868
Epoch 34/50
202/202 [==============================] - ETA: 0s - loss: 0.2758 - acc: 0.8823
Epoch 00034: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2758 - acc: 0.8823
Epoch 35/50
202/202 [==============================] - ETA: 0s - loss: 0.2724 - acc: 0.8798
Epoch 00035: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.2724 - acc: 0.8798
Epoch 36/50
202/202 [==============================] - ETA: 0s - loss: 0.2643 - acc: 0.8878
Epoch 00036: saving model to training_1/cp.ckpt
202/202 [==============================] - 50s 250ms/step - loss: 0.2643 - acc: 0.8878
Epoch 37/50
202/202 [==============================] - ETA: 0s - loss: 0.2605 - acc: 0.8848
Epoch 00037: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.2605 - acc: 0.8848
Epoch 38/50
202/202 [==============================] - ETA: 0s - loss: 0.2626 - acc: 0.8962
Epoch 00038: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2626 - acc: 0.8962
Epoch 39/50
202/202 [==============================] - ETA: 0s - loss: 0.2523 - acc: 0.8982
Epoch 00039: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2523 - acc: 0.8982
Epoch 40/50
202/202 [==============================] - ETA: 0s - loss: 0.2488 - acc: 0.8947
Epoch 00040: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.2488 - acc: 0.8947
Epoch 41/50
202/202 [==============================] - ETA: 0s - loss: 0.2471 - acc: 0.8937
Epoch 00041: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 255ms/step - loss: 0.2471 - acc: 0.8937
Epoch 42/50
202/202 [==============================] - ETA: 0s - loss: 0.2358 - acc: 0.8962
Epoch 00042: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 253ms/step - loss: 0.2358 - acc: 0.8962
Epoch 43/50
202/202 [==============================] - ETA: 0s - loss: 0.2445 - acc: 0.8987
Epoch 00043: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 252ms/step - loss: 0.2445 - acc: 0.8987
Epoch 44/50
202/202 [==============================] - ETA: 0s - loss: 0.2381 - acc: 0.9027
Epoch 00044: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 251ms/step - loss: 0.2381 - acc: 0.9027
Epoch 45/50
202/202 [==============================] - ETA: 0s - loss: 0.2232 - acc: 0.9121
Epoch 00045: saving model to training_1/cp.ckpt
202/202 [==============================] - 51s 254ms/step - loss: 0.2232 - acc: 0.9121
Epoch 46/50
202/202 [==============================] - ETA: 0s - loss: 0.2243 - acc: 0.9052
Epoch 00046: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 255ms/step - loss: 0.2243 - acc: 0.9052
Epoch 47/50
202/202 [==============================] - ETA: 0s - loss: 0.2200 - acc: 0.9121
Epoch 00047: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 257ms/step - loss: 0.2200 - acc: 0.9121
Epoch 48/50
202/202 [==============================] - ETA: 0s - loss: 0.2171 - acc: 0.9091
Epoch 00048: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 258ms/step - loss: 0.2171 - acc: 0.9091
Epoch 49/50
202/202 [==============================] - ETA: 0s - loss: 0.2184 - acc: 0.9116
Epoch 00049: saving model to training_1/cp.ckpt
202/202 [==============================] - 53s 260ms/step - loss: 0.2184 - acc: 0.9116
Epoch 50/50
202/202 [==============================] - ETA: 0s - loss: 0.2115 - acc: 0.9181
Epoch 00050: saving model to training_1/cp.ckpt
202/202 [==============================] - 52s 257ms/step - loss: 0.2115 - acc: 0.9181





<tensorflow.python.keras.callbacks.History at 0x7f11f8549e10>

保存为h5

model.save("weight.h5")

预测

def predict(image_batch):
    predict_logits=model.predict(image_batch)
    predict_label=[int(np.round(logit)) for logit in predict_logits]
    return predict_label

验证

model.evaluate(image_batch,label_batch)

1/1 [==============================] - 0s 2ms/step - loss: 0.3004 - acc: 0.8000





[0.3003780245780945, 0.800000011920929]

加载

load_model=create_model()
load_model.load_weights(checkpoint_path)

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
resnet50 (Model)             (None, 7, 7, 2048)        23587712  
_________________________________________________________________
flatten_1 (Flatten)          (None, 100352)            0         
_________________________________________________________________
dense_2 (Dense)              (None, 256)               25690368  
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 257       
=================================================================
Total params: 49,278,337
Trainable params: 25,690,625
Non-trainable params: 23,587,712
_________________________________________________________________





<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f11784fe910>

重新验证

load_model.evaluate(image_batch,label_batch)

1/1 [==============================] - 0s 6ms/step - loss: 0.3004 - acc: 0.8000





[0.3003780245780945, 0.800000011920929]