获取数据集，
数据集的下载地址为如下，
https://www.microsoft.com/en-us/download/details.aspx?id=54765
下载完成之后，没有实现测试集和训练集的分离，也没有做数据清洗。
使用下面的CPP工具做数据清理，
镜像
conanio/gcc9:latest
CMakeLists.txt
cmake_minimum_required(VERSION 3.3)
project(47_split_train_test_image)
set(CMAKE_CXX_STANDARD 17)
add_definitions(-g)
include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup()
include_directories(${INCLUDE_DIRS} /opt/pyenv/versions/3.7.13/include/python3.7m/)
LINK_DIRECTORIES(${LINK_DIRS} /opt/pyenv/versions/3.7.13/lib/)
file( GLOB main_file_list ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
file( GLOB source_files ${CMAKE_CURRENT_SOURCE_DIR}/*.cc)
foreach( main_file ${main_file_list} )
file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${main_file})
string(REPLACE ".cpp" "" file ${filename})
add_executable(${file} ${main_file} ${source_files})
target_link_libraries(${file} ${CONAN_LIBS} pthread python3.7m)
endforeach( main_file ${main_file_list})
conanfile.txt
[requires]
nlohmann_json/3.11.3
boost/1.72.0
pybind11/2.12.0
[generators]
cmake
pil.h
#ifndef _FREDRIC_MATPLOT_LIB_H_
#define _FREDRIC_MATPLOT_LIB_H_
#include <pybind11/embed.h>
#include <map>
#include <set>
#include <vector>
#include <string>
namespace py = pybind11;
using namespace py::literals;
namespace pil{
struct pil_t{
pil_t() {
py::initialize_interpreter();
pil_ = py::module_::import("PIL.Image");
}
~pil_t() {
pil_.release();
py::finalize_interpreter();
}
template<typename NumericX>
void to_py_list(const std::vector<NumericX>& x, py::list* result_x) {
for (const auto &elem : x) {
result_x->append(elem);
}
}
void to_kw_args(std::map<std::string, std::string> const& keywords, std::set<std::string> const& real_keys,
py::kwargs* kw_args) {
for (const auto& pair : keywords) {
if(real_keys.find(pair.first) != real_keys.end()) {
(*kw_args)[py::str(pair.first)] = std::atof(pair.second.data());
} else {
(*kw_args)[py::str(pair.first)] = pair.second;
}
}
}
bool open(std::string const& img_name) {
try {
py::object res = pil_.attr("open")(img_name.c_str());
if(res.is_none()) {
return false;
}
return true;
}catch (const py::error_already_set &e) {
// 处理异常
PyErr_Print();
return false;
}
}
private:
py::module_ pil_;
};
}
#endif
main.cpp
#include <filesystem>
#include <vector>
#include <iostream>
#include <sstream>
#include <dirent.h>
#include "pil.h"
namespace fs = std::filesystem;
struct train_test_split_t {
train_test_split_t(std::string const& src_folder_,
std::string const& dst_folder_, float train_split_rate_):
src_folder(src_folder_), dst_folder(dst_folder_),
train_split_rate(train_split_rate_),
pil_() {
}
void split() {
auto class_paths = get_all_class_paths();
make_dstinitation_dataset_dirs();
for(auto & class_path: class_paths) {
std::vector<fs::directory_entry> jpg_files;
int count = count_files(class_path, &jpg_files);
std::cout << "Finish count files, count: " << count << std::endl;
if(count > 0) {
int train_size = (int)(((float)count) * (train_split_rate));
int test_size = count - train_size;
std::string class_name = class_path.path().filename().string();
std::string train_data_class_dir = train_data_dir + "/" + class_name;
if(!fs::exists(train_data_class_dir)) {
fs::create_directories(train_data_class_dir);
}
std::string test_data_class_dir = test_data_dir + "/" + class_name;
if(!fs::exists(test_data_class_dir)) {
fs::create_directories(test_data_class_dir);
}
for(int i=0; i<train_size; ++i) {
std::string dst_path = train_data_class_dir + "/" + jpg_files[i].path().filename().string();
fs::copy_file(jpg_files[i].path(), dst_path);
}
for(int i=train_size; i<jpg_files.size(); ++i) {
std::string dst_path = test_data_class_dir + "/" + jpg_files[i].path().filename().string();
fs::copy_file(jpg_files[i].path(), dst_path);
}
}
}
}
int count_files(fs::path const& dir, std::vector<fs::directory_entry>* dir_files) {
int count = 0;
if (fs::exists(dir) && fs::is_directory(dir)) {
for (auto& entry : fs::directory_iterator(dir)) {
if (fs::is_regular_file(entry.path()) &&
entry.path().string().find(".jpg") != std::string::npos) {
bool success = pil_.open(entry.path().string());
if (!success) {
std::cerr << "Image [" << entry.path().string() << "] is not available, skipped!" << std::endl;
continue;
}
dir_files->push_back(entry);
++count;
}
}
}
return count;
}
void make_dstinitation_dataset_dirs() {
std::stringstream train_folder_ss, test_folder_ss;
train_folder_ss << dst_folder << "/train_dataset";
test_folder_ss << dst_folder << "/test_dataset";
std::cout << train_folder_ss.str() << std::endl;
std::cout << test_folder_ss.str() << std::endl;
if(fs::exists(train_folder_ss.str())) {
fs::remove_all(train_folder_ss.str());
}
fs::create_directory(train_folder_ss.str());
if(fs::exists(test_folder_ss.str())) {
fs::remove_all(test_folder_ss.str());
}
fs::create_directory(test_folder_ss.str());
train_data_dir = train_folder_ss.str();
test_data_dir = test_folder_ss.str();
}
std::vector<fs::directory_entry> get_all_class_paths() {
fs::path root_path(src_folder);
std::vector<fs::directory_entry> results;
if(fs::exists(root_path) && fs::is_directory(root_path)) {
fs::directory_iterator dir_it(root_path);
for(auto& entry: dir_it) {
results.push_back(entry);
}
}
return results;
}
std::string src_folder;
std::string dst_folder;
std::string train_data_dir;
std::string test_data_dir;
float train_split_rate;
pil::pil_t pil_;
};
int main(int argc, char* argv[]) {
if(argc < 3) {
std::cout << "Usage: ./bin/main {folder} {dst_folder} {train_split_rate}\n";
std::cout << "Example: ./bin/main ./datasets ./results_ds 0.8\n";
return EXIT_FAILURE;
}
std::string src_folder = std::string(argv[1]);
std::string dst_folder = std::string(argv[2]);
float train_split_rate = std::atof(argv[3]);
train_test_split_t train_test_split(src_folder, dst_folder, train_split_rate);
train_test_split.split();
return EXIT_SUCCESS;
}
运行方法
conan install ../
cmake ..
make
# 其中./PetImages 是 原始数据集目录
# ./dataset是目标数据集目录
# 0.8 是data split比例
./bin/main ./PetImages ./dataset 0.8
python 部分的代码，
docker环境, 切记这个环境不要自己用手去搭建，太麻烦了，会崩溃的，直接下载tensorflow官方镜像就可以了
tensorflow/tensorflow:2.5.1-jupyter
代码，
# 1. 问题陈述
# 猫狗分类
# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense, Activation
from tensorflow.keras.utils import to_categorical
import tensorflow as tf
# 构建神经网络
# 输入图像大小 64*64*3，
# 激活函数relu，大于0的要，小于0的不激活
# Conv2D需要复习下了
classifier = Sequential()
# Step1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape=(64, 64, 3), activation='relu'))
classifier.add(MaxPooling2D((2, 2)))
# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation='relu'))
classifier.add(MaxPooling2D((2, 2)))
classifier.add(Flatten())
classifier.add(Dense(512, activation='relu'))
classifier.add(Dense(2, activation='softmax'))
# 读取图片数据
# Fitting the CNN to the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
training_set = train_datagen.flow_from_directory('dataset/train_dataset',
target_size=(64, 64),
batch_size=32,
classes=['Cat', 'Dog'],
class_mode='categorical')
# 读取test_set
test_datagen = ImageDataGenerator(rescale=1./255)
test_set = test_datagen.flow_from_directory('dataset/test_dataset',
target_size=(64, 64),
batch_size=32,
class_mode='categorical',
classes=['Cat', 'Dog'])
# 定义一个回调函数来保存验证集上表现最好的模型
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath='best_model.h5',
monitor='val_acc', # 监控验证集上的精度函数
save_best_only=True, # 仅保存在验证集上表现最好的模型
save_weights_only=False, # 保存整个模型（包括模型架构）
verbose=1 # 打印保存信息
)
classifier.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# 训练
classifier.fit(training_set,
steps_per_epoch=80,
epochs=4,
validation_data=test_set,
validation_steps=2000,
callbacks=[checkpoint_callback])
# 使用单张图做预测
import numpy as np
from keras.preprocessing import image
# test_image = image.load_img('dataset/single_prediction/101.jfif', target_size=(64, 64))
test_image = image.load_img('dataset/single_prediction/2.jpg', target_size=(64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis=0)
result = classifier.predict(test_image)
training_set.class_indices
if result[0][0] == 1:
print("Cat")
else:
print("Dog")
# 5. 规律
# acc 如果一直下降。 val_acc一直上升。
# bias就越大。
# bias越大，就说明我正在记住我现在训练的图片，
# 我并不是在 识别 到底是猫还是狗。
程序的输出如下，
image.png
其中101.jiff确实是一张猫，
image.png
image.png
网友评论