系统环境 centos7
一、部署流程
1、研发push到github代码库
2、Jenkins 构建，pull git代码 使用maven进行编译打包
3、打包生成的代码，生成一个新版本的镜像，push到本地docker仓库harbor
4、发布，测试机器 pull 新版本的镜像，并删除原来的容器，重新运行新版本镜像。
二、环境说明
服务及服务器说明-Aliyun环境
1. 代码仓库
github 或者 git-server 或者 gitlab，本次使用github仓库
2、容器镜像仓库
ip：192.168.137.11
主机名：harbor
3、CI/CD服务器
ip：192.168.137.12
主机名：jenkins
软件：jdk，jenkins，git，maven，docker
4、应用服务器
ip：192.168.137.13
主机名：docker
软件：jq，docker或者 k8s集群
三、部署Harbor镜像仓库
1、下载安装
官方地址： https://github.com/goharbor/harbor/releases
#如果之前安装过旧版本的Docker，可以使用下面命令卸载：
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine \
docker-ce
#安装docker
#安装yum工具
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2 --skip-broken
#然后更新本地镜像源
yum-config-manager \
--add-repo \
https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i 's/download.docker.com/mirrors.aliyun.com\/docker-ce/g' /etc/yum.repos.d/docker-ce.repo
yum makecache fast
#docker-ce为社区免费版本
yum install -y docker-ce
#Docker应用需要用到各种端口，逐一去修改防火墙设置。因此先直接关闭防火墙！
# 关闭
systemctl stop firewalld
# 禁止开机启动防火墙
systemctl disable firewalld
# 通过命令启动docker：
systemctl start docker # 启动docker服务
systemctl stop docker # 停止docker服务
systemctl restart docker # 重启docker服务
systemctl enable docker
#然后输入命令，可以查看docker版本：
[root@localhost soft]# docker -v
Docker version 24.0.5, build ced0996
# 下载离线安装包：需要翻墙
yum -y install wget
yum -y install vim
yum -y install lrzsz
mkdir -p /data/soft && cd /data/soft
wget https://github.com/goharbor/harbor/releases/download/v2.4.3/harbor-offline-installer-v2.4.3.tgz
安装compose
curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
docker-compose version
#解压harbor
tar xf harbor-offline-installer-v1.8.0.tgz
# 配置harbor
cd harbor
cp harbor.yml.tmpl harbor.yml
vim harbor.yml // 主机名要可以解析(需要部署dns服务器，用/etc/hosts文件没有用)，如果不可以解析，可以使用IP地址,需要修改的内容如下:
#主机：
hostname = 192.168.137.11
#https 关闭：
# https related config
#https:
# https port for harbor, default is 443
#port: 443
# The path of cert and key files for nginx
#certificate: /your/certificate/path
#private_key: /your/private/key/path
# 修改完配置后启动
[root@localhost harbor]# ./install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 24.0.5
[Step 1]: checking docker-compose is installed ...
Note: docker-compose version: 1.29.2
......
[Step 5]: starting Harbor ...
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating registry ... done
Creating registryctl ... done
Creating harbor-db ... done
Creating harbor-portal ... done
Creating redis ... done
Creating harbor-core ... done
Creating nginx ... done
Creating harbor-jobservice ... done
✔ ----Harbor has been installed and started successfully.----
[root@localhost harbor]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
goharbor/harbor-exporter v2.4.3 776ac6ee91f4 12 months ago 81.5MB
goharbor/chartmuseum-photon v2.4.3 f39a9694988d 12 months ago 172MB
goharbor/redis-photon v2.4.3 b168e9750dc8 12 months ago 154MB
goharbor/trivy-adapter-photon v2.4.3 a406a715461c 12 months ago 251MB
goharbor/notary-server-photon v2.4.3 da89404c7cf9 12 months ago 109MB
goharbor/notary-signer-photon v2.4.3 38468ac13836 12 months ago 107MB
goharbor/harbor-registryctl v2.4.3 61243a84642b 12 months ago 135MB
goharbor/registry-photon v2.4.3 9855479dd6fa 12 months ago 77.9MB
goharbor/nginx-photon v2.4.3 0165c71ef734 12 months ago 44.4MB
goharbor/harbor-log v2.4.3 57ceb170dac4 12 months ago 161MB
goharbor/harbor-jobservice v2.4.3 7fea87c4b884 12 months ago 219MB
goharbor/harbor-core v2.4.3 d864774a3b8f 12 months ago 197MB
goharbor/harbor-portal v2.4.3 85f00db66862 12 months ago 53.4MB
goharbor/harbor-db v2.4.3 7693d44a2ad6 12 months ago 225MB
goharbor/prepare v2.4.3 c882d74725ee 12 months ago 268MB
[root@localhost harbor]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
08eba4e1786d goharbor/harbor-jobservice:v2.4.3 "/harbor/entrypoint.…" 2 minutes ago Up 2 minutes (healthy) harbor-jobservice
7c393d3db9b7 goharbor/nginx-photon:v2.4.3 "nginx -g 'daemon of…" 2 minutes ago Up 2 minutes (healthy) 0.0.0.0:80->8080/tcp, :::80->8080/tcp nginx
b032a4389ecb goharbor/harbor-core:v2.4.3 "/harbor/entrypoint.…" 2 minutes ago Up 2 minutes (healthy) harbor-core
a82fc3942b00 goharbor/redis-photon:v2.4.3 "redis-server /etc/r…" 2 minutes ago Up 2 minutes (healthy) redis
efa21347feb2 goharbor/harbor-portal:v2.4.3 "nginx -g 'daemon of…" 2 minutes ago Up 2 minutes (healthy) harbor-portal
cf7cad5d32a2 goharbor/harbor-db:v2.4.3 "/docker-entrypoint.…" 2 minutes ago Up 2 minutes (healthy) harbor-db
2b36cd6f9a47 goharbor/harbor-registryctl:v2.4.3 "/home/harbor/start.…" 2 minutes ago Up 2 minutes (healthy) registryctl
4f6309de2ca4 goharbor/registry-photon:v2.4.3 "/home/harbor/entryp…" 2 minutes ago Up 2 minutes (healthy) registry
4a623bd80ee0 goharbor/harbor-log:v2.4.3 "/bin/sh -c /usr/loc…" 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:1514->10514/tcp harbor-log
浏览器访问测试：http://192.168.137.11
默认管理员用户名密码如图
创建仓库
新建用户
项目授权
jenkins 服务器安装环境
jenkins（192.168.137.12）机器上操作
安装docker，和 192.168.137.11 安装一样版本的docker，步骤也一样
#如果之前安装过旧版本的Docker，可以使用下面命令卸载：
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine \
docker-ce
#安装docker
#安装yum工具
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2 --skip-broken
#然后更新本地镜像源
yum-config-manager \
--add-repo \
https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i 's/download.docker.com/mirrors.aliyun.com\/docker-ce/g' /etc/yum.repos.d/docker-ce.repo
yum makecache fast
#docker-ce为社区免费版本
yum install -y docker-ce
#Docker应用需要用到各种端口，逐一去修改防火墙设置。因此先直接关闭防火墙！
# 关闭
systemctl stop firewalld
systemctl disable firewalld
systemctl start docker
systemctl enable docker
#然后输入命令，可以查看docker版本：
[root@localhost ~]# docker -v
Docker version 24.0.5, build ced0996
安装git
更新至最新版本（yum）
添加 End Point 到 CentOS 7 仓库
sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
yum安装
yum -y install git
查看版本
[root@localhost ~]# git -v
git version 2.41.0
安装 java
卸载原来的java并yum安装指定的java版本，参考：https://cloud.tencent.com/developer/article/2192638
查看
rpm -qa | grep java
rpm -qa | grep jdk
批量卸载
rpm -qa | grep jdk | xargs rpm -e --nodeps
rpm -qa | grep java | xargs rpm -e --nodeps
单独卸载
rpm -e --nodeps java-11-openjdk
yum list | grep java #全部
yum search java-11-openjdk #指定版本
yum install -y java-11-openjdk
通过yum方式安装默认安装在/usr/lib/jvm文件下
[root@localhost soft]# which Java
/usr/bin/which: no Java in (/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin)
[root@localhost soft]# java -version
openjdk version "11.0.20" 2023-07-18 LTS
OpenJDK Runtime Environment (Red_Hat-11.0.20.0.8-1.el7_9) (build 11.0.20+8-LTS)
OpenJDK 64-Bit Server VM (Red_Hat-11.0.20.0.8-1.el7_9) (build 11.0.20+8-LTS, mixed mode, sharing)
安装maven
mkdir -p /data/soft && cd /data/soft
wget https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz --no-check-certificate
tar -zxvf apache-maven-3.8.8-bin.tar.gz
vi /etc/profile
#追加如下内容：
export MAVEN_HOME=/data/soft/apache-maven-3.8.8
export PATH=$MAVEN_HOME/bin:$PATH
#然后使文件生效并测试查看版本
source /etc/profile
mvn -v
安装启动jenkins
cd /data/soft/
wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo --no-check-certificate
# 安装公钥
rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io-2023.key
# 安装jenkins（安装的是最新的LTS版本）
yum install -y jenkins
# 查看yum都安装了哪些东西
rpm -ql jenkins
systemctl start jenkins
systemctl status jenkins
systemctl enable jenkins
页面访问 http://192.168.137.12:8080/，新建管理员账号，账号密码都为admin
由于在Jenkins机器上docker是使用root用户运行的，而Jenkins是使用普通用户jenkins运行的，所以要先配置下jenkins用户可以使用docker命令。
在root ALL=(ALL) ALL下一行追加jenkins ALL=(root) NOPASSWD: /usr/bin/docker
未行追加Defaults:jenkins !requiretty
如果不配置这个，在执行下面脚本时，会报错误：
- cp -f /home/jenkins/.jenkins/workspace/godseyeBranchForNov/godseye-container/target/godseye-container-wisedu.war /home/jenkins/docker-file/godseye_war/godseye.war
- sudo docker login -u jkzhao -p Wisedu123 -e 01115004@wisedu.com 172.16.206.32 sudo: sorry, you must have a tty to run sudo
tool 配置
maven home 查看
[root@localhost ~]# mvn -v
Apache Maven 3.8.8 (4c87b05d9aedce574290d1acc98575ed5eb6cd39)
Maven home: /data/soft/apache-maven-3.8.8
Java version: 11.0.20, vendor: Red Hat, Inc., runtime: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-1.el7_9.x86_64
Default locale: zh_CN, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-1127.el7.x86_64", arch: "amd64", family: "unix"
java home 查看
[root@localhost ~]# which java
/usr/bin/java
[root@localhost ~]# readlink -f /usr/bin/java | sed "s:bin/java::"
/usr/lib/jvm/java-11-openjdk-11.0.20.0.8-1.el7_9.x86_64/
git 安装路径查看
[root@localhost ~]# whereis git
git: /usr/bin/git /usr/share/man/man1/git.1.gz
安装插件
点击“系统管理”，点击“管理插件”，搜索插件"Maven Integration"和"SSH plugin"，进行安装。
配置远程机器
登录Jenkins，点击“Credentials”，点击“Add domain”。
点击“系统管理”，“系统配置”，找到“SSH remote hosts”。
2、测试harbor
[root@localhost soft]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
[root@localhost soft]# docker pull nginx
Using default tag: latest
latest: Pulling from library/nginx
52d2b7f179e3: Pull complete
fd9f026c6310: Pull complete
055fa98b4363: Pull complete
96576293dd29: Pull complete
a7c4092be904: Pull complete
e3b6889c8954: Pull complete
da761d9a302b: Pull complete
Digest: sha256:104c7c5c54f2685f0f46f3be607ce60da7085da3eaa5ad22d3d9f01594295e9c
Status: Downloaded newer image for nginx:latest
docker.io/library/nginx:latest
[root@localhost soft]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest eea7b3dcba7e 9 days ago 187MB
[root@localhost soft]# docker tag nginx:latest 192.168.137.11:80/jenkins/nginx:v0
[root@localhost soft]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
192.168.137.11:80/jenkins/nginx v0 eea7b3dcba7e 9 days ago 187MB
nginx latest eea7b3dcba7e 9 days ago 187MB
[root@localhost soft]# docker login 192.168.137.11
Username: test
Password:
Error response from daemon: Get "https://192.168.137.11/v2/": dial tcp 192.168.137.11:443: connect: connection refused
#私有仓库默认使用的是https，需要做处理
#写入cat的内容
[root@localhost soft]# vim /etc/docker/daemon.json
[root@localhost soft]# cat /etc/docker/daemon.json
{ "insecure-registries":["192.168.137.11:80"] }
[root@localhost soft]# docker login 192.168.137.11:80
Username: test
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@localhost soft]# docker push 192.168.137.11:80/jenkins/nginx:v0
The push refers to repository [192.168.137.11:80/jenkins/nginx]
563c64030925: Pushed
6fb960878295: Pushed
e161c3f476b5: Pushed
8a7e12012e6f: Pushed
d0a62f56ef41: Pushed
4713cb24eeff: Pushed
511780f88f80: Pushed
v0: digest: sha256:48a84a0728cab8ac558f48796f901f6d31d287101bc8b317683678125e0d2d35 size: 1778
显示镜像推送成功，harbor web页面查看该镜像，可以看到推送成功了
四、应用服务器部署
应用服务器（192.168.137.13）机器上操作
安装docker，和 192.168.137.11 安装一样版本的docker，步骤也一样
#如果之前安装过旧版本的Docker，可以使用下面命令卸载：
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine \
docker-ce
#安装docker
#安装yum工具
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2 --skip-broken
#然后更新本地镜像源
yum-config-manager \
--add-repo \
https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i 's/download.docker.com/mirrors.aliyun.com\/docker-ce/g' /etc/yum.repos.d/docker-ce.repo
yum makecache fast
#docker-ce为社区免费版本
yum install -y docker-ce
#Docker应用需要用到各种端口，逐一去修改防火墙设置。因此先直接关闭防火墙！
# 关闭
systemctl stop firewalld
systemctl start docker
systemctl enable docker
#然后输入命令，可以查看docker版本：
[root@localhost ~]# docker -v
Docker version 24.0.5, build ced0996
安装jq
后面的脚本会用到,jq类似于sed/awk专门处理json格式的文件
#查看是否有jq安装包
yum list| grep jq
#若有，直接安装jq
yum -y install jq
#若没有，用以下命令
yum -y install epel-release
#然后再yum安装
yum -y install jq
其他依赖
yum -y install yum-utils device-mapper-persistent-data lvm2
安装git
更新至最新版本（yum）
添加 End Point 到 CentOS 7 仓库
sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
yum安装
yum -y install git
查看版本
[root@localhost ~]# git -v
git version 2.41.0
修改配置：
visudo
追加 Defaults:root !requiretty
避免在机器业务机器上执行脚本时会报错：
[SSH] executing...
sudo: sorry, you must have a tty to run sudo
docker: invalid reference format.
私有仓库默认使用的是https，需要做处理
#写入cat的内容
[root@localhost soft]# vim /etc/docker/daemon.json
[root@localhost soft]# cat /etc/docker/daemon.json
{ "insecure-registries":["192.168.137.11:80"] }
五、Jenkins构建Job
1、构建Maven风格的Job
代码地址： https://github.com/yanqiang20172017/easy-springmvc-maven.git
Goals and options填写：clean package -Dmaven.test.skip=true
本次实验yum安装方式修改工作目录后的脚本：
# Jenkins机器：编译完成后，build生成一个新版本的镜像，push到远程docker仓库
# Variables
JENKINS_WAR_HOME='/var/lib/jenkins/workspace/maven-docker/target'
DOCKERFILE_HOME='/var/lib/jenkins/docker-file/maven-docker-test_war'
HARBOR_IP='192.168.137.11'
REPOSITORIES='jenkins/maven-docker'
HARBOR_USER='test'
HARBOR_USER_PASSWD='Harbor123456'
HARBOR_USER_EMAIL='test@qq.com'
# Copy the newest war to docker-file directory.
\cp -f ${JENKINS_WAR_HOME}/easy-springmvc-maven.war ${DOCKERFILE_HOME}/maven-docker.war
# Delete image early version.
sudo docker login -u ${HARBOR_USER} -p ${HARBOR_USER_PASSWD} ${HARBOR_IP}:80
IMAGE_ID=`sudo docker images | grep ${REPOSITORIES} | awk '{print $3}'`
if [ -n "${IMAGE_ID}" ];then
sudo docker rmi ${IMAGE_ID}
fi
# Build image.
cd ${DOCKERFILE_HOME}
TAG=`date +%Y%m%d-%H%M%S`
sudo docker build -t ${HARBOR_IP}:80/${REPOSITORIES}:${TAG} . &>/dev/null
# Push to the harbor registry.
sudo docker push ${HARBOR_IP}:80/${REPOSITORIES}:${TAG} &>/dev/null
提前建好 DOCKERFILE_HOME 的目录并编辑好Dockerfile文件
mkdir -p /var/lib/jenkins/docker-file/maven-docker-test_war cd /var/lib/jenkins/docker-file/maven-docker-test_war vim Dockerfile #写入以下内容
# Version 1.0
# Base images.
FROM tomcat:8.0.36-alpine
# Author.
MAINTAINER test <test@qq.com>
# Add war.
ADD maven-docker.war /usr/local/tomcat/webapps/
# Define working directory.
WORKDIR /usr/local/tomcat/bin/
# Define environment variables.
ENV PATH /usr/local/tomcat/bin:$PATH
# Define default command.
CMD ["catalina.sh", "run"]
# Expose ports.
EXPOSE 8080
注：war包的名字为git项目的名字
拉取镜像、发布
Harbor2.0的API发生了变化，此处是新版脚本方式，修改了镜像tag获取的方式
# 拉取镜像，发布
HARBOR_IP='192.168.137.11'
REPOSITORIES='jenkins/maven-docker'
HARBOR_USER='test'
HARBOR_USER_PASSWD='Harbor123456'
# 登录harbor
docker login -u ${HARBOR_USER} -p ${HARBOR_USER_PASSWD} ${HARBOR_IP}:80
# Stop container, and delete the container.
CONTAINER_ID=`docker ps | grep "maven-docker" | awk '{print $1}'`
if [ -n "$CONTAINER_ID" ]; then
docker stop $CONTAINER_ID
docker rm $CONTAINER_ID
else #如果容器启动时失败了，就需要docker ps -a才能找到那个容器
CONTAINER_ID=`docker ps -a | grep "maven-docker" | awk '{print $1}'`
if [ -n "$CONTAINER_ID" ]; then # 如果是第一次在这台机器上拉取运行容器，那么docker ps -a也是找不到这个容器的
docker rm $CONTAINER_ID
fi
fi
# Delet eeasy-springmvc-maven image early version.
IMAGE_ID=`sudo docker images | grep ${REPOSITORIES} | awk '{print $3}'`
if [ -n "${IMAGE_ID}" ];then
docker rmi ${IMAGE_ID}
fi
# Pull image.
# TAG=`curl -s http://${HARBOR_IP}/api/repositories/${REPOSITORIES}/tags | jq '.[-1]' | sed 's/\"//g'`
# TAG=`curl -s http://192.168.137.11/api/repositories/jenkins/maven-docker/tags | jq '.[-1]| {name:.name}' | awk -F '"' '/name/{print $4}'`
TAG=`curl -s http://192.168.137.11/api/v2.0/projects/jenkins/repositories/maven-docker/artifacts |awk -F'name":"' '{print $2}'|cut -d '"' -f1`
docker pull ${HARBOR_IP}:80/${REPOSITORIES}:${TAG} &>/dev/null
# Run.
docker run -d --name maven-docker -p 8080:8080 ${HARBOR_IP}:80/${REPOSITORIES}:${TAG}
最后保存，点击构建，构建完成，可以查看控制台输出
构建失败，控制台输出提示：
+ cp -f /var/lib/jenkins/workspace/maven-docker/target/easy-springmvc-maven.war /var/lib/jenkins/docker-file/maven-docker-test_war/maven-docker.war
cp: 无法创建普通文件"/var/lib/jenkins/docker-file/maven-docker-test_war/maven-docker.war": 权限不够
Build step 'Execute shell' marked build as failure
Finished: FAILURE
修改 jenkins(CI/CD服务器) 上面这个目录的权限
[root@localhost ~]# cd /var/lib/jenkins/
[root@localhost jenkins]# chmod -R 777 docker-file/
再次构建
我的前几次失败是因为jenkins的web界面上mvn的HOME环境配置漏配置了，后面太多是因为github访问超时导致构建失败了，第15次才成功......
docker应用服务器上查看docker镜像和容器验证下：
[root@localhost ~]# ip a | grep "eth0"
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
inet 192.168.137.13/24 brd 192.168.137.255 scope global noprefixroute eth0
[root@localhost ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
192.168.137.11:80/jenkins/maven-docker 20230826-023949 4640cc117faf 3 minutes ago 153MB
[root@localhost ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b235bd6b3fb3 192.168.137.11:80/jenkins/maven-docker:20230826-023949 "catalina.sh run" 3 minutes ago Up 3 minutes 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp maven-docker
harbor 页面查看jenkins项目相关也可以发现新增了maven-docker镜像
网友评论