搭建整体流程图

1640409529164.png

具体步骤

标题	url
基于Jenkins构建微服务发布平台流程（本文）	https://www.jianshu.com/p/43e3f2e3eee1
部署一套完整的K8s高可用集群（二进制）	https://www.jianshu.com/p/d574e1a9675d
Gitlab	https://www.jianshu.com/p/ea10df706808
企业级镜像仓库Harbor	https://www.jianshu.com/p/ac4a66bb4709
Helm应用包管理	https://www.jianshu.com/p/25ca410b1efd
k8s构建容器化微服务项目	https://www.jianshu.com/p/0f0b2c9ee415
k8s-Prometheus	https://www.jianshu.com/p/8a88f42a4d94
k8s-elk	https://www.jianshu.com/p/96ad780638fa

基于Jenkins构建微服务发布平台流程

1640310801682.png

配置PV持久化存储

部署NFS共享服务器

#在所有节点安装NFS软件包
yum install nfs-utils -y

#master节点作为NFS共享存储服务器，并授权网段
[root@k8s-m1 ~]#  vi /etc/exports
/ifs/kubernetes 192.168.153.0/24(rw,no_root_squash)

#启动nfs
[root@k8s-m1 ~]# systemctl start nfs
[root@k8s-m1 ~]# systemctl enable nfs

#找node挂载测试
[root@k8s-node1 ~]# mount -t nfs 192.168.153.25:/ifs/kubernetes /mnt
[root@k8s-node1 /]# umount /mnt

为Jenkins准备PV

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-jekins
spec:
  capacity:
    storage: 5Gi
  accessModes: ["ReadWriteOnce"]
  nfs:
    path: /ifs/kubernetes/jenkins-data
    server: 192.168.153.25
    
---------------------------------------------------------------------------------
[root@k8s-m1 jenkins]# kubectl apply -f pv-jekins.yaml 
persistentvolume/pv-jekins created
[root@k8s-m1 jenkins]# kubectl get pv
NAME        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     
pv-jekins   5Gi        RWO            Retain           Available                       

Jenkins及其组件安装

Jenkins安装

[root@k8s-m1 jenkins]# kubectl apply -f jenkins.yml 

Please use the following password to proceed to installation:

dc7d3ccd7b0749bbbd4e33134619140f

http://192.168.153.25:30006/

修改国内源

[root@k8s-m1 jenkins-data]#  cd /ifs/kubernetes/jenkins-data/updates/
[root@k8s-m1 updates]# sed -i 's/https:\/\/updates.jenkins.io\/download/https:\/\/mirrors.tuna.tsinghua.edu.cn\/jenkins/g' default.json
[root@k8s-m1 updates]# sed -i 's/http:\/\/www.google.com/https:\/\/www.baidu.com/g' default.json

#重启jenkins
[root@k8s-m1 jenkins]# kubectl delete pod jenkins-578b57ddcb-jchrb

安装插件

• Git：拉取代码
• Git Parameter：Git参数化构建
• Pipeline：流水线
• kubernetes：连接Kubernetes动态创建Slave代理
• Config File Provider：存储配置文件
• Extended Choice Parameter：扩展选择框参数，支持多选

Jenkins在K8s中动态创建代理

Jenkins主从架构介绍

001 当触发Jenkins任务时，Jenkins会调用Kubernetes API创建Slave Pod
002 Pod启动后会连接Jenkins，接受任务并处理

1640325338695.png

Kubernetes插件配置

Configure Clouds

#进入配置
Manage Jenkins -> Manage Nodes and Clouds -> Configure Clouds

[root@k8s-m1 ~]# kubectl get svc -n default
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                        
kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP     

https://kubernetes.default
http://jenkins.default

注意：一定是http://jenkins.default而不是https://jenkins.default

#测试连接
Connected to Kubernetes v1.20.4
 

自定义Jenkins Slave镜像

• Dockerfile：构建镜像
• jenkins-slave：shell脚本启动slave.jar
• settings.xml：修改maven官方源为阿里云源
• slave.jar：agent程序，接受master下发的任务，下载http://jenkinsip:port/jnlpJars/slave.jar
• helm和kubectl客户端工具

[root@k8s-m1 jenkins-slave]# ls
Dockerfile  helm  jenkins-slave  kubectl  settings.xml  slave.jar

构建并推送到镜像仓库：
docker build -t 192.168.153.20/library/jenkins-slave-jdk:1.8 .
docker push 192.168.153.20/library/jenkins-slave-jdk:1.8

测试主从架构是否正常

pipeline {
  agent {
    kubernetes {
        label "jenkins-slave"
        yaml """
kind: Pod
metadata:
  name: jenkins-slave
spec:
  containers:
  - name: jnlp
    image: "192.168.153.20/library/jenkins-slave-jdk:1.8"
"""
      }     
    }
    stages {
        stage('第一步测试'){
            steps {
                sh 'hostname'
            }
        }
    }
}


-----------------------------------------------------------------------------------
#运行过程时会启动slave的pod
[root@k8s-m1 jenkins-slave]# kubectl get pod
NAME                        READY   STATUS              RESTARTS   AGE
jenkins-578b57ddcb-7nwhj    1/1     Running             0          40m
jenkins-slave-354vz-0rcjt   0/1     ContainerCreating   0          13s

#运行结束后，slave的pod消失
[root@k8s-m1 jenkins-slave]# kubectl get pod
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-578b57ddcb-7nwhj   1/1     Running   0          41m

流水线自动发布微服务项目

准备工作

Gitlab

git config --global user.name "lql"
git config --global user.email "lql_h@163.com"

cd simple-microservice-dev3
git init
git remote add origin http://192.168.153.18/root/microservice.git
git add .
git commit -m "Initial commit"
git push -u origin master

http://192.168.153.18/root/microservice.git
http://192.168.153.18/root/microservice.git

Mysql

#Gitlab配置数据库配置文件application-fat.yml [product、order、stock]
mysql://192.168.153.27:3306/tb_product
mysql://192.168.153.27:3306/tb_stock
mysql://192.168.153.27:3306/tb_order

#启动mysql
[root@k8s-node1 ~]# docker start mysql

host

192.168.153.27 eureka.ctnrs.com
192.168.153.27 gateway.ctnrs.com

#ingresscontorl的pod在192.168.153.27上需要有

全局凭据

#凭据配置 
Configure credentials -> 全局凭据 (unrestricted)

d713a4b9-7938-4fc1-98f5-83da66273c91    root/****** (12345678)      git-auth
83ac8c1d-a5e6-4902-9e72-68b7ec6be75f    admin/****** (Harbor12345)  harbor-auth

Managed files

#Managed files
Managed files -> Custom file

[root@k8s-m1 jenkins-slave]#  cat /root/.kube/config 
apiVersion: v1
clusters:
- cluster:
    certificate-authority: /opt/kubernetes/ssl/ca.pem
    server: https://192.168.153.25:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: admin
  name: default
current-context: default
kind: Config
preferences: {}
users:
- name: admin
  user:
    client-certificate: /opt/kubernetes/ssl/admin.pem
    client-key: /opt/kubernetes/ssl/admin-key.pem


----------------------------------------------------------------------------
Custom file
MyCustom    aa483569-be58-4fc0-b9a3-070c4c9eef74

#取出(certificate-authority、client-certificate、client-key)的结果进行Base64编码（Encode），得到： 
certificate-authority-data
client-certificate-data
client-key-data

#测试配置文件
helm list  --kubeconfig helmconfig 

https://base64.us/

Harbor

#创建项目
microservice

#推送helm
helm push ms-0.1.0.tgz --username admin --password Harbor12345 http://192.168.153.20/chartrepo/microservice
 

部署ingress

#host（客户端）
192.168.153.27 eureka.ctnrs.com
192.168.153.27 gateway.ctnrs.com

[root@k8s-m1 k8s]# kubectl apply -f ingress-controller.yaml 

部署eurka

[root@k8s-m1 k8s]# kubectl apply -f eureka.yaml 

Pipeline脚本

#!/usr/bin/env groovy
// 所需插件: Git Parameter/Git/Pipeline/Config File Provider/kubernetes/Extended Choice Parameter
// 公共
def registry = "192.168.153.20"
// 项目
def project = "microservice"
def git_url = "http://192.168.153.18/root/microservice.git"
def gateway_domain_name = "gateway.ctnrs.com"
def portal_domain_name = "portal.ctnrs.com"
// 认证
def image_pull_secret = "registry-pull-secret"
def harbor_auth = "83ac8c1d-a5e6-4902-9e72-68b7ec6be75f"
def git_auth = "d713a4b9-7938-4fc1-98f5-83da66273c91"
// ConfigFileProvider ID
def k8s_auth = "aa483569-be58-4fc0-b9a3-070c4c9eef74"

pipeline {
  agent {
    kubernetes {
        label "jenkins-slave"
        yaml """
apiVersion: v1
kind: Pod
metadata:
  name: jenkins-slave
spec:
  containers:
  - name: jnlp
    image: "${registry}/library/jenkins-slave-jdk:1.8"
    imagePullPolicy: Always
    volumeMounts:
      - name: docker-cmd
        mountPath: /usr/bin/docker
      - name: docker-sock
        mountPath: /var/run/docker.sock
      - name: maven-cache
        mountPath: /root/.m2
  volumes:
    - name: docker-cmd
      hostPath:
        path: /usr/bin/docker
    - name: docker-sock
      hostPath:
        path: /var/run/docker.sock
    - name: maven-cache
      hostPath:
        path: /tmp/m2
"""
        }
      
      }
    parameters {
        gitParameter branch: '', branchFilter: '.*', defaultValue: 'origin/master', description: '选择发布的分支', name: 'Branch', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH'        
        extendedChoice defaultValue: 'none', description: '选择发布的微服务', \
          multiSelectDelimiter: ',', name: 'Service', type: 'PT_CHECKBOX', \
          value: 'gateway-service:9999,portal-service:8080,product-service:8010,order-service:8020,stock-service:8030'
        choice (choices: ['ms', 'demo'], description: '部署模板', name: 'Template')
        choice (choices: ['1', '3', '5', '7'], description: '副本数', name: 'ReplicaCount')
        choice (choices: ['ms'], description: '命名空间', name: 'Namespace')
    }
    stages {
        stage('拉取代码'){
            steps {
                checkout([$class: 'GitSCM', 
                branches: [[name: "${params.Branch}"]], 
                doGenerateSubmoduleConfigurations: false, 
                extensions: [], submoduleCfg: [], 
                userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_url}"]]
                ])
            }
        }
        stage('代码编译') {
            // 编译指定服务
            steps {
                sh """
                  mvn clean package -Dmaven.test.skip=true
                """
            }
        }
        stage('构建镜像') {
          steps {
              withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
                sh """
                 docker login -u ${username} -p '${password}' ${registry}
                 for service in \$(echo ${Service} |sed 's/,/ /g'); do
                    service_name=\${service%:*}
                    image_name=${registry}/${project}/\${service_name}:${BUILD_NUMBER}
                    cd \${service_name}
                    if ls |grep biz &>/dev/null; then
                        cd \${service_name}-biz
                    fi
                    docker build -t \${image_name} .
                    docker push \${image_name}
                    cd ${WORKSPACE}
                  done
                """
                configFileProvider([configFile(fileId: "${k8s_auth}", targetLocation: "admin.kubeconfig")]){
                    sh """
                    # 添加镜像拉取认证
                    kubectl create secret docker-registry ${image_pull_secret} --docker-username=${username} --docker-password=${password} --docker-server=${registry} -n ${Namespace} --kubeconfig admin.kubeconfig |true
                    # 添加私有chart仓库
                    helm repo add  --username ${username} --password ${password} myrepo http://${registry}/chartrepo/${project}
                    """
                }
              }
          }
        }
        stage('Helm部署到K8S') {
          steps {
              sh """
              common_args="-n ${Namespace} --kubeconfig admin.kubeconfig"
              
              for service in  \$(echo ${Service} |sed 's/,/ /g'); do
                service_name=\${service%:*}
                service_port=\${service#*:}
                image=${registry}/${project}/\${service_name}
                tag=${BUILD_NUMBER}
                helm_args="\${service_name} --set image.repository=\${image} --set image.tag=\${tag} --set replicaCount=${replicaCount} --set imagePullSecrets[0].name=${image_pull_secret} --set service.targetPort=\${service_port} --description=\${image}:\${tag}  myrepo/${Template}"

                # 判断是否为新部署
                if helm history \${service_name} \${common_args} &>/dev/null;then
                  action=upgrade
                else
                  action=install
                fi

                # 针对服务启用ingress
                if [ \${service_name} == "gateway-service" ]; then
                  helm \${action} \${helm_args} \
                  --set ingress.enabled=true \
                  --set ingress.host=${gateway_domain_name} \
                   \${common_args}
                elif [ \${service_name} == "portal-service" ]; then
                  helm \${action} \${helm_args} \
                  --set ingress.enabled=true \
                  --set ingress.host=${portal_domain_name} \
                   \${common_args}
                else
                  helm \${action} \${helm_args} \${common_args}
                fi
              done
              # 查看Pod状态
              sleep 10
              kubectl get pods \${common_args}
              """
          }
        }
    }
}

执行结果

[root@k8s-m1 ~]# helm list -n ms
NAME            NAMESPACE       REVISION        UPDATED                                 STATUS          CHART           APP VERSION
gateway-service ms              2               2021-12-25 02:49:52.32458612 +0000 UTC  deployed        ms-0.1.0        0.1.0      
order-service   ms              1               2021-12-24 14:34:25.413383174 +0000 UTC deployed        ms-0.1.0        0.1.0      
portal-service  ms              4               2021-12-25 04:54:16.041854433 +0000 UTC deployed        ms-0.1.0        0.1.0      
product-service ms              1               2021-12-24 14:27:54.785617184 +0000 UTC deployed        ms-0.1.0        0.1.0      
stock-service   ms              1               2021-12-24 14:34:27.396239729 +0000 UTC deployed        ms-0.1.0        0.1.0   

执行日志

kubectl create secret docker-registry registry-pull-secret --docker-username=admin --docker-password=**** --docker-server=192.168.153.20 -n ms --kubeconfig admin.kubeconfig

helm repo add --username admin --password **** myrepo http://192.168.153.20/chartrepo/microservice

helm upgrade portal-service --set image.repository=192.168.153.20/microservice/portal-service --set image.tag=11 --set replicaCount=1 --set 'imagePullSecrets[0].name=registry-pull-secret' --set service.targetPort=8080 --description=192.168.153.20/microservice/portal-service:11 myrepo/ms --set ingress.enabled=true --set ingress.host=portal.ctnrs.com -n ms --kubeconfig admin.kubeconfig

1640357371000.png

自动化部署效果

#网关测试
http://gateway.ctnrs.com/product/queryAllProduct?page=1&limit=10

{"status":200,"msg":"success","result":[{"id":1,"productName":"测试商品1","price":99.99,"stock":98},{"id":2,"productName":"美女","price":999.0,"stock":87},{"id":3,"productName":"Q币","price":100.0,"stock":77},{"id":4,"productName":"貂皮大衣很厚很厚的那种","price":9999.0,"stock":65}]}

#首页展示
http://portal.ctnrs.com/

1640356856850.png 1640356877654.png 1640356894602.png

项目回滚

整体流程

1640409785132.png

根据服务获取最近3次的版本号

[root@k8s-m1 ~]# curl -s -X GET -u admin:Harbor12345 http://192.168.153.20/v2/microservice/portal-service/tags/list

{"name":"microservice/portal-service","tags":["10"]}

----------------------------------------------------------------------------------
[root@k8s-m1 ~]# vi get_tag.sh                           
#!/bin/bash
Harbor_add=192.168.153.20
Username=admin
Password=Harbor12345
Project=$1
Service=$2

curl -s -X GET -u "${Username}:${Password}" "http://${Harbor_add}/v2/${Project}/${Service}/tags/list" |awk -F'[][]' '{split($2,a,",");for (v in a) print 
a[v]}' |sed 's/"//g' |sort -nr|head -n 3
------------------------------------------------------------------------------------

[root@k8s-m1 ~]# ./get_tag.sh microservice portal-service   
10
9
8

#脚本复制到jenkins容器中
[root@k8s-m1 ~]# kubectl get pod
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-578b57ddcb-7nwhj   1/1     Running   2          21h
[root@k8s-m1 ~]# kubectl cp get_tag.sh jenkins-578b57ddcb-7nwhj:/var/jenkins_home

配置插件

General ->This project is parameterized  ->Choice parameter
Name: Service
Choices: 
portal-service
gateway-service
product-service
order-service
stock-service

Description:请选择要回滚的服务

--------------------------------------------------------------------------
#安装插件Active Choices
Active Choices Reactive Parameter

Name ： Tag
Script： Groovy Script

Groovy Script：
cmd = "/bin/bash /var/jenkins_home/get_tag.sh microservice ${Service}"
tags_list = cmd.execute().text.tokenize()
return tags_list

Referenced parameters：Service

定位回滚版本号脚本

[root@k8s-m1 ~]# helm history portal-service -n ms
REVISION        UPDATED                         STATUS          CHART           APP VERSION     DESCRIPTION                                                            
2               Sat Dec 25 02:43:58 2021        deployed        ms-0.1.0        0.1.0           192.168.153.20/microservice/portal-service:11
3               Sat Dec 25 02:43:58 2021        deployed        ms-0.1.0        0.1.0           192.168.153.20/microservice/portal-service:12


[root@k8s-m1 ~]# helm history portal-service -n ms|awk '$NF~/portal-service:12/{print $1}'
3

Pipeline脚本

def registry = "192.168.153.20"
def namespace = "ms"
def k8s_auth = "aa483569-be58-4fc0-b9a3-070c4c9eef74"

pipeline {
  agent {
    kubernetes {
        label "jenkins-slave"
        yaml """
apiVersion: v1
kind: Pod
metadata:
  name: jenkins-slave
spec:
  containers:
  - name: jnlp
    image: "${registry}/library/jenkins-slave-jdk:1.8"
    imagePullPolicy: Always
    volumeMounts:
      - name: docker-cmd
        mountPath: /usr/bin/docker
      - name: docker-sock
        mountPath: /var/run/docker.sock
      - name: maven-cache
        mountPath: /root/.m2
  volumes:
    - name: docker-cmd
      hostPath:
        path: /usr/bin/docker
    - name: docker-sock
      hostPath:
        path: /var/run/docker.sock
    - name: maven-cache
      hostPath:
        path: /tmp/m2
"""
        }
      
      }
    stages {
        stage('执行回滚的操作') {
          steps {
                configFileProvider([configFile(fileId: "${k8s_auth}", targetLocation: "admin.kubeconfig")]){
                    sh """
                    #根据选择的服务名称拼接镜像地址
                    rollback_image=${Service}:${Tag}
                   revision=\$(helm history ${Service} --kubeconfig admin.kubeconfig  -n ms|awk '\$NF~/${Service}:${Tag}/{print \$1}')
                   helm  rollback ${Service} \$revision --kubeconfig admin.kubeconfig -n ${namespace} 

                    """
                }
              }
        }

    }
}

执行结果

[root@k8s-m1 ~]# helm list -n ms
NAME            NAMESPACE       REVISION        UPDATED                                 STATUS          CHART           APP VERSION
gateway-service ms              2               2021-12-25 02:49:52.32458612 +0000 UTC  deployed        ms-0.1.0        0.1.0  
......

执行日志

helm rollback portal-service 2 --kubeconfig admin.kubeconfig -n ms