Velero结合minio实现K8S业务数据备份与恢复；

环境：

主机名	系统	IP	功能
test-middleware01	ubuntu 20.04	172.16.0.14	部署minio服务；
test-deploy	ubuntu 20.04	172.16.0.3	用来部署velero服务，并充当velero客户端；
test-master01	ubuntu 20.04	172.16.0.6	K8S集群的master01节点；
test-ha-kp	ubuntu 20.04	172.16.0.5	K8S集群的负载均衡节点；

安装minio

创建minio数据目录

mkdir /mnt/data_disk/minio

通过docker进行安装

docker run --name minio \       # 运行container；
-p 9000:9000 \
-p 9999:9999 \
-d --restart=always \
-e "MINIO_ROOT_USER=admin" \        # 传输用户名变量，默认管理账号名，minioadmin；
-e "MINIO_ROOT_PASSWORD=12345678" \     # 传输用户密码，默认管理密码，minioadmin；
-v /mnt/data_disk/minio/data:/data \
minio/minio:latest server /data \
--console-address '0.0.0.0:9999'

查看安装结果

root@test-middleware01:~# docker ps
CONTAINER ID   IMAGE                COMMAND                  CREATED         STATUS         PORTS                                                                                  NAMES
be732ebbc1a1   minio/minio:latest   "/usr/bin/docker-ent…"   6 seconds ago   Up 3 seconds   0.0.0.0:9000->9000/tcp, :::9000->9000/tcp, 0.0.0.0:9999->9999/tcp, :::9999->9999/tcp   minio

访问地址：http://172.16.0.14:9999/login

创建Buckets

创建Buckets.png

部署velero

下载velero客户端文件

链接地址：https://github.com/vmware-tanzu/velero/releases/download/v1.8.1/velero-v1.8.1-linux-amd64.tar.gz

上传到deploy节点的/usr/local/src目录下；

安装velero客户端

cd /usr/local/src
tar -xvf velero-v1.8.1-linux-amd64.tar.gz
cp velero-v1.8.1-linux-amd64/velero /usr/local/bin/

配置velero认证

该认证配置分为两个层面，minio的认证 和 K8S的api-server的；

配置minio认证

mkdir /mnt/data_disk/velero
cd /mnt/data_disk/velero

# 编辑velero认证文件；
vim velero-auth.txt
[default]
aws_access_key_id = admin
aws_secret_access_key = 12345678

配置velero的api-server认证

该过程其实就是，在K8S中创建一个新的用户并赋权；由于用到ssl证书，所以要先进行证书的签发；

准备user-csr文件

目的：创建一个名为awsuser的K8S用户；

cd /mnt/data_disk/velero
vim awsuser-csr.json
{
  "CN": "awsuser",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

安装证书签发工具

apt install golang-cfssl
# 确认已经成功安装一下三个工具；
cfssl-certinfo --help
cfssl -h
cfssljson -h

准备相关文件

# ca-config.json文件；
cp /etc/kubeasz/clusters/test.cluster/ssl/ca-config.json /mnt/data_disk/velero/

# 登录master节点将ca.pem和ca-key.pem文件发送到deploy节点；
root@test-master01:~# scp -P 2200 /etc/kubernetes/ssl/ca.pem root@172.16.0.3:/mnt/data_disk/velero/

确认在/mnt/data_disk/velero/目录下有以下几个文件；

root@test-deploy:/mnt/data_disk/velero# ls
awsuser-csr.json  ca-config.json  ca-key.pem  ca.pem  velero-auth.txt

进行证书签发

cfssl gencert -ca=/mnt/data_disk/velero/ca.pem -ca-key=/mnt/data_disk/velero/ca-key.pem -config=/mnt/data_disk/velero/ca-config.json -profile=kubernetes ./awsuser-csr.json | cfssljson -bare awsuser

验证结果

root@test-deploy:/mnt/data_disk/velero# ll awsuser*
-rw-r--r-- 1 root root  997 Apr 20 11:03 awsuser.csr
-rw-r--r-- 1 root root  220 Apr 20 10:31 awsuser-csr.json
-rw------- 1 root root 1679 Apr 20 11:03 awsuser-key.pem
-rw-r--r-- 1 root root 1391 Apr 20 11:03 awsuser.pem

分发证书到api-server证书路径

将awsuser-key.pem和awsuser.pem复制到/etc/kubernetes/ssl目录下，如果没有创建即可；

mkdir -p /etc/kubernetes/ssl
root@test-deploy:/mnt/data_disk/velero# cp awsuser-key.pem awsuser.pem /etc/kubernetes/ssl/
root@test-deploy:/mnt/data_disk/velero# ls /etc/kubernetes/ssl/
awsuser-key.pem  awsuser.pem

生成用来访问K8S集群的，awsuser用户的认证文件

# 将K8S集群的apiserver地址声明为一个变量，可以是VIP；
export KUBE_APISERVER="https://172.16.0.6:6443"

# 使用之前生成的ca认证文件，生成awsuser的kubeconfig文件；
kubectl config set-cluster test.cluster \
--certificate-authority=/mnt/data_disk/velero/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=./awsuser.kubeconfig

root@test-deploy:/mnt/data_disk/velero# ls
awsuser.csr  awsuser-csr.json  awsuser-key.pem  awsuser.kubeconfig  awsuser.pem  ca-config.json  ca-key.pem  ca.pem  velero-auth.txt

这里生成的认证文件并不完全，需要补全客户端和上下文内容；

设置客户端证书认证

kubectl config set-credentials awsuser \
--client-certificate=/mnt/data_disk/velero/awsuser.pem \
--client-key=/mnt/data_disk/velero/awsuser-key.pem \
--embed-certs=true \
--kubeconfig=./awsuser.kubeconfig

设置上下文参数

kubectl config set-context test.cluster \
--cluster=test.cluster \
--user=awsuser \
--namespace=velero-system \
--kubeconfig=./awsuser.kubeconfig

设置默认上下文

root@test-deploy:/mnt/data_disk/velero# kubectl config use-context test.cluster --kubeconfig=awsuser.kubeconfig
Switched to context "test.cluster".

k8s集群中创建awsuser账户

注意如果veleor搭建在别的服务器上，此步骤需要切换到deploy来执行；

root@test-deploy:/mnt/data_disk/velero# kubectl create clusterrolebinding awsuser --clusterrole=cluster-admin --user=awsuser
clusterrolebinding.rbac.authorization.k8s.io/awsuser created

创建namespace

kubectl create ns velero-system

执行安装

如果以上部分实在无法执行，可以考虑使用/root/.kube/config文件代替此处的awsuser.kubeconfig

velero --kubeconfig  /mnt/data_disk/velero/awsuser.kubeconfig \
install \
--provider aws \
--plugins velero/velero-plugin-for-aws:v1.3.1 \
--bucket velerodata  \
--secret-file /mnt/data_disk/velero/velero-auth.txt \
--use-volume-snapshots=false \
--namespace velero-system \
--backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://172.16.0.14:9000

验证安装

# 使用deploy节点执行，可能会尝试下载镜像，需要等待一会；
kubectl logs deployment/velero -n velero-system

使用velero进行备份及恢复测试

备份测试

执行备份

# 声明一个变量，代表当前时间；
DATE=`date +%Y%m%d%H%M%S`

# 创建一个带有当前操作时间的备份文件，文件名为myserver-ns-backup-${DATE}；
# --include-cluster-resources=true，表示是否同时备份集群全局资源，如果备份目标是一个namespace，那么需要确认这个ns中是否关联了一些集群中的全局资源，如果有则需要同时备份，否则恢复时会出问题。可以使用kubectl api-resources查看输出结果
# --include-namespaces，指定备份特定的namespace；
# --kubeconfig=，使用哪个用户认证文件进行操作；
# --namespace，velero所在的namespace；

velero backup create myserver-ns-backup-${DATE} \
--include-cluster-resources=true \
--include-namespaces myserver \
--kubeconfig=/mnt/data_disk/velero/awsuser.kubeconfig \
--namespace velero-system

使用kubectl api-resources查看资源类型；

root@test-deploy:/mnt/data_disk/velero# kubectl api-resources
NAME                              SHORTNAMES   APIVERSION                             NAMESPACED   KIND
bindings                                       v1                                     true         Binding
componentstatuses                 cs           v1                                     false        ComponentStatus
configmaps                        cm           v1                                     true         ConfigMap
endpoints                         ep           v1                                     true         Endpoints
events                            ev           v1                                     true         Event
limitranges                       limits       v1                                     true         LimitRange
namespaces                        ns           v1                                     false        Namespace
nodes                             no           v1                                     false        Node
persistentvolumeclaims            pvc          v1                                     true         PersistentVolumeClaim
persistentvolumes                 pv           v1                                     false        PersistentVolume
pods                              po           v1                                     true         Pod
.
.

可以看到其中第4列NAMESPACED有两个参数

true：表示该资源绑定于指定namespace中，或者说其不具有其他的全局资源；

false：表示该资源并非绑定于某个特定的namespace中，或者说其是个全局性资源；

执行结果

root@test-deploy:/mnt/data_disk/velero# DATE=`date +%Y%m%d%H%M%S`
root@test-deploy:/mnt/data_disk/velero# echo $DATE
20230420155159
root@test-deploy:/mnt/data_disk/velero# velero backup create myserver-ns-backup-${DATE} \
> --include-cluster-resources=true \
> --include-namespaces myserver \
> --kubeconfig=/mnt/data_disk/velero/awsuser.kubeconfig \
> --namespace velero-system
Backup request "myserver-ns-backup-20230420155159" submitted successfully.
Run `velero backup describe myserver-ns-backup-20230420155159` or `velero backup logs myserver-ns-backup-20230420155159` for more details.

查看minio

velero-backup-01.png

备份恢复测试

删除数据

root@test-deploy:/mnt/data_disk/velero# kubectl get pod -n myserver
NAME        READY   STATUS    RESTARTS   AGE
net-test1   1/1     Running   0          3d3h
net-test2   1/1     Running   0          3d3h
net-test3   1/1     Running   0          3d3h
root@test-deploy:/mnt/data_disk/velero# kubectl delete pod/net-test3 -n myserver
pod "net-test3" deleted
root@test-deploy:/mnt/data_disk/velero# kubectl get pod -n myserver
NAME        READY   STATUS    RESTARTS   AGE
net-test1   1/1     Running   0          3d3h
net-test2   1/1     Running   0          3d3h

执行恢复

velero restore create --from-backup myserver-ns-backup-20230420155159 \
--wait \
--kubeconfig=/mnt/data_disk/velero/awsuser.kubeconfig \
--namespace velero-system

执行结果

root@test-deploy:/mnt/data_disk/velero# velero restore create --from-backup myserver-ns-backup-20230420155159 \
> --wait \
> --kubeconfig=/mnt/data_disk/velero/awsuser.kubeconfig \
> --namespace velero-system
Restore request "myserver-ns-backup-20230420155159-20230420160736" submitted successfully.
Waiting for restore to complete. You may safely press ctrl-c to stop waiting - your restore will continue in the background.
.................
Restore completed with status: Completed. You may check for more information using the commands `velero restore describe myserver-ns-backup-20230420155159-20230420160736` and `velero restore logs myserver-ns-backup-20230420155159-20230420160736`.
root@test-deploy:/mnt/data_disk/velero# kubectl get pod -n myserver
NAME        READY   STATUS    RESTARTS   AGE
net-test1   1/1     Running   0          3d3h
net-test2   1/1     Running   0          3d3h
net-test3   1/1     Running   0          62s

备份所有namespace脚本

#!/bin/bash

NS_NAME=`kubectl get namespace | awk '{if (NR>2){print}}' | awk '{print $1}'`
DATE=`date +%Y%m%d%_H%M%S`

for i in $NS_NAME; do
  velero backup create ${i}-${DATE} \
--include-cluster-resources=true \
--include-namespaces ${i} \
--kubeconfig=/mnt/data_disk/velero/awsuser.kubeconfig \
--namespace velero-system
done