kubespray + 1.7 kube-ovn偶发的一些怪异问

本文主要记录一些比较奇怪少见的现象

基于kubespray 部署之后，可以看到5个节点都是未ready的，但是部署完1.7之后竟然只剩两个master节点了，而管理网络都是通的

[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# k get node -A
NAME                           STATUS   ROLES                  AGE   VERSION
etcd-migration-test-master-1   Ready    control-plane,master   16h   v1.20.7
etcd-migration-test-master-3   Ready    control-plane,master   16h   v1.20.7


[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# ansible all -i inventory/env-inner-prod-on-prem/inventory.ini  -m ping
etcd-migration-test-worker-2 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
etcd-migration-test-master-3 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
etcd-migration-test-master-1 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
etcd-migration-test-master-2 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
etcd-migration-test-worker-1 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}

clean 掉kube-ovn后，重新部署发现还是只有两个节点

# 看下etcd的情况

[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# hostname=`hostname`
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# export ETCDCTL_API=3
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# export ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-$hostname.pem
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# export ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-$hostname-key.pem
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# export ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# export ETCDCTL_ENDPOINTS="https://10.122.130.11:2379,https://10.122.130.12:2379,https://10.122.130.13:2379"
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]#
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]#
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# etcd
etcd     etcdctl
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# etcd
etcd     etcdctl
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# etcdctl member list
5768444ba1836f2d, started, etcd3, https://10.122.130.13:2380, https://10.122.130.13:2379, false
b6643c183020dfa5, started, etcd2, https://10.122.130.12:2380, https://10.122.130.12:2379, false
e4558df5847b67c5, started, etcd1, https://10.122.130.11:2380, https://10.122.130.11:2379, false
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]#
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]#
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# etcdctl endpoint status -w table
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.122.130.11:2379 | e4558df5847b67c5 |  3.4.13 |   12 MB |     false |      false |         3 |     249869 |             249869 |        |
| https://10.122.130.12:2379 | b6643c183020dfa5 |  3.4.13 |   12 MB |     false |      false |         3 |     249869 |             249869 |        |
| https://10.122.130.13:2379 | 5768444ba1836f2d |  3.4.13 |   12 MB |      true |      false |         3 |     249869 |             249869 |        |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]#


# 看下k8s的基础组件的pod
[root@etcd-migration-test-master-1 env-inner-prod-on-prem]# k get po -A -o wide | grep -E "kube-api|kube-control|kube-sche"
kube-system   kube-apiserver-etcd-migration-test-master-1            1/1     Running             0          16h     10.122.130.11   etcd-migration-test-master-1   <none>           <none>
kube-system   kube-apiserver-etcd-migration-test-master-3            1/1     Running             0          16h     10.122.130.13   etcd-migration-test-master-3   <none>           <none>
kube-system   kube-controller-manager-etcd-migration-test-master-1   1/1     Running             0          16h     10.122.130.11   etcd-migration-test-master-1   <none>           <none>
kube-system   kube-controller-manager-etcd-migration-test-master-3   1/1     Running             0          16h     10.122.130.13   etcd-migration-test-master-3   <none>           <none>
kube-system   kube-scheduler-etcd-migration-test-master-1            1/1     Running             0          16h     10.122.130.11   etcd-migration-test-master-1   <none>           <none>
kube-system   kube-scheduler-etcd-migration-test-master-3            1/1     Running             0          16h     10.122.130.13   etcd-migration-test-master-3   <none>           <none>