版本信息
linux Centos7
kubernetes v1.12.2
docker v17.03.2-ce
节点信息
VIP:47.110.19.11 阿里云负载均衡
172.16.0.41 VPC-OPEN-MASTER001 kubelet,etcd,kube-controller-manager,kube-scheduler,kube-proxy,flannel
172.16.0.43 VPC-OPEN-MASTER002 kubelet,etcd,kube-controller-manager,kube-scheduler,kube-proxy,flannel
172.16.0.42 VPC-OPEN-MASTER003 kubelet,etcd,kube-controller-manager,kube-scheduler,kube-proxy,flannel
172.16.0.45 VPC-OPEN-NODE001 kubelet, kube-proxy, flannel
172.16.0.44 VPC-OPEN-NODE002 kubelet, kube-proxy, flannel
安装前准备
开始部署前确保所有节点网络正常，能访问公网。主要操作都在VPC-OPEN-MASTER001节点进行，设置VPC-OPEN-MASTER001可以免密码登陆其他节点。所有操作都使用root用户身份进行。
1、配置hosts解析
# vim /etc/hosts
172.16.0.41 vpc-open-master001 k8s-m1
172.16.0.43 vpc-open-master002 k8s-m2
172.16.0.42 vpc-open-master003 k8s-m3
2、安装docker
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum install -y https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm
systemctl start docker && systemctl enable docker
3、安装 kubeadm, kubelet 和 kubectl
# 配置yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum -y install kubectl-1.12.2 kubeadm-1.12.2 kubelet-1.12.2
查看安装情况
systemctl cat kubelet
# 可以看到kubelet以设置为系统服务，生成kubelet.service和10-kubeadm.conf两文件
# /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
# /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
4、配置系统相关参数
以下操作在所有节点操作
#!/bin/bash
#关闭Selinux
setenforce 0
sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/sysconfig/selinux
#关闭Swapp
swapoff -a
sed -i 's/.*swap.*/#&/' /etc/fstab
#修改转发配置
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness = 0
EOF
sysctl --system
#关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
# 开启forward
# Docker从1.13版本开始调整了默认的防火墙规则
# 禁用了iptables filter表中FOWARD链
# 这样会引起Kubernetes集群中跨Node的Pod无法通信
iptables -P FORWARD ACCEPT
# 加载ipvs相关内核模块
# 如果重新开机，需要重新加载
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
lsmod | grep ip_vs
5、配置阿里云负载均衡
6、配置kubelet
以下操作需要在所有节点上执行
# 配置kubelet使用国内阿里pause镜像，官方的镜像被墙，kubelet启动不了
cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
EOF
# 重新载入kubelet系统配置
systemctl daemon-reload
# 设置开机启动，暂时不启动kubelet
systemctl enable kubelet
配置master节点
- 配置第一个master节点
# 设置节点环境变量，后续ip,hostname信息都以环境变量表示
CP0_IP="172.16.0.41"
CP0_HOSTNAME="vpc-open-master001"
CP1_IP="172.16.0.43"
CP1_HOSTNAME="vpc-open-master002"
CP2_IP="172.16.0.42"
CP2_HOSTNAME="vpc-open-master003"
ADVERTISE_VIP="47.110.19.11"
# 生成kubeadm配置文件
cat > kubeadm-master.config <<EOF
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
# kubernetes版本
kubernetesVersion: v1.12.2
# 使用国内阿里镜像
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
apiServerCertSANs:
- "$CP0_HOSTNAME"
- "$CP0_IP"
- "$ADVERTISE_VIP"
- "127.0.0.1"
api:
advertiseAddress: $CP0_IP
controlPlaneEndpoint: $ADVERTISE_VIP:6443
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://$CP0_IP:2379"
advertise-client-urls: "https://$CP0_IP:2379"
listen-peer-urls: "https://$CP0_IP:2380"
initial-advertise-peer-urls: "https://$CP0_IP:2380"
initial-cluster: "$CP0_HOSTNAME=https://$CP0_IP:2380"
serverCertSANs:
- $CP0_HOSTNAME
- $CP0_IP
peerCertSANs:
- $CP0_HOSTNAME
- $CP0_IP
controllerManagerExtraArgs:
node-monitor-grace-period: 10s
pod-eviction-timeout: 10s
networking:
podSubnet: 10.244.0.0/16
kubeProxy:
config:
#mode: ipvs
mode: iptables
EOF
#提前拉取镜像
# kubeadm config images pull --config kubeadm-master.config
# 拉取的镜像如下
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.12.2
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.12.2
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.12.2
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.12.2
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.2.24
[config/images] Pulled registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.2.2
初始化
# kubeadm init --config kubeadm-master.config
[init] using Kubernetes version: v1.12.2
[preflight] running pre-flight checks
[preflight/images] Pulling images required for setting up a Kubernetes cluster
[preflight/images] This might take a minute or two, depending on the speed of your internet connection
[preflight/images] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[preflight] Activating the kubelet service
[certificates] Generated etcd/ca certificate and key.
[certificates] Generated etcd/healthcheck-client certificate and key.
[certificates] Generated apiserver-etcd-client certificate and key.
[certificates] Generated etcd/server certificate and key.
[certificates] etcd/server serving cert is signed for DNS names [vpc-open-master001 localhost vpc-open-master001] and IPs [127.0.0.1 ::1 172.16.0.41]
[certificates] Generated etcd/peer certificate and key.
[certificates] etcd/peer serving cert is signed for DNS names [vpc-open-master001 localhost vpc-open-master001] and IPs [172.16.0.41 127.0.0.1 ::1 172.16.0.41]
[certificates] Generated ca certificate and key.
[certificates] Generated apiserver certificate and key.
[certificates] apiserver serving cert is signed for DNS names [vpc-open-master001 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local vpc-open-master001] and IPs [10.96.0.1 172.16.0.41 47.110.19.11 172.16.0.41 47.110.19.11 127.0.0.1]
[certificates] Generated apiserver-kubelet-client certificate and key.
[certificates] Generated front-proxy-ca certificate and key.
[certificates] Generated front-proxy-client certificate and key.
[certificates] valid certificates and keys now exist in "/etc/kubernetes/pki"
[certificates] Generated sa key and public key.
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"
[controlplane] wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[etcd] Wrote Static Pod manifest for a local etcd instance to "/etc/kubernetes/manifests/etcd.yaml"
[init] waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] this might take a minute or longer if the control plane images have to be pulled
[apiclient] All control plane components are healthy after 22.504801 seconds
[uploadconfig] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.12" in namespace kube-system with the configuration for the kubelets in the cluster
[markmaster] Marking the node vpc-open-master001 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node vpc-open-master001 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "vpc-open-master001" as an annotation
[bootstraptoken] using token: jjv5r2.g448y7z9bxzdwhls
[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 47.110.19.11:6443 --token jjv5r2.g448y7z9bxzdwhls --discovery-token-ca-cert-hash sha256:939b311021acd36ce6f16875bf25201bf6e664c4aadf94b0246ef7bc054535e4
此时运行命令查看节点信息，发现节点状态为NotReady
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
vpc-open-master001 NotReady master 2m54s v1.12.2
安装网络插件flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
稍等一会，部署网络，查看运行pod
# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6c66ffc55b-6cr7h 1/1 Running 0 5m24s
kube-system coredns-6c66ffc55b-7664m 1/1 Running 0 5m24s
kube-system etcd-vpc-open-master001 1/1 Running 0 4m37s
kube-system kube-apiserver-vpc-open-master001 1/1 Running 0 4m52s
kube-system kube-controller-manager-vpc-open-master001 1/1 Running 0 4m47s
kube-system kube-flannel-ds-amd64-kl79l 1/1 Running 0 77s
kube-system kube-proxy-zqdkm 1/1 Running 0 5m24s
kube-system kube-scheduler-vpc-open-master001 1/1 Running 0 4m49s
上传文件至其他节点
# 打包ca相关文件上传至其他master节点
cd /etc/kubernetes && tar cvzf k8s-key.tgz pki/ca.* pki/sa.* pki/front-proxy-ca.* pki/etcd/ca.*
scp /etc/kubernetes/k8s-key.tgz $CP1_IP:/etc/kubernetes
ssh $CP1_IP 'tar xf /etc/kubernetes/k8s-key.tgz -C /etc/kubernetes/'
scp /etc/kubernetes/k8s-key.tgz $CP2_IP:/etc/kubernetes
ssh $CP2_IP 'tar xf /etc/kubernetes/k8s-key.tgz -C /etc/kubernetes/'
- 配置第二个master节点
#!/bin/bash
# 设置节点环境变量，后续ip,hostname信息都以环境变量表示
CP0_IP="172.16.0.41"
CP0_HOSTNAME="vpc-open-master001"
CP1_IP="172.16.0.43"
CP1_HOSTNAME="vpc-open-master002"
CP2_IP="172.16.0.42"
CP2_HOSTNAME="vpc-open-master003"
ADVERTISE_VIP="47.110.19.11"
# 生成kubeadm配置文件
cat > kubeadm-master.config <<EOF
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
# kubernetes版本
kubernetesVersion: v1.12.2
# 使用国内阿里镜像
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
apiServerCertSANs:
- "$CP1_HOSTNAME"
- "$CP1_IP"
- "$ADVERTISE_VIP"
- "127.0.0.1"
api:
advertiseAddress: $CP1_IP
controlPlaneEndpoint: $ADVERTISE_VIP:6443
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://$CP1_IP:2379"
advertise-client-urls: "https://$CP1_IP:2379"
listen-peer-urls: "https://$CP1_IP:2380"
initial-advertise-peer-urls: "https://$CP1_IP:2380"
initial-cluster: "$CP0_HOSTNAME=https://$CP0_IP:2380,$CP1_HOSTNAME=https://$CP1_IP:2380"
initial-cluster-state: existing
serverCertSANs:
- $CP1_HOSTNAME
- $CP1_IP
peerCertSANs:
- $CP1_HOSTNAME
- $CP1_IP
controllerManagerExtraArgs:
node-monitor-grace-period: 10s
pod-eviction-timeout: 10s
networking:
podSubnet: 10.244.0.0/16
kubeProxy:
config:
# mode: ipvs
mode: iptables
EOF
# 提前拉取镜像
kubeadm config images pull --config kubeadm-master.config
# 配置kubelet
# 生成证书
kubeadm alpha phase certs all --config kubeadm-master.config
# 生成kubelet相关配置文件
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config
kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config
kubeadm alpha phase kubeconfig kubelet --config kubeadm-master.config
# 启动kubelet
systemctl restart kubelet
# 部署 controlplane，即kube-apiserver, kube-controller-manager, kube-scheduler等各组件
# 生成controlplane的配置文件
kubeadm alpha phase kubeconfig all --config kubeadm-master.config
# 设置kubectl 默认配置文件
mkdir ~/.kube
cp /etc/kubernetes/admin.conf ~/.kube/config
查看节点情况
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
vpc-open-master001 Ready master 6m59s v1.12.2
vpc-open-master002 Ready <none> 69s v1.12.2
# 现在master002可以算作一个node节点
# 查看此节点上的pod运行情况
# kubectl get pods --all-namespaces -o wide |grep master002
kube-system kube-flannel-ds-amd64-jwrks 1/1 Running 0 2m38s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-proxy-97mfb 1/1 Running 0 2m38s 172.16.0.43 vpc-open-master002 <none>
# 添加etcd到集群中
# 向添加etcd集群中添加成员，此时如果使用kubectl命令会出错"Unable to connect to the server: unexpected EOF"，这是etcd添加第二个节点的机制导致的，先启动etcd就可以了
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
# 部署etcd静态pod
kubeadm alpha phase etcd local --config kubeadm-master.config
# 查看ectd节点
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member list
# 部署controlplane静态pod文件，kubelet会自动启动各组件
kubeadm alpha phase controlplane all --config kubeadm-master.config
# 此时查看节点及pod运行情况
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
vpc-open-master001 Ready master 11m v1.12.2
vpc-open-master002 Ready <none> 5m11s v1.12.2
# kubectl get pods --all-namespaces -o wide |grep master002
kube-system etcd-vpc-open-master002 1/1 Running 0 42s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-apiserver-vpc-open-master002 1/1 Running 0 23s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-controller-manager-vpc-open-master002 1/1 Running 0 23s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-flannel-ds-amd64-jwrks 1/1 Running 0 5m23s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-proxy-97mfb 1/1 Running 0 5m23s 172.16.0.43 vpc-open-master002 <none>
kube-system kube-scheduler-vpc-open-master002 1/1 Running 0 23s 172.16.0.43 vpc-open-master002 <none>
# 标记为master节点，只是在此节点上添加了label和taint
# kubeadm alpha phase mark-master --config kubeadm-master.config
[markmaster] Marking the node vpc-open-master002 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node vpc-open-master002 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
# 查看节点信息
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
vpc-open-master001 Ready master 12m v1.12.2
vpc-open-master002 Ready master 6m41s v1.12.2
- 配置第三个master节点
#!/bin/bash
# 设置节点环境变量，后续ip,hostname信息都以环境变量表示
CP0_IP="172.16.0.41"
CP0_HOSTNAME="vpc-open-master001"
CP1_IP="172.16.0.43"
CP1_HOSTNAME="vpc-open-master002"
CP2_IP="172.16.0.42"
CP2_HOSTNAME="vpc-open-master003"
ADVERTISE_VIP="47.110.19.11"
# 生成kubeadm配置文件，与第一个master节点的区别除了修改ip外，主要是etcd增加节点的配置
cat > kubeadm-master.config <<EOF
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
# kubernetes版本
kubernetesVersion: v1.12.2
# 使用国内阿里镜像
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
apiServerCertSANs:
- "$CP2_HOSTNAME"
- "$CP2_IP"
- "$ADVERTISE_VIP"
- "127.0.0.1"
api:
advertiseAddress: $CP2_IP
controlPlaneEndpoint: $ADVERTISE_VIP:6443
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://$CP2_IP:2379"
advertise-client-urls: "https://$CP2_IP:2379"
listen-peer-urls: "https://$CP2_IP:2380"
initial-advertise-peer-urls: "https://$CP2_IP:2380"
initial-cluster: "$CP0_HOSTNAME=https://$CP0_IP:2380,$CP1_HOSTNAME=https://$CP1_IP:2380,$CP2_HOSTNAME=https://$CP2_IP:2380"
initial-cluster-state: existing
serverCertSANs:
- $CP2_HOSTNAME
- $CP2_IP
peerCertSANs:
- $CP2_HOSTNAME
- $CP2_IP
controllerManagerExtraArgs:
node-monitor-grace-period: 10s
pod-eviction-timeout: 10s
networking:
podSubnet: 10.244.0.0/16
kubeProxy:
config:
# mode: ipvs
mode: iptables
EOF
# 提前拉取镜像
kubeadm config images pull --config kubeadm-master.config
# 配置kubelet
# 生成证书
kubeadm alpha phase certs all --config kubeadm-master.config
# 生成kubelet相关配置文件
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config
kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config
kubeadm alpha phase kubeconfig kubelet --config kubeadm-master.config
# 启动kubelet
systemctl restart kubelet
# 部署 controlplane，即kube-apiserver, kube-controller-manager, kube-scheduler等各组件
# 生成controlplane的配置文件
kubeadm alpha phase kubeconfig all --config kubeadm-master.config
# 设置kubectl 默认配置文件
mkdir ~/.kube
cp /etc/kubernetes/admin.conf ~/.kube/config
# 添加etcd到集群中
# 向添加etcd集群中添加成员，此时如果使用kubectl命令不会出错
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP2_HOSTNAME} https://${CP2_IP}:2380
# 部署etcd静态pod
kubeadm alpha phase etcd local --config kubeadm-master.config
# 查看ectd节点
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member list
# 部署controlplane静态pod文件，kubelet会自动启动各组件
kubeadm alpha phase controlplane all --config kubeadm-master.config
# 标记为master节点，只是在此节点上添加了label和taint
kubeadm alpha phase mark-master --config kubeadm-master.config
查看节点运行情况
# kubectl get nodes -w
NAME STATUS ROLES AGE VERSION
vpc-open-master001 Ready master 25m v1.12.2
vpc-open-master002 Ready master 19m v1.12.2
vpc-open-master003 Ready master 98s v1.12.2
# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6c66ffc55b-mrsqg 1/1 Running 0 25m
kube-system coredns-6c66ffc55b-nl686 1/1 Running 0 25m
kube-system etcd-vpc-open-master001 1/1 Running 0 24m
kube-system etcd-vpc-open-master002 1/1 Running 0 15m
kube-system etcd-vpc-open-master003 1/1 Running 0 52s
kube-system kube-apiserver-vpc-open-master001 1/1 Running 0 24m
kube-system kube-apiserver-vpc-open-master002 1/1 Running 0 14m
kube-system kube-apiserver-vpc-open-master003 1/1 Running 0 32s
kube-system kube-controller-manager-vpc-open-master001 1/1 Running 1 24m
kube-system kube-controller-manager-vpc-open-master002 1/1 Running 0 14m
kube-system kube-controller-manager-vpc-open-master003 1/1 Running 0 32s
kube-system kube-flannel-ds-amd64-jwrks 1/1 Running 0 19m
kube-system kube-flannel-ds-amd64-nxrx2 1/1 Running 0 108s
kube-system kube-flannel-ds-amd64-zmwbx 1/1 Running 0 24m
kube-system kube-proxy-97mfb 1/1 Running 0 19m
kube-system kube-proxy-h8ctq 1/1 Running 0 25m
kube-system kube-proxy-vf2k6 1/1 Running 0 108s
kube-system kube-scheduler-vpc-open-master001 1/1 Running 1 24m
kube-system kube-scheduler-vpc-open-master002 1/1 Running 0 14m
kube-system kube-scheduler-vpc-open-master003 1/1 Running 0 32s
配置node节点
# 初始化操作同master一样，不能省略，必须要做
# 软件安装及镜像下载
yum -y install kubectl-1.12.2 kubelet-1.12.2 kubeadm-1.12.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy-amd64:v1.12.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy-amd64:v1.12.2 k8s.gcr.io/kube-proxy-amd64:v1.12.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1 k8s.gcr.io/pause:3.1
# 加入集群
# kubeadm join 47.110.19.11:6443 --token jjv5r2.g448y7z9bxzdwhls --discovery-token-ca-cert-hash sha256:939b311021acd36ce6f16875bf25201bf6e664c4aadf94b0246ef7bc054535e4
[preflight] running pre-flight checks
[WARNING RequiredIPVSKernelModulesAvailable]: the IPVS proxier will not be used, because the following required kernel modules are not loaded: [ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh] or no builtin kernel ipvs support: map[nf_conntrack_ipv4:{} ip_vs:{} ip_vs_rr:{} ip_vs_wrr:{} ip_vs_sh:{}]
you can solve this problem with following methods:
1. Run 'modprobe -- ' to load missing kernel modules;
2. Provide the missing builtin kernel ipvs support
[WARNING Hostname]: hostname "vpc-open-node002" could not be reached
[WARNING Hostname]: hostname "vpc-open-node002" lookup vpc-open-node002 on 100.100.2.138:53: no such host
[discovery] Trying to connect to API Server "47.110.19.11:6443"
[discovery] Created cluster-info discovery client, requesting info from "https://47.110.19.11:6443"
[discovery] Requesting info from "https://47.110.19.11:6443" again to validate TLS against the pinned public key
[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "47.110.19.11:6443"
[discovery] Successfully established connection with API Server "47.110.19.11:6443"
[kubelet] Downloading configuration for the kubelet from the "kubelet-config-1.12" ConfigMap in the kube-system namespace
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[preflight] Activating the kubelet service
[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "vpc-open-node002" as an annotation
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the master to see this node join the cluster.
如果后面要加入多台机器，都是同一个命令
kubeadm join 47.110.19.11:6443 --token jjv5r2.g448y7z9bxzdwhls --discovery-token-ca-cert-hash sha256:939b311021acd36ce6f16875bf25201bf6e664c4aadf94b0246ef7bc054535e4
此命令创建的令牌默认有效期24h，如果过期或者忘记，需要使用如下命令创建新的token
kubeadm alpha phase bootstrap-token create --config kubeadm-master.config
在master节点上查看当前token列表
kubeadm token list
在master上查看当前节点及pod情况
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
vpc-open-master001 Ready master 75m v1.12.2
vpc-open-master002 Ready master 69m v1.12.2
vpc-open-master003 Ready master 51m v1.12.2
vpc-open-node001 Ready <none> 18m v1.12.2
vpc-open-node002 Ready <none> 3m55s v1.12.2
# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6c66ffc55b-mrsqg 1/1 Running 0 74m
kube-system coredns-6c66ffc55b-nl686 1/1 Running 0 74m
kube-system etcd-vpc-open-master001 1/1 Running 0 74m
kube-system etcd-vpc-open-master002 1/1 Running 0 64m
kube-system etcd-vpc-open-master003 1/1 Running 0 50m
kube-system kube-apiserver-vpc-open-master001 1/1 Running 0 74m
kube-system kube-apiserver-vpc-open-master002 1/1 Running 0 64m
kube-system kube-apiserver-vpc-open-master003 1/1 Running 0 49m
kube-system kube-controller-manager-vpc-open-master001 1/1 Running 1 74m
kube-system kube-controller-manager-vpc-open-master002 1/1 Running 0 64m
kube-system kube-controller-manager-vpc-open-master003 1/1 Running 0 49m
kube-system kube-flannel-ds-amd64-dzq5b 1/1 Running 0 4m5s
kube-system kube-flannel-ds-amd64-hnf2f 1/1 Running 0 18m
kube-system kube-flannel-ds-amd64-jwrks 1/1 Running 0 69m
kube-system kube-flannel-ds-amd64-nxrx2 1/1 Running 0 51m
kube-system kube-flannel-ds-amd64-zmwbx 1/1 Running 0 73m
kube-system kube-proxy-97mfb 1/1 Running 0 69m
kube-system kube-proxy-h8ctq 1/1 Running 0 74m
kube-system kube-proxy-mw2l7 1/1 Running 0 18m
kube-system kube-proxy-qxztv 1/1 Running 0 4m5s
kube-system kube-proxy-vf2k6 1/1 Running 0 51m
kube-system kube-scheduler-vpc-open-master001 1/1 Running 1 74m
kube-system kube-scheduler-vpc-open-master002 1/1 Running 0 64m
kube-system kube-scheduler-vpc-open-master003 1/1 Running 0 49m
kubeadm初始化过程分步操作
kubeadm init命令实际是由的原子工作任务组成的，详见http://docs.kubernetes.org.cn/829.html
# 配置kubelet
# 生成证书
kubeadm alpha phase certs all --config kubeadm-master.config
# 生成kubelet相关配置文件
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config
kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config
kubeadm alpha phase kubeconfig kubelet --config kubeadm-master.config
# 启动kubelet
systemctl restart kubelet
# 部署 controlplane，即kube-apiserver, kube-controller-manager, kube-scheduler等各组件
# 生成controlplane的配置文件
kubeadm alpha phase kubeconfig all --config kubeadm-master.config
# 设置kubectl 默认配置文件
mkdir ~/.kube
cp /etc/kubernetes/admin.conf ~/.kube/config
# 添加etcd到集群中
# 向添加etcd集群中添加成员，此时如果使用kubectl命令不会出错
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP2_HOSTNAME} https://${CP2_IP}:2380
# 部署etcd静态pod
kubeadm alpha phase etcd local --config kubeadm-master.config
# 查看ectd节点
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member list
# 部署controlplane静态pod文件，kubelet会自动启动各组件
kubeadm alpha phase controlplane all --config kubeadm-master.config
# 生成添加节点命令
# 上传配置到configMap中
kubeadm alpha phase upload-config --config kubeadm-master.config
kubeadm alpha phase kubelet config upload --config kubeadm-master.config
# 配置节点的 TLS 引导
kubeadm alpha phase bootstrap-token all --config kubeadm-master.config
# 单独创建token
kubeadm alpha phase bootstrap-token create --config kubeadm-master.config
# 单独安装插件，通过 API server 安装内部 coreDNS 服务和 kube-proxy 插件组件
kubeadm alpha phase addon all --config kubeadm-master.config
# 标记为master节点，只是在此节点上添加了label和taint
kubeadm alpha phase mark-master --config kubeadm-master.config
# 清除docker容器，方便反复试验
docker ps -a | awk 'NR!=1{print $1}' | xargs docker rm -f
umount /var/lib/kubelet/pods/*/volumes/kubernetes.io~secret/*
rm -rf /var/lib/kubelet/
rm -rf /var/lib/etcd/
rm -rf /var/log/pods/
# 重新生成删除的配置文件
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config
kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config
# kubeconfig中的client-certificate-data字段值是证书的base64编码后的文本，还原为证书格式
cat /etc/kubernetes/admin.conf | grep client-certificate-data | awk -F ': ' '{print $2}' | base64 -d > /etc/kubernetes/pki/client.crt
部署kubernetes-dashboard
在master节点上操作
- 生成kubernetes-dashboard.yaml文件
# wget https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml
# 生成kubernetes-dashboard.yaml，把镜像地址改为国内阿里镜像，设置service的type为NodePort，nodePort为30001对集群外访问提供端口
# cat > kubernetes-dashboard.yaml << EOF
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------- Dashboard Secret ------------------- #
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque
---
# ------------------- Dashboard Service Account ------------------- #
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Role & Role Binding ------------------- #
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
rules:
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create"]
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create"]
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics from heapster.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard-minimal
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Deployment ------------------- #
kind: Deployment
apiVersion: apps/v1beta2
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: registry.cn-hangzhou.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:v1.10.0
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
# ------------------- Dashboard Service ------------------- #
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30001
selector:
k8s-app: kubernetes-dashboard
EOF
- 部署dashboard
kubectl create -f kubernetes-dashboard.yaml
查看Pod及Service运行情况
# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
coredns-6c66ffc55b-mrsqg 1/1 Running 0 16h 10.244.0.4 vpc-open-master001 <none>
coredns-6c66ffc55b-nl686 1/1 Running 0 16h 10.244.0.5 vpc-open-master001 <none>
etcd-vpc-open-master001 1/1 Running 0 16h 172.16.0.41 vpc-open-master001 <none>
etcd-vpc-open-master002 1/1 Running 0 15h 172.16.0.43 vpc-open-master002 <none>
etcd-vpc-open-master003 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kube-apiserver-vpc-open-master001 1/1 Running 0 16h 172.16.0.41 vpc-open-master001 <none>
kube-apiserver-vpc-open-master002 1/1 Running 0 15h 172.16.0.43 vpc-open-master002 <none>
kube-apiserver-vpc-open-master003 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kube-controller-manager-vpc-open-master001 1/1 Running 1 16h 172.16.0.41 vpc-open-master001 <none>
kube-controller-manager-vpc-open-master002 1/1 Running 0 15h 172.16.0.43 vpc-open-master002 <none>
kube-controller-manager-vpc-open-master003 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kube-flannel-ds-amd64-dzq5b 1/1 Running 0 14h 172.16.0.44 vpc-open-node002 <none>
kube-flannel-ds-amd64-hnf2f 1/1 Running 0 15h 172.16.0.45 vpc-open-node001 <none>
kube-flannel-ds-amd64-jwrks 1/1 Running 0 16h 172.16.0.43 vpc-open-master002 <none>
kube-flannel-ds-amd64-nxrx2 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kube-flannel-ds-amd64-zmwbx 1/1 Running 0 16h 172.16.0.41 vpc-open-master001 <none>
kube-proxy-97mfb 1/1 Running 0 16h 172.16.0.43 vpc-open-master002 <none>
kube-proxy-h8ctq 1/1 Running 0 16h 172.16.0.41 vpc-open-master001 <none>
kube-proxy-mw2l7 1/1 Running 0 15h 172.16.0.45 vpc-open-node001 <none>
kube-proxy-qxztv 1/1 Running 0 14h 172.16.0.44 vpc-open-node002 <none>
kube-proxy-vf2k6 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kube-scheduler-vpc-open-master001 1/1 Running 1 16h 172.16.0.41 vpc-open-master001 <none>
kube-scheduler-vpc-open-master002 1/1 Running 0 15h 172.16.0.43 vpc-open-master002 <none>
kube-scheduler-vpc-open-master003 1/1 Running 0 15h 172.16.0.42 vpc-open-master003 <none>
kubernetes-dashboard-85477d54d7-72bjj 1/1 Running 0 22s 10.244.3.2 vpc-open-node001 <none>
- 创建一个管理员用户
# 生成配置文件
cat > kubernetes-dashboard-rbac.yaml << EOF
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: admin
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
EOF
# 创建用户
kubectl create -f kubernetes-dashboard-rbac.yaml
-
登录dashboard
在浏览器中访问地址：https://47.110.19.11/ #这块在阿里云做负载均衡了
图片.png
# 运行命令查看secret列表
# kubectl get secrets -n kube-system
NAME TYPE DATA AGE
admin-token-9v6ql kubernetes.io/service-account-token 3 5m32s
attachdetach-controller-token-5b48q kubernetes.io/service-account-token 3 16h
bootstrap-signer-token-cd8pr kubernetes.io/service-account-token 3 16h
bootstrap-token-jjv5r2 bootstrap.kubernetes.io/token 6 16h
certificate-controller-token-8fwzr kubernetes.io/service-account-token 3 16h
clusterrole-aggregation-controller-token-6bw5h kubernetes.io/service-account-token 3 16h
coredns-token-z8fqw kubernetes.io/service-account-token 3 16h
cronjob-controller-token-bszlt kubernetes.io/service-account-token 3 16h
daemon-set-controller-token-k9x27 kubernetes.io/service-account-token 3 16h
default-token-n6wsp kubernetes.io/service-account-token 3 16h
deployment-controller-token-zd5hr kubernetes.io/service-account-token 3 16h
disruption-controller-token-clbvw kubernetes.io/service-account-token 3 16h
endpoint-controller-token-df2nv kubernetes.io/service-account-token 3 16h
expand-controller-token-xrq62 kubernetes.io/service-account-token 3 16h
flannel-token-btkhc kubernetes.io/service-account-token 3 16h
generic-garbage-collector-token-9klpk kubernetes.io/service-account-token 3 16h
horizontal-pod-autoscaler-token-vqjhc kubernetes.io/service-account-token 3 16h
job-controller-token-nph48 kubernetes.io/service-account-token 3 16h
kube-proxy-token-7vzj6 kubernetes.io/service-account-token 3 16h
kubernetes-dashboard-certs Opaque 0 9m50s
kubernetes-dashboard-key-holder Opaque 2 9m44s
kubernetes-dashboard-token-r9bbl kubernetes.io/service-account-token 3 9m51s
namespace-controller-token-jhxxt kubernetes.io/service-account-token 3 16h
node-controller-token-cwndg kubernetes.io/service-account-token 3 16h
persistent-volume-binder-token-2lv28 kubernetes.io/service-account-token 3 16h
pod-garbage-collector-token-m9kln kubernetes.io/service-account-token 3 16h
pv-protection-controller-token-s4hnb kubernetes.io/service-account-token 3 16h
pvc-protection-controller-token-r5cgj kubernetes.io/service-account-token 3 16h
replicaset-controller-token-4xv99 kubernetes.io/service-account-token 3 16h
replication-controller-token-tkcck kubernetes.io/service-account-token 3 16h
resourcequota-controller-token-pm258 kubernetes.io/service-account-token 3 16h
service-account-controller-token-p86s7 kubernetes.io/service-account-token 3 16h
service-controller-token-blf7k kubernetes.io/service-account-token 3 16h
statefulset-controller-token-b48r9 kubernetes.io/service-account-token 3 16h
token-cleaner-token-zh8rd kubernetes.io/service-account-token 3 16h
ttl-controller-token-x7kz6 kubernetes.io/service-account-token 3 16h
# 第一个为admin-token-9v6ql的secret，运行命令查看secret详情
# kubectl describe secret admin-token-9v6ql -n kube-system
Name: admin-token-9v6ql
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: admin
kubernetes.io/service-account.uid: 43ee5cd4-f2b6-11e8-b0d3-00163e199148
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi05djZxbCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjQzZWU1Y2Q0LWYyYjYtMTFlOC1iMGQzLTAwMTYzZTE5OTE0OCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.uj1kkIy39qvJFQO-Aagwd73oZNJT5sg9Xc0lDrcMtFEXkrYCwIbsV0ecF412EZ-gVVGZPBYVh1TRt2_Ffmv5XZd65F-E6O9V-eps0rhdlTuOHyipCLqrO2-4DOUMG4H3Pu6Lraz_fPekhjc__AgzKS882kBdJPLWM5vbRys6j3MtStNXoentWwNpUkiL_ezxHrI2Du4Xm6paZMw9O_JKRkDX-h_vx1ik6NyPD71-_6JJwZO3vx40ByN7xhM3aJ3eRyCf0yFnagAOg5SoLShjsJs_FTCeDM8PQuvxGiAoOT_y2rfiH75UJd-7fZNyZTKhnjm47yeDvjs3JbB3gBE5Pw
到dashboard登录界面，选择“令牌”，复制token的值到“输入令牌”文本框中，登录
图片.png
部署metrics-server
# 新建目录
# mkdir /data/kubernetes/metrics-server
# cd /data/kubernetes/metrics-server
# 下载文件到本地
# for file in aggregated-metrics-reader.yaml auth-delegator.yaml auth-reader.yaml metrics-apiservice.yaml metrics-server-deployment.yaml metrics-server-service.yaml resource-reader.yaml; do wget https://raw.githubusercontent.com/kubernetes-incubator/metrics-server/master/deploy/1.8%2B/$file;done
# 修改images地址
# cat aggregated-metrics-reader.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: system:aggregated-metrics-reader
labels:
rbac.authorization.k8s.io/aggregate-to-view: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules:
- apiGroups: ["metrics.k8s.io"]
resources: ["pods"]
verbs: ["get", "list", "watch"]
# cat auth-reader.yaml
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
# cat metrics-server-deployment.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: metrics-server
namespace: kube-system
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: metrics-server
namespace: kube-system
labels:
k8s-app: metrics-server
spec:
selector:
matchLabels:
k8s-app: metrics-server
template:
metadata:
name: metrics-server
labels:
k8s-app: metrics-server
spec:
serviceAccountName: metrics-server
volumes:
# mount in tmp so we can safely use from-scratch images and/or read-only containers
- name: tmp-dir
emptyDir: {}
containers:
- name: metrics-server
image: hub.huoban.com/k8s/metrics-server-amd64:v0.2.1
imagePullPolicy: IfNotPresent
command:
- /metrics-server
- --source=kubernetes.summary_api:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
volumeMounts:
- name: tmp-dir
mountPath: /tmp
# cat resource-reader.yaml
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- deployments
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
# cat auth-delegator.yaml
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
# cat metrics-apiservice.yaml
---
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
name: v1beta1.metrics.k8s.io
spec:
service:
name: metrics-server
namespace: kube-system
group: metrics.k8s.io
version: v1beta1
insecureSkipTLSVerify: true
groupPriorityMinimum: 100
versionPriority: 100
# cat metrics-server-service.yaml
---
apiVersion: v1
kind: Service
metadata:
name: metrics-server
namespace: kube-system
labels:
kubernetes.io/name: "Metrics-server"
spec:
selector:
k8s-app: metrics-server
ports:
- port: 443
protocol: TCP
targetPort: 443
# 启动
# kubectl apply -f ./
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
serviceaccount/metrics-server created
deployment.extensions/metrics-server created
service/metrics-server created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
# 查看pod状态
# kubectl get pods -n kube-system -o wide|grep metrics-server
metrics-server-667fc65b-mw6lh 1/1 Running 0 19s 10.244.3.5 vpc-open-node001 <none>
等待几分钟，然后查看收集的信息
# kubectl top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
vpc-open-master001 162m 8% 1546Mi 41%
vpc-open-master002 132m 6% 1471Mi 39%
vpc-open-master003 79m 3% 1427Mi 38%
vpc-open-node001 42m 2% 853Mi 23%
vpc-open-node002 26m 1% 759Mi 20%
# kubectl top pods -n kube-system
NAME CPU(cores) MEMORY(bytes)
coredns-6c66ffc55b-mrsqg 1m 12Mi
coredns-6c66ffc55b-nl686 1m 10Mi
etcd-vpc-open-master001 29m 273Mi
etcd-vpc-open-master002 18m 260Mi
etcd-vpc-open-master003 17m 264Mi
kube-apiserver-vpc-open-master001 29m 505Mi
kube-apiserver-vpc-open-master002 62m 472Mi
kube-apiserver-vpc-open-master003 16m 433Mi
kube-controller-manager-vpc-open-master001 27m 72Mi
kube-controller-manager-vpc-open-master002 0m 12Mi
kube-controller-manager-vpc-open-master003 0m 14Mi
kube-flannel-ds-amd64-dzq5b 1m 18Mi
kube-flannel-ds-amd64-hnf2f 1m 14Mi
kube-flannel-ds-amd64-jwrks 1m 14Mi
kube-flannel-ds-amd64-nxrx2 1m 14Mi
kube-flannel-ds-amd64-zmwbx 1m 12Mi
kube-proxy-97mfb 2m 12Mi
kube-proxy-h8ctq 2m 12Mi
kube-proxy-mw2l7 2m 11Mi
kube-proxy-qxztv 2m 12Mi
kube-proxy-vf2k6 2m 12Mi
kube-scheduler-vpc-open-master001 7m 13Mi
kube-scheduler-vpc-open-master002 6m 11Mi
kube-scheduler-vpc-open-master003 7m 11Mi
kubernetes-dashboard-85477d54d7-72bjj 3m 12Mi
metrics-server-667fc65b-mw6lh 1m 9Mi
部署ingress-nginx
# 下载官方提供的安装文件
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml
# 修改文件，网络模式修改成hostNetwork
# vim mandatory.yaml
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
data:
proxy-body-size: "200m"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "<election-id>-<ingress-class>"
# Here: "<ingress-controller-leader>-<nginx>"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
hostNetwork: true
serviceAccountName: nginx-ingress-serviceaccount
containers:
- name: nginx-ingress-controller
image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0
args:
- /nginx-ingress-controller
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
hostPort: 80
containerPort: 80
- name: https
hostPort: 443
containerPort: 443
volumeMounts:
- name: ssl
mountPath: /etc/ingress-controller/ssl
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
volumes:
- name: ssl
nfs:
path: /conf/global_sign_ssl
server: 0c54248c72-vok17.cn-hangzhou.nas.aliyuncs.com
---
apiVersion: v1
kind: Service
metadata:
name: ingress-nginx
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
- name: https
port: 443
targetPort: 443
protocol: TCP
selector:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
# kubectl apply -f mandatory.yaml
# 查看运行的pod
# kubectl get pods -n ingress-nginx -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-ingress-controller-5c4679db66-vgdxp 1/1 Running 0 24m 172.16.0.45 vpc-open-node001 <none>
nginx-ingress-controller-5c4679db66-wpjqz 1/1 Running 0 24m 172.16.0.44 vpc-open-node002 <none>
网友评论