ubuntu 18.04
docker 18.06
k8s       1.13.3

docker image:
# k8s v1.13.1
k8s.gcr.io/kube-apiserver:v1.13.1
k8s.gcr.io/kube-controller-manager:v1.13.1
k8s.gcr.io/kube-scheduler:v1.13.1
k8s.gcr.io/kube-proxy:v1.13.1
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.2.24
k8s.gcr.io/coredns:1.2.6

# calico v3.4.0
quay.io/calico/cni:v3.4.0
quay.io/calico/node:v3.4.0
quay.io/calico/kube-controllers:v3.4.0

# flannel
quay.io/coreos/flannel:v0.10.0-amd64

apt update

apt-get install ipvsadm docker-ce

apt install -y apt-transport-https

curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -

tee /etc/apt/sources.list.d/kubernetes.list <<EOF
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

apt update

/etc/keepalived/keepalived.conf

global_defs {
  router_id LVS_DEVEL
  script_user root
  enable_script_security
}


vrrp_script chk_process {
    script "killall -0 kube-apiserver"
    interval 1
    weight -20
    timeout 2
    fall 2
    rise 1
}
vrrp_script chk_tcp_port {
    script "/usr/bin/timeout 1 /bin/bash -c '</dev/tcp/127.0.0.1/6443'"
    interval 1
    weight -20
    timeout 2
    fall 2
    rise 1
}
vrrp_instance VI_1 {
  state BACKUP
  virtual_router_id 200
  advert_int 1
  authentication {
     auth_type PASS
     auth_pass password
  }
  track_script {
    chk_tcp_port
  }
  interface eth0
  nopreempt
  virtual_ipaddress {           # VIP
    10.10.6.30/24 dev eth0 label eth0:1     
  }
  priority 101                      # 101 on master, 99,97 on backup
  unicast_src_ip 10.10.6.31       # My IP
  unicast_peer {                                  # Peer IP
    10.10.6.32
    10.10.6.33
  }
}

# /etc/haproxy/haproxy.cfg

defaults
    log /dev/log local0
    #log 127.0.0.1 local0 debug
    mode tcp
    retries 1
    #option httplog                  # 日志类别http日志格式
    option tcplog
    option dontlognull            # 不记录健康检查的日志信息
    option redispatch            # serverid对应服务器宕掉后,强制定向到其他健康的服务器
    option abortonclose         #当服务器负载很高的话,自动结束到当前处理比较久的连接
    maxconn 2000000
    timeout connect 3s
    timeout client  6s
    timeout server  6s
listen admin_stats
    bind 0.0.0.0:1080
    mode http
    option httplog
    maxconn 1000
    stats refresh 30s
    stats uri /haproxy.status    # curl '127.0.0.1:1080/haproxy.status;csv'
    stats realm Haproxy

listen kube 
    bind 0.0.0.0:8443
    mode tcp
    retries 3
    option tcplog
    option tcp-check
    server kube-1 192.168.10.244:6443 maxconn 10240 check inter 1s
    server kube-2 192.168.10.245:6443 maxconn 10240 check inter 1s
    server kube-3 192.168.10.246:6443 maxconn 10240 check inter 1s

/bin/mkdir -p /var/lib/etcd/default
id etcd || useradd -d /var/lib/etcd -s /usr/sbin/nologin etcd
/bin/chown etcd:etcd -R /var/lib/etcd/

# apt-get install etcd -y

# download etcd version for k8s, 3.2.18+
ETCD_VER=v3.2.26
ETCD_VER=v3.3.13
test -f etcd-${ETCD_VER}-linux-amd64.tar.gz || wget -c https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar zxfv etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/local/bin --strip-components=1 --exclude=README* --exclude=Documentation

mkdir -p /etc/default
cat > /etc/default/etcd.env << EOF
ETCD1=10.10.6.31
ETCD2=10.10.6.32
ETCD3=10.10.6.33
EOF

mkdir -p /etc/systemd/system/etcd.service.d
cat > /etc/systemd/system/etcd.service.d/override.conf <<-"EOF" 
[Unit]
Description=etcd - highly-available key value store
Documentation=https://github.com/coreos/etcd
After=network.target
Wants=network-online.target

[Service]
Environment=DAEMON_ARGS=
#Environment=ETCD_NAME=%H
Environment=ETCD_DATA_DIR=/var/lib/etcd/default
EnvironmentFile=-/etc/default/%p.env
Type=notify
User=etcd
PermissionsStartOnly=true

Restart=on-abnormal
RestartSec=5
LimitNOFILE=1048576
TimeoutStartSec=0
StartLimitInterval=0
ExecStartPre=/bin/mkdir -p /var/lib/etcd/default
ExecStartPre=/bin/chown etcd:etcd -R /var/lib/etcd/

Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001
Environment=ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380
Environment=ETCD_INITIAL_CLUSTER_STATE=new
Environment=ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"

ExecStartPre=/bin/bash -c "/bin/systemctl set-environment DEFAULT_IPV4=$(ip route get 8.8.8.8 | head -1 | awk '{print $7}')"
ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${DEFAULT_IPV4}:2380"
ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD_ADVERTISE_CLIENT_URLS=http://${DEFAULT_IPV4}:2379,http://${DEFAULT_IPV4}:4001"


#ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD1=10.10.6.31"
#ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD2=10.10.6.32"
#ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD3=10.10.6.33"

ExecStartPre=/bin/bash -c "/bin/systemctl set-environment ETCD_INITIAL_CLUSTER=${ETCD1}=http://${ETCD1}:2380,${ETCD2}=http://${ETCD2}:2380,${ETCD3}=http://${ETCD3}:2380"

ExecStart=
ExecStart=/usr/local/bin/etcd $DAEMON_ARGS -name ${DEFAULT_IPV4}

[Install]
WantedBy=multi-user.target

EOF

test -f /etc/systemd/system/etcd.service || \cp -f /etc/systemd/system/etcd.service.d/override.conf  /etc/systemd/system/etcd.service
systemctl daemon-reload

systemctl restart etcd
systemctl status etcd

etcdctl member list
etcdctl cluster-health

# master node 
apt install -y kubeadm
#apt-mark hold kubelet kubeadm

add vip address

ip addr add 10.10.6.30/24 dev eth0

kubeadm list and pull images

kubeadm config images list

k8s.gcr.io/kube-apiserver:v1.13.3
k8s.gcr.io/kube-controller-manager:v1.13.3
k8s.gcr.io/kube-scheduler:v1.13.3
k8s.gcr.io/kube-proxy:v1.13.3
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.2.24
k8s.gcr.io/coredns:1.2.6

kubeadm config images pull

default config:

kubeadm config print init-defaults > kubeadm.conf

# init
cat > kube.yaml <<-EOF
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controllerManager:
  extraArgs:
    address: 0.0.0.0
scheduler:
  extraArgs:
    address: 0.0.0.0
networking:
  podSubnet: 30.0.0.0/10
  serviceSubnet: 10.96.0.0/12
apiServer:
  certSANs:
  - kubernetes
  - kubernetes.default
  - kubernetes.default.svc
  - kubernetes.default.svc.cluster
  - kubernetes.default.svc.cluster.local
  - 127.0.0.1
  - 30.0.0.1
  - 22.0.0.1
  - 33.0.0.1
  - 55.0.0.1
  - 10.96.0.1
  - 10.10.6.30
  - 10.10.6.31
  - 10.10.6.32
  - 10.10.6.33
controlPlaneEndpoint: "10.10.6.30:6443"
imageRepository: "gcr.azk8s.cn/google_containers"
etcd:
    external:
        endpoints:
        - http://10.10.6.31:2379
        - http://10.10.6.32:2379
        - http://10.10.6.33:2379
EOF


# docker pull images 
s=$(kubeadm config images list)
n=$(echo $s | sed -r 's,k8s.gcr.io/,,g')
#for i in $n;  do docker pull gcrxio/$i ; docker tag gcrxio/$i  k8s.gcr.io/$i ; done
for i in $n;  do docker pull gcr.azk8s.cn/google_containers/$i ; docker tag gcr.azk8s.cn/google_containers/$i  k8s.gcr.io/$i ; done
# test
kubeadm init \
--ignore-preflight-errors="Swap,NumCPU" \
--config kube.yaml \
--dry-run 

# run
kubeadm init  --config kube.yaml

# config
  #mkdir -p $HOME/.kube
  #sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  #sudo chown $(id -u):$(id -g) $HOME/.kube/config
mkdir -p /root/.kube
ln -s /etc/kubernetes/admin.conf  /root/.kube/config

# 不允许 master 执行 shedule
kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule-


# cni 
[https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/)

kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule-


# calico
curl https://docs.projectcalico.org/v3.7/manifests/calico-etcd.yaml -O
POD_CIDR="33.0.0.0/8" 
sed -i -e "s?192.168.0.0/16?$POD_CIDR?g" calico-etcd.yaml

change etcd_endpoints

kubectl apply -f calico-etcd.yaml

kubectl create -f https://raw.githubusercontent.com/cilium/cilium/v1.4/examples/kubernetes/1.13/cilium.yaml

# other master join
s="10.1.6.102 10.1.6.103"
for i in $s ; do 
rsync -Pav -e "ssh -p 60022" /etc/kubernetes/{pki,admin.conf} --exclude=pki/apiserver.*  root@${i}:/etc/kubernetes ;
done


# 新 mster 加入集群的命令：
echo "$(kubeadm token create --print-join-command)  --experimental-control-plane"
 

# 允许master部署pod allow master run pods
kubectl taint nodes --all node-role.kubernetes.io/master-
kubectl taint node k8s-master-1 node-role.kubernetes.io/master-

# 禁止master部署pod
kubectl taint nodes k8s-master-1 node-role.kubernetes.io/master=true:NoSchedule

# worker 加入集群的命令
swapoff
apt-get install -y docker-ce kubeadm

# docker pull images
s=$(kubeadm config images list)
n=$(echo $s | sed -r 's,k8s.gcr.io/,,g')
#for i in $n;  do docker pull gcrxio/$i ; docker tag gcrxio/$i  k8s.gcr.io/$i ; done
for i in $n;  do docker pull gcr.azk8s.cn/google_containers/$i ; docker tag gcr.azk8s.cn/google_containers/$i  k8s.gcr.io/$i ; done
echo $(kubeadm token create --print-join-command) 

# 集群正常后，保持默认配置：
kubectl taint nodes <node1> node-role.kubernetes.io/master=:NoSchedule


## 销毁集群
kubectl drain <node name> --delete-local-data --force --ignore-daemonsets
kubectl delete node <node name>
kubeadm reset

# iptables to ipvs
kubectl edit configmap kube-proxy -n kube-system

mode: ""   =>  mode: "ipvs" 
 


# api cert: 365 days
openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout | grep GMT

# cert renew
kubeadm alpha certs renew all

# https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/

# Mounted BPF filesystem mounted on all worker nodes
# mount bpffs /sys/fs/bpf -t bpf

cat <<EOF | sudo tee /etc/systemd/system/sys-fs-bpf.mount
[Unit]
Description=Cilium BPF mounts
Documentation=http://docs.cilium.io/
DefaultDependencies=no
Before=local-fs.target umount.target
After=swap.target

[Mount]
What=bpffs
Where=/sys/fs/bpf
Type=bpf

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable sys-fs-bpf.mount
systemctl start sys-fs-bpf.mount

cni install /w NetworkManager

mkdir -p /etc/NetworkManager/conf.d
test -d /etc/NetworkManager/conf.d && cat > /etc/NetworkManager/conf.d/cni.conf <<-"EOF"
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:flannel*
EOF

wget -O calico-etcd.yaml -c https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/etcd.yaml
wget -O calico-install.yaml https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml

# replace 10.96.232.136   with service_ip
# replace 192.168.0.0/16  with pod_cidr 

## flannel
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

metallb

kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.7.3/manifests/metallb.yaml

kubectl get pods -n metallb-system  -o wide

cat <<EOF | tee metallb-conf.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 10.10.6.240-10.10.6.250
EOF
kubectl apply -f metallb-conf.yaml

docker pull gcrxio/pause:3.1

cni.go:203] Unable to update cni config: No networks found in /etc/cni/net.d
Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
解决办法：
安装 cni 插件。

# nginx-deployment-service.yaml
---
apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2
kind: Deployment
metadata:
  name: nginx
spec:
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nginx
  replicas: 3 # tells deployment to run 1 pods matching the template
  template: # create pods using pod definition in this template
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: default
  labels:
    app: nginx
spec:
  externalTrafficPolicy: Local
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx
  type: LoadBalancer

# ingress 
[https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/static-ip](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/static-ip)


wget -O ingress-nginx-base.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml
kubectl apply -f ingress-nginx-base.yaml

kubect apply -f  ingress-nginx-loadbalancer.yaml
# ingress-nginx-loadbalancer.yaml
apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: LoadBalancer
  #loadBalancerIP: 10.10.6.243   # 指定固定地址, 不指定的话，会随机分配一个
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kubectl get pods -n ingress-nginx
kubectl get all -n ingress-nginx

istio

curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.1.7 sh -

cp istioctl to /usr/local/bin

n=2}'|awk -F'"' '{print n; do docker pull $i ; done

helm init
helm repo add istio.io https://storage.googleapis.com/istio-release/releases/1.1.7/charts/

升级

apt-get update && apt-get dist-upgrade -y

# docker pull images 
s=$(kubeadm config images list)
n=$(echo $s | sed -r 's,k8s.gcr.io/,,g')
#for i in $n;  do docker pull gcrxio/$i ; docker tag gcrxio/$i  k8s.gcr.io/$i ; done

for i in $n;  do docker pull gcr.azk8s.cn/google_containers/$i ; docker tag gcr.azk8s.cn/google_containers/$i  k8s.gcr.io/$i ; done