安装nginx并配置成L4代理

安装节点

node1 node4

安装nginx

配置nginx

在nginx配置文件末尾添加，以下内容，stream是L4反向代理：

stream {
    upstream kube-apiserver {
        server 172.16.6.182:6443    max_fails=3 fail_timeout=30s;
        server 172.16.6.183:6443    max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}
[root@node4 yum.repos.d]# systemctl restart nginx
[root@node4 yum.repos.d]# systemctl enable nginx
# 验证
[root@node4 yum.repos.d]# curl 127.0.0.1:7443
Client sent an HTTP request to an HTTPS server.

安装keepalived

• 安装

yum install keepalived -y

• keepalived监听脚本

vim /etc/keepalived/check_port.sh
#!/bin/bash
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
        PORT_PROCESS=`netstat -lntp | grep $CHK_PORT | wc -l `
        if [ $PORT_PROCESS -eq 0 ];then
                echo "Port $CHEK_PORT is not userd.end."
                exit 1
        fi
else
        echo "Check port cant be empty!"
fi

# chmod +x /etc/keepalived/check_port.sh

## keepalived主配置文件
+ 需要修改的为 ip地址  网卡。
+ keepalived主节点中，必须加上 nopreempt，nopreempt是非抢占机制。
+ 在生产中vip是不能轻易动的，vip飘逸属于重大生产事故。

[root@node1 yum.repos.d]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 172.16.6.181
}

vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}

vrrp_instance VI_1 {
state MASTER
interface eth1
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 172.16.6.181
nopreempt
authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
172.16.6.180
}
}

## keepalived 从

[root@node4 yum.repos.d]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 172.16.6.184
}

vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state BACKUP
interface eth1
virtual_router_id 251
mcast_src_ip 172.16.6.184
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
172.16.6.180
}
}

+ 查看

[root@node1 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:d0:c1:d2 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.181/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet 172.16.6.180/32 scope global eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fed0:c1d2/64 scope link
valid_lft forever preferred_lft forever

[root@node4 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:99:55:60 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.184/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe99:5560/64 scope link
valid_lft forever preferred_lft forever

## 测试

+ 主节点停掉nginx，vip漂移到从节点。当主节点nginx又起来之后vip不会自动飘回来
+ 原因是主节点配置了nopreempt，非抢占机制

主节点

[root@node1 yum.repos.d]# nginx -s stop
[root@node1 yum.repos.d]# netstat -lntp | grep 7443
[root@node1 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:d0:c1:d2 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.181/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fed0:c1d2/64 scope link
valid_lft forever preferred_lft forever

从节点查看ip

[root@node4 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:99:55:60 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.184/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet 172.16.6.180/32 scope global eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe99:5560/64 scope link
valid_lft forever preferred_lft forever

主节点又启动了nginx，vip不会自动回来

[root@node1 yum.repos.d]# nginx
[root@node1 yum.repos.d]# netstat -lntp | grep 7443
tcp 0 0 0.0.0.0:7443 0.0.0.0:* LISTEN 32318/nginx: master

主节点查看vip

[root@node1 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:d0:c1:d2 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.181/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fed0:c1d2/64 scope link
valid_lft forever preferred_lft forever

+ vip飘回来需要做好万全的准备并且在流量低谷的时候，还需要多方确认后再操作。

[root@node1 yum.repos.d]# netstat -lntp | grep 7443
tcp 0 0 0.0.0.0:7443 0.0.0.0:* LISTEN 32318/nginx: master
[root@node1 yum.repos.d]# systemctl restart keepalived
[root@node1 yum.repos.d]# ip add show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 08:00:27:d0:c1:d2 brd ff:ff:ff:ff:ff:ff
inet 172.16.6.181/24 brd 172.16.6.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet 172.16.6.180/32 scope global eth1
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fed0:c1d2/64 scope link
valid_lft forever preferred_lft forever

[root@node4 yum.repos.d]# systemctl restart keepalived