keepalived+nginx高可用搭建

keepalived + nginx负载均衡搭建

1.实验环境

ip	主机名	系统	作用
192.168.1.61	mytest1	centos7.5	keepalive_master+nginx
192.168.1.62	mytest2	centos7.5	keepalive_backup+nginx
192.168.1.63	mytest3	centos7.5	web1服务器（nginx实现）
192.168.1.64	mytest4	centos7.5	web2服务器（nginx实现）
192.168.1.99	vip		用于访问的vip

2.解析图

keepalived_nginx.png

3.系统设置

为了测试方便，直接关掉防火墙

systemctl stop firewalld
systemctl disable firewalld

4.keepalive+nginx服务器的安装和配置

4.1.keepalive安装和配置

yum安装，一般yum源都有

yum install -y keepalived

keepalived的设定，这里为了管理方便，建议采用include方式，以后添加vip只要在目录里添加相关vip文件即可

master机上的命令

# 备份原文件
cp -ip /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
# 清空原配置
cat /dev/null>/etc/keepalived/keepalived.conf
# 写普通的配置，用了include方法，指定/etc/keepalived/vip目录为vip文件存放地址
cat <<EOF>/etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state MASTER
    interface eno16777736
    virtual_router_id 99
    priority 100
    advert_int 1
       authentication {
          auth_type PASS
          auth_pass 1111
       }
       include /etc/keepalived/vip/*.vip
}
EOF
# 创建vip目录
[ -d /etc/keepalived/vip ] || mkdir -p /etc/keepalived/vip
# 写vip文件
cat <<EOF>/etc/keepalived/vip/192.168.1.99.vip
virtual_ipaddress {
    192.168.1.99
}
EOF
# 启动和加入开机启动
systemctl start keepalived && systemctl enable keepalived

backup机上的设定

# 备份原文件
cp -ip /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
# 清空原配置
cat /dev/null>/etc/keepalived/keepalived.conf
# 写普通的配置，用了include方法，指定/etc/keepalived/vip目录为vip文件存放地址
cat <<EOF>/etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state MASTER
    interface enp0s8
    virtual_router_id 99
    priority 80
    advert_int 1
       authentication {
          auth_type PASS
          auth_pass 1111
       }
       include /etc/keepalived/vip/*.vip
}
EOF
# 创建vip目录
[ -d /etc/keepalived/vip ] || mkdir -p /etc/keepalived/vip
# 写vip文件
cat <<EOF>/etc/keepalived/vip/192.168.1.99.vip
virtual_ipaddress {
    192.168.1.99
}
EOF
# 启动和加入开机启动
systemctl start keepalived && systemctl enable keepalived

配置文件说明

vrrp_instance VI_1 {
    state MASTER #说明keepalived中的角色，backup机请写BACKUP
    interface enp0s8 #要使用的网络设备，请ip a确认使用网卡
    virtual_router_id 99 #VRRP组名，两个节点设置一样，以指明各个节点同属一VRRP组，建议不要使用默认的51
    priority 100 #主节点的优先级，数值在1~254，注意从节点必须比主节点的优先级别低
    advert_int 1 #组播信息发送间隔，两个节点需一致
       authentication { #设置验证信息，两个节点需一致
          auth_type PASS
          auth_pass 1111
       }
       include /etc/keepalived/vip/*.vip
}

验证vip

# ping vip值
ping 192.168.1.99
# 查看网络信息是否有vip，如下
[root@mytest1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:9b:81:77 brd ff:ff:ff:ff:ff:ff
    inet 10.0.2.15/24 brd 10.0.2.255 scope global noprefixroute dynamic enp0s3
       valid_lft 82937sec preferred_lft 82937sec
    inet6 fe80::13db:1226:b988:8892/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:4f:cd:c0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.61/24 brd 192.168.1.255 scope global noprefixroute enp0s8
       valid_lft forever preferred_lft forever
    inet 192.168.1.99/32 scope global enp0s8
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe4f:cdc0/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever

4.2 本地回路

1.为什么要添加本地回路，因为nginx监听地址为vip，而vip只在一台机器上，所以为了其他机器能够监听vip，所以添加本地回路。

ip addr add 192.168.1.99/32 dev lo
# 并且为了系统重启后不消失，加入开机启动的rc.local或者其他方法
echo "ip addr add 192.168.1.99/32 dev lo">>/etc/rc.local && chmod 755 /etc/rc.local

4.3.nginx配置和转发

nginx一般yum源没有，以下是源码安装方法

安装编译环境

yum -y install make zlib zlib-devel gcc-c++ libtool  openssl openssl-devel

下载nginx源码包并安装，这里要用到upstream这个模块，地址为http://nginx.org/en/download.html

要注意这个nginx是只做转发，偷懒安装命令只加了--with-stream，如果要作为http，有些东西是少的

tar -xzvf nginx-1.15.6.tar.gz
cd nginx-1.15.6
./configure --with-stream && make && make install
ln -s /usr/local/nginx/sbin/nginx /usr/bin/nginx
nginx -v

配置文件修改

这里nginx只做转发，其他的不要，跟keepalived一样为了管理方便，用include方法

cat <<EOF>/usr/local/nginx/conf/nginx.conf
worker_processes 1;
worker_rlimit_nofile 200000;
events {
    worker_connections 120000;
}
stream {
    include /etc/nginx/stream/*.stream;
}
EOF

[ -d /etc/nginx/stream ] || mkdir -p /etc/nginx/stream

cat <<EOF>/etc/nginx/stream/192.168.1.99_80.stream
upstream 192.168.1.99_80 {
    server 192.168.1.63:80 weight=10;
    server 192.168.1.64:80 weight=10;
}
server {
    listen 192.168.1.99:80;
    proxy_connect_timeout 1s;
    proxy_timeout 30s;
    proxy_pass 192.168.1.99_80;
}
EOF

语法检查和启动

nginx -t
# 如果没有问题，显示效果如下
[root@mytest1 ~]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
# 启动nginx
nginx

# 顺便说一句，请添加开机启动nginx
echo 'nginx'>>/etc/rc.local && chmod 755 /etc/rc.local

5.验证

验证方式在于keepalived是否高可用

5.1 实验一

关掉mytest1的keepalived，看vip是否跳到mytest2上

# mytest1执行
systemctl stop keepalived
# mytest2验证
ip a
# 第五台主机用curl验证vip是否能访问web服务
curl 192.168.1.99:80

5.2 实验二

启动mytest1的keepalived，看vip是否回到mytest1上

# mytest1执行
systemctl start keepalived
ip a
# 第五台主机用curl验证vip是否能访问web服务
curl 192.168.1.99:80

后端服务是否轮循

这个不用多说，第五台服务器跑个循环

for i in `seq 1 4`;do curl 192.168.1.99;done
# 正常结果如下，我这里后端是两台http服务器，为了区分index写了服务器名
szlyq@szlyq-PC:~$ for i in `seq 1 4`;do curl 192.168.1.99;done
myhostname is mytest3
myhostname is mytest4
myhostname is mytest3
myhostname is mytest4