二、架构01-LVS的NAT模式用法

一、LVS实现NAT（nginx）

1、架构图

2、RS配置NGINX

[root@node02 ~]# yum install -y nginx 

[root@node02 ~]# vim /usr/share/nginx/html/test01.html

<h1>RS1,172.24.8.22</h1>

[root@node02 ~]# systemctl start nginx

[root@node02 ~]# ss -tnl

State      Recv-Q Send-Q                                                  Local Address:Port                                                                 Peer Address:Port

LISTEN     0      128                                                                 *:111                                                                             *:*

LISTEN     0      128                                                                 *:80                                                                              *:*

LISTEN     0      5                                                       192.168.122.1:53                                                                              *:*

LISTEN     0      128                                                                 *:22                                                                              *:*

LISTEN     0      128                                                         127.0.0.1:631                                                                             *:*

LISTEN     0      100                                                         127.0.0.1:25                                                                              *:*

LISTEN     0      128                                                         127.0.0.1:6010                                                                            *:*

LISTEN     0      128                                                         127.0.0.1:6011                                                                            *:*

LISTEN     0      128                                                                :::111                                                                            :::*

LISTEN     0      128                                                                :::80                                                                             :::*

LISTEN     0      128                                                                :::22                                                                             :::*

LISTEN     0      128                                                               ::1:631                                                                            :::*

LISTEN     0      100                                                               ::1:25                                                                             :::*

LISTEN     0      128                                                               ::1:6010                                                                           :::*

LISTEN     0      128                                                               ::1:6011                                                                           :::*

[root@node02 ~]# systemctl enable nginx

Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.

[root@node03 ~]# yum install -y nginx

[root@node03 ~]# vim /usr/share/nginx/html/test01.html

<h1>RS2,172.24.8.33</h1>

[root@node03 ~]# systemctl start nginx

[root@node03 ~]# ss -tnl

State      Recv-Q Send-Q                                                  Local Address:Port                                                                 Peer Address:Port

LISTEN     0      128                                                                 *:111                                                                             *:*

LISTEN     0      128                                                                 *:80                                                                              *:*

LISTEN     0      128                                                                 *:22                                                                              *:*

LISTEN     0      100                                                         127.0.0.1:25                                                                              *:*

LISTEN     0      128                                                                :::111                                                                            :::*

LISTEN     0      128                                                                :::80                                                                             :::*

LISTEN     0      128                                                                :::22                                                                             :::*

LISTEN     0      100                                                               ::1:25                                                                             :::*

[root@node03 ~]#systemctl enable nginx

Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.

3、配置VS

[root@node01 ~]# curl http://172.24.8.22/test01.html

<h1>RS1,172.24.8.22</h1>

[root@node01 ~]# curl http://172.24.8.33/test01.html

<h1>RS2,172.24.8.33</h1>

[root@node01 ~]# yum install -y ipvsadm

已加载插件：fastestmirror, langpacks

Loading mirror speeds from cached hostfile

epel/x86_64/metalink                                                                                                                                                   | 8.6 kB  00:00:00

* base: mirrors.huaweicloud.com

* epel: mirrors.aliyun.com

* extras: mirrors.huaweicloud.com

* updates: mirror.jdcloud.com

base                                                                                                                                                                   | 3.6 kB  00:00:00

epel                                                                                                                                                                   | 4.7 kB  00:00:00

extras                                                                                                                                                                 | 3.4 kB  00:00:00

updates                                                                                                                                                                | 3.4 kB  00:00:00

(1/2): epel/x86_64/updateinfo                                                                                                                                          | 957 kB  00:00:00

(2/2): epel/x86_64/primary_db                                                                                                                                          | 6.6 MB  00:00:01

软件包 ipvsadm-1.27-7.el7.x86_64 已安装并且是最新版本

无须任何处理

[root@node01 ~]#   ipvsadm -A -t 192.168.32.132:80 -s rr

[root@node01 ~]# ipvsadm -Ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 rr

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:80 -r 172.24.8.22 -m

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:80 -r 172.24.8.33 -m

[root@node01 ~]# ipvsadm -Ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 rr

  -> 172.24.8.22:80               Masq    1      0          0

  -> 172.24.8.33:80               Masq    1      0          0

[root@node01 ~]#

[root@node01 ~]#

[root@node01 ~]# sysctl -w net.ipv4.ip_forward=1

net.ipv4.ip_forward = 1

[root@node01 ~]# curl http://192.168.32.132/test01.html

^C

[root@node01 ~]# sysctl -w net.ipv4.ip_forward=1

net.ipv4.ip_forward = 1

[root@node01 ~]# ipvsadm -Ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 rr

  -> 172.24.8.22:80               Masq    1      0          1

  -> 172.24.8.33:80               Masq    1      0          1

4、验证

[40796.DESKTOP-Q4O3FV5] ➤ curl http://192.168.32.132/test01.html

<h1>RS1,172.24.8.22</h1>

                                                                                                                                                                                             ✔

──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

[2019-01-31 14:01.30]  ~

[40796.DESKTOP-Q4O3FV5] ➤ curl http://192.168.32.132/test01.html

<h1>RS2,172.24.8.33</h1>

                                                                                                                                                                                             ✔

──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

[2019-01-31 14:01.31]  ~

[40796.DESKTOP-Q4O3FV5] ➤ for i in {1..10}; do curl http://192.168.32.132/test01.html; done

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

5、示例

    示例01  修改算法权重

[root@node01 ~]# ipvsadm -e -t 192.168.32.132:80 -r 172.24.8.22 -m -w 2

[root@node01 ~]# ipvsadm -e -t 192.168.32.132:80 -r 172.24.8.33 -m -w 3

[root@node01 ~]# ipvsadm -Ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 rr

  -> 172.24.8.22:80               Masq    2      0          0

  -> 172.24.8.33:80               Masq    3      0          0

[root@node01 ~]# ipvsadm -Ln --rate

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port                 CPS    InPPS   OutPPS    InBPS   OutBPS

  -> RemoteAddress:Port

TCP  192.168.32.132:80                   0        0        0        0        0

  -> 172.24.8.22:80                      0        0        0        0        0

  -> 172.24.8.33:80                      0        0        0        0        0

[root@node01 ~]# ipvsadm -Ln --stats

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes

  -> RemoteAddress:Port

TCP  192.168.32.132:80                  23      103       56     6194     6150

  -> 172.24.8.22:80                     11       46       28     2791     3075

  -> 172.24.8.33:80                     12       57       28     3403     3075

[root@node01 ~]# ipvsadm -E -t 192.168.32.132:80 -s wrr

[root@node01 ~]# ipvsadm -Ln --stats

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes

  -> RemoteAddress:Port

TCP  192.168.32.132:80                  23      103       56     6194     6150

  -> 172.24.8.22:80                     11       46       28     2791     3075

  -> 172.24.8.33:80                     12       57       28     3403     3075

[root@node01 ~]# ipvsadm -Ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 wrr

  -> 172.24.8.22:80               Masq    2      0          0

  -> 172.24.8.33:80               Masq    3      0          0

验证

[40796.DESKTOP-Q4O3FV5] ➤ for i in {1..10}; do curl http://192.168.32.132/test01.html; done

<h1>RS2,172.24.8.33</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS2,172.24.8.33</h1>

<h1>RS1,172.24.8.22</h1>

示例02  删除一个RS

[root@node01 ~]# ipvsadm -d -t 192.168.32.132:80 -r 172.24.8.33

验证

[40796.DESKTOP-Q4O3FV5] ➤ for i in {1..10}; do curl http://192.168.32.132/test01.html; done

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

示例03 删除两个RS

[root@node01 ~]# ipvsadm -d -t 192.168.32.132:80 -r 172.24.8.33

验证

[root@node01 ~]# for i in {1..10}; do curl http://192.168.32.132/test01.html; done

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

curl: (7) Failed connect to 192.168.32.132:80; 拒绝连接

示例04 自定义找不到RS服务器页面

[root@node01 ~]# yum install -y nginx

[root@node01 ~]# vim /usr/share/nginx/html/index.html

<h1>Sorry</h1>

[root@node01 ~]# nginx -t

nginx: the configuration file /etc/nginx/nginx.conf syntax is ok

nginx: configuration file /etc/nginx/nginx.conf test is successful

[root@node01 ~]# nginx -s reload

[root@node01 ~]# curl http://172.24.8.11

<h1>Sorry</h1>

[root@node01 ~]# ipvsadm -e -t 192.168.32.132:80 -r 127.0.0.1 -g

验证

[40796.DESKTOP-Q4O3FV5] ➤ for i in {1..10}; do curl http://192.168.32.132/index.html; done

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

<h1>Sorry</h1>

[root@node01 ~]# ipvsadm -C  #清空配置项

[root@node01 ~]# ipvsadm -Ln #查看配置项

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

[root@node01 ~]# ipvsadm -A -t 192.168.32.132:80 -s rr

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:80 -r 172.24.8.22 -m

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:80 -r 172.24.8.33 -m

示例05 sh算法 源目标地址哈希

[root@node01 ~]# ipvsadm -E -t 192.168.32.132:80 -s sh

[root@node01 ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:80 sh

  -> 172.24.8.22:80               Masq    1      0          0

  -> 172.24.8.33:80               Masq    1      0          0

验证

[40796.DESKTOP-Q4O3FV5] ➤ for i in {1..10}; do curl http://192.168.32.132/test01.html; done

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

<h1>RS1,172.24.8.22</h1>

示例06 telnet服务

[root@node02 ~]# systemctl start telnet.socket

[root@node02 ~]# useradd centos ; echo "xiaoshi22" | passwd --stdin centos

更改用户 centos 的密码 。

passwd：所有的身份验证令牌已经成功更新。

[root@node03 ~]#  systemctl start telnet.socket

[root@node03 ~]# useradd centos ; echo "xiaoshi22" | passwd --stdin centos

更改用户 centos 的密码 。

passwd：所有的身份验证令牌已经成功更新。

[root@node01 ~]# ipvsadm -A -t 192.168.32.132:23 -s wlc

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:23 -r 172.24.8.22 -m -w 1

[root@node01 ~]# ipvsadm -a -t 192.168.32.132:23 -r 172.24.8.33 -m -w 2

[root@node01 ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:23 wlc

  -> 172.24.8.22:23               Masq    1      0          0

  -> 172.24.8.33:23               Masq    2      1          0

TCP  192.168.32.132:80 sh

  -> 172.24.8.22:80               Masq    1      0          0

  -> 172.24.8.33:80               Masq    1      0          0

[root@node01 ~]# ipvsadm -ln --stats（状态）

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port               Conns（连接数）   InPkts（报文数）  OutPkts  InBytes（字节数） OutBytes

  -> RemoteAddress:Port

TCP  192.168.32.132:23                   1       53       36     2281     1818

  -> 172.24.8.22:23                      0        0        0        0        0

  -> 172.24.8.33:23                      1       53       36     2281     1818

TCP  192.168.32.132:80                  40      200      160    12040    17320

  -> 172.24.8.22:80                     35      175      140    10535    15155

  -> 172.24.8.33:80                      5       25       20     1505     2165

[root@node01 ~]# ipvsadm -ln --rate（速率）

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port                 CPS    InPPS   OutPPS    InBPS   OutBPS

  -> RemoteAddress:Port

TCP  192.168.32.132:23                   0        0        0        0        0

  -> 172.24.8.22:23                      0        0        0        0        0

  -> 172.24.8.33:23                      0        0        0        0        0

TCP  192.168.32.132:80                   0        0        0        0        0

  -> 172.24.8.22:80                      0        0        0        0        0

  -> 172.24.8.33:80                      0        0        0        0        0

[root@node01 ~]# ipvsadm -ln -c（查看连接）

IPVS connection entries

pro expire state       source             virtual            destination

TCP 09:39  ESTABLISHED 192.168.32.1:50890 192.168.32.132:23  172.24.8.33:23

示例07 保存配置

[root@node01 ~]# ipvsadm -S

-A -t node01:telnet -s wlc

-a -t node01:telnet -r 172.24.8.22:telnet -m -w 1

-a -t node01:telnet -r 172.24.8.33:telnet -m -w 2

-A -t node01:http -s sh

-a -t node01:http -r 172.24.8.22:http -m -w 1

-a -t node01:http -r 172.24.8.33:http -m -w 1

[root@node01 ~]# ipvsadm -S -n

-A -t 192.168.32.132:23 -s wlc

-a -t 192.168.32.132:23 -r 172.24.8.22:23 -m -w 1

-a -t 192.168.32.132:23 -r 172.24.8.33:23 -m -w 2

-A -t 192.168.32.132:80 -s sh

-a -t 192.168.32.132:80 -r 172.24.8.22:80 -m -w 1

-a -t 192.168.32.132:80 -r 172.24.8.33:80 -m -w 1

[root@node01 ~]# ipvsadm -S -n > /etc/sysconfig/ipvsadm-config

anaconda-ks.cfg  .bash_history    .bash_profile    .cache/          .cshrc           .rnd             .tcshrc

.ansible/        .bash_logout     .bashrc          .config/         .pki/            .ssh/            .viminfo

[root@node01 ~]# ipvsadm -S -n > /etc/sysconfig/ipvsadm

[root@node01 ~]# ipvsadm -C

[root@node01 ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

[root@node01 ~]# ipvsadm -R < /etc/sysconfig/ipvsadm

[root@node01 ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:23 wlc

  -> 172.24.8.22:23               Masq    1      0          0

  -> 172.24.8.33:23               Masq    2      1          0

TCP  192.168.32.132:80 sh

  -> 172.24.8.22:80               Masq    1      0          0

  -> 172.24.8.33:80               Masq    1      0          0

[root@node01 ~]# ipvsadm -C

[root@node01 ~]# systemctl restart ipvsadm.service

[root@node01 ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.32.132:23 wlc

  -> 172.24.8.22:23               Masq    1      0          0

  -> 172.24.8.33:23               Masq    2      1          0

TCP  192.168.32.132:80 sh

  -> 172.24.8.22:80               Masq    1      0          0

  -> 172.24.8.33:80               Masq    1      0          0

[root@node01 ~]# rm /etc/sysconfig/ip

ip6tables-config  iptables-config   ipvsadm           ipvsadm-config

[root@node01 ~]# rm /etc/sysconfig/ip

ip6tables-config  iptables-config   ipvsadm           ipvsadm-config

[root@node01 ~]# rm /etc/sysconfig/ipvsadm

rm：是否删除普通文件 "/etc/sysconfig/ipvsadm"？y

[root@node01 ~]# systemctl stop  ipvsadm.service

[root@node01 ~]# more  /etc/sysconfig/ipvsadm

-A -t 192.168.32.132:23 -s wlc

-a -t 192.168.32.132:23 -r 172.24.8.22:23 -m -w 1

-a -t 192.168.32.132:23 -r 172.24.8.33:23 -m -w 2

-A -t 192.168.32.132:80 -s sh

-a -t 192.168.32.132:80 -r 172.24.8.22:80 -m -w 1

-a -t 192.168.32.132:80 -r 172.24.8.33:80 -m -w 1

收工