网络丢包排查三

一 负载均衡

发现我们设置负载均衡后，网卡队列的数字相差还是有些大：

队列数字不均衡

所以重新设置下flow-hash

#ethtool -N eno2 rx-flow-hash tcp4 sdfn
#ethtool -N eno2 rx-flow-hash udp4 sdfn

[root@localhost ~]# ethtool -n eno2 rx-flow-hash udp4
UDP over IPV4 flows use these fields for computing Hash flow key:
IP SA
IP DA
L4 bytes 0 & 1 [TCP/UDP src port]
L4 bytes 2 & 3 [TCP/UDP dst port]

[root@localhost ~]# ethtool -n eno2 rx-flow-hash tcp4
TCP over IPV4 flows use these fields for computing Hash flow key:
IP SA
IP DA
L4 bytes 0 & 1 [TCP/UDP src port]
L4 bytes 2 & 3 [TCP/UDP dst port]

二 网卡优化设置

i40e驱动的网卡优化，驱动更新到最新

rmmod i40e && modprobe i40e
ifconfig eth1 down
/usr/local/sbin/ethtool -L eth1 combined 16
/usr/local/sbin/ethtool -K eth1 rxhash on
/usr/local/sbin/ethtool -K eth1 ntuple on
ifconfig eth1 up
/usr/local/sbin/ethtool -X eth1 hkey 6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A equal 16
/usr/local/sbin/ethtool -A eth1 rx off
/usr/local/sbin/ethtool -C eth1 adaptive-rx off adaptive-tx off rx-usecs 125
/usr/local/sbin/ethtool -G eth1 rx 1024

多核心的软中断查看：

watch -d "/bin/cat /proc/softirqs | /usr/bin/awk 'NR == 1{printf \"%-15s %-15s %-15s %-15s %-15s\n\",\" \",\$1,\$2,\$3,\$4}; NR > 1{printf \"%-15s %-15s %-15s %-15s %-15s\n\",\$1,\$2,\$3,\$4,\$5}'"

sar -u ALL -P ALL 1

RPS调优脚本：

#!/bin/bash  
# Enable RPS (Receive Packet Steering)  
      
rfc=4096  
cc=$(grep -c processor /proc/cpuinfo)  
rsfe=$(echo $cc*$rfc | bc)  
sysctl -w net.core.rps_sock_flow_entries=$rsfe  
for fileRps in $(ls /sys/class/net/eth*/queues/rx-*/rps_cpus)  
do
    echo fff > $fileRps  
done
      
for fileRfc in $(ls /sys/class/net/eth*/queues/rx-*/rps_flow_cnt)  
do
    echo $rfc > $fileRfc  
done
      
tail /sys/class/net/eth*/queues/rx-*/{rps_cpus,rps_flow_cnt}

·

#netmap参数
cat /sys/module/netmap/parameters/buf_curr_num

简单代码参考:

#include <stdio.h>
#include <poll.h>
 
#define NETMAP_WITH_LIBS
#include <net/netmap_user.h>
 
unsigned long pps = 0;
 
static void receive_packets(struct netmap_ring *ring)
{
    int i;
    char *buf;
 
    while (!nm_ring_empty(ring)) {
        i   = ring->cur;
        buf = NETMAP_BUF(ring, ring->slot[i].buf_idx);
        pps++;

        ring->head = ring->cur = nm_ring_next(ring, i); 
    }
}
 
int main(void)
{
    struct nm_desc *d;
    struct pollfd fds;
    struct netmap_ring *ring;
    int i; 
 
    d = nm_open("netmap:eth1", NULL, 0, 0); 
  
    fds.fd     = d->fd;
    fds.events = POLLIN;
 
    while (1) {
        if (poll(&fds, 1, 1) < 0) {
            perror("poll()");
            exit(1);
        }
 
        for (i = d->first_rx_ring; i <= d->last_rx_ring; i++) {
            ring = NETMAP_RXRING(d->nifp, i);
            receive_packets(ring);
        }
    }

    return 0;
}