kube-ovn underlay vlan 模式默认网络使用

作者: cloudFans | 来源:发表于2021-12-21 14:02 被阅读0次

kube-ovn underlay vlan 模式默认网络使用
kube-ovn 1.7 偶现一个node上的pod 无法和其他
kube-ovn 1.7.1 ping丢包率简测
OpenStack neutron相关概念阐述
docker网络
vtp协议
简单区分Vmware的三种网络连接模式(bridged、NAT、
linux.network.overlay
neutron ovn 中的路由对照 kube-ovn
Docker五种网络管理

关于lb一般要支持以下功能：

健康检查
后端热更新
protocol proxy

在当前服务架构中，负载均衡设备(或者代理设备)常被使用，当我们使用了负载均衡/代理设备之后，服务器端通常看到的是代理设备的IP地址。在HTTP协议下，通常在代理设备上会追加X-Forward-For头，用于存放原始客户端的IP地址，这样服务器端可以看到原始客户端IP。但是在UDP/TCP四层情况下，不存在可以追加Client IP信息的字段。Proxy Protocol是可以用来解决UDP/TCP四层情况下服务器端获取IP地址的问题。他工作在4层和7层之间，追加了一个Proxy Protocol头部，用于记录客户端地址信息。

代理协议(Proxy protocol)，是HAProxy的作者Willy Tarreau于2010年开发和设计的一个Internet协议，通过为tcp添加一个很小的头信息，来方便的传递客户端信息（协议栈、源IP、目的IP、源端口、目的端口等)，在网络情况复杂又需要获取用户真实IP时非常有用。

代理协议分为V1和V2两个版本，V1是人类易读的，V2是二进制格式的。

注意点：
Proxy protocol需要两个角色sender和receiver,
sender在与receiver之间建立连接后，会先发送一个带有客户信息的tcp header,
因为更改了tcp协议，需receiver也支持Proxy protocol，否则不能识别tcp包头，导致无法成功建立连接。

二、 kube-ovn 关于lb的使用

ovn 关于lb的设计有两种，使用方式：https://hustcat.github.io/ovn-lb-practice/
，

router-lb
switch-lb

# 查看kube-ovn lb

kubectl ko nbctl  list load_balancer 

[root@deployer ~]# kubectl ko nbctl  list load_balancer 
_uuid               : fb4e6fb8-94c1-401f-b7a7-a65aaabfc334
external_ids        : {}
health_check        : []
ip_port_mappings    : {}
name                : cluster-udp-session-loadbalancer
protocol            : udp
selection_fields    : [ip_src]
vips                : {}

_uuid               : 25dc76d0-9691-42a3-8a0a-4c5026b3ac0c
external_ids        : {}
health_check        : []
ip_port_mappings    : {}
name                : cluster-udp-loadbalancer
protocol            : udp
selection_fields    : []
vips                : {"10.96.0.3:53"="10.120.36.26:53,10.120.36.32:53"}

_uuid               : 7471e2d9-3a59-4969-a6c5-069e7f21982e
external_ids        : {}
health_check        : []
ip_port_mappings    : {}
name                : cluster-tcp-loadbalancer
protocol            : tcp
selection_fields    : []
vips                : {"10.100.178.221:9115"="10.120.37.111:9115", "10.100.31.63:9000"="10.120.36.213:9000", "10.101.129.124:8080"="10.120.36.42:8080", "10.101.167.157:10903"="10.120.37.109:10903", "10.101.167.157:10904"="10.120.37.109:10904", "10.101.202.243:9402"="10.120.36.91:9402", "10.101.212.65:9200"="10.120.36.111:9200", "10.101.212.65:9300"="10.120.36.111:9300", "10.101.62.245:8080"="10.120.36.14:8080", "10.101.67.230:9997"="10.120.36.124:9997", "10.102.250.80:9998"="10.120.36.130:9998", "10.103.114.69:443"="10.120.36.98:443", "10.103.166.184:3000"="10.120.36.125:3000", "10.103.177.135:80"="10.120.36.2:80,10.120.36.4:80", "10.103.210.215:9476"="10.120.36.13:9476", "10.103.24.98:10665"="10.120.32.245:10665,10.120.33.146:10665,10.120.33.245:10665,10.120.33.72:10665,10.120.34.229:10665,10.120.34.53:10665,10.120.34.95:10665,10.120.35.101:10665,10.120.35.103:10665,10.120.35.189:10665,10.120.35.225:10665,10.120.35.243:10665,10.120.35.93:10665", "10.103.71.163:9999"="10.120.36.84:9999,10.120.36.85:9999,10.120.36.86:9999", "10.104.103.122:5601"="10.120.36.18:5601", "10.104.104.158:10090"="10.120.36.145:10090", "10.104.181.95:8895"="10.120.36.132:8895", "10.104.181.95:9996"="10.120.36.132:9996", "10.104.198.68:5044"="10.120.36.44:5044,10.120.36.48:5044", "10.104.198.68:9304"="10.120.36.44:9304,10.120.36.48:9304", "10.104.198.68:9600"="10.120.36.44:9600,10.120.36.48:9600", "10.104.214.130:10661"="10.120.36.17:10661,10.120.36.27:10661,10.120.36.28:10661", "10.104.217.37:9200"="10.120.36.190:9200,10.120.36.191:9200,10.120.36.192:9200", "10.104.217.37:9300"="10.120.36.190:9300,10.120.36.191:9300,10.120.36.192:9300", "10.104.254.192:6643"="10.120.33.146:6643", "10.105.117.70:10000"="10.120.36.139:10000", "10.105.194.111:6641"="10.120.33.146:6641", "10.105.237.81:443"="10.120.36.94:10250", "10.105.48.218:5601"="10.120.36.181:5601", "10.106.138.162:2181"="10.120.36.11:2181,10.120.36.19:2181,10.120.36.8:2181", "10.106.17.113:9997"="10.120.36.50:9997", "10.106.172.124:9093"="10.120.36.22:9093,10.120.36.33:9093,10.120.36.34:9093", "10.106.97.109:9999"="10.120.36.117:9999,10.120.36.35:9999,10.120.36.36:9999", "10.107.232.157:9092"="10.120.36.22:9092,10.120.36.33:9092,10.120.36.34:9092", "10.107.26.156:9476"="10.120.36.40:9476", "10.109.151.20:9200"="10.120.36.151:9200,10.120.36.152:9200,10.120.36.153:9200", "10.109.151.20:9300"="10.120.36.151:9300,10.120.36.152:9300,10.120.36.153:9300", "10.109.182.170:80"="10.120.36.37:9111", "10.110.194.192:8080"="10.120.36.164:8080,10.120.36.170:8080,10.120.36.20:8080,10.120.36.21:8080,10.120.36.239:8080,10.120.36.24:8080,10.120.36.25:8080,10.120.36.29:8080,10.120.36.30:8080,10.120.36.31:8080,10.120.37.54:8080,10.120.37.60:8080,10.120.37.62:8080", "10.110.245.115:8443"="10.120.36.53:8443", "10.110.52.142:9999"="10.120.36.128:9999", "10.110.93.238:6642"="10.120.33.146:6642", "10.111.248.135:443"="10.120.37.110:6443", "10.111.46.107:10002"="10.120.36.127:10002", "10.96.0.1:443"="10.120.33.146:6443,10.120.34.53:6443,10.120.35.101:6443", "10.96.0.37:9640"="10.120.36.141:9640", "10.96.0.3:53"="10.120.36.26:53,10.120.36.32:53", "10.96.0.3:9153"="10.120.36.26:9153,10.120.36.32:9153", "10.96.40.232:3000"="10.120.36.131:3000", "10.96.90.209:8895"="10.120.36.135:8895", "10.96.90.209:9996"="10.120.36.135:9996", "10.97.100.253:443"="10.120.37.91:8443", "10.97.134.11:10913"="10.120.37.124:10913", "10.97.15.102:9999"="10.120.36.134:9999", "10.97.155.49:80"="10.120.36.38:8080", "10.97.236.187:10660"="10.120.33.245:10660,10.120.33.72:10660,10.120.34.95:10660", "10.97.51.229:5601"="10.120.36.45:5601", "10.97.72.9:2222"="10.120.36.119:2222", "10.97.72.9:5000"="10.120.36.119:5000", "10.98.104.134:443"="10.120.36.12:443,10.120.36.5:443,10.120.36.7:443", "10.98.104.134:80"="10.120.36.12:80,10.120.36.5:80,10.120.36.7:80", "10.98.104.134:8080"="10.120.36.12:8080,10.120.36.5:8080,10.120.36.7:8080", "10.98.153.50:25482"="10.120.36.120:25482", "10.98.153.50:8123"="10.120.36.120:8123", "10.98.189.7:10903"="10.120.37.108:10903", "10.98.189.7:10904"="10.120.37.108:10904", "10.98.219.15:9998"="10.120.36.123:9998", "10.98.81.52:10902"="10.120.37.48:10902", "10.99.65.148:9200"="10.120.36.55:9200", "10.99.65.148:9300"="10.120.36.55:9300", "10.99.7.73:10002"="10.120.36.102:10002", "10.99.89.196:8443"="10.120.37.112:8443", "10.99.89.196:9443"="10.120.37.112:9443"}

_uuid               : 52b2a1b5-fe81-49bc-b68a-1282bac29e45
external_ids        : {}
health_check        : []
ip_port_mappings    : {}
name                : cluster-tcp-session-loadbalancer
protocol            : tcp
selection_fields    : [ip_src]
vips                : {"10.107.140.182:8080"="10.120.36.54:8080"}


来自于同一个源ip的访问应该由同一个lb 后端server来提供服务，比如访问jumperver guacamole

[root@deployer ~]# kubectl get pod -A -o wide | grep 10.120.36.54
jumpserver      guacamole-7db557fccb-482b6                       1/1     Running     1          98d     10.120.36.54    inner-prod-common-c6-4xl-asg-ofg-2or-s57-server-m3e


[root@deployer ~]# kubectl get svc -A -o wide | grep guacamole
jumpserver      guacamole                                 ClusterIP      10.107.140.182   <none>          8080/TCP                                       83d    app=guacamole,release=guacamole

[root@deployer ~]# kubectl get svc -n jumpserver      guacamole -o yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: '{"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"guacamole","release":"guacamole"},"name":"guacamole","namespace":"jumpserver"},"spec":{"ports":[{"name":"guacamole","port":8080,"protocol":"TCP","targetPort":"guacamole"}],"publishNotReadyAddresses":false,"selector":{"app":"guacamole","release":"guacamole"},"sessionAffinity":"ClientIP","type":"ClusterIP"}}'
  creationTimestamp: "2021-09-28T06:22:48Z"
  labels:
    app: guacamole
    release: guacamole
  name: guacamole
  namespace: jumpserver
  resourceVersion: "17044007"
  uid: 401c5005-e95b-45cd-ae9a-b30374c971e4
spec:
  clusterIP: 10.107.140.182
  clusterIPs:
  - 10.107.140.182
  ports:
  - name: guacamole
    port: 8080
    protocol: TCP
    targetPort: guacamole
  selector:
    app: guacamole
    release: guacamole
  sessionAffinity: ClientIP
  sessionAffinityConfig:  // 会话保持
    clientIP:
      timeoutSeconds: 10800
  type: ClusterIP
status:
  loadBalancer: {}

ovn lb的初始化


func (c Client) AddLbToLogicalSwitch(tcpLb, tcpSessLb, udpLb, udpSessLb, ls string) error {
    if err := c.addLoadBalancerToLogicalSwitch(tcpLb, ls); err != nil {
        klog.Errorf("failed to add tcp lb to %s, %v", ls, err)
        return err
    }

    if err := c.addLoadBalancerToLogicalSwitch(udpLb, ls); err != nil {
        klog.Errorf("failed to add udp lb to %s, %v", ls, err)
        return err
    }

    if err := c.addLoadBalancerToLogicalSwitch(tcpSessLb, ls); err != nil {
        klog.Errorf("failed to add tcp session lb to %s, %v", ls, err)
        return err
    }

    if err := c.addLoadBalancerToLogicalSwitch(udpSessLb, ls); err != nil {
        klog.Errorf("failed to add udp session lb to %s, %v", ls, err)
        return err
    }

    return nil
}

如上 kube-ovn underlay vlan default网络pod访问svc直接走的是logic switch lb 走dnat 访问到 svc对应的后端 pod 的endpoint

当前基于provider ovs 网桥多加一个host ns上的网卡也可以用来中转访问svc，就像ipvlan的hostns中的子接口的作用一样。

扩展：

ovn lb 支持udp 和 tcp，ipv6应该也已经支持了，但是不支持proxy protocol

注意点：

ovn router lb 需要额外的vpc subnet ip 来构造tcp udp的健康检查包，所以会多消耗一倍的vpc 后端ip