[root@k8s-ctrl-1 01-test-old-enable-eip-snat]# kubectl ko vsctl k8s-ctrl-3 show
525ba0d9-2abf-4fe3-a8fc-c7d98454359a
Bridge br-int
fail_mode: secure
datapath_type: system
Port "1ddc84ce_net2_h"
Interface "1ddc84ce_net2_h"
Port "2f6bc0320676_h"
Interface "2f6bc0320676_h"
Port ed17244b_net1_h
Interface ed17244b_net1_h
Port mirror0
Interface mirror0
type: internal
Port ab1ae5dc_net2_h
Interface ab1ae5dc_net2_h
Port e2b7d6e8_net2_h
Interface e2b7d6e8_net2_h
Port "79bb4961_net2_h"
Interface "79bb4961_net2_h"
Port ovn-678b8a-0
Interface ovn-678b8a-0
type: geneve
options: {csum="true", key=flow, remote_ip="10.5.205.21"}
Port cad08081_net1_h
Interface cad08081_net1_h
Port ede35a15fe5f_h
Interface ede35a15fe5f_h
Port "2f6bc032_net2_h"
Interface "2f6bc032_net2_h"
Port "6723c5021eb4_h"
Interface "6723c5021eb4_h"
Port "2f6bc032_net1_h"
Interface "2f6bc032_net1_h"
Port cad08081_net2_h
Interface cad08081_net2_h
Port c350047e7c84_h
Interface c350047e7c84_h
Port e2b7d6e8e620_h
Interface e2b7d6e8e620_h
Port "47ac21f7da3c_h"
Interface "47ac21f7da3c_h"
Port "79bb4961_net1_h"
Interface "79bb4961_net1_h"
Port "92b57ffcf5ba_h"
Interface "92b57ffcf5ba_h"
Port "92d860f12f47_h"
Interface "92d860f12f47_h"
Port "04d074d31fee_h"
Interface "04d074d31fee_h"
Port br-int
Interface br-int
type: internal
Port cad080818770_h
Interface cad080818770_h
Port "3db344ebebe2_h"
Interface "3db344ebebe2_h"
Port "72b4a9e47adc_h"
Interface "72b4a9e47adc_h"
Port "8005f7ed10ff_h"
Interface "8005f7ed10ff_h"
Port ovn-c815f1-0
Interface ovn-c815f1-0
type: geneve
options: {csum="true", key=flow, remote_ip="10.5.205.22"}
Port e2b7d6e8_net1_h
Interface e2b7d6e8_net1_h
Port "55cb646f_net2_h"
Interface "55cb646f_net2_h"
Port ed17244b6d7d_h
Interface ed17244b6d7d_h
Port "55cb646f0034_h"
Interface "55cb646f0034_h"
Port ovn0
Interface ovn0
type: internal
Port e24ffbe318ff_h
Interface e24ffbe318ff_h
Port c350047e_net1_h
Interface c350047e_net1_h
Port "1ddc84ce_net1_h"
Interface "1ddc84ce_net1_h"
Port ed17244b_net2_h
Interface ed17244b_net2_h
Port "79bb4961e4bc_h"
Interface "79bb4961e4bc_h"
Port "55cb646f_net1_h"
Interface "55cb646f_net1_h"
Port "1cf4b0139718_h"
Interface "1cf4b0139718_h"
Port "7cd8c467bc27_h"
Interface "7cd8c467bc27_h"
Port "067a32e50ff4_h"
Interface "067a32e50ff4_h"
Port patch-br-int-to-localnet.vlan-203
Interface patch-br-int-to-localnet.vlan-203
type: patch
options: {peer=patch-localnet.vlan-203-to-br-int}
Port "41cb6b651a77_h"
Interface "41cb6b651a77_h"
Port "3db344eb_net2_h"
Interface "3db344eb_net2_h"
Port ab1ae5dcb654_h
Interface ab1ae5dcb654_h
Port "1ddc84ce1a87_h"
Interface "1ddc84ce1a87_h"
Port b115aa521eb4_h
Interface b115aa521eb4_h
Port "820be1c2ff93_h"
Interface "820be1c2ff93_h"
Port ab1ae5dc_net1_h
Interface ab1ae5dc_net1_h
Port c350047e_net2_h
Interface c350047e_net2_h
Port "3db344eb_net1_h"
Interface "3db344eb_net1_h"
Bridge br-vlan
Port vlan
Interface vlan
Port patch-localnet.vlan-203-to-br-int
Interface patch-localnet.vlan-203-to-br-int
type: patch
options: {peer=patch-br-int-to-localnet.vlan-203}
Port br-vlan
Interface br-vlan
type: internal
ovs_version: "2.17.5"
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]# kubectl ko vsctl k8s-ctrl-2 show
af19c729-981e-4327-9c34-2d76f9f61b31
Bridge br-int
fail_mode: secure
datapath_type: system
Port ovn-678b8a-0
Interface ovn-678b8a-0
type: geneve
options: {csum="true", key=flow, remote_ip="10.5.205.21"}
Port patch-br-int-to-localnet.underlay
Interface patch-br-int-to-localnet.underlay
type: patch
options: {peer=patch-localnet.underlay-to-br-int}
Port br-int
Interface br-int
type: internal
Bridge br-vlan
Port br-vlan
Interface br-vlan
type: internal
Port vlan
Interface vlan
Port patch-localnet.underlay-to-br-int
Interface patch-localnet.underlay-to-br-int
type: patch
options: {peer=patch-br-int-to-localnet.underlay}
ovs_version: "2.17.5"
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]# kubectl ko vsctl k8s-ctrl-1 show
c572565b-e7e0-4ab1-8946-e07b4be0af75
Bridge br-int
fail_mode: secure
datapath_type: system
Port mirror0
Interface mirror0
type: internal
Port ovn-22956a-0
Interface ovn-22956a-0
type: geneve
options: {csum="true", key=flow, remote_ip="10.5.205.23"}
Port ovn0
Interface ovn0
type: internal
Port ovn-c815f1-0
Interface ovn-c815f1-0
type: geneve
options: {csum="true", key=flow, remote_ip="10.5.205.22"}
Bridge br-vlan
Port patch-localnet.underlay-to-br-int
Interface patch-localnet.underlay-to-br-int
type: patch
options: {peer=patch-br-int-to-localnet.underlay}
Port vlan
Interface vlan
Port br-vlan
Interface br-vlan
type: internal
ovs_version: "2.17.5"
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]#
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]# k get provider-networks -A -owide
NAME DEFAULTINTERFACE READY
vlan vlan true
# k get vlan -A
NAME ID PROVIDER
underlay 0 vlan
vlan-202 202 vlan
vlan-203 203 vlan
vlan-207 207 vlan
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]# k get subnet -A | grep vlan
vlan-202 ovn ovn-cluster IPv4 10.5.202.0/24 false false false distributed 0 154 0 0 ["10.5.202.1..10.5.202.99","10.5.202.254"]
vlan-203 ovn ovn-cluster IPv4 10.5.203.0/24 false false false distributed 1 153 0 0 ["10.5.203.1..10.5.203.99","10.5.203.254"]
vlan-207 ovn ovn-cluster IPv4 10.5.207.0/24 false false false distributed 0 154 0 0 ["10.5.207.1..10.5.207.99","10.5.207.254"]
[root@k8s-ctrl-1 01-test-old-enable-eip-snat]#
# 目前的实现是一个provider network，多个vlan，多个underlay subnet
目前认为这种方式应该是有问题的，至少目前看多个节点的ovs网桥初始化不一致。
而且根据kube-ovn 源码搜下tag= 看一下就知道vlan tag在哪里加的，也就是说，一张网卡，只能对应一个ovs网桥，因为目前代码只有一个localnet端口
小结: 存在一张两个网桥 且nat gw 使用该网桥时，所有pod 网卡无法通网关，所有iptables eip 无法二层直通，可以看到流量出去但没有回包，当然snat后的流量(基于iptables eip)也是如此
网友评论