openstack 网桥 & 流表

openstack的网桥

image.png

网桥名称	作用
物理网桥	* flat、vpc、public网络跨主机流量转发物理网桥，外接eth口或者bond口； * 每Ethernet/Bond接口一个； * 既可以接收带vlan tag的网络报文，也可以接收不带vlan tag的网络报文。
tunnel网桥	* 作用同物理网桥，提供主机间互联，只不过物理网桥提供underlay方案，而tunnel网桥提供overlay方案； * 所有tunnel口使用一个tunnel网桥；
集成网桥	* vm、router、service netns互联内部网桥，不同网络内部vlan隔离； * 本机的网络功能和本机内流量转发全部在集成网桥上实现。

其中物理网桥和tunnel网桥一起可以看成是外部网桥，用来完成进出节点流量转发。
markdown的表格编辑不友好，这里直接截图了。

流表分析

这里只看较为复杂的集成网桥br-int的流表，物理网桥和tunnel网桥较为简单，且网上资料较多。

image.png image.png

大概的分层设计：

image.png

流表解读

### 
#  tap939d0ce5-a8   network-vlan1001 dhcp ns tap nic ,   tag=2,
#  tap8d442e03-f5   vm3 vnic                         ,   tag=2,
#  tap44a2f223-19   the other network dhcp ns tap nic,   tag=1,
#  int-br-eth0      vlan-br 接入口
#  reg5=in_port or out_port, reg6=vlan tag
 1(patch-tun): addr:4a:8a:5f:df:61:75
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 3(int-br-eth0): addr:72:12:5b:05:d9:7e
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 4(tap44a2f223-19): addr:fa:16:3e:f6:98:b3
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 5(int-br-eth2): addr:46:6e:da:f6:0e:aa
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 6(tap939d0ce5-a8): addr:fa:16:3e:49:7f:80
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 7(tap8d442e03-f5): addr:fe:16:3e:43:03:5a
     config:     0
     state:      0
     current:    10MB-FD COPPER
     speed: 10 Mbps now, 0 Mbps max
###


[root@controller ~]# ovs-ofctl dump-flows br-int
 #### 1、有效vlan是1-4094，4095无效，drop
 cookie=0xcedae18f46b0a9c1, duration=15025.933s, table=0, n_packets=0, n_bytes=0, priority=65535,vlan_tci=0x0fff/0x1fff actions=drop
 #### 2、vlan bridge过来的报文，做vlan转换，外转内，未知vlan报文drop
 cookie=0xcedae18f46b0a9c1, duration=15024.747s, table=0, n_packets=14, n_bytes=2668, priority=3,in_port="int-br-eth0",dl_vlan=1001 actions=mod_vlan_vid:2,resubmit(,60)
 cookie=0xcedae18f46b0a9c1, duration=15025.914s, table=0, n_packets=0, n_bytes=0, priority=2,in_port="int-br-eth0" actions=drop
 #### 3、其他接口直接送 table60
 cookie=0xcedae18f46b0a9c1, duration=15025.935s, table=0, n_packets=35, n_bytes=5634, priority=0 actions=resubmit(,60)
 cookie=0xcedae18f46b0a9c1, duration=15025.936s, table=23, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0xcedae18f46b0a9c1, duration=15025.934s, table=24, n_packets=0, n_bytes=0, priority=0 actions=drop

 #### 4、dhcp ns出来的包，设置reg5（inport）和reg6（vlan tag），resubmit 73
 cookie=0xcedae18f46b0a9c1, duration=15023.082s, table=60, n_packets=0, n_bytes=0, priority=100,in_port="tap44a2f223-19" actions=load:0x4->NXM_NX_REG5[],load:0x1->NXM_NX_REG6[],resubmit(,73)
 cookie=0xcedae18f46b0a9c1, duration=15023.082s, table=60, n_packets=9, n_bytes=2352, priority=100,in_port="tap939d0ce5-a8" actions=load:0x6->NXM_NX_REG5[],load:0x2->NXM_NX_REG6[],resubmit(,73)
 #### 5、vm出来的包，设置reg5（inport）和reg6（vlan tag），resubmit 71
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=60, n_packets=8, n_bytes=1160, priority=100,in_port="tap8d442e03-f5" actions=load:0x7->NXM_NX_REG5[],load:0x2->NXM_NX_REG6[],resubmit(,71)
 
 #### 6、外部网桥进来的流量，直接设置了out put，送table 81转发了，去掉vlan（为什么？？？）
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=60, n_packets=5, n_bytes=454, priority=90,dl_vlan=2,dl_dst=fa:16:3e:43:03:5a actions=load:0x7->NXM_NX_REG5[],load:0x2->NXM_NX_REG6[],strip_vlan,resubmit(,81)
 ###  6.1 其他节点发过来的未知单播、广播，全部 NORMAL
 cookie=0xcedae18f46b0a9c1, duration=15025.934s, table=60, n_packets=33, n_bytes=4588, priority=3 actions=NORMAL
 
 ####  #### 一个保护，table 71之前不会有进入ct的情况，如果有，ct_clear 重新来一次
 cookie=0xcedae18f46b0a9c1, duration=15025.428s, table=71, n_packets=0, n_bytes=0, priority=110,ct_state=+trk actions=ct_clear,resubmit(,71)
 
 #### 7、vm 出来的arp报文，resubmit 94，table94中做normal处理，vlan类型的网络没有arp代答功能； （reg5=0x7,in_port="tap8d442e03-f5"为重复匹配，没有必要）
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=71, n_packets=2, n_bytes=84, priority=95,arp,reg5=0x7,in_port="tap8d442e03-f5",dl_src=fa:16:3e:43:03:5a,arp_spa=10.1.1.113 actions=resubmit(,94)
 
 #### 8、ct建好了走ct，zone id使用vlan tag
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=71, n_packets=4, n_bytes=392, priority=65,ip,reg5=0x7,in_port="tap8d442e03-f5",dl_src=fa:16:3e:43:03:5a,nw_src=10.1.1.113 actions=ct(table=72,zone=NXM_NX_REG6[0..15])
 
 #### 9、dhcp报文 request
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=71, n_packets=2, n_bytes=684, priority=80,udp,reg5=0x7,in_port="tap8d442e03-f5",tp_src=68,tp_dst=67 actions=resubmit(,73)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=71, n_packets=0, n_bytes=0, priority=70,udp,reg5=0x7,in_port="tap8d442e03-f5",tp_src=67,tp_dst=68 actions=resubmit(,93)

 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=71, n_packets=0, n_bytes=0, priority=10,reg5=0x7,in_port="tap8d442e03-f5" actions=ct_clear,resubmit(,93)
 cookie=0xcedae18f46b0a9c1, duration=15025.524s, table=71, n_packets=0, n_bytes=0, priority=0 actions=drop
 
 #### 10、本地vm发出的数据报文
 ### 10.1、正向，非首包
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=3, n_bytes=294, priority=74,ct_state=+est-rel-rpl,ip,reg5=0x7 actions=resubmit(,73)
 ### 10.2、正向，首包
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=1, n_bytes=98, priority=74,ct_state=+new-est,ip,reg5=0x7 actions=resubmit(,73)
 ### 10.3 无效包、非法的包（ct_mark=1)丢弃。
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=50,ct_state=+inv+trk actions=resubmit(,93)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=50,ct_mark=0x1,reg5=0x7 actions=resubmit(,93)
 
 #### 10.4、回程反向
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=50,ct_state=+est-rel+rpl,ct_zone=2,ct_mark=0,reg5=0x7 actions=resubmit(,94)
 
 #### 10.5、ct关联流，比如icmp不可达消息或者ftp的数据流，走NORMAL
 cookie=0xceae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=50,ct_state=-new-est+rel-inv,ct_zone=2,ct_mark=0,reg5=0x7 actions=resubmit(,94)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=40,ct_state=-est,reg5=0x7 actions=resubmit(,93)
 #### 10.6、+est+rel 等，标记ct_mark=1后，这种状态是非法的，由于没有后续处理，本次报文drop，同时由于ct commit了，下次可以匹配到mark标记
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=72, n_packets=0, n_bytes=0, priority=40,ct_state=+est,ip,reg5=0x7 actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))
 cookie=0xcedae18f46b0a9c1, duration=15025.505s, table=72, n_packets=0, n_bytes=0, priority=0 actions=drop
 
 
 
 #### 11、本节点上vm、ns发往本地vm的包，这里确认out_port，进入 table 81，egress
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=73, n_packets=3, n_bytes=784, priority=100,reg6=0x2,dl_dst=fa:16:3e:43:03:5a actions=load:0x7->NXM_NX_REG5[],resubmit(,81)
 #### 12、本节点上vm 发往remote 或 local ns，首包，ct commit 之后，走table91，94 NORMAL发出
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=73, n_packets=1, n_bytes=98, priority=90,ct_state=+new-est,ip,reg5=0x7 actions=ct(commit,zone=NXM_NX_REG6[0..15]),resubmit(,91)
 #### 13、本节点上 ns 发往remote，非首包；发往local的未知单播（非vm mac）和广播，NORMAL
 cookie=0xcedae18f46b0a9c1, duration=15023.082s, table=73, n_packets=0, n_bytes=0, priority=80,reg5=0x4 actions=resubmit(,94)
 cookie=0xcedae18f46b0a9c1, duration=15023.082s, table=73, n_packets=6, n_bytes=1568, priority=80,reg5=0x6 actions=resubmit(,94)
 #### 13.1、本节点上，vm 发往 remote，发往local的未知单播（非vm mac）和广播，非首包；，NORMAL
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=73, n_packets=5, n_bytes=978, priority=80,reg5=0x7 actions=resubmit(,94)
 cookie=0xcedae18f46b0a9c1, duration=15025.487s, table=73, n_packets=0, n_bytes=0, priority=0 actions=drop



#### table81 开始做egress 方向转发处理
#### 确认了出接口的控制报文，arp reply、dhcp（offer+ack) 回程包
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=81, n_packets=2, n_bytes=88, priority=100,arp,reg5=0x7 actions=output:"tap8d442e03-f5"
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=81, n_packets=2, n_bytes=742, priority=95,udp,reg5=0x7,tp_src=67,tp_dst=68 actions=output:"tap8d442e03-f5"
#### 13.5 其他节点通过物理 bridge（vlan-br）进来的包 或者 本机ns进来的包，match ct_state=-trk
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=81, n_packets=4, n_bytes=408, priority=90,ct_state=-trk,ip,reg5=0x7 actions=ct(table=82,zone=NXM_NX_REG6[0..15])
 
 #### 14、local to local，从 11 过来
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=81, n_packets=0, n_bytes=0, priority=80,ct_state=+trk,reg5=0x7 actions=resubmit(,82)
 cookie=0xcedae18f46b0a9c1, duration=15025.467s, table=81, n_packets=0, n_bytes=0, priority=0 actions=drop
 
 
 
 
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+est-rel-rpl,ip,reg5=0x7 actions=conjunction(16,2/2)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+new-est,ip,reg5=0x7 actions=conjunction(17,2/2)
 
 #### conjunctive flow的意义：两个纬度 [(from)vm ip, (to)vm port] 的任意组合，本文档由于本机只有一台vm，所以纬度2不明显。
 #### 将流表数量从 [ m * n ] 减少为 [ 2*m + n ]，n为 port数量，n为vm ip数量
 #### 15、1）local to local，转发，非首包 2）外部进来的包，转发，首包
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=70,conj_id=16,ct_state=+est-rel-rpl,ip,reg5=0x7 actions=load:0x10->NXM_NX_REG7[],output:"tap8d442e03-f5"
 
 #### 16、1）local to local，转发，首包； 2）外部进来的包，转发，首包。
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=70,conj_id=17,ct_state=+new-est,ip,reg5=0x7 actions=load:0x11->NXM_NX_REG7[],ct(commit,zone=NXM_NX_REG6[0..15]),output:"tap8d442e03-f5",resubmit(,92)
 
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+est-rel-rpl,ip,reg6=0x2,nw_src=10.1.1.125 actions=conjunction(16,1/2)
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+est-rel-rpl,ip,reg6=0x2,nw_src=10.1.1.113 actions=conjunction(16,1/2)
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+est-rel-rpl,ip,reg6=0x2,nw_src=10.1.1.123 actions=conjunction(16,1/2)
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+new-est,ip,reg6=0x2,nw_src=10.1.1.125 actions=conjunction(17,1/2)
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+new-est,ip,reg6=0x2,nw_src=10.1.1.113 actions=conjunction(17,1/2)
 cookie=0xcedae18f46b0a9c1, duration=1807.543s, table=82, n_packets=0, n_bytes=0, priority=70,ct_state=+new-est,ip,reg6=0x2,nw_src=10.1.1.123 actions=conjunction(17,1/2)
 #### 17、无效和非法的包，drop
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=50,ct_state=+inv+trk actions=resubmit(,93)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=50,ct_mark=0x1,reg5=0x7 actions=resubmit(,93)
 #### 18、这里应该是local vm<-->remote 的反向回程流表，local vm<-->local vm 在 10.4 完成。
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=4, n_bytes=392, priority=50,ct_state=+est-rel+rpl,ct_zone=2,ct_mark=0,reg5=0x7 actions=output:"tap8d442e03-f5"
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=50,ct_state=-new-est+rel-inv,ct_zone=2,ct_mark=0,reg5=0x7 actions=output:"tap8d442e03-f5"
 
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=40,ct_state=-est,reg5=0x7 actions=resubmit(,93)
 cookie=0xcedae18f46b0a9c1, duration=1805.563s, table=82, n_packets=0, n_bytes=0, priority=40,ct_state=+est,ip,reg5=0x7 actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))
 cookie=0xcedae18f46b0a9c1, duration=15025.448s, table=82, n_packets=0, n_bytes=0, priority=0 actions=drop
 
 
 cookie=0xcedae18f46b0a9c1, duration=15025.392s, table=91, n_packets=1, n_bytes=98, priority=1 actions=resubmit(,94)
 cookie=0xcedae18f46b0a9c1, duration=15025.374s, table=92, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0xcedae18f46b0a9c1, duration=15025.355s, table=93, n_packets=0, n_bytes=0, priority=0 actions=drop
 cookie=0xcedae18f46b0a9c1, duration=15025.410s, table=94, n_packets=14, n_bytes=2728, priority=1 actions=NORMAL