结合源码理解flannel vxlan

简单总结

同节点通信

image.png

不同节点通信

image.png

相关源码

pkg/backend/vxlan/vxlan.go中
初始化

func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
  ...
        dev, err = newVXLANDevice(&devAttrs)
  ...
    return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease, cfg.MTU)
}

pkg/backend/vxlan/device.go中

创建vxlan网络设备
func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
  ...

    link := &netlink.Vxlan{
        LinkAttrs: netlink.LinkAttrs{
            Name:         devAttrs.name,
            HardwareAddr: hardwareAddr,
            MTU:          devAttrs.MTU - 50,
        },
        VxlanId:      int(devAttrs.vni),
        VtepDevIndex: devAttrs.vtepIndex,
        SrcAddr:      devAttrs.vtepAddr,
        Port:         devAttrs.vtepPort,
        Learning:     devAttrs.learning,
        GBP:          devAttrs.gbp,
    }

    link, err = ensureLink(link)
    if err != nil {
        return nil, err
    }

    _, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", devAttrs.name), "0")

  ...
}
func ensureLink(vxlan *netlink.Vxlan) (*netlink.Vxlan, error) {
    err := netlink.LinkAdd(vxlan)
  ...
    return vxlan, nil
}

pkg/backend/vxlan/vxlan_network.go中

监听子网变更事件

func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, v6Dev *vxlanDevice, _ ip.IP4Net, lease *lease.Lease, mtu int) (*network, error) {
    nw := &network{
        SimpleNetwork: backend.SimpleNetwork{
            SubnetLease: lease,
            ExtIface:    extIface,
        },
        subnetMgr: subnetMgr,
        dev:       dev,
        v6Dev:     v6Dev,
        mtu:       mtu,
    }

    return nw, nil
}

func (nw *network) Run(ctx context.Context) {
  ...
        nw.handleSubnetEvents(evtBatch)
  ...
}

func (nw *network) handleSubnetEvents(batch []lease.Event) {
  ...
        if event.Lease.EnableIPv4 && nw.dev != nil {
            if err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {
                log.Error("error decoding subnet lease JSON: ", err)
                continue
            }

            // This route is used when traffic should be vxlan encapsulated
            vxlanRoute = netlink.Route{
                LinkIndex: nw.dev.link.Attrs().Index,
                Scope:     netlink.SCOPE_UNIVERSE,
                Dst:       sn.ToIPNet(),
                Gw:        sn.IP.ToIP(),
            }
            vxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)

            // directRouting is where the remote host is on the same subnet so vxlan isn't required.
            directRoute = netlink.Route{
                Dst: sn.ToIPNet(),
                Gw:  attrs.PublicIP.ToIP(),
            }
            if nw.dev.directRouting {
                if dr, err := ip.DirectRouting(attrs.PublicIP.ToIP()); err != nil {
                    log.Error(err)
                } else {
                    directRoutingOK = dr
                }
            }
        }
  ...
}

pkg/backend/vxlan/device.go中

添加arp(用于知道对端端mac地址)和fdb(用于知道对端的ip地址)
func (dev *vxlanDevice) AddFDB(n neighbor) error {
    log.V(4).Infof("calling AddFDB: %v, %v", n.IP, n.MAC)
    return netlink.NeighSet(&netlink.Neigh{
        LinkIndex:    dev.link.Index,
        State:        netlink.NUD_PERMANENT,
        Family:       syscall.AF_BRIDGE,
        Flags:        netlink.NTF_SELF,
        IP:           n.IP.ToIP(),
        HardwareAddr: n.MAC,
    })
}

func (dev *vxlanDevice) AddARP(n neighbor) error {
    log.V(4).Infof("calling AddARP: %v, %v", n.IP, n.MAC)
    return netlink.NeighSet(&netlink.Neigh{
        LinkIndex:    dev.link.Index,
        State:        netlink.NUD_PERMANENT,
        Type:         syscall.RTN_UNICAST,
        IP:           n.IP.ToIP(),
        HardwareAddr: n.MAC,
    })
}

补充

查看bridge以及bridge上网络设备

brctl show

查看对端ip

bridge fdb show dev

查看对端mac

ip neigh