k8s cni bridge

bridge 是 cni一个比较重要的实现,基本原理就是 linux的bridge, bridge 源码比较短，本文对 bridge 源码做分析。

目标效果

配置的网络图如下

image.png

代码入口

//plugins/main/bridge/bridge.go
func main() {
    skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("bridge"))
}

main函数是标准的CNI插件开局，这里只分析cmdAdd

func cmdAdd(args *skel.CmdArgs) error {
        //加载配置
    n, cniVersion, err := loadNetConf(args.StdinData, args.Args)
    isLayer3 := n.IPAM.Type != ""
        //添加bridge
        br, brInterface, err := setupBridge(n)
       //添加veth对
    hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan, n.mac)
       //调用IPAM  插件
       if isLayer3 {
        // run the IPAM plugin and get back the config to apply
        r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
     }
       //收集网关信息
    gwsV4, gwsV6, err := calcGateways(result, n)
//为容器配置ip和mac地址
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
                return err
            }
 //最后输出日志
return types.PrintResult(result, cniVersion)
  }

加载配置

type NetConf struct {
    types.NetConf
    BrName       string `json:"bridge"`
    IsGW         bool   `json:"isGateway"`
    IsDefaultGW  bool   `json:"isDefaultGateway"`
    ForceAddress bool   `json:"forceAddress"`
    IPMasq       bool   `json:"ipMasq"`
    MTU          int    `json:"mtu"`
    HairpinMode  bool   `json:"hairpinMode"`
    PromiscMode  bool   `json:"promiscMode"`
    Vlan         int    `json:"vlan"`
    MacSpoofChk  bool   `json:"macspoofchk,omitempty"`
    EnableDad    bool   `json:"enabledad,omitempty"`

    Args struct {
        Cni BridgeArgs `json:"cni,omitempty"`
    } `json:"args,omitempty"`
    RuntimeConfig struct {
        Mac string `json:"mac,omitempty"`
    } `json:"runtimeConfig,omitempty"`

    mac string
}

配置的struct 是 NetConf

{
    "name": "mynet",
    "type": "bridge",
    "mtu": 1472,
    "ipMasq": false,
    "isGateway": true,
    "ipam": {
        "type": "host-local",
        "subnet": "203.0.113.0/24"
    }
}

传入的配置示例，当 ipam 指定时 isLayer3 时true，会调用IPAM 插件

添加bridge

func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) {
    vlanFiltering := false
    if n.Vlan != 0 {
        vlanFiltering = true
    }
    // create bridge if necessary
    br, err := ensureBridge(n.BrName, n.MTU, n.PromiscMode, vlanFiltering)
    if err != nil {
        return nil, nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
    }

    return br, &current.Interface{
        Name: br.Attrs().Name,
        Mac:  br.Attrs().HardwareAddr.String(),
    }, nil
}

调用的是 ensureBridge

func ensureBridge(brName string, mtu int, promiscMode, vlanFiltering bool) (*netlink.Bridge, error) {
    br := &netlink.Bridge{
        LinkAttrs: netlink.LinkAttrs{
            Name: brName,
            MTU:  mtu,
            // Let kernel use default txqueuelen; leaving it unset
            // means 0, and a zero-length TX queue messes up FIFO
            // traffic shapers which use TX queue length as the
            // default packet limit
            TxQLen: -1,
        },
    }
    if vlanFiltering {
        br.VlanFiltering = &vlanFiltering
    }

    err := netlink.LinkAdd(br)
    if err != nil && err != syscall.EEXIST {
        return nil, fmt.Errorf("could not add %q: %v", brName, err)
    }

    if promiscMode {
        if err := netlink.SetPromiscOn(br); err != nil {
            return nil, fmt.Errorf("could not set promiscuous mode on %q: %v", brName, err)
        }
    }

    // Re-fetch link to read all attributes and if it already existed,
    // ensure it's really a bridge with similar configuration
    br, err = bridgeByName(brName)
    if err != nil {
        return nil, err
    }

    // we want to own the routes for this interface
    _, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", brName), "0")

    if err := netlink.LinkSetUp(br); err != nil {
        return nil, err
    }

    return br, nil
}

ensureBridge 通过系统调用 LinkAdd 添加bridge,添加后 bridgeByName 重新查询出brige,确保存在。
accept_ra 0 关闭路由建议,最后 LinkSetUp 启用 bridge设备,返回mac地址。

添加Veth对(linux虚拟网络设备)

func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int, mac string) (*current.Interface, *current.Interface, error) {
    contIface := &current.Interface{}
    hostIface := &current.Interface{}

    err := netns.Do(func(hostNS ns.NetNS) error {
        // create the veth pair in the container and move host end into host netns
        hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, mac, hostNS)
        if err != nil {
            return err
        }
        contIface.Name = containerVeth.Name
        contIface.Mac = containerVeth.HardwareAddr.String()
        contIface.Sandbox = netns.Path()
        hostIface.Name = hostVeth.Name
        return nil
    })
    if err != nil {
        return nil, nil, err
    }

    // need to lookup hostVeth again as its index has changed during ns move
    hostVeth, err := netlink.LinkByName(hostIface.Name)
    if err != nil {
        return nil, nil, fmt.Errorf("failed to lookup %q: %v", hostIface.Name, err)
    }
    hostIface.Mac = hostVeth.Attrs().HardwareAddr.String()

    // connect host veth end to the bridge
    if err := netlink.LinkSetMaster(hostVeth, br); err != nil {
        return nil, nil, fmt.Errorf("failed to connect %q to bridge %v: %v", hostVeth.Attrs().Name, br.Attrs().Name, err)
    }

    // set hairpin mode
    if err = netlink.LinkSetHairpin(hostVeth, hairpinMode); err != nil {
        return nil, nil, fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVeth.Attrs().Name, err)
    }
      ....
}

1：hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, mac, hostNS)
创建Veth对,一端用于host（hostVeth），一端用于容器（containerVeth）

2：netlink.LinkSetMaster(hostVeth, br)
将hostVeth挂在前面创建的bridge上

3：netlink.LinkSetHairpin(hostVeth, hairpinMode)
bridge的某个端口打开hairpin mode后允许从这个端口收到的包仍然从这个端口发出

调用IPAM 插件

r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)

以ipam 的实现 host-local为例，host-local在 subnet 地址范围内分配ip,分配的ip记录到本地文件。
返回的结果包含 ip4,ip6

//返回结果示例
{
    "ips": [
        {
            "version": "4",
            "address": "203.0.113.2/24",
            "gateway": "203.0.113.1"
        },
        {
            "version": "6",
            "address": "2001:db8:1::2/64",
            "gateway": "2001:db8:1::1"
        }
    ],
}

收集网关信息

gwsV4, gwsV6, err := calcGateways(result, n)
gwsV4:203.0.113.1
gwsV6:2001:db8:1::1

为容器设置ip

func ConfigureIface(ifName string, res *current.Result) error {
        for _, ipc := range res.IPs {
            addr := &netlink.Addr{IPNet: &ipc.Address, Label: ""}
        if err = netlink.AddrAdd(link, addr); err != nil {
            return fmt.Errorf("failed to add IP addr %v to %q: %v", ipc, ifName, err)
        }

    if err := netlink.LinkSetUp(link); err != nil {
        return fmt.Errorf("failed to set %q UP: %v", ifName, err)
    }
}

通过 netlink.AddrAdd 系统调用设置ip和LinkSetUp 启用ip

为bridge设置ip和mac地址

//err = ensureAddr(br, gws.family, &gw, n.ForceAddress)

func ensureAddr(br netlink.Link, family int, ipn *net.IPNet, forceAddress bool) error {
addr := &netlink.Addr{IPNet: ipn, Label: ""}
    if err := netlink.AddrAdd(br, addr); err != nil && err != syscall.EEXIST {
        return fmt.Errorf("could not add IP address to %q: %v", br.Attrs().Name, err)
    }

    // Set the bridge's MAC to itself. Otherwise, the bridge will take the
    // lowest-numbered mac on the bridge, and will change as ifs churn
    if err := netlink.LinkSetHardwareAddr(br, br.Attrs().HardwareAddr); err != nil {
        return fmt.Errorf("could not set bridge's mac: %v", err)
    }

通过AddrAdd 设置ip,通过 LinkSetHardwareAddr 设置mac地址为自身的mac地址(作者时说不这样做的话，mac地址会变化，变化原因未知)
ip地址就是前面gateway的地址。

输出日志

return types.PrintResult(result, cniVersion)

最后会将配置的结果输出日志