接 k8s cni
前面说到CRI负责 CNI的调用，以containerd为例子，sandbox_run.go 的 RunPodSandbox 函数会调用 setupPodNetwork 配置网络。
本文主要结合 flannel -cni 介绍 setupPodNetwork 背后的cni逻辑。
CNI
//pkg/cri/server/sandbox_run.go
func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
// Create initial internal sandbox object.
sandbox := sandboxstore.NewSandbox(
sandboxstore.Metadata{
ID: id,
Name: name,
Config: config,
RuntimeHandler: r.GetRuntimeHandler(),
},
sandboxstore.Status{
State: sandboxstore.StateUnknown,
},
)
}
sandbox.NetNS, err = netns.NewNetNS(netnsMountDir)
if err := c.setupPodNetwork(ctx, &sandbox); err != nil
先创建了一个sandbox(对应Pod),然后创建了一个 net namespace（linux namespaces隔离网络），调用 setupPodNetwork 设置网络
//pkg/cri/server/sandbox_run.go
func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.Sandbox) error {
var (
id = sandbox.ID
config = sandbox.Config
path = sandbox.NetNSPath
netPlugin = c.getNetworkPlugin(sandbox.RuntimeHandler)
)
result, err := netPlugin.Setup(ctx, id, path, opts...)
}
获取CNI插件，然后调用插件设置。
//pkg/cri/server/sandbox_run.go
//获取插件
func (c *criService) getNetworkPlugin(runtimeClass string) cni.CNI {
i, ok := c.netPlugin[runtimeClass]
}
所有插件是保存在一个 map里，map是如何初始化的？
//pkg/cri/server/service_linux.go
func (c *criService) initPlatform() (err error) {
i, err := cni.New(cni.WithMinNetworkCount(networkAttachCount),
cni.WithPluginConfDir(dir),
cni.WithPluginMaxConfNum(max),
cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
c.netPlugin[name] = i
}
//containerd/go-cni/cni.go
func New(config ...Opt) (CNI, error) {
cni := defaultCNIConfig()
func defaultCNIConfig() *libcni {
return &libcni{
config: config{
pluginDirs: []string{DefaultCNIDir},
pluginConfDir: DefaultNetDir,
pluginMaxConfNum: DefaultMaxConfNum,
prefix: DefaultPrefix,
},
cniConfig: cnilibrary.NewCNIConfig(
[]string{
DefaultCNIDir,
},
&invoke.DefaultExec{
RawExec: &invoke.RawExec{Stderr: os.Stderr},
PluginDecoder: version.PluginDecoder{},
},
),
networkCount: 1,
}
}
Setup的调用
////containerd/go-cni/cni.go
// Setup setups the network in the namespace and returns a Result
func (c *libcni) Setup(ctx context.Context, id string, path string, opts ...NamespaceOpts) (*Result, error) {
if err := c.Status(); err != nil {
return nil, err
}
//创建 ns对象
ns, err := newNamespace(id, path, opts...)
if err != nil {
return nil, err
}
//attachNetworks
result, err := c.attachNetworks(ctx, ns)
if err != nil {
return nil, err
}
return c.createResult(result)
}
func (c *libcni) attachNetworks(ctx context.Context, ns *Namespace) ([]*types100.Result, error) {
var wg sync.WaitGroup
var firstError error
results := make([]*types100.Result, len(c.Networks()))
rc := make(chan asynchAttachResult)
//遍历 NetWork
for i, network := range c.Networks() {
wg.Add(1)
//attach NetWork
go asynchAttach(ctx, i, network, ns, &wg, rc)
}
}
func asynchAttach(ctx context.Context, index int, n *Network, ns *Namespace, wg *sync.WaitGroup, rc chan asynchAttachResult) {
defer wg.Done()
r, err := n.Attach(ctx, ns)
rc <- asynchAttachResult{index: index, res: r, err: err}
}
func (n *Network) Attach(ctx context.Context, ns *Namespace) (*types100.Result, error) {
r, err := n.cni.AddNetworkList(ctx, n.config, ns.config(n.ifName))
if err != nil {
return nil, err
}
return types100.NewResultFromResult(r)
}
SetUp遍历 Networks ，add 每个Network
github.com/containernetworking/cni/libcni/api.go// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
var err error
var result types.Result
for _, net := range list.Plugins {
result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt)
if err != nil {
return nil, fmt.Errorf("plugin %s failed (add): %w", pluginDescription(net.Network), err)
}
}
return result, nil
}
func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec)
}
addNetwork 实际就是调用 DefaultCNIDir = "/opt/cni/bin" 目录下的CNI插件实际实现(bridge,flannel等)
参数 NetworkConfigList 到 NetworkConfig 的关系如下
image.png
上面还都是是containerd 中
flannel-cni
flannel 实现了 CNI 插件
//flannel.go
func main() {
fullVer := fmt.Sprintf("CNI Plugin %s version %s (%s/%s) commit %s built on %s", Program, Version, runtime.GOOS, runtime.GOARCH, Commit, buildDate)
skel.PluginMain(cmdAdd, cmdCheck, cmdDel, cni.All, fullVer)
}
可以看到CNI 的标准实现，cmdAdd，cmdCheck....，我们分析 cmdAdd
func cmdAdd(args *skel.CmdArgs) error {
//加载 NetworkConfig 配置，设置默认的 SubnetFile(/run/flannel/subnet.env)
n, err := loadFlannelNetConf(args.StdinData)
//加载 /run/flannel/subnet.env
fenv, err := loadFlannelSubnetEnv(n.SubnetFile)
return doCmdAdd(args, n, fenv)
}
//run/flannel/subnet.env 文件内容示例
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
/run/flannel/subnet.env 文件是flannel 启动时输出,用于后面的配置转换。
//flannel_linux.go
func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error {
return delegateAdd(args.ContainerID, n.DataDir, n.Delegate)
}
//flannel.go
func delegateAdd(cid, dataDir string, netconf map[string]interface{}) error {
if err = saveScratchNetConf(cid, dataDir, netconfBytes); err != nil {
return err
}
result, err := invoke.DelegateAdd(context.TODO(), netconf["type"].(string), netconfBytes, nil)
}
flannel cni插件会做配置转换，并调用转换后的插件。
//saveScratchNetConf + un/flannel/subnet.env 的内容
{
"name": "mynet",
"type": "flannel"
}
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
转换为
{
"name": "mynet",
"type": "bridge",
"mtu": 1472,
"ipMasq": false,
"isGateway": true,
"ipam": {
"type": "host-local",
"subnet": "10.244.0.1/24"
}
}
bridge 插件完成后,运行在host上的 flannel 接管了网络。
上面 flannel cni插件会做配置转换 很重要，flannel cni插件通过重新调用转换后的配置文件的插件来配置网络(比如bridge)。
网友评论