kubelet启动流程解析（一）启动方式及初始化解析

我们一般以systemd系统守护进程的方式启动kubelet

$ systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node Agent
   Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled)
  Drop-In: /etc/systemd/system/kubelet.service.d
           └─10-kubeadm.conf
   Active: active (running) since Sat 2021-11-06 16:05:42 CST; 4 days ago
     Docs: http://kubernetes.io/docs/
  Process: 2153 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
  Process: 2150 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/systemd/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
  Process: 2143 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/memory/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
  Process: 2111 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpuset/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
  Process: 2105 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpuacct/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
  Process: 2061 ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpu/system.slice/kubelet.service (code=exited, status=0/SUCCESS)
 Main PID: 2168 (kubelet)
    Tasks: 230
   Memory: 867.4M
   CGroup: /system.slice/kubelet.service
           └─2168 /usr/local/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.co...

Nov 11 10:20:57 node1 kubelet[2168]: W1111 10:20:57.208900    2168 volume_linux.go:49] Setting volume ownership for /var/lib/kubelet/pods/5a15...
Nov 11 10:20:57 node1 kubelet[2168]: W1111 10:20:57.210533    2168 volume_linux.go:49] Setting volume ownership for /var/lib/kubelet/pods/5a15...
Nov 11 10:20:57 node1 kubelet[2168]: W1111 10:20:57.210833    2168 volume_linux.go:49] Setting volume ownership for /var/lib/kubelet/pods/5a15...
Nov 11 10:20:57 node1 kubelet[2168]: E1111 10:20:57.767389    2168 summary_sys_containers.go:47] Failed to get system container stats for "/sy...
Nov 11 10:20:58 node1 kubelet[2168]: I1111 10:20:58.064091    2168 topology_manager.go:219] [topologymanager] RemoveContainer - Contai...8291c7ef
Nov 11 10:20:58 node1 kubelet[2168]: I1111 10:20:58.064592    2168 topology_manager.go:219] [topologymanager] RemoveContainer - Contai...78e11ac7
Nov 11 10:20:58 node1 kubelet[2168]: E1111 10:20:58.074045    2168 pod_workers.go:191] Error syncing pod 2f3115c0-22d2-4094-a467-f543cd34da3f ...
Nov 11 10:20:58 node1 kubelet[2168]: E1111 10:20:58.082305    2168 pod_workers.go:191] Error syncing pod ea9d254d-3d05-4b69-b1ff-c986454078d8 ...
Nov 11 10:20:58 node1 kubelet[2168]: E1111 10:20:58.085689    2168 kuberuntime_manager.go:801] container start failed: CreateContainer...ot found
Nov 11 10:20:58 node1 kubelet[2168]: E1111 10:20:58.085713    2168 pod_workers.go:191] Error syncing pod ececebcf-daeb-45e1-8516-06a95d706293 ...
Hint: Some lines were ellipsized, use -l to show in full.

我们看下service配置

$ cat /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/

[Service]
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpu/system.slice/kubelet.service
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpuacct/system.slice/kubelet.service
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpuset/system.slice/kubelet.service
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/memory/system.slice/kubelet.service
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/systemd/system.slice/kubelet.service
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice/kubelet.service
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

其中启动部分执行了/usr/local/bin/kubelet二进制文件，启动的参数我们暂且不关注。

/usr/local/bin/kubelet二进制文件是由kubernetes源码编译而来，接下来让我们从kubelet源码角度分析它的启动流程

主函数入口

启动主函数位于: kubernetes/cmd/kubelet/kubelet.go

package main

import (
    "math/rand"
    "os"
    "time"

    "k8s.io/component-base/logs"
    _ "k8s.io/component-base/metrics/prometheus/restclient"
    _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
    "k8s.io/kubernetes/cmd/kubelet/app"
)
func main() {
    rand.Seed(time.Now().UnixNano())

    // 初始化kubelet指令
    command := app.NewKubeletCommand()
    logs.InitLogs()
    defer logs.FlushLogs()

    if err := command.Execute(); err != nil {
        os.Exit(1)
    }
}

• rand.Seed(time.Now().UnixNano()): 定义了全局随机数种子
• command := app.NewKubeletCommand(): 根据启动参数初始化了kubelet指令
• logs.InitLogs(): 初始化日志控制器
• err := command.Execute(): 执行启动流程

接下来我们来分析kubelet初始化流程

kubelet指令初始化解析

源码位置: kubernetes/cmd/kubelet/app/server.go

// NewKubeletCommand creates a *cobra.Command object with default parameters
func NewKubeletCommand() *cobra.Command {
    cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)
    cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
    kubeletFlags := options.NewKubeletFlags()
    kubeletConfig, err := options.NewKubeletConfiguration()
    // programmer error
    if err != nil {
        klog.Fatal(err)
    }

    cmd := &cobra.Command{
        Use: componentKubelet,
        Long: `The kubelet is the primary "node agent" that runs on each
node. It can register the node with the apiserver using one of: the hostname; a flag to
override the hostname; or specific logic for a cloud provider.

The kubelet works in terms of a PodSpec. A PodSpec is a YAML or JSON object
that describes a pod. The kubelet takes a set of PodSpecs that are provided through
various mechanisms (primarily through the apiserver) and ensures that the containers
described in those PodSpecs are running and healthy. The kubelet doesn't manage
containers which were not created by Kubernetes.

Other than from an PodSpec from the apiserver, there are three ways that a container
manifest can be provided to the Kubelet.

File: Path passed as a flag on the command line. Files under this path will be monitored
periodically for updates. The monitoring period is 20s by default and is configurable
via a flag.

HTTP endpoint: HTTP endpoint passed as a parameter on the command line. This endpoint
is checked every 20 seconds (also configurable with a flag).

HTTP server: The kubelet can also listen for HTTP and respond to a simple API
(underspec'd currently) to submit a new manifest.`,
        // The Kubelet has special flag parsing requirements to enforce flag precedence rules,
        // so we do all our parsing manually in Run, below.
        // DisableFlagParsing=true provides the full set of flags passed to the kubelet in the
        // `args` arg to Run, without Cobra's interference.
        DisableFlagParsing: true,
        Run: func(cmd *cobra.Command, args []string) {
            // initial flag parse, since we disable cobra's flag parsing
            if err := cleanFlagSet.Parse(args); err != nil {
                cmd.Usage()
                klog.Fatal(err)
            }

            // check if there are non-flag arguments in the command line
            cmds := cleanFlagSet.Args()
            if len(cmds) > 0 {
                cmd.Usage()
                klog.Fatalf("unknown command: %s", cmds[0])
            }

            // short-circuit on help
            help, err := cleanFlagSet.GetBool("help")
            if err != nil {
                klog.Fatal(`"help" flag is non-bool, programmer error, please correct`)
            }
            if help {
                cmd.Help()
                return
            }

            // short-circuit on verflag
            verflag.PrintAndExitIfRequested()
            utilflag.PrintFlags(cleanFlagSet)

            // set feature gates from initial flags-based config
            if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
                klog.Fatal(err)
            }

            // validate the initial KubeletFlags
            if err := options.ValidateKubeletFlags(kubeletFlags); err != nil {
                klog.Fatal(err)
            }

            if kubeletFlags.ContainerRuntime == "remote" && cleanFlagSet.Changed("pod-infra-container-image") {
                klog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead")
            }

            // load kubelet config file, if provided
            if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
                kubeletConfig, err = loadConfigFile(configFile)
                if err != nil {
                    klog.Fatal(err)
                }
                // We must enforce flag precedence by re-parsing the command line into the new object.
                // This is necessary to preserve backwards-compatibility across binary upgrades.
                // See issue #56171 for more details.
                if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil {
                    klog.Fatal(err)
                }
                // update feature gates based on new config
                if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
                    klog.Fatal(err)
                }
            }

            // We always validate the local configuration (command line + config file).
            // This is the default "last-known-good" config for dynamic config, and must always remain valid.
            if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig); err != nil {
                klog.Fatal(err)
            }

            // use dynamic kubelet config, if enabled
            var kubeletConfigController *dynamickubeletconfig.Controller
            if dynamicConfigDir := kubeletFlags.DynamicConfigDir.Value(); len(dynamicConfigDir) > 0 {
                var dynamicKubeletConfig *kubeletconfiginternal.KubeletConfiguration
                dynamicKubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(dynamicConfigDir,
                    func(kc *kubeletconfiginternal.KubeletConfiguration) error {
                        // Here, we enforce flag precedence inside the controller, prior to the controller's validation sequence,
                        // so that we get a complete validation at the same point where we can decide to reject dynamic config.
                        // This fixes the flag-precedence component of issue #63305.
                        // See issue #56171 for general details on flag precedence.
                        return kubeletConfigFlagPrecedence(kc, args)
                    })
                if err != nil {
                    klog.Fatal(err)
                }
                // If we should just use our existing, local config, the controller will return a nil config
                if dynamicKubeletConfig != nil {
                    kubeletConfig = dynamicKubeletConfig
                    // Note: flag precedence was already enforced in the controller, prior to validation,
                    // by our above transform function. Now we simply update feature gates from the new config.
                    if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
                        klog.Fatal(err)
                    }
                }
            }

            // construct a KubeletServer from kubeletFlags and kubeletConfig
            kubeletServer := &options.KubeletServer{
                KubeletFlags:         *kubeletFlags,
                KubeletConfiguration: *kubeletConfig,
            }

            // use kubeletServer to construct the default KubeletDeps
            kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate)
            if err != nil {
                klog.Fatal(err)
            }

            // add the kubelet config controller to kubeletDeps
            kubeletDeps.KubeletConfigController = kubeletConfigController

            // set up stopCh here in order to be reused by kubelet and docker shim
            stopCh := genericapiserver.SetupSignalHandler()

            // start the experimental docker shim, if enabled
            if kubeletServer.KubeletFlags.ExperimentalDockershim {
                if err := RunDockershim(&kubeletServer.KubeletFlags, kubeletConfig, stopCh); err != nil {
                    klog.Fatal(err)
                }
                return
            }

            // run the kubelet
            klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration)
            if err := Run(kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate, stopCh); err != nil {
                klog.Fatal(err)
            }
        },
    }

    // keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
    kubeletFlags.AddFlags(cleanFlagSet)
    options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
    options.AddGlobalFlags(cleanFlagSet)
    cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))

    // ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
    const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
    cmd.SetUsageFunc(func(cmd *cobra.Command) error {
        fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
        return nil
    })
    cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
        fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
    })

    return cmd
}