kubelet创建容器之运行容器启动后的钩子

比如：调用注册中心注册自己

概述

kubelet通过以下四个步骤，来启动pod容器：

1. 拉取镜像
2. 创建容器
3. 启动容器
4. 执行容器启动后的钩子

本文主要解析执行容器启动后的钩子阶段kubelet所做工作，对应pod的配置声明为:

apiVersion: v1
kind: Pod
metadata:
  name: lifecycle-demo
spec:
  containers:
  - name: lifecycle-demo-container
    image: nginx
    lifecycle:
      postStart:
        exec:
          command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]

Kubernetes在容器创建后立即发送postStart事件。
然而，postStart处理函数的调用不保证在容器的入口点（entrypoint） 之前执行。
postStart处理函数与容器的代码是异步执行的，但Kubernetes的容器管理逻辑会一直阻塞等待postStart处理函数执行完毕。 只有postStart处理函数执行完毕，容器的状态才会变成RUNNING。

接下来我们对下述执行容器启动后的钩子阶段的代码逻辑进行解析:

源码实现

func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {
...
    if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
        kubeContainerID := kubecontainer.ContainerID{
            Type: m.runtimeName,
            ID:   containerID,
        }
        msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)
        if handlerErr != nil {
            m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)
            if err := m.killContainer(pod, kubeContainerID, container.Name, "FailedPostStartHook", nil); err != nil {
                klog.Errorf("Failed to kill container %q(id=%q) in pod %q: %v, %v",
                    container.Name, kubeContainerID.String(), format.Pod(pod), ErrPostStartHook, err)
            }
            return msg, fmt.Errorf("%s: %v", ErrPostStartHook, handlerErr)
        }
    }

    return "", nil

流程解析

1. 判断pod是否配置了postStart钩子，若未配置跳过该逻辑
2. 调用postStart钩子处理函数，执行成功返回空异常，执行失败调用pre-stop钩子处理函数并kill掉容器

postStart钩子处理函数实现

当postStart钩子为命令类型时，容器内执行postStart钩子指令，若执行结果非空退出码返回异常

当postStart钩子为http请求类型时，按http://host:port/path格式发送请求，返回请求响应结果

kubernetes\pkg\kubelet\lifecycle\handlers.go

func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *v1.Pod, container *v1.Container, handler *v1.Handler) (string, error) {
    switch {
    case handler.Exec != nil:
        var msg string
        // TODO(tallclair): Pass a proper timeout value.
        // 容器内执行`postStart`钩子指令，若执行结果非空退出码返回异常
        output, err := hr.commandRunner.RunInContainer(containerID, handler.Exec.Command, 0)
        if err != nil {
            msg = fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - error: %v, message: %q", handler.Exec.Command, container.Name, format.Pod(pod), err, string(output))
            klog.V(1).Infof(msg)
        }
        return msg, err
    case handler.HTTPGet != nil:
        msg, err := hr.runHTTPHandler(pod, container, handler)
        if err != nil {
            msg = fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - error: %v, message: %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), err, msg)
            klog.V(1).Infof(msg)
        }
        return msg, err
    default:
        err := fmt.Errorf("invalid handler: %v", handler)
        msg := fmt.Sprintf("Cannot run handler: %v", err)
        klog.Errorf(msg)
        return msg, err
    }
}

postStart钩子处理函数执行失败后的逻辑

postStart钩子处理函数执行失败后，将执行preStop钩子处理函数。等待preStop事件处理程序结束或者Pod的--grace-period超时，删除容器

kubernetes\pkg\kubelet\kuberuntime\kuberuntime_container.go

func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubecontainer.ContainerID, containerName string, message string, gracePeriodOverride *int64) error {
    var containerSpec *v1.Container
    if pod != nil {
        if containerSpec = kubecontainer.GetContainerSpec(pod, containerName); containerSpec == nil {
            return fmt.Errorf("failed to get containerSpec %q(id=%q) in pod %q when killing container for reason %q",
                containerName, containerID.String(), format.Pod(pod), message)
        }
    } else {
        // Restore necessary information if one of the specs is nil.
        restoredPod, restoredContainer, err := m.restoreSpecsFromContainerLabels(containerID)
        if err != nil {
            return err
        }
        pod, containerSpec = restoredPod, restoredContainer
    }

    // From this point, pod and container must be non-nil.
    gracePeriod := int64(minimumGracePeriodInSeconds)
    switch {
    case pod.DeletionGracePeriodSeconds != nil:
        gracePeriod = *pod.DeletionGracePeriodSeconds
    case pod.Spec.TerminationGracePeriodSeconds != nil:
        gracePeriod = *pod.Spec.TerminationGracePeriodSeconds
    }

    if len(message) == 0 {
        message = fmt.Sprintf("Stopping container %s", containerSpec.Name)
    }
    m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeNormal, events.KillingContainer, message)

    // Run internal pre-stop lifecycle hook
    if err := m.internalLifecycle.PreStopContainer(containerID.ID); err != nil {
        return err
    }

    // Run the pre-stop lifecycle hooks if applicable and if there is enough time to run it
    if containerSpec.Lifecycle != nil && containerSpec.Lifecycle.PreStop != nil && gracePeriod > 0 {
        gracePeriod = gracePeriod - m.executePreStopHook(pod, containerID, containerSpec, gracePeriod)
    }
    // always give containers a minimal shutdown window to avoid unnecessary SIGKILLs
    if gracePeriod < minimumGracePeriodInSeconds {
        gracePeriod = minimumGracePeriodInSeconds
    }
    if gracePeriodOverride != nil {
        gracePeriod = *gracePeriodOverride
        klog.V(3).Infof("Killing container %q, but using %d second grace period override", containerID, gracePeriod)
    }

    klog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod)

    err := m.runtimeService.StopContainer(containerID.ID, gracePeriod)
    if err != nil {
        klog.Errorf("Container %q termination failed with gracePeriod %d: %v", containerID.String(), gracePeriod, err)
    } else {
        klog.V(3).Infof("Container %q exited normally", containerID.String())
    }

    m.containerRefManager.ClearRef(containerID)

    return err
}