k8s pod qos

简单总结

qos分类

如果pod中所有容器都未设置request和liimit,则为besteffort
如果pod中所有容器必须设置了cpu和memory的limit,且request和limit数量相等(个数相等非设置的具体值相等,即设置了3个requst,2个limit则不满足),则为guaranteed
否则为burstable

qos处理

cpu

借助于cgroup和CFS进行管理,超额会限流
https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html

memory

pod级别超额了,会根据cgroup的memory.oom_control中的oom_kill_disable有不同表现(这个特性后面被移除了）

为1则kill
为0则程序会暂停等待额外的内存释放

node级别内存超了,则根据设置的oom_score来进行内存释放，数值越小越不会被kill

如果pod为static或者mirror或者pod的priority大于等于2000000000,且Pod的priorityClassName为system-node-critical,则oomscore为-997
如果pod qos为guaranteed,则oomscrore为-997
如果pod qos为besteffort,则oomscore为1000
否则根据内存request和limit的比值*1000计算,最终值在3到999间

相关源码

pkg/apis/core/v1/helper/qos/qos.go中

func GetPodQOS(pod *v1.Pod) v1.PodQOSClass {
    requests := v1.ResourceList{}
    limits := v1.ResourceList{}
    zeroQuantity := resource.MustParse("0")
    isGuaranteed := true
    allContainers := []v1.Container{}
    allContainers = append(allContainers, pod.Spec.Containers...)
    allContainers = append(allContainers, pod.Spec.InitContainers...)
    for _, container := range allContainers {
        // process requests
        for name, quantity := range container.Resources.Requests {
            if !isSupportedQoSComputeResource(name) {
                continue
            }
            if quantity.Cmp(zeroQuantity) == 1 {
                delta := quantity.DeepCopy()
                if _, exists := requests[name]; !exists {
                    requests[name] = delta
                } else {
                    delta.Add(requests[name])
                    requests[name] = delta
                }
            }
        }
        // process limits
        qosLimitsFound := sets.NewString()
        for name, quantity := range container.Resources.Limits {
            if !isSupportedQoSComputeResource(name) {
                continue
            }
            if quantity.Cmp(zeroQuantity) == 1 {
                qosLimitsFound.Insert(string(name))
                delta := quantity.DeepCopy()
                if _, exists := limits[name]; !exists {
                    limits[name] = delta
                } else {
                    delta.Add(limits[name])
                    limits[name] = delta
                }
            }
        }

        if !qosLimitsFound.HasAll(string(v1.ResourceMemory), string(v1.ResourceCPU)) {
            isGuaranteed = false
        }
    }
    if len(requests) == 0 && len(limits) == 0 {
        return v1.PodQOSBestEffort
    }
    // Check is requests match limits for all resources.
    if isGuaranteed {
        for name, req := range requests {
            if lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 {
                isGuaranteed = false
                break
            }
        }
    }
    if isGuaranteed &&
        len(requests) == len(limits) {
        return v1.PodQOSGuaranteed
    }
    return v1.PodQOSBurstable
}

pkg/kubelet/qos/policy.go中

func GetContainerOOMScoreAdjust(pod *v1.Pod, container *v1.Container, memoryCapacity int64) int {
    if types.IsNodeCriticalPod(pod) {
        // Only node critical pod should be the last to get killed.
        return guaranteedOOMScoreAdj
    }

    switch v1qos.GetPodQOS(pod) {
    case v1.PodQOSGuaranteed:
        // Guaranteed containers should be the last to get killed.
        return guaranteedOOMScoreAdj
    case v1.PodQOSBestEffort:
        return besteffortOOMScoreAdj
    }

    // Burstable containers are a middle tier, between Guaranteed and Best-Effort. Ideally,
    // we want to protect Burstable containers that consume less memory than requested.
    // The formula below is a heuristic. A container requesting for 10% of a system's
    // memory will have an OOM score adjust of 900. If a process in container Y
    // uses over 10% of memory, its OOM score will be 1000. The idea is that containers
    // which use more than their request will have an OOM score of 1000 and will be prime
    // targets for OOM kills.
    // Note that this is a heuristic, it won't work if a container has many small processes.
    memoryRequest := container.Resources.Requests.Memory().Value()
    oomScoreAdjust := 1000 - (1000*memoryRequest)/memoryCapacity
    // A guaranteed pod using 100% of memory can have an OOM score of 10. Ensure
    // that burstable pods have a higher OOM score adjustment.
    if int(oomScoreAdjust) < (1000 + guaranteedOOMScoreAdj) {
        return (1000 + guaranteedOOMScoreAdj)
    }
    // Give burstable pods a higher chance of survival over besteffort pods.
    if int(oomScoreAdjust) == besteffortOOMScoreAdj {
        return int(oomScoreAdjust - 1)
    }
    return int(oomScoreAdjust)
}