httpserver 代码

package main

import (
    "context"
    "fmt"
    "github.com/fsnotify/fsnotify"
    log "github.com/sirupsen/logrus"
    "github.com/spf13/viper"
    "io"
    "net"
    "net/http"
    "os"
    "os/signal"
    "strconv"
    "strings"
    "syscall"
    "time"
)
var logLevel string
var viperInstance *viper.Viper

func main() {
    httpserverConf := os.Getenv("HTTPSERVER_CONF")
    log.Info("configFile from env is " + httpserverConf)
    if httpserverConf == "" {
        httpserverConf = "/etc/httpserver/httpserver.properties"
    }
    log.Info("confFile is " + httpserverConf)
    viperInstance = viper.New() // viper实例
    viperInstance.SetConfigFile(httpserverConf) // 指定配置文件路径

    err := viperInstance.ReadInConfig()
    if err != nil { // 处理读取配置文件的错误
        panic(fmt.Errorf("Fatal error config file: %s \n", err))
    }
    viperInstance.WatchConfig()
    viperInstance.OnConfigChange(func(e fsnotify.Event) {
        fmt.Println("Detect config change: %s \n", e.String())
        log.Warn("Config file updated.")
        viperLoadConf(viperInstance)  // 加载配置的方法
    })
    viperLoadConf(viperInstance)

    port := 8080
    portstr := ":" + strconv.Itoa(port)
    log.Info("httpserver listend port: ", port)

    mux := http.NewServeMux()
    mux.HandleFunc("/", HttpHandler)
    mux.HandleFunc("/healthz", HealthZ)
    mux.HandleFunc("/sleep", SleepTest)
    server := &http.Server{
        Addr:         portstr,
        Handler:      mux,
    }
    go server.ListenAndServe()
    listenSignal(context.Background(), server)
}

func dynamicConfig() {
    viperInstance.WatchConfig()
    viperInstance.OnConfigChange(func(e fsnotify.Event) {
        fmt.Println("Detect config change: %s \n", e.String())
        log.Warn("Config file updated.")
        viperLoadConf(viperInstance)  // 加载配置的方法
    })
}

func viperLoadConf(viperInstance *viper.Viper) {
    logLevel = viperInstance.GetString("log_level")
    level, err := log.ParseLevel(logLevel)
    if err != nil {
        level = log.GetLevel()
    }
    log.SetLevel(level)
    myconf := viperInstance.GetString("my_conf")
    log.Trace(myconf + " in viperLoadConf")
    log.Debug(myconf + " in viperLoadConf")
    log.Info(myconf + " in viperLoadConf")
    log.Warn(myconf + " in viperLoadConf")
    log.Error(myconf + " in viperLoadConf")
}

func HealthZ(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(200)
}
func HttpHandler(w http.ResponseWriter, r *http.Request) {
    myconf := viperInstance.GetString("my_conf")
    log.Trace(myconf)
    log.Debug(myconf)
    log.Info(myconf)
    log.Warn(myconf)
    log.Error(myconf)
    ip := ClientIP(r)
    httpcode := 200
    log.Info("reqLog: clientIP : " + ip + " httpcode " + strconv.Itoa(httpcode))
    headers := r.Header
    log.Info("req headers : %+v", headers)
    for k, v := range headers {
        for _, val := range v {
            w.Header().Set(k, val)
        }
    }
    version := os.Getenv("VERSION")
    if version != "" {
        w.Header().Set("VERSION", version)
    }
    w.WriteHeader(httpcode)
    io.WriteString(w, "ok")
}

func ClientIP(r *http.Request) string {
    xForwardedFor := r.Header.Get("X-Forwarded-For")
    ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0])
    if ip != "" {
        return ip
    }
    ip = strings.TrimSpace(r.Header.Get("X-Real-Ip"))
    if ip != "" {
        return ip
    }
    if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil {
        return ip
    }
    return ""
}

func SleepTest(w http.ResponseWriter, r *http.Request) {
    values := r.URL.Query()
    sleeptimeStr := values.Get("sleep")
    sleeptime, err := strconv.Atoi(sleeptimeStr)
    if err != nil {
        sleeptime = 1
    }
    time.Sleep(time.Duration(sleeptime) * time.Second)
    fmt.Fprintln(w, "Hello world, sleep " + strconv.Itoa(sleeptime) + "s")
    log.Printf( "Hello world, sleep %+vs", sleeptime)
}

// 优雅终止
func listenSignal(ctx context.Context, httpSrv *http.Server) {
    sigs := make(chan os.Signal, 1)
    signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

    select {
    case <-sigs:
        fmt.Println("notify sigs")
        httpSrv.Shutdown(ctx)
        fmt.Println("http shutdown gracefully")
    }
}

Deployment Yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: httpserver
  namespace: mxs
  labels:
    app: httpserver
spec:
  replicas: 2
  strategy:
    type: RollingUpdate
    rollingUpdate:
      # maxSurge: 最大激增数, 指更新过程中, 最多可以比replicas预先设定值多出的pod数量, 可以为固定值或百分比(默认25%), 更新过程中最多会有replicas + maxSurge个pod
      maxSurge: 2
      # maxUnavailable: 最大无效数, 指更新过程中, 最多有几个pod处于无法服务状态, 当maxSurge不为0时, 此栏位也不可为0, 整个更新过程中, 会有maxUnavailable个pod处于Terminating状态
      maxUnavailable: 1
  # minReadySeconds: 容器内应用的启动时间, pod变为run状态, 会在minReadySeconds后继续更新下一个pod. 如果不设置该属性, pod会在run成功后, 立即更新下一个pod.
  minReadySeconds: 15
  selector:
    matchLabels:
      app: httpserver
  template:
    metadata:
      labels:
        app: httpserver
    spec:
      containers:
        - name: httpserver
          image: tjudream/httpserver:v5
          command: [/httpserver]
          envFrom:
          - configMapRef: 
              name: httpserver-env-cm
          volumeMounts:
          - name: config-volume
            mountPath: /etc/httpserver/
          resources:
            limits:
              cpu: 500m
              memory: 512Mi
            requests:
              cpu: 500m
              memory: 512Mi
          # 优雅启动
          livenessProbe:
            httpGet:
              ### this probe will fail with 404 error code
              ### only httpcode between 200-400 is retreated as success
              path: /healthz
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5
          # 探活
          readinessProbe:
            httpGet:
              ### this probe will fail with 404 error code
              ### only httpcode between 200-400 is retreated as success
              path: /healthz
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 5
            successThreshold: 2
      volumes:
        - name: config-volume
          configMap:
          # Provide the name of the ConfigMap containing the files you want
          # to add to the container
            name: httpserver-conf-cm

ConfigMap

apiVersion: v1
kind: ConfigMap
metadata:
  name: httpserver-env-cm
  namespace: mxs
data:
  VERSION: v1.0 from cm
  HTTPSERVER_CONF: /etc/httpserver/httpserver.properties
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: httpserver-conf-cm
  namespace: mxs
data:
  httpserver.properties: |
  log_level: debug
  my_conf: myconf_v1

优雅启动

deploy 中的 livenessProbe 提供优雅启动功能，只有当 8080 端口的 http get 请求 url 127.0.0.1/healthz 返回 200 时，pod 才会被 k8s 设置成 ready 状态，然后才会接收外部流量

livenessProbe:
          httpGet:
            ### this probe will fail with 404 error code
            ### only httpcode between 200-400 is retreated as success
            path: /healthz
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5

优雅终止

go 代码中的 listenSignal 函数提供优雅终止功能，当 go 的 httpserver 服务接收到 k8s 的终止信号时，会调用 httpserver 的 Shutdown 函数，不再接收新的请求，将现在正在处理中的请求处理完成后会主动退出

// 优雅终止
func listenSignal(ctx context.Context, httpSrv *http.Server) {
    sigs := make(chan os.Signal, 1)
    signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

    select {
    case <-sigs:
        fmt.Println("notify sigs")
        httpSrv.Shutdown(ctx)
        fmt.Println("http shutdown gracefully")
    }
}

资源需求和 QoS 保证

k8s 的 QoS 保证：

• pod中所有容器都必须统一设置limits，并且设置参数都一致，如果有一个容器要设置requests，那么所有容器都要设置，并设置参数同limits一致，那么这个pod的QoS就是Guaranteed级别
• Burstable: pod中只要有一个容器的requests和limits的设置不相同，该pod的QoS即为Burstable
• Best-Effort：如果对于全部的resources来说requests与limits均未设置，该pod的QoS即为Best-Effort

3种QoS优先级从有低到高（从左向右）：Best-Effort pods -> Burstable pods -> Guaranteed pods

httpserver 这个容器设置了一样的 requests 和 limit 且只有这一个容器，所以是最高 QoS 保证，即 Guaranteed

探活

deploy 中配置的 readinessProbe 用于探活，一旦 /healthz 这个接口请求不通，或者返回非 200~400 的状态码，则 k8s 就会将这个pod调度为非 ready 状态，流量就不会请求到这个 pod 上了

 readinessProbe:
          httpGet:
            ### this probe will fail with 404 error code
            ### only httpcode between 200-400 is retreated as success
            path: /healthz
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 5
          successThreshold: 2

日常运维需求，日志等级

日常日志查看，其中 httpserver-565798b9f9-4rghf 是 pod 的名称，由于 httpserver 将日志打印到控制台中所以可以直接通过 logs 命令查看

kubectl logs -f httpserver-565798b9f9-4rghf

但是在pod出现异常退出时，日志也就销毁了，这种方法就无法查看到日志了

常用的日志解决方案是采用 ELK 方案，搭建 ElasticSearch 集群，采用 filebeat（最初是 logstash，由于内存消耗比较大，所以现在一般用 filebeat 替代）采集日志存储至 ES 中，最后通过 Kibana 查询和展示。

filebeat 可以采用 sidecar 的方式挂载到业务容器中，也可通过 DeamonSet 的方式启动之后进行收集。

配置和代码分离

通过 ConfigMap 配置 httpserver 的配置

envFrom:
- configMapRef: 
    name: httpserver-env-cm
volumeMounts:
- name: config-volume
  mountPath: /etc/httpserver/httpserver.properties

ConfigMap + viper 实现 httpserver 热加载配置

    viperInstance.WatchConfig()
    viperInstance.OnConfigChange(func(e fsnotify.Event) {
        fmt.Println("Detect config change: %s \n", e.String())
        log.Warn("Config file updated.")
        viperLoadConf(viperInstance)  // 加载配置的方法
    })

当 ConfigMap 发生变化时，会被 viperInstance.WatchConfig() 监控到，然后会调用 OnConfigChange 函数中的匿名函数，重新加载配置

HPA 动态扩缩容

HPA的全称为Horizontal Pod Autoscaling，它可以根据当前pod资源的使用率（如CPU、磁盘、内存等），进行副本数的动态的扩容与缩容，以便减轻各个pod的压力。
安装 meritcs

wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

在 image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1 上边加入一行 - --kubelet-insecure-tls

image.png

kubectl apply -f components.yaml

创建 hpa，在 cpu 使用率高于 80% 或者内存使用率高于 70% 的时候进行扩容，最少 2 个实例，最多 5 个实例

apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
  name: httpserver
  namespace: mxs
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: httpserver
  minReplicas: 2
  maxReplicas: 5
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 80
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 70