前置条件

Kubernetes 1.13 或更高版本的集群
kubectl 1.13 或者更高版本
Helm v3 或更高版本

参考配置

1. 配置 ingress 支持 websocket 协议

cat > ingress-inner-test.yaml << EOF
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx-inner
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_set_header Upgrade "websocket";
      proxy_set_header Connection "Upgrade";
  labels:
    app: rancher
  name: rancher-inner-domain
  namespace: cattle-system
spec:
  rules:
  - host: racher.inner.admin.com
    http:
      paths:
      - backend:
          serviceName: rancher
          servicePort: 80
        pathType: ImplementationSpecific
  tls:
  - hosts:
    - racher.inner.admin.com
    secretName: tls-rancher-ingress
EOF
kubectl  apply -f ingress-inner-test.yaml

验证ingress创建状态

kubectl get ingress -n cattle-system rancher-inner-domain
kubectl get svc -n ingress-inner

image

验证服务运行状态

修改 /etc/hosts 添加 10.7.10.40 racher.inner.admin.com 解析记录,
curl https://racher.inner.admin.com -k
kubectl logs -f ingress-inner-ingress-nginx-inner-controller-57d46bb54b-nfszp -n ingress-inner 日志返回 HTTP 200 确认ingress配置正确，且可以正常请求后端服务

10.8.32.0 - - [18/Mar/2021:16:14:51 +0000] "GET / HTTP/1.1" 200 9421 "-" "curl/7.29.0" 86 0.003 [cattle-system-rancher-80] [] 10.8.147.24:80 9407 0.003 200 0d1f631906950ce9355486403c2d6f69

2. 配置 Ingress 支持 tcp/udp 转发

更改ingress-nginx的deployment启动参数

添加--tcp-services-configmap和--udp-services-configmap参数，开启tcp与udp的支持

containers:
- args:
  - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
  - --udp-services-configmap=$(POD_NAMESPACE)/udp-services

更改ingress-nginx的service，声明tcp和udp用的端口号

  ports:
  - name: proxied-tcp
    nodePort: 31800 
    port: 11800
    protocol: TCP
    targetPort: 11800
  - name: proxied-udp
    nodePort: 30091
    port: 9001
    protocol: UDP
    targetPort: 9001

新建一个名为tcp-services的configmap

格式为<ingress-controller-svc-port>:"<namespace>/<service-name>:<port>"

例如下面表示将monitoring命名空间下的hetaoskywalking-oap服务的11800端口映射到ingress-controller service的5044 端口

cat > tcp-services.yaml << EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: tcp-services
  namespace: ingress-inner
data:
  11800: "monitoring/hetaoskywalking-oap:11800"
EOF
kubectl apply -f tcp-services.yaml

3. 配置 Ingress 绑定ULB 实现 http 跳转https功能

创建一个ingress，需要在annotations中定义nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
本地测试修改 /etc/hosts 添加解析记录: ingress_lb_ip svc.domain
curl http://svc.domain 确认服务是否正常
确认ingress服务正常，可以为svc.domain添加DNS解析记录

完整验证示例

准备SSL/TLS证书，这里使用的自签名证书

#!/bin/bash
openssl req -newkey rsa:2048       \
            -keyout ca.key         \
            -out ca.crt            \
            -days 3650             \
            -x509                  \
            -passout pass:ca_key_xxxxx \
            -subj '/C=CN/ST=beijing/L=BJ/O=RD/OU=RDTEAM/CN=admin.com'

for cert_name in nginx
do
   openssl genrsa -out ${cert_name}.key 2048             \
                  -passout pass:111111
   openssl req -new -key ${cert_name}.key                \
                    -out ${cert_name}.csr                \
                    -passin pass:111111                  \
                    -subj "/C=CN/ST=beijing/L=BJ/O=RD/OU=RDTEAM/CN=${cert_name}.admin.com"
   openssl x509 -req -sha256                             \
                 -extensions v3_req                      \
                 -days 3650                              \
                 -in ${cert_name}.csr                    \
                 -CAkey ca.key                           \
                 -CA ca.crt                              \
                 -CAcreateserial                         \
                 -passin pass:ca_key_xxxxx               \
                 -out ${cert_name}.crt
done

创建 nginx 需要的 configmap

cat > default.conf << EOF
server {

    listen 80 default_server;
    server_name _;
    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm; 
    }
}
server {
    listen       443 ssl;
    listen  [::]:443 ssl;
    server_name _;

    ssl_certificate /etc/nginx/ssl/tls.crt;
    ssl_certificate_key /etc/nginx/ssl/tls.key;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm; 
    }

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}
EOF
kubectl create ns nginx
kubectl delete configmap nginx-configmap -n nginx
kubectl create configmap nginx-configmap --from-file=default.conf -n nginx

创建 nginx 需要的 secret

kubectl create ns nginx
kubectl create secret tls nginx-secret --cert=nginx.crt --key=nginx.key -n nginx

创建 nginx 服务

cat > nginx-deploy-svc.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: nginx
  labels:
    app: nginx
spec:
  ports:
  - port: 443
    protocol: TCP
    targetPort: 443
    name: https
  - port: 80
    protocol: TCP
    targetPort: 80
    name: http
  selector:
    app: nginx
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: nginx 
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      volumes:
      - name: secret-volume
        secret:
           secretName: nginx-secret 
      - name: configmap-volume
        configMap:
          name: nginx-configmap 
      containers:
      - name: nginx
        image: uhub.service.ucloud.cn/ucloud/nginx:latest
        ports:
        - containerPort: 443
        - containerPort: 80
        volumeMounts:
        - mountPath: /etc/nginx/ssl
          name: secret-volume
        - mountPath: /etc/nginx/conf.d
          name: configmap-volume
EOF
kubectl apply -f nginx-deploy-svc.yaml

创建 ingress 示例

cat > nginx-svc-ingress.yaml << EOF
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: nginx-svc
  namespace: nginx
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
  tls:
  - hosts:
    - nginx.admin.com
    secretName: nginx-secret
  rules:
  - host: nginx.admin.com
    http:
      paths:
      - backend:
          serviceName: nginx
          servicePort: 80
        path: /
EOF
kubectl apply -f nginx-svc-ingress.yaml