关于kube-ovn NetworkAttachmentDefi

https://github.com/kubeovn/kube-ovn/blob/master/docs/multi-nic.md

[root@hci-dev-mst-1 01-ext-macvlan]# cat 00-nad-macvlan.yaml
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: ovn-vpc-external-network
  namespace: kube-system
spec:
  config: '{
      "cniVersion": "0.3.0",
      "type": "macvlan",
      "master": "eth3",
      "mode": "bridge",
      "ipam": {
        "type": "kube-ovn",
        "server_socket": "/run/openvswitch/kube-ovn-daemon.sock",
        "provider": "ovn-vpc-external-network.kube-system"
      }
    }'
[root@hci-dev-mst-1 01-ext-macvlan]# cat 01-subnet.yaml
apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
  name: ovn-vpc-external-network
spec:
  protocol: IPv4
  provider: ovn-vpc-external-network.kube-system
  cidrBlock: 172.20.10.0/24
  gateway: 172.20.10.2
  excludeIps:
  - 172.20.10.1..172.20.10.200

root@hci-dev-mst-1 01-ext-macvlan]# k get subnet
NAME                       PROVIDER                               VPC           PROTOCOL   CIDR             PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS
join                       ovn                                    ovn-cluster   IPv4       100.64.0.0/16    false     false   false     distributed   5        65528         0        0             ["100.64.0.1"]
ovn-default                ovn                                    ovn-cluster   IPv4       10.16.0.0/16     false     true    true      distributed   10       65523         0        0             ["10.16.0.1"]
ovn-vpc-external-network   ovn-vpc-external-network.kube-system   ovn-cluster   IPv4       172.20.10.0/24   false     false   false     distributed   1        53            0        0             ["172.20.10.1..172.20.10.200"]


# 可以看到这种用法，nad 和 subnet 一一对应，可以清晰的在subnet中看到和nad provider是一致的

还有另外一种用法，就是仅基于nad当一个占位符，在pod中用的时候才指定对应的subnet

[centos@hci-ctrl-1 ~]$ k get network-attachment-definitions.k8s.cni.cncf.io  -n yealink           net1  -o yaml
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"k8s.cni.cncf.io/v1","kind":"NetworkAttachmentDefinition","metadata":{"annotations":{},"name":"net1","namespace":"yealink"},"spec":{"config":"{ \"cniVersion\": \"0.3.1\", \"type\": \"kube-ovn\", \"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\", \"provider\": \"net1.yealink.ovn\" }"}}
  creationTimestamp: "2022-07-19T08:10:26Z"
  generation: 1
  name: net1
  namespace: yealink
  resourceVersion: "5471432"
  uid: 4204380f-e09f-47d9-aea1-d9186074f35d
spec:
  config: '{ "cniVersion": "0.3.1", "type": "kube-ovn", "server_socket": "/run/openvswitch/kube-ovn-daemon.sock",
    "provider": "net1.yealink.ovn" }'


# 双网卡 - 相同子网
apiVersion: v1
kind: Pod
metadata:
  namespace: yealink
  name: shanghai-double-net-same
  annotations:
    # 主网卡使用【shanghai-vm】子网
    ovn.kubernetes.io/logical_switch: shanghai-vm
    # 附加一张网卡
    k8s.v1.cni.cncf.io/networks: net1
    # 次网卡使用【shanghai-pod】子网
    net1.yealink.ovn.kubernetes.io/logical_switch: shanghai-vm
spec:
  containers:
  - name: alpine-sshd


# 对照nat gw的实现

[centos@hci-ctrl-1 nfs]$ k get deployment -n kube-system            vpc-nat-gw-yealink-singapore-vm  -o yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "8"
  creationTimestamp: "2022-07-19T09:52:10Z"
  generation: 8
  name: vpc-nat-gw-yealink-singapore-vm
  namespace: kube-system
  resourceVersion: "43875792"
  uid: 3d7b29ff-8196-4a31-ad86-affa560f903c
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: vpc-nat-gw-yealink-singapore-vm
      ovn.kubernetes.io/vpc-nat-gw: "true"
  strategy:
    type: Recreate
  template:
    metadata:
      annotations:
        deployment.kubernetes.io/revision: "6"
        k8s.v1.cni.cncf.io/networks: kube-system/ovn-vpc-external-network # 可以看到只需要配置改行即可启用net1网卡
        ovn.kubernetes.io/ip_address: 10.115.128.2
        ovn.kubernetes.io/logical_switch: yealink-singapore-vm
        ovn.kubernetes.io/vpc_nat_gw: yealink-singapore-vm
      creationTimestamp: null
      labels:
        app: vpc-nat-gw-yealink-singapore-vm
        ovn.kubernetes.io/vpc-nat-gw: "true"
    spec:
      containers:
      - args:
        - -c
        - while true; do sleep 10000; done
        command:
        - bash
        image: cr.yealinkops.com/iaas/kube-ovn/vpc-nat-gateway:v1.10.3.1
        imagePullPolicy: IfNotPresent
        name: vpc-nat-gw
        resources: {}
        securityContext:
          allowPrivilegeEscalation: true
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      nodeSelector:
        node.kubevirt.io/dedicated: xnet
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        key: node.kubevirt.io/dedicated
        operator: Equal
        value: xnet
status:
  availableReplicas: 1
  conditions:
  - lastTransitionTime: "2022-08-24T12:14:19Z"
    lastUpdateTime: "2022-08-24T12:14:19Z"
    message: Deployment has minimum availability.
    reason: MinimumReplicasAvailable
    status: "True"
    type: Available
  - lastTransitionTime: "2022-08-24T12:07:12Z"
    lastUpdateTime: "2022-08-24T12:14:19Z"
    message: ReplicaSet "vpc-nat-gw-yealink-singapore-vm-5f46d4d974" has successfully
      progressed.
    reason: NewReplicaSetAvailable
    status: "True"
    type: Progressing
  observedGeneration: 8
  readyReplicas: 1
  replicas: 1
  updatedReplicas: 1

这种用法的缺点就是没法基于pod 的annotation中的provider字段和subnet 建立对应关系。不好维护。 这也说明一点，multus nad 的作用主要是是一个键的作用。 其值甚至可以随意指定。

结论: 最好将nad 和 subnet 一一对应，建立稳定的映射关系，可以保证在subnet 和 nad provider一一对应，且可以避免每次手动指定logical_switch 出现错误。