kube-proxy 源码分析之proxy服务框架

author:XiaoYang

程序入口与初始化

kube-proxy使用通用Cobra框架构建一个标准的CLI应用.同kubernets基础组件(如:scheduler)一样的方式来创建应用和运行应用，因此应用command构建层不将对其深入分析，我们将重心聚焦在后面的proxyserver创建与proxyserver run代码分析。

!FILENAME cmd/kube-proxy/proxy.go:35

func main() {
    rand.Seed(time.Now().UnixNano())

    command := app.NewProxyCommand()  //Cobra命令风格应用对象创建
  //......
  
  //对前面实例化Command对象的执行
    if err := command.Execute(); err != nil {
        fmt.Fprintf(os.Stderr, "error: %v\n", err)
        os.Exit(1)
    }
}

NewProxyCommand()返回命令command对象,command.Execute()则调用定义的Run: func{},执行内部的opts.Run().

!FILENAME cmd/kube-proxy/app/server.go:370

func NewProxyCommand() *cobra.Command {
  // 创建options对象(全局的应用配置对象)
    opts := NewOptions()

    cmd := &cobra.Command{
        Use: "kube-proxy",
        Long: `The Kubernetes network proxy runs on each node...略`,
        Run: func(cmd *cobra.Command, args []string) {   //Command应用RUN命令func定义
            verflag.PrintAndExitIfRequested()
            utilflag.PrintFlags(cmd.Flags())
      
            if err := initForOS(opts.WindowsService); err != nil {
                klog.Fatalf("failed OS init: %v", err)
            }
      // 完成所有需求的options配置(外置配置文件、自定义主机名处理、特征功能开关设置等)
            if err := opts.Complete(); err != nil {
                klog.Fatalf("failed complete: %v", err)
            }
      // 校验kube-proxy配置的有效性
            if err := opts.Validate(args); err != nil {
                klog.Fatalf("failed validate: %v", err)
            }
      klog.Fatal(opts.Run())  //应用真正执行Run(),后面进行分析
        },
    }

    var err error
    opts.config, err = opts.ApplyDefaults(opts.config)   //应用默认配置，完成配置的初始化工作
    if err != nil {
        klog.Fatalf("unable to create flag defaults: %v", err)
    }

    opts.AddFlags(cmd.Flags())

    cmd.MarkFlagFilename("config", "yaml", "yml", "json")

  return cmd  //返回命令command对象
}

NewOptions() 全局配置对象创建

!FILENAME cmd/kube-proxy/app/server.go:184

func NewOptions() *Options {
    return &Options{
        config:      new(kubeproxyconfig.KubeProxyConfiguration), //实例化配置config对象
        healthzPort: ports.ProxyHealthzPort,                      //Healthz端口
        metricsPort: ports.ProxyStatusPort,                       //metrics端口
        scheme:      scheme.Scheme,
        codecs:      scheme.Codecs,
        CleanupIPVS: true,
    }
}

//完整的配置结构体与命令执行时用户传递的命令选项相对应
type KubeProxyConfiguration struct {
    metav1.TypeMeta
    FeatureGates map[string]bool              //功能特征模块开关
    BindAddress string                        //默认所有接口0.0.0.0 
    HealthzBindAddress string                 //默认0.0.0.0:10256
    MetricsBindAddress string                 //默认127.0.0.1:10249
    EnableProfiling bool                      //"/debug/pprof"
    ClusterCIDR string                        //ClusterCIDR
    HostnameOverride string                   //自定义Hostname
    ClientConnection apimachineryconfig.ClientConnectionConfiguration  //apiserver client
  IPTables KubeProxyIPTablesConfiguration   //IPTABLES配置项(地址伪装、同步周期等)
  IPVS KubeProxyIPVSConfiguration           //IPVS配置项(同步周期、调度器等)
    OOMScoreAdj *int32                        //修改OOMScoreAdj分值
    Mode ProxyMode                            //proxy模式
    PortRange string                          //端口range
    ResourceContainer string                  //"Default: /kube-proxy"
    UDPIdleTimeout metav1.Duration            //UDP空闲超时
    Conntrack KubeProxyConntrackConfiguration //Conntrack对象
    ConfigSyncPeriod metav1.Duration          //同步周期
    NodePortAddresses []string                //Node地址
}

const (
    ProxyHealthzPort = 10256
  ProxyStatusPort = 10249
)

opts.Run() 创建proxy服务并启动

!FILENAME cmd/kube-proxy/app/server.go:250

func (o *Options) Run() error {
    if len(o.WriteConfigTo) > 0 {
        return o.writeConfigFile()           //写配置文件
    }

    proxyServer, err := NewProxyServer(o)  //基于配置创建proxy服务
    if err != nil {
        return err
    }

    return proxyServer.Run()              //运行proxy服务，后续详细分析proxyserver执行代码逻辑
}

上面已完成对kube-proxy第一层(CLI创建、配置项初始化、启动)分析,下面进入proxy Server创建与启动运行代码分析( kube-proxy第二层 )。

Proxy Server创建

NewProxyServer() 非windows版本代码

!FILENAME cmd/kube-proxy/app/server_others.go:55

func NewProxyServer(o *Options) (*ProxyServer, error) {
    return newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)
}

func newProxyServer(
    config *proxyconfigapi.KubeProxyConfiguration,
    cleanupAndExit bool,
    cleanupIPVS bool,
    scheme *runtime.Scheme,
    master string) (*ProxyServer, error) {

    if config == nil {
        return nil, errors.New("config is required")
    }

    if c, err := configz.New(proxyconfigapi.GroupName); err == nil {
        c.Set(config)
    } else {
        return nil, fmt.Errorf("unable to register configz: %s", err)
    }

  //协议IPV4 or IPV6
    protocol := utiliptables.ProtocolIpv4
    if net.ParseIP(config.BindAddress).To4() == nil {
        klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress)
        protocol = utiliptables.ProtocolIpv6
    }

  // 关键依赖工具实现的api接口iptables/ipvs/ipset/dbus
  // 从此处则可以看出kube-proxy底层技术的依赖（当然不同的proxy-mode，实现技术也不一样 ）
    var iptInterface utiliptables.Interface
    var ipvsInterface utilipvs.Interface
    var kernelHandler ipvs.KernelHandler
    var ipsetInterface utilipset.Interface
    var dbus utildbus.Interface


  // exec命令执行器对象创建
    execer := exec.New()
  // dbus对象创建（linux实现进程间通信机制）
    dbus = utildbus.New()
  // iptables操作对象创建
    iptInterface = utiliptables.New(execer, dbus, protocol)
  // IPVS
    kernelHandler = ipvs.NewLinuxKernelHandler()
  // ipset
    ipsetInterface = utilipset.New(execer)
  // IPVS环境检测
    canUseIPVS, _ := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface)
    if canUseIPVS {
        ipvsInterface = utilipvs.New(execer)
    }

    // We omit creation of pretty much everything if we run in cleanup mode
    if cleanupAndExit {
        return &ProxyServer{
            execer:         execer,
            IptInterface:   iptInterface,
            IpvsInterface:  ipvsInterface,
            IpsetInterface: ipsetInterface,
            CleanupAndExit: cleanupAndExit,
        }, nil
    }

  //api client
    client, eventClient, err := createClients(config.ClientConnection, master)
    if err != nil {
        return nil, err
    }

    //主机名
    hostname, err := utilnode.GetHostname(config.HostnameOverride)
    if err != nil {
        return nil, err
    }
  // 事件广播器
    eventBroadcaster := record.NewBroadcaster()
  // Create event recorder
    recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname})

    nodeRef := &v1.ObjectReference{
        Kind:      "Node",
        Name:      hostname,
        UID:       types.UID(hostname),
        Namespace: "",
    }

    var healthzServer *healthcheck.HealthzServer
    var healthzUpdater healthcheck.HealthzUpdater
   
  //创建默认的healthzServer服务对象
    if len(config.HealthzBindAddress) > 0 {
        healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef)
        healthzUpdater = healthzServer
    }

    var proxier proxy.ProxyProvider
    var serviceEventHandler proxyconfig.ServiceHandler
    var endpointsEventHandler proxyconfig.EndpointsHandler

  // proxyMode模式配置获取
    proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
  // 节点绑定IP
    nodeIP := net.ParseIP(config.BindAddress)
    if nodeIP.IsUnspecified() {
        nodeIP = utilnode.GetNodeIP(client, hostname)
    }
  
    if proxyMode == proxyModeIPTables {                   // proxyMode为"IPTables"
        klog.V(0).Info("Using iptables Proxier.")
        if config.IPTables.MasqueradeBit == nil {
            // MasqueradeBit must be specified or defaulted.
            return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config")
        }

        //创建iptables proxier对象
        proxierIPTables, err := iptables.NewProxier(
            iptInterface,
            utilsysctl.New(),
            execer,
            config.IPTables.SyncPeriod.Duration,
            config.IPTables.MinSyncPeriod.Duration,
            config.IPTables.MasqueradeAll,
            int(*config.IPTables.MasqueradeBit),
            config.ClusterCIDR,
            hostname,
            nodeIP,
            recorder,
            healthzUpdater,
            config.NodePortAddresses,
        )
        if err != nil {
            return nil, fmt.Errorf("unable to create proxier: %v", err)
        }
        metrics.RegisterMetrics()
    //iptables proxier对象和事件处理
        proxier = proxierIPTables               
        serviceEventHandler = proxierIPTables
        endpointsEventHandler = proxierIPTables
        // No turning back. Remove artifacts that might still exist from the userspace Proxier.
        klog.V(0).Info("Tearing down inactive rules.")
        //模式转换清理userspace/ipvs模式所创建iptables规则
        userspace.CleanupLeftovers(iptInterface)
        if canUseIPVS {
            ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
        }
    } else if proxyMode == proxyModeIPVS {                  // proxyMode为"IPVS"
        klog.V(0).Info("Using ipvs Proxier.")
    //ipvs proxier对象创建
        proxierIPVS, err := ipvs.NewProxier(
            iptInterface,
            ipvsInterface,
            ipsetInterface,
            utilsysctl.New(),
            execer,
            config.IPVS.SyncPeriod.Duration,
            config.IPVS.MinSyncPeriod.Duration,
            config.IPVS.ExcludeCIDRs,
            config.IPTables.MasqueradeAll,
            int(*config.IPTables.MasqueradeBit),
            config.ClusterCIDR,
            hostname,
            nodeIP,
            recorder,
            healthzServer,
            config.IPVS.Scheduler,
            config.NodePortAddresses,
        )
        if err != nil {
            return nil, fmt.Errorf("unable to create proxier: %v", err)
        }
        metrics.RegisterMetrics()
    //ipvs proxier对象和事件处理
        proxier = proxierIPVS
        serviceEventHandler = proxierIPVS
        endpointsEventHandler = proxierIPVS
        klog.V(0).Info("Tearing down inactive rules.")
    //模式转换清理userspace/iptables模式规则
        userspace.CleanupLeftovers(iptInterface)
        iptables.CleanupLeftovers(iptInterface)
    } else {                                              // proxyMode为"userspace"
        klog.V(0).Info("Using userspace Proxier.")
    //创建RR模式负载均衡
        loadBalancer := userspace.NewLoadBalancerRR()
    //设置EndpointsConfigHandler(endpoints事件处理)
        endpointsEventHandler = loadBalancer
    //创建userspace proxier对象
        proxierUserspace, err := userspace.NewProxier(
            loadBalancer,
            net.ParseIP(config.BindAddress),
            iptInterface,
            execer,
            *utilnet.ParsePortRangeOrDie(config.PortRange),
            config.IPTables.SyncPeriod.Duration,
            config.IPTables.MinSyncPeriod.Duration,
            config.UDPIdleTimeout.Duration,
            config.NodePortAddresses,
        )
        if err != nil {
            return nil, fmt.Errorf("unable to create proxier: %v", err)
        }
    //userspace proxier对象和service事件处理
        serviceEventHandler = proxierUserspace
        proxier = proxierUserspace

        klog.V(0).Info("Tearing down inactive rules.")
        //模式转换清理iptables/ipvs模式所创建iptables规则
        iptables.CleanupLeftovers(iptInterface)
        if canUseIPVS {
            ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
        }
    }
 
  //注册reloadfunc为proxier的sync()同步方法
    iptInterface.AddReloadFunc(proxier.Sync)
  
  // 构建ProxyServer对象
    return &ProxyServer{
        Client:                 client,                  //apiServer client
        EventClient:            eventClient,             //事件client
        IptInterface:           iptInterface,            //iptables接口
        IpvsInterface:          ipvsInterface,           //ipvs接口
        IpsetInterface:         ipsetInterface,          //ipset接口
        execer:                 execer,                  //exec命令执行器   
        Proxier:                proxier,                 //proxier创建对象
        Broadcaster:            eventBroadcaster,        //事件广播器
        Recorder:               recorder,                //事件记录器
        ConntrackConfiguration: config.Conntrack,        //Conntrack配置
        Conntracker:            &realConntracker{},      //Conntrack对象
        ProxyMode:              proxyMode,               //proxy模式
        NodeRef:                nodeRef,                 //node节点reference信息
        MetricsBindAddress:     config.MetricsBindAddress,        //metric服务地址配置
        EnableProfiling:        config.EnableProfiling,           //debug/pprof配置
        OOMScoreAdj:            config.OOMScoreAdj,               //OOMScoreAdj值配置
        ResourceContainer:      config.ResourceContainer,         //容器资源配置
        ConfigSyncPeriod:       config.ConfigSyncPeriod.Duration, //同步周期配置
        ServiceEventHandler:    serviceEventHandler,              //处理service事件proxier对象
        EndpointsEventHandler:  endpointsEventHandler,            //处理endpoints事件proxier对象
        HealthzServer:          healthzServer,                    //健康检测服务
    }, nil
}

Proxy Server运行

!FILENAME cmd/kube-proxy/app/server.go:481

func (s *ProxyServer) Run() error {
    klog.Infof("Version: %+v", version.Get())

  // 如果CleanupAndExit设置为true,则删除存在的所有iptables规则项，然后应用退出。
    if s.CleanupAndExit {
        encounteredError := userspace.CleanupLeftovers(s.IptInterface)
        encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
        encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface, s.IpsetInterface, s.CleanupIPVS) || encounteredError
        if encounteredError {
            return errors.New("encountered an error while tearing down rules.")
        }
        return nil
    }

  // 根据启动参数配置"oom-score-adj"分值调整，取值区间[-1000, 1000]
    var oomAdjuster *oom.OOMAdjuster
    if s.OOMScoreAdj != nil {
        oomAdjuster = oom.NewOOMAdjuster()
        if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.OOMScoreAdj)); err != nil {
            klog.V(2).Info(err)
        }
    }

  //"resource-container"设置是否运行在容器里
    if len(s.ResourceContainer) != 0 {
        if err := resourcecontainer.RunInResourceContainer(s.ResourceContainer); err != nil {
            klog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err)
        } else {
            klog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer)
        }
    }

  //事件广播器
    if s.Broadcaster != nil && s.EventClient != nil {
        s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
    }

    // 根据配置启动healthz健康检测服务
    if s.HealthzServer != nil {
        s.HealthzServer.Run()
    }

  // 根据配置启动metrics服务。URI: "/proxyMode" 与 "/metrics"
    if len(s.MetricsBindAddress) > 0 {
        mux := mux.NewPathRecorderMux("kube-proxy")
        healthz.InstallHandler(mux)
        mux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) {
            fmt.Fprintf(w, "%s", s.ProxyMode)
        })
        mux.Handle("/metrics", prometheus.Handler())
        if s.EnableProfiling {
            routes.Profiling{}.Install(mux)
        }
        configz.InstallHandler(mux)
        go wait.Until(func() {
            err := http.ListenAndServe(s.MetricsBindAddress, mux)
            if err != nil {
                utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err))
            }
        }, 5*time.Second, wait.NeverStop)
    }

  // 如果需要(命令选项或配置项)调节conntrack配置值
    if s.Conntracker != nil {
        max, err := getConntrackMax(s.ConntrackConfiguration)
        if err != nil {
            return err
        }
        if max > 0 {
            err := s.Conntracker.SetMax(max)
            if err != nil {
                if err != readOnlySysFSError {
                    return err
                }
        
                const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: " +
                    "cannot modify conntrack limits, problems may arise later."
                s.Recorder.Eventf(s.NodeRef, api.EventTypeWarning, err.Error(), message)
            }
        }
     //设置conntracker的TCPEstablishedTimeout
        if s.ConntrackConfiguration.TCPEstablishedTimeout != nil && s.ConntrackConfiguration.TCPEstablishedTimeout.Duration > 0 {
            timeout := int(s.ConntrackConfiguration.TCPEstablishedTimeout.Duration / time.Second)
            if err := s.Conntracker.SetTCPEstablishedTimeout(timeout); err != nil {
                return err
            }
        }
    //设置conntracker的TCPCloseWaitTimeout
        if s.ConntrackConfiguration.TCPCloseWaitTimeout != nil && s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration > 0 {
            timeout := int(s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration / time.Second)
            if err := s.Conntracker.SetTCPCloseWaitTimeout(timeout); err != nil {
                return err
            }
        }
    }
 
   // informer机制获取与监听Services和Endpoints的配置与事件信息
   // 注册ServiceEventHandler服务事件的处理
   // 注册EndpointsEventHandler端点事件的处理
    informerFactory := informers.NewSharedInformerFactory(s.Client, s.ConfigSyncPeriod)
    serviceConfig := config.NewServiceConfig(informerFactory.Core().V1().Services(), s.ConfigSyncPeriod)
    serviceConfig.RegisterEventHandler(s.ServiceEventHandler)
    go serviceConfig.Run(wait.NeverStop)

    endpointsConfig := config.NewEndpointsConfig(informerFactory.Core().V1().Endpoints(), s.ConfigSyncPeriod)
    endpointsConfig.RegisterEventHandler(s.EndpointsEventHandler)
    go endpointsConfig.Run(wait.NeverStop)
  
    go informerFactory.Start(wait.NeverStop)

  // "新生儿降生的哭声"，作者命名比较有生活情趣^_^
  //  服务成功启动，将启动事件广播。
  //   s.Recorder.Eventf(s.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.")
  //    nodeRef := &v1.ObjectReference{
    //      Kind:      "Node",
    //      Name:      hostname,
    //      UID:       types.UID(hostname),
    //      Namespace: "",
    //   }
    s.birthCry()

  // Proxier(代理服务提供者)进行循环配置同步与处理proxy逻辑（本文聚焦应用主框架，后有专篇分析Proxier）
    // 此时将进入了proxier运行，默认使用的iptables模式的proxier对象
    s.Proxier.SyncLoop()
    return nil
}

s.Proxier.SyncLoop()的运行将进入kube-proxy第三层(service实现机制层),Proxier实例化对象是在proxy server对象创建时通过config配置文件或"-proxy-mode"指定（userspace / iptables / ipvs模式），而默认使用的iptables模式proxier对象。第三层的代码分析将针对三种模式设立专篇分析，此处为关键部分请记住此处，在后面proxier的分析文章重点关注。

在第二层的框架层我们还须关注kube-proxy与kubernetes集群同步信息的机制informer。kube-proxy组件在proxy server的run()运行创建service、endpoints的informer对其list同步数据和watch监听事件(add、delete、update)，调用注册的proxier handler进行处理(第三层proxier的同步处理机制调用与触发)。

同步规则机制(Informer)

kube-proxy同样使用client-go标准的ApiServer同步方式，创建informer,注册事件处理器handler，持续监控watch事件并调用handler处理事件add/update/delete，后端处理则由proxier(userspace/iptables/ipvs)实现。因为endpoints的同步与service同步方式一致，则下面仅说明service代码实现。

!FILENAME pkg/proxy/config/config.go:174

func NewServiceConfig(serviceInformer coreinformers.ServiceInformer, resyncPeriod time.Duration) *ServiceConfig {
    result := &ServiceConfig{
        lister:       serviceInformer.Lister(),                //监听器
        listerSynced: serviceInformer.Informer().HasSynced,    //监听器同步状态值
    }

  //在服务informer上添加了资源事件的处理器handleFunc。（Add/update/delete）
    serviceInformer.Informer().AddEventHandlerWithResyncPeriod(
        cache.ResourceEventHandlerFuncs{
            AddFunc:    result.handleAddService,
            UpdateFunc: result.handleUpdateService,
            DeleteFunc: result.handleDeleteService,
        },
        resyncPeriod,     
    )

    return result
}

!FILENAME pkg/proxy/config/config.go:119

func (c *ServiceConfig) Run(stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()

    klog.Info("Starting service config controller")
    defer klog.Info("Shutting down service config controller")

    if !controller.WaitForCacheSync("service config", stopCh, c.listerSynced) {
        return
    }

    for i := range c.eventHandlers {
        klog.V(3).Info("Calling handler.OnServiceSynced()")
        c.eventHandlers[i].OnServiceSynced()               //服务与proxier处理同步
    }

    <-stopCh
}

HandleAddService()新增事件处理

!FILENAME pkg/proxy/config/config.go:217

func (c *ServiceConfig) handleAddService(obj interface{}) {
    service, ok := obj.(*v1.Service)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj))
        return
    }
    for i := range c.eventHandlers {
        klog.V(4).Info("Calling handler.OnServiceAdd")
        c.eventHandlers[i].OnServiceAdd(service)            //服务新增事件与proxier处理同步
    }
}

HandleUpdateService()更新事件处理

!FILENAME pkg/proxy/config/config.go:229

func (c *ServiceConfig) handleUpdateService(oldObj, newObj interface{}) {
    oldService, ok := oldObj.(*v1.Service)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", oldObj))
        return
    }
    service, ok := newObj.(*v1.Service)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", newObj))
        return
    }
    for i := range c.eventHandlers {
        klog.V(4).Info("Calling handler.OnServiceUpdate")
        c.eventHandlers[i].OnServiceUpdate(oldService, service) //服务更新事件与proxier处理同步 
    }
}

HandleDeleteService()删除事件处理

!FILENAME pkg/proxy/config/config.go:246

func (c *ServiceConfig) handleDeleteService(obj interface{}) {
    service, ok := obj.(*v1.Service)
    if !ok {
        tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
        if !ok {
            utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj))
            return
        }
        if service, ok = tombstone.Obj.(*v1.Service); !ok {
            utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj))
            return
        }
    }
    for i := range c.eventHandlers {
        klog.V(4).Info("Calling handler.OnServiceDelete")
        c.eventHandlers[i].OnServiceDelete(service)         //服务删除事件与proxier处理同步
    }
}

第三层proxier分析，请参看iptables、ipvs、userspace-mode proxier分析文档。

〜本文END〜