kubernetes源码阅读—— kube-apiserver

介绍

• kube-apiserver组件负责将Kubernetes的“资源组、资源版本、资源”以RESTful风格的形式对外暴露并提供服务。该组件是Kubernetes系统集群中所有组件沟通的桥梁。
• kube-apiserver基于go-restful框架
• kube-apiserver提供了3种HTTP Server:

1. KubeAPIServer：提供了Kubernetes内置核心资源服务。该服务通过Master对象进行管理，并通过legacyscheme.Scheme资源注册表管理Master相关资源。
2. APIExtensionsServer：提供了CRD自定义资源服务，开发者可通过CRD对Kubernetes资源进行扩展。该服务通过CustomResourceDefinitions对象进行管理，并通过extensionsapiserver.Scheme资源注册表管理CRD相关资源。
3. AggregatorServer：提供了AA （APIAggregator）聚合服务，开发者可通过AA对Kubernetes聚合服务进行扩展。API聚合服务通过APIAggregator对象进行管理，并通过aggregatorscheme.Scheme资源注册表管理AA相关资源。

kube-apiserver启动流程

1. 资源注册。
2. Cobra命令行参数解析。
3. 创建APIServer通用配置。
4. 创建APIExtensionsServer。
5. 创建KubeAPIServer。
6. 创建AggregatorServer。
7. 创建GenericAPIServer。
8. 启动HTTP服务。
9. 启动HTTPS服务。

• 初始化全局变量：

// pkg\api\legacyscheme\scheme.go
var (
    // Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered.
    // NOTE: If you are copying this file to start a new api group, STOP! Copy the
    // extensions group instead. This Scheme is special and should appear ONLY in
    // the api group, unless you really know what you're doing.
    // TODO(lavalamp): make the above error impossible.
    Scheme = runtime.NewScheme()

    // Codecs provides access to encoding and decoding for the scheme
    Codecs = serializer.NewCodecFactory(Scheme)

    // ParameterCodec handles versioning of objects that are converted to query parameters.
    ParameterCodec = runtime.NewParameterCodec(Scheme)
)

• 由import机制，pkg/apis下的每个资源组会进行资源注册，如core:

// pkg\apis\core\install\install.go
func init() {
    Install(legacyscheme.Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
    utilruntime.Must(core.AddToScheme(scheme))
    utilruntime.Must(v1.AddToScheme(scheme))
    utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion))
}

• 程序入口：

// cmd\kube-apiserver\apiserver.go
func main() {
    rand.Seed(time.Now().UnixNano())

    pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)

    //构建命令行对象，返回*cobra.Command
    command := app.NewAPIServerCommand()

    logs.InitLogs()
    defer logs.FlushLogs()

    //command.Execute()最终会调用 Command结构体中定义的Run函数
    if err := command.Execute(); err != nil {
        os.Exit(1)
    }
}

// cmd\kube-apiserver\app\server.go
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
    // 初始化各个模块的默认配置
    s := options.NewServerRunOptions()
    cmd := &cobra.Command{
        Use: "kube-apiserver",
        Long: `The Kubernetes API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,

        // stop printing usage when the command errors
        SilenceUsage: true,
        PersistentPreRunE: func(*cobra.Command, []string) error {
            // silence client-go warnings.
            // kube-apiserver loopback clients should not log self-issued warnings.
            rest.SetDefaultWarningHandler(rest.NoWarnings{})
            return nil
        },
        RunE: func(cmd *cobra.Command, args []string) error {
            verflag.PrintAndExitIfRequested()
            fs := cmd.Flags()
            cliflag.PrintFlags(fs)

            err := checkNonZeroInsecurePort(fs)
            if err != nil {
                return err
            }
            // set default options
            // 完成参数的配置
            completedOptions, err := Complete(s)
            if err != nil {
                return err
            }

            // validate options
            if errs := completedOptions.Validate(); len(errs) != 0 {
                return utilerrors.NewAggregate(errs)
            }
            // 使用配置运行apiserver
            return Run(completedOptions, genericapiserver.SetupSignalHandler())
        },
        Args: func(cmd *cobra.Command, args []string) error {
            for _, arg := range args {
                if len(arg) > 0 {
                    return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
                }
            }
            return nil
        },
    }

    fs := cmd.Flags()
    namedFlagSets := s.Flags()
    verflag.AddFlags(namedFlagSets.FlagSet("global"))
    globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
    options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
    for _, f := range namedFlagSets.FlagSets {
        fs.AddFlagSet(f)
    }

    usageFmt := "Usage:\n  %s\n"
    cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
    cmd.SetUsageFunc(func(cmd *cobra.Command) error {
        fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
        cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
        return nil
    })
    cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
        fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
        cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
    })

    return cmd
}


// Run runs the specified APIServer.  This should never exit.
func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error {
    // To help debugging, immediately log version
    klog.Infof("Version: %+v", version.Get())

    // 创建server
    server, err := CreateServerChain(completeOptions, stopCh)
    if err != nil {
        return err
    }

    prepared, err := server.PrepareRun()
    if err != nil {
        return err
    }

    // 启动server
    return prepared.Run(stopCh)
}

• CreateServerChain先调用CreateKubeAPIServerConfig函数，根据配置参数创建了运行api server所需要的资源。再依次创建apiExtensionsServer，kubeAPIServer和aggregatorServer三个server。

// CreateServerChain creates the apiservers connected via delegation.
func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*aggregatorapiserver.APIAggregator, error) {
    nodeTunneler, proxyTransport, err := CreateNodeDialer(completedOptions)
    if err != nil {
        return nil, err
    }

    kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions, nodeTunneler, proxyTransport)
    if err != nil {
        return nil, err
    }

    // If additional API servers are added, they should be gated.
    apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
        serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig))
    if err != nil {
        return nil, err
    }
    apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
    if err != nil {
        return nil, err
    }

    kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
    if err != nil {
        return nil, err
    }

    // aggregator comes last in the chain
    aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, proxyTransport, pluginInitializer)
    if err != nil {
        return nil, err
    }
    aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
    if err != nil {
        // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
        return nil, err
    }

    return aggregatorServer, nil
}

• CreateKubeAPIServerConfig创建了kube-apiserver不同模块实例化所需的配置：

// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
func CreateKubeAPIServerConfig(
    s completedServerRunOptions,
    nodeTunneler tunneler.Tunneler,
    proxyTransport *http.Transport,
) (
    *controlplane.Config,
    aggregatorapiserver.ServiceResolver,
    []admission.PluginInitializer,
    error,
) {
    genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
    if err != nil {
        return nil, nil, nil, err
    }

    if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.Transport.ServerList[0]); err == nil && port != "0" && len(port) != 0 {
        if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.Transport.ServerList}.CheckEtcdServers); err != nil {
            return nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
        }
    }

    capabilities.Initialize(capabilities.Capabilities{
        AllowPrivileged: s.AllowPrivileged,
        // TODO(vmarmol): Implement support for HostNetworkSources.
        PrivilegedSources: capabilities.PrivilegedSources{
            HostNetworkSources: []string{},
            HostPIDSources:     []string{},
            HostIPCSources:     []string{},
        },
        PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
    })

    s.Metrics.Apply()
    serviceaccount.RegisterMetrics()

    s.Logs.Apply()

    serviceIPRange, apiServerServiceIP, err := controlplane.ServiceIPRange(s.PrimaryServiceClusterIPRange)
    if err != nil {
        return nil, nil, nil, err
    }

    // defaults to empty range and ip
    var secondaryServiceIPRange net.IPNet
    // process secondary range only if provided by user
    if s.SecondaryServiceClusterIPRange.IP != nil {
        secondaryServiceIPRange, _, err = controlplane.ServiceIPRange(s.SecondaryServiceClusterIPRange)
        if err != nil {
            return nil, nil, nil, err
        }
    }

    config := &controlplane.Config{
        GenericConfig: genericConfig,
        ExtraConfig: controlplane.ExtraConfig{
            APIResourceConfigSource: storageFactory.APIResourceConfigSource,
            StorageFactory:          storageFactory,
            EventTTL:                s.EventTTL,
            KubeletClientConfig:     s.KubeletConfig,
            EnableLogsSupport:       s.EnableLogsHandler,
            ProxyTransport:          proxyTransport,

            Tunneler: nodeTunneler,

            ServiceIPRange:          serviceIPRange,
            APIServerServiceIP:      apiServerServiceIP,
            SecondaryServiceIPRange: secondaryServiceIPRange,

            APIServerServicePort: 443,

            ServiceNodePortRange:      s.ServiceNodePortRange,
            KubernetesServiceNodePort: s.KubernetesServiceNodePort,

            EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType),
            MasterCount:            s.MasterCount,

            ServiceAccountIssuer:        s.ServiceAccountIssuer,
            ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration,
            ExtendExpiration:            s.Authentication.ServiceAccounts.ExtendExpiration,

            VersionedInformers: versionedInformers,

            IdentityLeaseDurationSeconds:      s.IdentityLeaseDurationSeconds,
            IdentityLeaseRenewIntervalSeconds: s.IdentityLeaseRenewIntervalSeconds,
        },
    }

    clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider()
    if err != nil {
        return nil, nil, nil, err
    }
    config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider

    requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
    if err != nil {
        return nil, nil, nil, err
    }
    if requestHeaderConfig != nil {
        config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
        config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
        config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
        config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
        config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
    }

    if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil {
        return nil, nil, nil, err
    }

    if nodeTunneler != nil {
        // Use the nodeTunneler's dialer to connect to the kubelet
        config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial
    }
    if config.GenericConfig.EgressSelector != nil {
        // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet
        config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup

        // Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources.
        networkContext := egressselector.Cluster.AsNetworkContext()
        dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext)
        if err != nil {
            return nil, nil, nil, err
        }
        c := proxyTransport.Clone()
        c.DialContext = dialer
        config.ExtraConfig.ProxyTransport = c
    }

    // Load the public keys.
    var pubKeys []interface{}
    for _, f := range s.Authentication.ServiceAccounts.KeyFiles {
        keys, err := keyutil.PublicKeysFromFile(f)
        if err != nil {
            return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
        }
        pubKeys = append(pubKeys, keys...)
    }
    // Plumb the required metadata through ExtraConfig.
    config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuer
    config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI
    config.ExtraConfig.ServiceAccountPublicKeys = pubKeys

    return config, serviceResolver, pluginInitializers, nil
}

• CreateKubeAPIServerConfig首先通过buildGenericConfig实例化genericConfig对象，并为genericConfig对象设置默认值：

// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
func buildGenericConfig(
    s *options.ServerRunOptions,
    proxyTransport *http.Transport,
) (
    genericConfig *genericapiserver.Config,
    versionedInformers clientgoinformers.SharedInformerFactory,
    serviceResolver aggregatorapiserver.ServiceResolver,
    pluginInitializers []admission.PluginInitializer,
    admissionPostStartHook genericapiserver.PostStartHookFunc,
    storageFactory *serverstorage.DefaultStorageFactory,
    lastErr error,
) {
    genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
    // 用于设置启用/禁用Group、Version和Resource。如果未在命令行参数中指定启用/禁用的GV，则启动默认的GVR。
    // 默认启用Stable和Beta版本的资源，不启用Alpha版本的资源。
    genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()

    if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
        return
    }

    if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
        return
    }
    if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
        return
    }
    if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
        return
    }
    if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
        return
    }
    //openapi规范相关配置
    genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme))
    genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
    genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
        sets.NewString("watch", "proxy"),
        sets.NewString("attach", "exec", "proxy", "log", "portforward"),
    )

    kubeVersion := version.Get()
    genericConfig.Version = &kubeVersion

    // 与apiserver交互的etcd的相关配置
    storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
    storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
    completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd)
    if err != nil {
        lastErr = err
        return
    }
    storageFactory, lastErr = completedStorageFactoryConfig.New()
    if lastErr != nil {
        return
    }
    if genericConfig.EgressSelector != nil {
        storageFactory.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup
    }
    if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
        return
    }

    // Use protobufs for self-communication.
    // Since not every generic apiserver has to support protobufs, we
    // cannot default to it in generic apiserver and need to explicitly
    // set it in kube-apiserver.
    genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf"
    // Disable compression for self-communication, since we are going to be
    // on a fast local network
    genericConfig.LoopbackClientConfig.DisableCompression = true

    kubeClientConfig := genericConfig.LoopbackClientConfig
    clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
    if err != nil {
        lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
        return
    }
    versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)

    // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
    if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
        return
    }

    // 授权相关的配置。BuildAuthorizer实例化认证器
    genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers)
    if err != nil {
        lastErr = fmt.Errorf("invalid authorization config: %v", err)
        return
    }
    if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) {
        genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
    }

    lastErr = s.Audit.ApplyTo(genericConfig)
    if lastErr != nil {
        return
    }

    admissionConfig := &kubeapiserveradmission.Config{
        ExternalInformers:    versionedInformers,
        LoopbackClientConfig: genericConfig.LoopbackClientConfig,
        CloudConfigFile:      s.CloudProvider.CloudConfigFile,
    }
    serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
    pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver)
    if err != nil {
        lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
        return
    }

    err = s.Admission.ApplyTo(
        genericConfig,
        versionedInformers,
        kubeClientConfig,
        feature.DefaultFeatureGate,
        pluginInitializers...)
    if err != nil {
        lastErr = fmt.Errorf("failed to initialize admission: %v", err)
    }

    if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness {
        genericConfig.FlowControl = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers)
    }

    return
}

• CreateServerChain创建完通用配置后创建了APIExtensionsServer和KubeAPIServer。先看CreateKubeAPIServer：

// cmd\kube-apiserver\app\server.go
// CreateKubeAPIServer creates and wires a workable kube-apiserver
func CreateKubeAPIServer(kubeAPIServerConfig *controlplane.Config, delegateAPIServer genericapiserver.DelegationTarget) (*controlplane.Instance, error) {
    kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer)
    if err != nil {
        return nil, err
    }

    return kubeAPIServer, nil
}

• kubeAPIServer先创建genericAPIServer。实例化Instance。通过InstallLegacyAPI函数将没有Group的Resource注册到/api前缀的路径下，路径为/api/<version>/<resource>。通过InstallAPIs函数将有Group的esource注册到/apis前缀的路径下，路径为/apis/<group>/<version>/<resource>：

// pkg\controlplane\instance.go
// New returns a new instance of Master from the given config.
// Certain config fields will be set to a default value if unset.
// Certain config fields must be specified, including:
//   KubeletClientConfig
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
    if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
        return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
    }
    // 创建名为kube-apiserver的服务
    s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
    if err != nil {
        return nil, err
    }

    if c.ExtraConfig.EnableLogsSupport {
        routes.Logs{}.Install(s.Handler.GoRestfulContainer)
    }

    // Metadata and keys are expected to only change across restarts at present,
    // so we just marshal immediately and serve the cached JSON bytes.
    md, err := serviceaccount.NewOpenIDMetadata(
        c.ExtraConfig.ServiceAccountIssuerURL,
        c.ExtraConfig.ServiceAccountJWKSURI,
        c.GenericConfig.ExternalAddress,
        c.ExtraConfig.ServiceAccountPublicKeys,
    )
    if err != nil {
        // If there was an error, skip installing the endpoints and log the
        // error, but continue on. We don't return the error because the
        // metadata responses require additional, backwards incompatible
        // validation of command-line options.
        msg := fmt.Sprintf("Could not construct pre-rendered responses for"+
            " ServiceAccountIssuerDiscovery endpoints. Endpoints will not be"+
            " enabled. Error: %v", err)
        if c.ExtraConfig.ServiceAccountIssuerURL != "" {
            // The user likely expects this feature to be enabled if issuer URL is
            // set and the feature gate is enabled. In the future, if there is no
            // longer a feature gate and issuer URL is not set, the user may not
            // expect this feature to be enabled. We log the former case as an Error
            // and the latter case as an Info.
            klog.Error(msg)
        } else {
            klog.Info(msg)
        }
    } else {
        routes.NewOpenIDMetadataServer(md.ConfigJSON, md.PublicKeysetJSON).
            Install(s.Handler.GoRestfulContainer)
    }

    // 实例化Instance
    m := &Instance{
        GenericAPIServer:          s,
        ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo,
    }

    // 添加/api/* 路由，注册没有Group的资源
    // install legacy rest storage
    if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {
        legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{
            StorageFactory:              c.ExtraConfig.StorageFactory,
            ProxyTransport:              c.ExtraConfig.ProxyTransport,
            KubeletClientConfig:         c.ExtraConfig.KubeletClientConfig,
            EventTTL:                    c.ExtraConfig.EventTTL,
            ServiceIPRange:              c.ExtraConfig.ServiceIPRange,
            SecondaryServiceIPRange:     c.ExtraConfig.SecondaryServiceIPRange,
            ServiceNodePortRange:        c.ExtraConfig.ServiceNodePortRange,
            LoopbackClientConfig:        c.GenericConfig.LoopbackClientConfig,
            ServiceAccountIssuer:        c.ExtraConfig.ServiceAccountIssuer,
            ExtendExpiration:            c.ExtraConfig.ExtendExpiration,
            ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,
            APIAudiences:                c.GenericConfig.Authentication.APIAudiences,
        }
        if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil {
            return nil, err
        }
    }

    // 添加/apis/* 路由，注册有Group的资源
    // The order here is preserved in discovery.
    // If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
    // the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
    // This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
    // with specific priorities.
    // TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
    // handlers that we have.
    restStorageProviders := []RESTStorageProvider{
        apiserverinternalrest.StorageProvider{},
        authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences},
        authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
        autoscalingrest.RESTStorageProvider{},
        batchrest.RESTStorageProvider{},
        certificatesrest.RESTStorageProvider{},
        coordinationrest.RESTStorageProvider{},
        discoveryrest.StorageProvider{},
        extensionsrest.RESTStorageProvider{},
        networkingrest.RESTStorageProvider{},
        noderest.RESTStorageProvider{},
        policyrest.RESTStorageProvider{},
        rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer},
        schedulingrest.RESTStorageProvider{},
        storagerest.RESTStorageProvider{},
        flowcontrolrest.RESTStorageProvider{},
        // keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
        // See https://github.com/kubernetes/kubernetes/issues/42392
        appsrest.StorageProvider{},
        admissionregistrationrest.RESTStorageProvider{},
        eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL},
    }
    if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {
        return nil, err
    }

    if c.ExtraConfig.Tunneler != nil {
        m.installTunneler(c.ExtraConfig.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes())
    }

    m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error {
        kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
        if err != nil {
            return err
        }
        controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, kubeClient)

        // prime values and start listeners
        if m.ClusterAuthenticationInfo.ClientCA != nil {
            if notifier, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.Notifier); ok {
                notifier.AddListener(controller)
            }
            if controller, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.ControllerRunner); ok {
                // runonce to be sure that we have a value.
                if err := controller.RunOnce(); err != nil {
                    runtime.HandleError(err)
                }
                go controller.Run(1, hookContext.StopCh)
            }
        }
        if m.ClusterAuthenticationInfo.RequestHeaderCA != nil {
            if notifier, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.Notifier); ok {
                notifier.AddListener(controller)
            }
            if controller, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.ControllerRunner); ok {
                // runonce to be sure that we have a value.
                if err := controller.RunOnce(); err != nil {
                    runtime.HandleError(err)
                }
                go controller.Run(1, hookContext.StopCh)
            }
        }

        go controller.Run(1, hookContext.StopCh)
        return nil
    })

    if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.APIServerIdentity) {
        m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error {
            kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
            if err != nil {
                return err
            }
            controller := lease.NewController(
                clock.RealClock{},
                kubeClient,
                m.GenericAPIServer.APIServerID,
                int32(c.ExtraConfig.IdentityLeaseDurationSeconds),
                nil,
                time.Duration(c.ExtraConfig.IdentityLeaseRenewIntervalSeconds)*time.Second,
                metav1.NamespaceSystem,
                labelAPIServerHeartbeat)
            go controller.Run(wait.NeverStop)
            return nil
        })
        m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error {
            kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
            if err != nil {
                return err
            }
            go apiserverleasegc.NewAPIServerLeaseGC(
                kubeClient,
                time.Duration(c.ExtraConfig.IdentityLeaseDurationSeconds)*time.Second,
                metav1.NamespaceSystem,
                KubeAPIServerIdentityLeaseLabelSelector,
            ).Run(wait.NeverStop)
            return nil
        })
    }

    return m, nil
}

• InstallLegacyAPI注册没有Group的资源：

// InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled.
func (m *Instance) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) error {
    legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter)
    if err != nil {
        return fmt.Errorf("error building core storage: %v", err)
    }

    controllerName := "bootstrap-controller"
    coreClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
    bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient, coreClient, coreClient.RESTClient())
    m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook)
    m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook)

    if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil {
        return fmt.Errorf("error in registering group versions: %v", err)
    }
    return nil
}

• 首先，legacyRESTStorageProvider.NewLegacyRESTStorage函数实例化APIGroupInfo。然后通过调用NewREST方法创建了podTemplate, event，secret，persistentVolume，endpoints等多个资源的资源存储对象RESTStorage。最后将这些资源的路由和其资源存储对象RESTStorage的映射关系存入APIGroupInfo的VersionedResourcesStorageMap字段。：

func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generic.RESTOptionsGetter) (LegacyRESTStorage, genericapiserver.APIGroupInfo, error) {
    apiGroupInfo := genericapiserver.APIGroupInfo{
        PrioritizedVersions:          legacyscheme.Scheme.PrioritizedVersionsForGroup(""),
        VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, //第一个string存储version，第二个string存储资源名
        Scheme:                       legacyscheme.Scheme,
        ParameterCodec:               legacyscheme.ParameterCodec,
        NegotiatedSerializer:         legacyscheme.Codecs,
    }
    ......
    restStorage := LegacyRESTStorage{}
    
    podTemplateStorage, err := podtemplatestore.NewREST(restOptionsGetter)
    ......
    serviceClusterIPAllocator, err := ipallocator.NewAllocatorCIDRRange(&serviceClusterIPRange, func(max int, rangeSpec string) (allocator.Interface, error) {
        mem := allocator.NewAllocationMap(max, rangeSpec)
        // TODO etcdallocator package to return a storage interface via the storageFactory
        etcd, err := serviceallocator.NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), serviceStorageConfig)
        if err != nil {
            return nil, err
        }
        serviceClusterIPRegistry = etcd
        return etcd, nil
    })
    if err != nil {
        return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, fmt.Errorf("cannot create cluster IP allocator: %v", err)
    }
    restStorage.ServiceClusterIPAllocator = serviceClusterIPRegistry
    .....
    restStorageMap := map[string]rest.Storage{
        "pods":             podStorage.Pod,
        "pods/attach":      podStorage.Attach,
        "pods/status":      podStorage.Status,
        "pods/log":         podStorage.Log,
        "pods/exec":        podStorage.Exec,
        "pods/portforward": podStorage.PortForward,
        "pods/proxy":       podStorage.Proxy,
        "pods/binding":     podStorage.Binding,
        "bindings":         podStorage.LegacyBinding,

        "podTemplates": podTemplateStorage,
    ......
    apiGroupInfo.VersionedResourcesStorageMap["v1"] = restStorageMap
    return restStorage, apiGroupInfo, nil
    
// LegacyRESTStorage returns stateful information about particular instances of REST storage to
// master.go for wiring controllers.
// TODO remove this by running the controller as a poststarthook
type LegacyRESTStorage struct {
    ServiceClusterIPAllocator          rangeallocation.RangeRegistry
    SecondaryServiceClusterIPAllocator rangeallocation.RangeRegistry
    ServiceNodePortAllocator           rangeallocation.RangeRegistry
}

• 每个资源都通过NewREST创建了RESTStorage。RESTStorage包含NewFunc, NewListFunc, CreateStrategy, UpdateStrategy, DeleteStrategy等对应增删改查等动作的处理方法。例如podTemplate：

// pkg\registry\core\podtemplate\storage\storage.go
// NewREST returns a RESTStorage object that will work against pod templates.
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, error) {
    store := &genericregistry.Store{
        NewFunc:                  func() runtime.Object { return &api.PodTemplate{} },
        NewListFunc:              func() runtime.Object { return &api.PodTemplateList{} },
        DefaultQualifiedResource: api.Resource("podtemplates"),

        CreateStrategy: podtemplate.Strategy,
        UpdateStrategy: podtemplate.Strategy,
        DeleteStrategy: podtemplate.Strategy,

        ReturnDeletedObject: true,

        TableConvertor: printerstorage.TableConvertor{TableGenerator: printers.NewTableGenerator().With(printersinternal.AddHandlers)},
    }
    options := &generic.StoreOptions{RESTOptions: optsGetter}
    if err := store.CompleteWithOptions(options); err != nil {
        return nil, err
    }
    return &REST{store}, nil
}

// REST implements a RESTStorage for pod templates.
type REST struct {
    *genericregistry.Store
}

• InstallLegacyAPI最后通过m.GenericAPIServer.InstallLegacyAPIGroup函数将APIGroupInfo对象中的<资源组>/<资源版本>/<资源>/<子资源>（包括资源存储对象）注册到KubeAPIServer Handlers方法。

// staging\src\k8s.io\apiserver\pkg\server\genericapiserver.go
func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
    if !s.legacyAPIGroupPrefixes.Has(apiPrefix) {
        return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List())
    }

    openAPIModels, err := s.getOpenAPIModels(apiPrefix, apiGroupInfo)
    if err != nil {
        return fmt.Errorf("unable to get openapi models: %v", err)
    }

    if err := s.installAPIResources(apiPrefix, apiGroupInfo, openAPIModels); err != nil {
        return err
    }

    // Install the version handler.
    // Add a handler at /<apiPrefix> to enumerate the supported api versions.
    s.Handler.GoRestfulContainer.Add(discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix).WebService())

    return nil
}

// getOpenAPIModels is a private method for getting the OpenAPI models
func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (openapiproto.Models, error) {
    if s.openAPIConfig == nil {
        return nil, nil
    }
    pathsToIgnore := openapiutil.NewTrie(s.openAPIConfig.IgnorePrefixes)
    resourceNames := make([]string, 0)
    for _, apiGroupInfo := range apiGroupInfos {
        groupResources, err := getResourceNamesForGroup(apiPrefix, apiGroupInfo, pathsToIgnore)
        if err != nil {
            return nil, err
        }
        resourceNames = append(resourceNames, groupResources...)
    }

    // Build the openapi definitions for those resources and convert it to proto models
    openAPISpec, err := openapibuilder.BuildOpenAPIDefinitionsForResources(s.openAPIConfig, resourceNames...)
    if err != nil {
        return nil, err
    }
    for _, apiGroupInfo := range apiGroupInfos {
        apiGroupInfo.StaticOpenAPISpec = openAPISpec
    }
    return utilopenapi.ToProtoModels(openAPISpec)
}

// getResourceNamesForGroup is a private method for getting the canonical names for each resource to build in an api group
func getResourceNamesForGroup(apiPrefix string, apiGroupInfo *APIGroupInfo, pathsToIgnore openapiutil.Trie) ([]string, error) {
    // Get the canonical names of every resource we need to build in this api group
    resourceNames := make([]string, 0)
    for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
        for resource, storage := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
            path := gpath.Join(apiPrefix, groupVersion.Group, groupVersion.Version, resource)
            if !pathsToIgnore.HasPrefix(path) {
                kind, err := genericapi.GetResourceKind(groupVersion, storage, apiGroupInfo.Scheme)
                if err != nil {
                    return nil, err
                }
                sampleObject, err := apiGroupInfo.Scheme.New(kind)
                if err != nil {
                    return nil, err
                }
                name := openapiutil.GetCanonicalTypeName(sampleObject)
                resourceNames = append(resourceNames, name)
            }
        }
    }

    return resourceNames, nil
}

// installAPIResources is a private method for installing the REST storage backing each api groupversionresource
func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, openAPIModels openapiproto.Models) error {
    var resourceInfos []*storageversion.ResourceInfo
    // 遍历apiGroupInfo的每个groupVersion
    for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
        if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
            klog.Warningf("Skipping API %v because it has no resources.", groupVersion)
            continue
        }
        apiGroupVersion := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
        if apiGroupInfo.OptionsExternalVersion != nil {
            apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
        }
        apiGroupVersion.OpenAPIModels = openAPIModels

        if openAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
            typeConverter, err := fieldmanager.NewTypeConverter(openAPIModels, false)
            if err != nil {
                return err
            }
            apiGroupVersion.TypeConverter = typeConverter
        }

        apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
        
        // 将每个groupVersion的Resource路径和增删改查动作的处理注册进restful Container
        r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
        if err != nil {
            return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
        }
        resourceInfos = append(resourceInfos, r...)
    }

    if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
        utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
        // API installation happens before we start listening on the handlers,
        // therefore it is safe to register ResourceInfos here. The handler will block
        // write requests until the storage versions of the targeting resources are updated.
        s.StorageVersionManager.AddResourceInfo(resourceInfos...)
    }

    return nil
}

func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) *genericapi.APIGroupVersion {
    storage := make(map[string]rest.Storage)
    for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
        storage[strings.ToLower(k)] = v
    }
    version := s.newAPIGroupVersion(apiGroupInfo, groupVersion)
    version.Root = apiPrefix
    version.Storage = storage
    return version
}

• installAPIResources中从restStorage知道每个资源可以执行哪些操作，存入action，再将action对应为REST action，并对每一个操作注册handler，注册到route，再将route注册到webservice。

// staging\src\k8s.io\apiserver\pkg\endpoints\groupversion.go
// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
// It is expected that the provided path root prefix will serve all operations. Root MUST NOT end
// in a slash.
func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]*storageversion.ResourceInfo, error) {
    prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version)
    installer := &APIInstaller{
        group:             g,
        prefix:            prefix,
        minRequestTimeout: g.MinRequestTimeout,
    }

    apiResources, resourceInfos, ws, registrationErrors := installer.Install()
    versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, staticLister{apiResources})
    versionDiscoveryHandler.AddToWebService(ws)
    container.Add(ws)
    return removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors)
}

// staging\src\k8s.io\apiserver\pkg\endpoints\installer.go
// Install handlers for API resources.
func (a *APIInstaller) Install() ([]metav1.APIResource, []*storageversion.ResourceInfo, *restful.WebService, []error) {
    var apiResources []metav1.APIResource
    var resourceInfos []*storageversion.ResourceInfo
    var errors []error
    // 创建前缀为prefix/group/version的newWebService
    ws := a.newWebService()

    // Register the paths in a deterministic (sorted) order to get a deterministic swagger spec.
    paths := make([]string, len(a.group.Storage))
    var i int = 0
    for path := range a.group.Storage {
        paths[i] = path
        i++
    }
    sort.Strings(paths)
    // 遍历指定Version, Group下的Resource路径，注册其hander
    for _, path := range paths {
        apiResource, resourceInfo, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
        if err != nil {
            errors = append(errors, fmt.Errorf("error in registering resource: %s, %v", path, err))
        }
        if apiResource != nil {
            apiResources = append(apiResources, *apiResource)
        }
        if resourceInfo != nil {
            resourceInfos = append(resourceInfos, resourceInfo)
        }
    }
    return apiResources, resourceInfos, ws, errors
}



// staging\src\k8s.io\apiserver\pkg\endpoints\installer.go
// newWebService creates a new restful webservice with the api installer's prefix and version.
func (a *APIInstaller) newWebService() *restful.WebService {
    ws := new(restful.WebService)
    ws.Path(a.prefix)
    // a.prefix contains "prefix/group/version"
    ws.Doc("API at " + a.prefix)
    // Backwards compatibility, we accepted objects with empty content-type at V1.
    // If we stop using go-restful, we can default empty content-type to application/json on an
    // endpoint by endpoint basis
    ws.Consumes("*/*")
    mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
    ws.Produces(append(mediaTypes, streamMediaTypes...)...)
    ws.ApiVersion(a.group.GroupVersion.String())

    return ws
}


func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, *storageversion.ResourceInfo, error) {
    ......
    // what verbs are supported by the storage, used to know what verbs we support per path
    creater, isCreater := storage.(rest.Creater)
    namedCreater, isNamedCreater := storage.(rest.NamedCreater)
    lister, isLister := storage.(rest.Lister)
    getter, isGetter := storage.(rest.Getter)
    getterWithOptions, isGetterWithOptions := storage.(rest.GetterWithOptions)
    gracefulDeleter, isGracefulDeleter := storage.(rest.GracefulDeleter)
    collectionDeleter, isCollectionDeleter := storage.(rest.CollectionDeleter)
    updater, isUpdater := storage.(rest.Updater)
    patcher, isPatcher := storage.(rest.Patcher)
    watcher, isWatcher := storage.(rest.Watcher)
    connecter, isConnecter := storage.(rest.Connecter)
    storageMeta, isMetadata := storage.(rest.StorageMetadata)
    storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider)
    ......
    // Handler for standard REST verbs (GET, PUT, POST and DELETE).
    // Add actions at the resource path: /api/apiVersion/resource
    actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)
    actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)
    actions = appendIf(actions, action{"DELETECOLLECTION", resourcePath, resourceParams, namer, false}, isCollectionDeleter)
    // DEPRECATED in 1.11
    actions = appendIf(actions, action{"WATCHLIST", "watch/" + resourcePath, resourceParams, namer, false}, allowWatchList)

    // Add actions at the item path: /api/apiVersion/resource/{name}
    actions = appendIf(actions, action{"GET", itemPath, nameParams, namer, false}, isGetter)
    if getSubpath {
        actions = appendIf(actions, action{"GET", itemPath + "/{path:*}", proxyParams, namer, false}, isGetter)
    }
    actions = appendIf(actions, action{"PUT", itemPath, nameParams, namer, false}, isUpdater)
    actions = appendIf(actions, action{"PATCH", itemPath, nameParams, namer, false}, isPatcher)
    actions = appendIf(actions, action{"DELETE", itemPath, nameParams, namer, false}, isGracefulDeleter)
    ......
    for _, action := range actions {
        ......
        switch action.Verb {
        case "GET": // Get a resource.
            var handler restful.RouteFunction
            if isGetterWithOptions {
                handler = restfulGetResourceWithOptions(getterWithOptions, reqScope, isSubresource)
            } else {
                handler = restfulGetResource(getter, reqScope)
            }

            if needOverride {
                // need change the reported verb
                handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
            } else {
                handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
            }
            if enableWarningHeaders {
                handler = utilwarning.AddWarningsHandler(handler, warnings)
            }

            doc := "read the specified " + kind
            if isSubresource {
                doc = "read " + subresource + " of the specified " + kind
            }
            route := ws.GET(action.Path).To(handler).
                Doc(doc).
                Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
                Operation("read"+namespaced+kind+strings.Title(subresource)+operationSuffix).
                Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
                Returns(http.StatusOK, "OK", producedObject).
                Writes(producedObject)
            if isGetterWithOptions {
                if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
                    return nil, nil, err
                }
            }
            addParams(route, action.Params)
            routes = append(routes, route)
        case "LIST": // List all resources of a kind.
            ......
        case "PUT": // Update a resource.
            ......
        case "PATCH": // Partially update a resource
            ......
        case "POST": // Create a resource.
            ......
        case "DELETE": // Delete a resource.
            ......
        ......
        for _, route := range routes {
            route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{
                Group:   reqScope.Kind.Group,
                Version: reqScope.Kind.Version,
                Kind:    reqScope.Kind.Kind,
            })
            route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb))
            ws.Route(route)
        }
    }
    ......
    return &apiResource, resourceInfo, nil