ovs支持qos功能，其根据datapath类型调用不同的接口，对于kernel datapath，qos使用kernel提供的tc模块实现了入方向限速和出方向流量整型。对于userspace datapath，qos使用了dpdk提供的qos lib实现比较简单的入方向和出方向限速。

kernel qos

kernel的tc提供了多种调度机制，但是ovs目前只支持如下两种：htb和hfsc，更多qos参数可参考ovs db中qos和queue的定义。

image.png
也可以参考这篇文章，专门讲述qos。

userspace qos

先看一下代码中配置qos的地方，配置这部分是不区分哪种qos的，适用于kernel和userspace qos，区别是根据类型调用不同的api。

userspace qos配置的例子可以参数官网。

不管哪种qos，配置都会下到最底层接口收发包模块，不会在datapath或者slow path中。

遍历所有bridge上所有port下的interface，执行iface_configure_qos

static void
bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg)
    HMAP_FOR_EACH (br, node, &all_bridges) {
        struct port *port;

        /* We need the datapath ID early to allow LACP ports to use it as the
         * default system ID. */
        bridge_configure_datapath_id(br);

        HMAP_FOR_EACH (port, hmap_node, &br->ports) {
            struct iface *iface;

            LIST_FOR_EACH (iface, port_elem, &port->ifaces) {
                ...
                iface_configure_qos(iface, port->cfg->qos);
                ...
            }
        }
        ...
    }

配置接口的qos，入方向，出方向，队列上的配置都在此函数完成。对于usespace qos来说，调用dpdk提供的接口，入方向qos配置最终调用 netdev_dpdk_set_policing，出方向qos配置调用 netdev_dpdk_set_qos

static void
iface_configure_qos(struct iface *iface, const struct ovsrec_qos *qos)
{
    struct ofpbuf queues_buf;

    ofpbuf_init(&queues_buf, 0);

    if (!qos || qos->type[0] == '\0') {
        netdev_set_qos(iface->netdev, NULL, NULL);
    } else {
        const struct ovsdb_datum *queues;
        struct netdev_queue_dump dump;
        unsigned int queue_id;
        struct smap details;
        bool queue_zero;
        size_t i;

        /* Configure top-level Qos for 'iface'. */
        //class->set_qos(netdev, type, details) -- netdev_dpdk_set_qos
        netdev_set_qos(iface->netdev, qos->type, &qos->other_config);

        /* Deconfigure queues that were deleted. */
        queues = ovsrec_qos_get_queues(qos, OVSDB_TYPE_INTEGER, OVSDB_TYPE_UUID);
        smap_init(&details);
        NETDEV_QUEUE_FOR_EACH (&queue_id, &details, &dump, iface->netdev) {
            if (!queue_ids_include(queues, queue_id)) {
                netdev_delete_queue(iface->netdev, queue_id);
            }
        }
        smap_destroy(&details);

        /* Configure queues for 'iface'. */
        queue_zero = false;
        for (i = 0; i < qos->n_queues; i++) {
            const struct ovsrec_queue *queue = qos->value_queues[i];
            queue_id = qos->key_queues[i];

            if (queue_id == 0) {
                queue_zero = true;
            }

            if (queue->n_dscp == 1) {
                struct ofproto_port_queue *port_queue;

                port_queue = ofpbuf_put_uninit(&queues_buf,
                                               sizeof *port_queue);
                port_queue->queue = queue_id;
                port_queue->dscp = queue->dscp[0];
            }

            //class->set_queue(netdev, queue_id, details)
            netdev_set_queue(iface->netdev, queue_id, &queue->other_config);
        }
        if (!queue_zero) {
            smap_init(&details);
            netdev_set_queue(iface->netdev, 0, &details);
            smap_destroy(&details);
        }
    }

    if (iface->ofp_port != OFPP_NONE) {
        const struct ofproto_port_queue *port_queues = queues_buf.data;
        size_t n_queues = queues_buf.size / sizeof *port_queues;

        ofproto_port_set_queues(iface->port->bridge->ofproto, iface->ofp_port,
                                port_queues, n_queues);
    }

    //netdev->netdev_class->set_policing(netdev, kbits_rate, kbits_burst) -- netdev_dpdk_set_qos
    netdev_set_policing(iface->netdev,
                        MIN(UINT32_MAX, iface->cfg->ingress_policing_rate),
                        MIN(UINT32_MAX, iface->cfg->ingress_policing_burst));

    ofpbuf_uninit(&queues_buf);
}

入方向设置 netdev_dpdk_set_policing

static int
netdev_dpdk_set_policing(struct netdev* netdev, uint32_t policer_rate,
                         uint32_t policer_burst)
{
    struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
    struct ingress_policer *policer;

    /* Force to 0 if no rate specified,
     * default to 8000 kbits if burst is 0,
     * else stick with user-specified value.
     */
    policer_burst = (!policer_rate ? 0
                     : !policer_burst ? 8000
                     : policer_burst);

    ovs_mutex_lock(&dev->mutex);

    policer = ovsrcu_get_protected(struct ingress_policer *,
                                    &dev->ingress_policer);

    if (dev->policer_rate == policer_rate &&
        dev->policer_burst == policer_burst) {
        /* Assume that settings haven't changed since we last set them. */
        ovs_mutex_unlock(&dev->mutex);
        return 0;
    }

    /* Destroy any existing ingress policer for the device if one exists */
    if (policer) {
        ovsrcu_postpone(free, policer);
    }

    if (policer_rate != 0) {
        //函数内部调用dpdk lib提供的api进行配置，返回policer
        policer = netdev_dpdk_policer_construct(policer_rate, policer_burst);
    } else {
        policer = NULL;
    }
    //将policer赋值到接口设备 dev->ingress_policer, 从此接口收包后，会使用policer测速。
    ovsrcu_set(&dev->ingress_policer, policer);
    dev->policer_rate = policer_rate;
    dev->policer_burst = policer_burst;
    ovs_mutex_unlock(&dev->mutex);

    return 0;
}
//rte_meter_srtcm_config 为dpdk提供的api
static struct ingress_policer *
netdev_dpdk_policer_construct(uint32_t rate, uint32_t burst)
{
    struct ingress_policer *policer = NULL;
    uint64_t rate_bytes;
    uint64_t burst_bytes;
    int err = 0;

    policer = xmalloc(sizeof *policer);
    rte_spinlock_init(&policer->policer_lock);

    /* rte_meter requires bytes so convert kbits rate and burst to bytes. */
    rate_bytes = rate * 1000ULL / 8;
    burst_bytes = burst * 1000ULL / 8;

    policer->app_srtcm_params.cir = rate_bytes;
    policer->app_srtcm_params.cbs = burst_bytes;
    policer->app_srtcm_params.ebs = 0;
    err = rte_meter_srtcm_config(&policer->in_policer,
                                    &policer->app_srtcm_params);
    if (err) {
        VLOG_ERR("Could not create rte meter for ingress policer");
        free(policer);
        return NULL;
    }

    return policer;
}

出方向设置 netdev_dpdk_set_qos
当前看的是2.8.2版本的ovs，目前只支持一种出方向限速egress_policer_ops 。最终也是调用dpdk的lib api rte_meter_srtcm_config进行配置，和入方向配置使用的函数相同。

static const struct dpdk_qos_ops egress_policer_ops = {
    "egress-policer",    /* qos_name */
    egress_policer_qos_construct,
    egress_policer_qos_destruct,
    egress_policer_qos_get,
    egress_policer_qos_is_equal,
    egress_policer_run
};

static int
netdev_dpdk_set_qos(struct netdev *netdev, const char *type,
                    const struct smap *details)
{
    struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
    const struct dpdk_qos_ops *new_ops = NULL;
    struct qos_conf *qos_conf, *new_qos_conf = NULL;
    int error = 0;

    ovs_mutex_lock(&dev->mutex);

    qos_conf = ovsrcu_get_protected(struct qos_conf *, &dev->qos_conf);

    //目前只有一种：egress_policer_ops，name为egress-policer
    new_ops = qos_lookup_name(type);

    if (!new_ops || !new_ops->qos_construct) {
        new_qos_conf = NULL;
        if (type && type[0]) {
            error = EOPNOTSUPP;
        }
    } else if (qos_conf && qos_conf->ops == new_ops
               && qos_conf->ops->qos_is_equal(qos_conf, details)) {
        new_qos_conf = qos_conf;
    } else {
        //egress_policer_qos_construct
        error = new_ops->qos_construct(details, &new_qos_conf);
    }

    if (error) {
        VLOG_ERR("Failed to set QoS type %s on port %s: %s",
                 type, netdev->name, rte_strerror(error));
    }

    if (new_qos_conf != qos_conf) {
        ovsrcu_set(&dev->qos_conf, new_qos_conf);
        if (qos_conf) {
            ovsrcu_postpone(qos_conf->ops->qos_destruct, qos_conf);
        }
    }

    ovs_mutex_unlock(&dev->mutex);

    return error;
}

static int
egress_policer_qos_construct(const struct smap *details,
                             struct qos_conf **conf)
{
    struct egress_policer *policer;
    int err = 0;

    policer = xmalloc(sizeof *policer);
    qos_conf_init(&policer->qos_conf, &egress_policer_ops);
    //解析配置
    egress_policer_details_to_param(details, &policer->app_srtcm_params);
    err = rte_meter_srtcm_config(&policer->egress_meter,
                                 &policer->app_srtcm_params);
    if (!err) {
        *conf = &policer->qos_conf;
    } else {
        free(policer);
        *conf = NULL;
        err = -err;
    }

    return err;
}

报文处理

入方向qos处理
ovs dpdk中收包函数有两种，一种是vhostuser类型，调用netdev_dpdk_vhost_rxq_recv，一种是物理网卡类型的，调用netdev_dpdk_rxq_recv

netdev_dpdk_vhost_rxq_recv
netdev_dpdk_rxq_recv
    struct netdev_rxq_dpdk *rx = netdev_rxq_dpdk_cast(rxq);
    struct netdev_dpdk *dev = netdev_dpdk_cast(rxq->netdev);
    //从接口设备取出policer
    struct ingress_policer *policer = netdev_dpdk_get_ingress_policer(dev);
    int nb_rx;
    int dropped = 0;
    //调用dpdk函数rte_eth_rx_burst收包
    nb_rx = rte_eth_rx_burst(rx->port_id, rxq->queue_id,
                             (struct rte_mbuf **) batch->packets,
                             NETDEV_MAX_BURST);
    //如果policer不为空，说明配置了qos，开始处理报文。              
    if (policer) {
        dropped = nb_rx;
        nb_rx = ingress_policer_run(policer, (struct rte_mbuf **) batch->packets, nb_rx);
            rte_spinlock_lock(&policer->policer_lock);
            cnt = netdev_dpdk_policer_run(&policer->in_policer, pkts, pkt_cnt);
                int i = 0;
                int cnt = 0;
                struct rte_mbuf *pkt = NULL;
                uint64_t current_time = rte_rdtsc();

                for (i = 0; i < pkt_cnt; i++) {
                    pkt = pkts[i];
                    /* Handle current packet */
                    if (netdev_dpdk_policer_pkt_handle(meter, pkt, current_time)) {
                        if (cnt != i) {
                            pkts[cnt] = pkt;
                        }
                        cnt++;
                    } else {
                        rte_pktmbuf_free(pkt);
                    }
                }

                return cnt;
            rte_spinlock_unlock(&policer->policer_lock);
        dropped -= nb_rx;
    }
//调用dpdk函数rte_meter_srtcm_color_blind_check对每个报文进行着色处理，绿色表示允许报文通过，红色和黄色表示报文超速，需要丢弃。
static inline bool
netdev_dpdk_policer_pkt_handle(struct rte_meter_srtcm *meter,
                               struct rte_mbuf *pkt, uint64_t time)
{
    uint32_t pkt_len = rte_pktmbuf_pkt_len(pkt) - sizeof(struct ether_hdr);

    return rte_meter_srtcm_color_blind_check(meter, time, pkt_len) ==
                                                e_RTE_METER_GREEN;
}

出方向qos处理
出方向处理也是调用rte_meter_srtcm_color_blind_check对报文着色，根据颜色类型判断是否丢包。

netdev_dpdk_send__
__netdev_dpdk_vhost_send
    cnt = netdev_dpdk_qos_run(dev, pkts, cnt);
        struct qos_conf *qos_conf = ovsrcu_get(struct qos_conf *, &dev->qos_conf);

        if (qos_conf) {
            rte_spinlock_lock(&qos_conf->lock);
            //egress_policer_run
            cnt = qos_conf->ops->qos_run(qos_conf, pkts, cnt);
                int cnt = 0;
                struct egress_policer *policer =
                    CONTAINER_OF(conf, struct egress_policer, qos_conf);

                cnt = netdev_dpdk_policer_run(&policer->egress_meter, pkts, pkt_cnt);
                    int i = 0;
                    int cnt = 0;
                    struct rte_mbuf *pkt = NULL;
                    uint64_t current_time = rte_rdtsc();

                    for (i = 0; i < pkt_cnt; i++) {
                        pkt = pkts[i];
                        /* Handle current packet */
                        if (netdev_dpdk_policer_pkt_handle(meter, pkt, current_time)) {
                            if (cnt != i) {
                                pkts[cnt] = pkt;
                            }
                            cnt++;
                        } else {
                            rte_pktmbuf_free(pkt);
                        }
                    }

                    return cnt;
                return cnt;
            rte_spinlock_unlock(&qos_conf->lock);
        }

        return cnt;