文档中心

背景

随着计算机网络带宽不断提升，单个CPU处理网络中断存在瓶颈，不能完全满足网卡的需求，在BCC云服务器存在多个CPU的情况下，通过开启网卡多队列，可以将云服务器中的网卡中断分散给不同的CPU进行处理，从而提升网络PPS和带宽性能。

支持网卡多队列的镜像列表

网卡多队列的支持情况和实例规格、镜像的操作系统有关。

手动配置网卡多队列

查看网卡队列数

[root@instance-0wazpxeh ~]# ethtool -l eth0  #查询网卡eth0的队列数
Channel parameters for eth0:
Pre-set maximums:
RX:               0
TX:               0
Other:            0
Combined:         4  #表示此网卡最多支持设置开启4个队列
Current hardware settings:
RX:               0
TX:               0
Other:            0
Combined:         4 #表示当前开启的是4个队列

如果查询结果中，两个Combined字段取值相同，则表示弹性网卡已开启支持多队列。

否则，使用以下命令开启多队列：

ethtool -L eth0 combined 4

自动配置网卡多队列

以centos 7 为例，创建网卡多队列服务

新建 /usr/lib/systemd/system/virt-net-init.service

usr/lib/systemd/system/virt-net-init.service
[Unit]
Description=Virt IO Network Init
After=local-fs.target
Before=network-pre.target
 
[Service]
Type=oneshot
ExecStart=/usr/libexec/virt-net-init.sh
RemainAfterExit=True
 
[Install]
WantedBy=multi-user.target

创建网卡多队列配置服本

新建/usr/libexec/virt-net-init.sh

#!/bin/bash
 
rps_flow_cnt=4096
#get specified mask
function get_specified_cpumask(){
    local cpu_num=$1
    quotient=$((${cpu_num}/32))
    remainder=$((${cpu_num}-32*$quotient))
    if [ ${quotient} -gt 0 ]; then
        res_tail=""
        res_head="80000000"
        while [ $quotient -gt 1 ]
        do
            res_tail="${res_tail},00000000"
            ((quotient--))
        done
        if [ $remainder -ne 0 ];then
            res_tail="${res_tail},00000000"
            res_head=$((1<<($remainder-1)))
            res_head=`printf "%x" ${res_head}`
        fi
        result="${res_head}${res_tail}"
    else
        result=$((1<<(${cpu_num}-1)))
        result=`printf "%x" $result`
    fi
    echo $result
}
 
#get all mask for rps/xps
function get_all_cpus() {
    local cpu_nums=$(grep -c processor /proc/cpuinfo)
    quotient=$((${cpu_nums}/32))
    remainder=$((${cpu_nums}-32*$quotient))
    if [ ${quotient} -gt 0 ];then
        res_head="ffffffff"
        res_tail=""
        while [ $quotient -gt 1 ]
        do
            res_tail="${res_tail},ffffffff"
            ((quotient--))
        done
        if [ $remainder -ne 0 ];then
            res_tail="${res_tail},ffffffff"
            res_head=$(((1<<$remainder)-1))
            res_head=`printf "%x" ${res_head}`
        fi
        result="${res_head}${res_tail}"
    else
        result=$(((1<<${cpu_nums})-1))
        result=`printf "%x" $result`
    fi
    echo $result
}
 
#set xps
function set_xps() {
    dev=$1
    for xps_file in `ls /sys/class/net/$dev/queues/tx-*/xps_cpus`
    do
        xps_cpus=$(get_all_cpus)
        echo "[INFO] set ${xps_cpus} into ${xps_file}." | logger -i -t 'virt-net-init'
        echo ${xps_cpus} > ${xps_file}
    done
}
 
#set rps/rfs for multicore when multiqueue is disbale
function set_rps_and_rfs(){
    rps_cpus=$(get_all_cpus)
    dev=$1
    queues=`ls -ld /sys/class/net/$dev/queues/rx-* | wc -l`
    num=0
    while [ $num -lt $queues ]
    do
        echo ${rps_cpus} > /sys/class/net/$dev/queues/rx-$num/rps_cpus
        echo "[INFO] set ${rps_cpus} into /sys/class/net/$dev/queues/rx-$num/rps_cpus." | logger -i -t 'virt-net-init'
        echo ${rps_flow_cnt} > /sys/class/net/$dev/queues/rx-$num/rps_flow_cnt
        echo "[INFO] set ${rps_flow_cnt} into /sys/class/net/$dev/queues/rx-$num/rps_flow_cnt." | logger -i -t 'virt-net-init'
        ((num++))
    done
 
}
 
function set_smp_affinity(){
    local dev=$1
    pci_dbdf=$(ethtool -i $dev | grep bus-info | cut -d' ' -f2)
    if [ -z $pci_dbdf ]; then
        echo "[ERR] No LiquidIO NIC detected" | logger -i -t 'virt-net-init'
    else
        dir=$(find /sys/devices/ -type d | grep $pci_dbdf | grep "/net$")
        result=${dir%/*}
        # get virtioX
        virtio_num=${result##*/}
 
        cpunums=`cat /proc/cpuinfo | grep processor | wc -l`
        if [ ${cpunums} -lt 2 ];then
            echo "[INFO] don't need set affinity for net interrupt!!" | logger -i -t 'virt-net-init'
        else
            irq_input=$(grep ${virtio_num}-input /proc/interrupts | sed "s/: .*//g" | sed "s/^ *//g")
            cur_cpunum=0
            for irqnum in ${irq_input}; do
                num=$((${cur_cpunum}%$cpunums+1))
                mask=`get_specified_cpumask $num`
                #echo "[INFO] irq:${irqnum} bind to cpu:$mask." | logger -i -t 'virt-net-init'
                echo "[INFO] echo $mask > /proc/irq/$irqnum/smp_affinity" | logger -i -t 'virt-net-init'
                echo $mask > /proc/irq/$irqnum/smp_affinity
                ((cur_cpunum++))
            done
             
            irq_output=$(grep ${virtio_num}-output /proc/interrupts | sed "s/: .*//g" | sed "s/^ *//g")
            cur_cpunum=0
            for irqnum in ${irq_output}; do
                num=$((${cur_cpunum}%$cpunums+1))
                mask=`get_specified_cpumask $num`
                #echo "[INFO] irq:${irqnum} bind to cpu:$mask." | logger -i -t 'virt-net-init'
                echo "[INFO] echo $mask > /proc/irq/$irqnum/smp_affinity" | logger -i -t 'virt-net-init'
                echo $mask > /proc/irq/$irqnum/smp_affinity
                ((cur_cpunum++))
            done
        fi
    fi
}
 
#set multiqueue
function set_multiqueue() {
    local dev=$1
    local pre_set=`ethtool -l ${dev} | grep Combined | head -n 1 | cut -f 2`
    local cur_set=`ethtool -l ${dev} | grep Combined | tail -n 1 | cut -f 2`
    #virtio-net will enable multiqueue by default
    if [ ${pre_set} -gt ${cur_set} ];then
        ethtool -L ${dev} combined ${pre_set} 2>&1 | logger -i -t 'virt-net-init'
        if [ "${PIPESTATUS[0]}" -ne 0 ]; then
            echo "[WARN] set multiqueue[chnum = ${cur_set}] for $dev failed. Ignore 

this if multiqueue is already set or chnum = 1." | logger -i -t 'virt-net-init'
        fi
    else
        echo "[INFO] multiqueue[chnum= ${cur_set}] for $dev is enabled." | logger -i -t 'virt-net-init'
    fi
}
 
function main(){
    for (( i = 0; i < 5; i++ )); do
        dir_list=$(find /sys/devices/ -type d | grep virtio | grep "/net$")
        if [[ ! -z "${dir_list}" ]]; then
            break
        fi
        sleep 0.5
    done
 
    total_queues=0
    for dir in ${dir_list[*]}; do
        dev=$(ls $dir)
        if [[ -n "$dev" ]]; then
            #check if multiqueue is available
            queue_nums=`ethtool -l $dev 2>&- | grep "Combined" | head -n 1 | awk '{print $2}'`
            if [ "$?" == "0" ] && [ ${queue_nums} -gt 1 ];then
                #set multiqueue and affinity
                #echo "set multiqueue for ${dev},and rps/rfs don't need to set."
                set_multiqueue ${dev}
                set_smp_affinity ${dev}
                #continue
            else
                #set rps and rfs
                set_rps_and_rfs ${dev}
                queues=`ls -ld /sys/class/net/$dev/queues/rx-* | wc -l`
                total_queues=$((${total_queues}+$queues))
                set_xps ${dev}
            fi
        else
            echo "[WARN] net device isn't exist." | logger -i -t 'virt-net-init'
        fi
    done
 
    if [ ${total_queues} -ne 0 ];then
        rps_sock_flow_entries=$((total_queues*${rps_flow_cnt}))
        echo ${rps_sock_flow_entries} > /proc/sys/net/core/rps_sock_flow_entries
        echo "[INFO] set ${rps_sock_flow_entries} for rfs." | logger -i -t 'virt-net-init'
    fi
}
main

为多队列配置脚本添加可执行权限

chmod + x /usr/libexec/virt-net-init.sh

增加网卡多队列配置开机自动启动

systemctl enable virt-net-init.service | logger -i -t 'virt-net-init'