网络性能调优

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
{
	u32 i, j;
	u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; // rss_i是使能的队列个数

	/* Program table for at least 4 queues w/ SR-IOV so that VFs can
	 * make full use of any rings they may have.  We will use the
	 * PSRTYPE register to control how many rings we use within the PF.
	 */
	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
		rss_i = 4;

	/* Fill out hash function seeds */
	ixgbe_store_key(adapter);

	/* Fill out redirection table */
	memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));

	for (i = 0, j = 0; i < reta_entries; i++, j++) {
		if (j == rss_i) // j表示队列id，达到最大值 rss_i 后，从0开始
			j = 0;

		adapter->rss_indir_tbl[i] = j;
	}

	ixgbe_store_reta(adapter);
}

1
2

$ cat /proc/irq/18/smp_affinity
ff

1
2

$ cat /proc/irq/18/smp_affinity_list
0-7

1

$ echo "20" > /proc/irq/18/smp_affinity_list

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# cat /proc/interrupts
           CPU0       CPU1
  0:  918926335          0    IO-APIC-edge  timer
  1:          2          0    IO-APIC-edge  i8042
  8:          0          0    IO-APIC-edge  rtc
  9:          0          0   IO-APIC-level  acpi
 12:          4          0    IO-APIC-edge  i8042
 14:    8248017          0    IO-APIC-edge  ide0
 50:        194          0   IO-APIC-level  ohci_hcd:usb2
 58:      31673          0   IO-APIC-level  sata_nv
 90:    1070374          0         PCI-MSI  eth0
233:         10          0   IO-APIC-level  ehci_hcd:usb1
NMI:       5077       2032
LOC:  918809969  918809894
ERR:          0
MIS:          0

1
2
3
4
5

$ readlink -e /sys/class/net/eth1
/sys/devices/pci0000:00/0000:00:03.0/virtio0/net/eth1

$ ls $(readlink -e /sys/class/net/eth1)/../../../msi_irqs
26  27  28

1
2
3

$ nsenter -t 944372 -n -m
$ ls $(readlink -e /sys/class/net/eth0)/../../../msi_irqs
291  292  293  294  295  296  297  298	299  300  301  302  303  304  305  306	307  308  309  310  311  312  313  314	315  316  317  318  319  320  321  322	323

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

//设备接收队列，和 RPS 相关成员是 rps_map
/* This structure contains an instance of an RX queue. */
struct netdev_rx_queue {
#ifdef CONFIG_RPS
    struct rps_map __rcu        *rps_map;
    struct rps_dev_flow_table __rcu *rps_flow_table;
#endif
    struct kobject          kobj;
    struct net_device       *dev;
} ____cacheline_aligned_in_smp;

//存放配置RPS的值，假如/sys/class/net/(dev)/queues/rx-(n)/rps_cpus=f，则
//len=4, cpus指向额外分配的内存数组，每个元素保存一个cpu值
struct rps_map {
    unsigned int len;
    struct rcu_head rcu;
    u16 cpus[0];
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

store_rps_map
    for_each_cpu_and(cpu, mask, cpu_online_mask)
      map->cpus[i++] = cpu;
    if (i)
      map->len = i;
   //将map赋值到queue->rps_map中，在get_rps_cpu中会使用到
   rcu_assign_pointer(queue->rps_map, map);

    if (map)
        //使能RPS
        static_key_slow_inc(&rps_needed);

1
2
3
4
5
6
7

static inline __u32 skb_get_hash(struct sk_buff *skb)
{
	if (!skb->l4_hash && !skb->sw_hash)
		__skb_get_hash(skb);

	return skb->hash;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

/*
 * get_rps_cpu is called from netif_receive_skb and returns the target
 * CPU from the RPS map of the receiving queue for a given skb.
 * rcu_read_lock must be held on entry.
 */
static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
               struct rps_dev_flow **rflowp)
{
    struct netdev_rx_queue *rxqueue;
    struct rps_map *map;
    struct rps_dev_flow_table *flow_table;
    struct rps_sock_flow_table *sock_flow_table;
    int cpu = -1;
    u16 tcpu;
    u32 hash;

    //skb->queue_mapping+1 记录了skb从哪个队列接收上来
    if (skb_rx_queue_recorded(skb)) {
        //获取接收skb队列index
        u16 index = skb_get_rx_queue(skb);
        if (unlikely(index >= dev->real_num_rx_queues)) {
            WARN_ONCE(dev->real_num_rx_queues > 1,
                  "%s received packet on queue %u, but number "
                  "of RX queues is %u\n",
                  dev->name, index, dev->real_num_rx_queues);
            goto done;
        }
        //根据index，获取rxqueue
        rxqueue = dev->_rx + index;
    } else
        //这里应该是queue 0吧
        rxqueue = dev->_rx;

    map = rcu_dereference(rxqueue->rps_map);
    if (map) {
        //如果rps_map只配置了一个CPU，并且没有配置rps_flow_table，
        //并且rps_sock_flow_entries 配置的这个cpu在线，则直接使用这个cpu。
        //如果这个cpu不在线，则返回-1.
        if (map->len == 1 &&
            !rcu_access_pointer(rxqueue->rps_flow_table)) {
            tcpu = map->cpus[0];
            if (cpu_online(tcpu))
                cpu = tcpu;
            goto done;
        }
    //没有配置 rps_map，也没有配置 rps_flow_table
    } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
        goto done;
    }

    skb_reset_network_header(skb);
    //根据skb获取hash值。如果在RSS模式下，可以直接使用网
    //卡计算的hash值，否则需要根据数据包信息计算一个
    hash = skb_get_hash(skb);
    if (!hash)
        goto done;
    //flow_table 和 sock_flow_table 是RFS的流程，暂时忽略
    flow_table = rcu_dereference(rxqueue->rps_flow_table);
    sock_flow_table = rcu_dereference(rps_sock_flow_table);
    if (flow_table && sock_flow_table) {
       /* ... */
    }
    //如果没有在设备流表rps_flow_table和全局流表
    //rps_sock_flow_table中找到目标cpu，则使用hash在
    //rps_map中找一个cpu即可。
    if (map) {
        tcpu = map->cpus[reciprocal_scale(hash, map->len)];
        if (cpu_online(tcpu)) {
            cpu = tcpu;
            goto done;
        }
    }

done:
    return cpu;
}

1

 /sys/class/net/<dev>/queues/rx-<n>/rps_cpus

1

if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {

1

 /proc/sys/net/core/flow_limit_cpu_bitmap

1

$ sudo sysctl -w net.core.netdev_max_backlog=3000

1

$ sudo sysctl -w net.core.flow_limit_table_len=8192

1
2
3
4
5
6
7
8

/*
 * The rps_sock_flow_table contains mappings of flows to the last CPU
 * on which they were processed by the application (set in recvmsg).
 */
struct rps_sock_flow_table {
	unsigned int mask;
	u16 ents[0]; // 弹性数组，key 是 flow hash，value 是流渴望被处理的CPU，也就是应用所在的CPU
};

1

$ sudo sysctl -w net.core.rps_sock_flow_entries=32768

1
2
3
4
5
6
7
8

// 设置 /proc/sys/net/core/rps_sock_flow_entries 时，调用如下函数
// 设置这个表其实就是分配内存数组，并将所有ents初始化为 `RPS_NO_CPU`。
rps_sock_flow_sysctl
    for (i = 0; i < size; i++)
        sock_table->ents[i] = RPS_NO_CPU;

    if (sock_table)
        static_key_slow_inc(&rps_needed);

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
					u32 hash)
{
	if (table && hash) {
		unsigned int cpu, index = hash & table->mask;

		/* We only give a hint, preemption can change cpu under us */
      	/*当前CPU*/
		cpu = raw_smp_processor_id();
		/*ents存放当前cpu*/
		if (table->ents[index] != cpu)
			table->ents[index] = cpu;
	}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

struct netdev_rx_queue {
	struct rps_map __rcu		*rps_map;
  	/*设备流表*/
	struct rps_dev_flow_table __rcu	*rps_flow_table;
	struct kobject			kobj;
	struct net_device		*dev;
} ____cacheline_aligned_in_smp;

struct rps_dev_flow_table {
	unsigned int mask;
	struct rcu_head rcu;
	struct rps_dev_flow flows[0];	//弹性数组
};

struct rps_dev_flow {
	u16 cpu;	/* 处理该流的cpu */
	u16 filter;
	unsigned int last_qtail;	/* sd->input_pkt_queue队列的尾部索引，即该队列长度 */
};

1
2
3
4
5

//设置  /sys/class/net/<dev>/queues/rx-<n>/rps_flow_cnt 时
store_rps_dev_flow_table_cnt
    table->mask = mask;
    for (count = 0; count <= mask; count++)
        table->flows[count].cpu = RPS_NO_CPU;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

/*
 * get_rps_cpu is called from netif_receive_skb and returns the target
 * CPU from the RPS map of the receiving queue for a given skb.
 * rcu_read_lock must be held on entry.
 */
static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
               struct rps_dev_flow **rflowp)
{
    struct netdev_rx_queue *rxqueue;
    struct rps_map *map;
    struct rps_dev_flow_table *flow_table;
    struct rps_sock_flow_table *sock_flow_table;
    int cpu = -1;
    u16 tcpu;
    u32 hash;

    /* ... */

    map = rcu_dereference(rxqueue->rps_map);

    /* ... */

    skb_reset_network_header(skb);
    hash = skb_get_hash(skb);
    if (!hash)
        goto done;

    flow_table = rcu_dereference(rxqueue->rps_flow_table);
    sock_flow_table = rcu_dereference(rps_sock_flow_table);
    if (flow_table && sock_flow_table) {
        u16 next_cpu;
        struct rps_dev_flow *rflow;
        // tcpu记录的是处理数据包的cpu
        rflow = &flow_table->flows[hash & flow_table->mask];
        tcpu = rflow->cpu;
        // next_cpu 记录的是运行 application 的 cpu
        next_cpu = sock_flow_table->ents[hash & sock_flow_table->mask];

        /*
         * If the desired CPU (where last recvmsg was done) is
         * different from current CPU (one in the rx-queue flow
         * table entry), switch if one of the following holds:
         *   - Current CPU is unset (equal to RPS_NO_CPU).
         *   - Current CPU is offline.
         *   - The current CPU's queue tail has advanced beyond the
         *     last packet that was enqueued using this table entry.
         *     This guarantees that all previous packets for the flow
         *     have been dequeued, thus preserving in order delivery.
         */
        if (unlikely(tcpu != next_cpu) &&
            (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
             ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
              rflow->last_qtail)) >= 0)) {
            tcpu = next_cpu;
            rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
        }

        if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
            *rflowp = rflow;
            cpu = tcpu;
            goto done;
        }
    }

    // 如果没有在设备流表rps_flow_table和全局流表rps_sock_flow_table中找到目标cpu，
    // 则使用hash在rps_map中找一个cpu即可。
    if (map) {
        tcpu = map->cpus[reciprocal_scale(hash, map->len)];
        if (cpu_online(tcpu)) {
            cpu = tcpu;
            goto done;
        }
    }

done:
    return cpu;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
        struct rps_dev_flow *rflow, u16 next_cpu)
{
    if (next_cpu != RPS_NO_CPU) {
        rflow->last_qtail =
            per_cpu(softnet_data, next_cpu).input_queue_head;
    }

    rflow->cpu = next_cpu;
    return rflow;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
        struct rps_dev_flow *rflow, u16 next_cpu)
{
    if (next_cpu != RPS_NO_CPU) {
#ifdef CONFIG_RFS_ACCEL
        struct netdev_rx_queue *rxqueue;
        struct rps_dev_flow_table *flow_table;
        struct rps_dev_flow *old_rflow;
        u32 flow_id;
        u16 rxq_index;
        int rc;

        /* Should we steer this flow to a different hardware queue? */
        if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
            !(dev->features & NETIF_F_NTUPLE))
            goto out;
        rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
        if (rxq_index == skb_get_rx_queue(skb))
            goto out;

        rxqueue = dev->_rx + rxq_index;
        flow_table = rcu_dereference(rxqueue->rps_flow_table);
        if (!flow_table)
            goto out;
        flow_id = skb_get_hash(skb) & flow_table->mask;
        rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
                            rxq_index, flow_id);
        if (rc < 0)
            goto out;
        old_rflow = rflow;
        rflow = &flow_table->flows[flow_id];
        rflow->filter = rc;
        if (old_rflow->filter == rflow->filter)
            old_rflow->filter = RPS_NO_FILTER;
    out:
#endif
        rflow->last_qtail =
            per_cpu(softnet_data, next_cpu).input_queue_head;
    }

    rflow->cpu = next_cpu;
    return rflow;
}

1

echo f > /sys/class/net/<dev>/queues/tx-<n>/xps_cpus

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

/*
 * This structure holds an XPS map which can be of variable length.  The
 * map is an array of queues.
 */
//percpu的结构，queues用来保存此cpu可以使用哪几个queue
struct xps_map {
    unsigned int len;
    unsigned int alloc_len;
    struct rcu_head rcu;
    u16 queues[0];
};

/*
 * This structure holds all XPS maps for device.  Maps are indexed by CPU.
 */
//cpu_map大小为cpu个数
struct xps_dev_maps {
    struct rcu_head rcu;
    struct xps_map __rcu *cpu_map[0];
};

//perdevice结构，xps_maps 用来保存设备 tx queue 和 cpu 的对应关系
struct net_device {
#ifdef CONFIG_XPS
    struct xps_dev_maps __rcu *xps_maps;
#endif

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155

static ssize_t store_xps_map(struct netdev_queue *queue,
              struct netdev_queue_attribute *attribute,
              const char *buf, size_t len)
{
    struct net_device *dev = queue->dev;
    unsigned long index;
    cpumask_var_t mask;
    int err;

    if (!capable(CAP_NET_ADMIN))
        return -EPERM;

    if (!alloc_cpumask_var(&mask, GFP_KERNEL))
        return -ENOMEM;
    //获取设置的队列索引
    index = get_netdev_queue_index(queue);

    //获取设置的cpu列表
    err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits);
    if (err) {
        free_cpumask_var(mask);
        return err;
    }

    //更新queue和cpu的映射关系
    err = netif_set_xps_queue(dev, mask, index);

    free_cpumask_var(mask);

    return err ? : len;
}

int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
            u16 index)
{
    struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
    struct xps_map *map, *new_map;
    int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
    int cpu, numa_node_id = -2;
    bool active = false;

    mutex_lock(&xps_map_mutex);

    dev_maps = xmap_dereference(dev->xps_maps);

    /* allocate memory for queue storage */
    for_each_online_cpu(cpu) {
        if (!cpumask_test_cpu(cpu, mask))
            continue;

        if (!new_dev_maps)
            new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
        if (!new_dev_maps) {
            mutex_unlock(&xps_map_mutex);
            return -ENOMEM;
        }

        map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
                 NULL;

        map = expand_xps_map(map, cpu, index);
        if (!map)
            goto error;

        RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
    }

    if (!new_dev_maps)
        goto out_no_new_maps;

    for_each_possible_cpu(cpu) {
        if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
            /* add queue to CPU maps */
            int pos = 0;

            map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
            while ((pos < map->len) && (map->queues[pos] != index))
                pos++;

            if (pos == map->len)
                map->queues[map->len++] = index;
#ifdef CONFIG_NUMA
            if (numa_node_id == -2)
                numa_node_id = cpu_to_node(cpu);
            else if (numa_node_id != cpu_to_node(cpu))
                numa_node_id = -1;
#endif
        } else if (dev_maps) {
            /* fill in the new device map from the old device map */
            map = xmap_dereference(dev_maps->cpu_map[cpu]);
            RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
        }

    }

    rcu_assign_pointer(dev->xps_maps, new_dev_maps);

    /* Cleanup old maps */
    if (dev_maps) {
        for_each_possible_cpu(cpu) {
            new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
            map = xmap_dereference(dev_maps->cpu_map[cpu]);
            if (map && map != new_map)
                kfree_rcu(map, rcu);
        }

        kfree_rcu(dev_maps, rcu);
    }

    dev_maps = new_dev_maps;
    active = true;

out_no_new_maps:
    /* update Tx queue numa node */
    netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
                     (numa_node_id >= 0) ? numa_node_id :
                     NUMA_NO_NODE);

    if (!dev_maps)
        goto out_no_maps;

    /* removes queue from unused CPUs */
    for_each_possible_cpu(cpu) {
        if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
            continue;

        if (remove_xps_queue(dev_maps, cpu, index))
            active = true;
    }

    /* free map if not active */
    if (!active) {
        RCU_INIT_POINTER(dev->xps_maps, NULL);
        kfree_rcu(dev_maps, rcu);
    }

out_no_maps:
    mutex_unlock(&xps_map_mutex);

    return 0;
error:
    /* remove any maps that we added */
    for_each_possible_cpu(cpu) {
        new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
        map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
                 NULL;
        if (new_map && new_map != map)
            kfree(new_map);
    }

    mutex_unlock(&xps_map_mutex);

    kfree(new_dev_maps);
    return -ENOMEM;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

//__dev_queue_xmit为通用发送函数，其中会调用netdev_pick_tx选择合适的tx queue
static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
      txq = netdev_pick_tx(dev, skb, accel_priv);

struct netdev_queue *netdev_pick_tx(struct net_device *dev,
                    struct sk_buff *skb,
                    void *accel_priv)
{
    int queue_index = 0;
    //设备tx queue个数大于1才需要选择
    if (dev->real_num_tx_queues != 1) {
        const struct net_device_ops *ops = dev->netdev_ops;

        //如果网卡驱动提供了ndo_select_queue，则使用
        //ndo_select_queue选择queue。没有合适的还得调用
        //__netdev_pick_tx。
        if (ops->ndo_select_queue)
            queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
                                __netdev_pick_tx);
        else
            queue_index = __netdev_pick_tx(dev, skb);

        if (!accel_priv)
            queue_index = netdev_cap_txqueue(dev, queue_index);
    }
    //将tx queue索引保存到skb->queue_mapping
    skb_set_queue_mapping(skb, queue_index);
    //根据索引获取指定的netdev_queue
    return netdev_get_tx_queue(dev, queue_index);
}

static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
{
    struct sock *sk = skb->sk;
    //取出sk中保存的tx queue
    int queue_index = sk_tx_queue_get(sk);

    //在以下三种情况下才会更新tx queue
    //a. queue_index小于0，这是初始情况
    //b. skb->ooo_okay(ooo全称out of order)，一个标志位，只
    //有在socket发送完数据后才会设置，表示可以切换queue(假
    //如初始设置的queue是1，突然改成3了，为了防
    //止报文乱序，要等socket中的数据都发送完成后，设置
    //ooo_okay标志后，才可以将  发送queue改成3)，只对tcp
    //socket有效
    //c. queue_index大于设备queue总数
    if (queue_index < 0 || skb->ooo_okay ||
        queue_index >= dev->real_num_tx_queues) {
        //从xps_cpu中选择queue
        int new_index = get_xps_queue(dev, skb);
        if (new_index < 0)
            //如果没有设置，就根据hash和设备queue总数计算出一个
            new_index = skb_tx_hash(dev, skb);

        if (queue_index != new_index && sk &&
            rcu_access_pointer(sk->sk_dst_cache))
            //将queue索引更新到 sk->sk_tx_queue_mapping
            sk_tx_queue_set(sk, new_index);

        queue_index = new_index;
    }

    return queue_index;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
{
#ifdef CONFIG_XPS
    struct xps_dev_maps *dev_maps;
    struct xps_map *map;
    int queue_index = -1;

    rcu_read_lock();
    dev_maps = rcu_dereference(dev->xps_maps);
    if (dev_maps) {
        map = rcu_dereference(
            dev_maps->cpu_map[raw_smp_processor_id()]);
        if (map) {
            if (map->len == 1)
                queue_index = map->queues[0];
            else
                queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
                                       map->len)];
            if (unlikely(queue_index >= dev->real_num_tx_queues))
                queue_index = -1;
        }
    }
    rcu_read_unlock();

    return queue_index;
#else
    return -1;
#endif
}

1
2

# 1010101
# echo 85 > /sys/class/net/eth0/queues/tx-0/xps_cpus

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
{
	struct sock *sk = skb->sk;
	int queue_index = sk_tx_queue_get(sk);
	/*发送队列的index不合法 或者
	ooo_okay 不为0时重新获取发送队列
	*/
  	/*ooo是 out of order，
  	ooo_okay 标志表示流中没有未完成的数据包，所以发送队列可以改变而没有产生乱序数据包的风险。
    传输层负责适当地设置ooo_okay。 例如，TCP在连接的所有数据已被确认时设置标志。
    */
	if (queue_index < 0 || skb->ooo_okay ||
	    queue_index >= dev->real_num_tx_queues) {
		int new_index = get_xps_queue(dev, skb);
		if (new_index < 0)
			new_index = skb_tx_hash(dev, skb);

		if (queue_index != new_index && sk &&
		    rcu_access_pointer(sk->sk_dst_cache))
			sk_tx_queue_set(sk, new_index);

		queue_index = new_index;
	}
	return queue_index;
}

1
2

$ ethtool -k eth0 | grep generic-receive-offload
generic-receive-offload: on

1

$ sudo ethtool -K ens33 gro on

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#define NETIF_F_SOFT_FEATURES	(NETIF_F_GSO | NETIF_F_GRO)

int register_netdevice(struct net_device *dev)
{
  /* ... */

	dev->hw_features |= NETIF_F_SOFT_FEATURES;
	dev->features |= NETIF_F_SOFT_FEATURES;
	dev->wanted_features = dev->features & dev->hw_features;

  /* ... */
}

1
2
3
4
5
6
7
8

// intel e1000 网卡
static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
  /* ... */
  if ((hw->mac_type >= e1000_82544) && (hw->mac_type != e1000_82547))
		netdev->hw_features |= NETIF_F_TSO;
  /* ... */
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

/* This will initiate an outgoing connection. */
int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{
  /* ... */

	/* OK, now commit destination to socket.  */
	sk->sk_gso_type = SKB_GSO_TCPV4;
	sk_setup_caps(sk, &rt->dst);

  /* ... */
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
{
	__sk_dst_set(sk, dst);
	sk->sk_route_caps = dst->dev->features;
	if (sk->sk_route_caps & NETIF_F_GSO)
		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
	sk->sk_route_caps &= ~sk->sk_route_nocaps;
	if (sk_can_gso(sk)) {
		if (dst->header_len) {
			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
		} else {
			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
			sk->sk_gso_max_size = dst->dev->gso_max_size;
			sk->sk_gso_max_segs = dst->dev->gso_max_segs;
		}
	}
}

1
2
3
4
5

static inline bool sk_can_gso(const struct sock *sk)
{
  	/*对于tcp，在tcp_v4_connect中被设置：sk->sk_gso_type = SKB_GSO_TCPV4*/
	  return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
}

1
2
3
4
5
6
7

static inline bool net_gso_ok(netdev_features_t features, int gso_type)
{
	netdev_features_t feature = gso_type << NETIF_F_GSO_SHIFT;

  /* ... */
	return (features & feature) == feature;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
{
	int mss_now;
	/*通过ip option，SACKs及pmtu确定当前的mss*/
	mss_now = tcp_current_mss(sk);
  	/*tcp_xmit_size_goal获取发送数据报到达网络设备时数据段的最大长度，该长度用来分割数据，TCP发送报文时，	   *每个SKB的大小不能超过该值。
	 *在此传入是否标识MSG_OOB(out-of-band,比普通数据更高的优先级传送的带外数据)位，这是因为MSG_OOB是判断  		*是否支持GSO的条件之一，而紧急数据不支持GSO。
	 *在不支持GSO的情况下，size_goal就等于mss_now，而如果支持GSO，则size_goal会是MSS的整数倍。数据报发送	 *到网络设备后再由网络设备根据MSS进行分割。*/
	*size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
	return mss_now;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, int large_allowed)
{
	struct tcp_sock *tp = tcp_sk(sk);
	u32 xmit_size_goal, old_size_goal;

	xmit_size_goal = mss_now;
	/*这里large_allowed表示是否是紧急数据；
	large_allowed为真表示无带外数据，可以大包发送*/
	if (large_allowed && sk_can_gso(sk)) {
		u32 gso_size, hlen;

		/* Maybe we should/could use sk->sk_prot->max_header here ? */
		hlen = inet_csk(sk)->icsk_af_ops->net_header_len +
		       inet_csk(sk)->icsk_ext_hdr_len +
		       tp->tcp_header_len;

		/* 目标是每ms发送至少一个数据包，而不是每100 ms发送一个大的TSO数据包。
		   sk_pacing_rate为 每秒的bytes。
		   这保留了ACK时钟，并且与tcp_tso_should_defer（）启发式一致。
		   sysctl_tcp_min_tso_segs 为 sysctl控制的系统变量。我的系统环境中值为2。
		 */
		gso_size = sk->sk_pacing_rate / (2 * MSEC_PER_SEC);
		gso_size = max_t(u32, gso_size,
				 sysctl_tcp_min_tso_segs * mss_now);
		/*xmit_size_goal为
		gso最大分段大小减去tcp和ip头部长度 与
		gso_size中比较小的值
		*/
		xmit_size_goal = min_t(u32, gso_size,
				       sk->sk_gso_max_size - 1 - hlen);
		/*最多达到收到的最大rwnd窗口通告的一半*/
		xmit_size_goal = tcp_bound_to_half_wnd(tp, xmit_size_goal);

		/* We try hard to avoid divides here */
		old_size_goal = tp->xmit_size_goal_segs * mss_now;

		if (likely(old_size_goal <= xmit_size_goal &&
			   old_size_goal + mss_now > xmit_size_goal)) {
			xmit_size_goal = old_size_goal;
		} else {
			tp->xmit_size_goal_segs =
				min_t(u16, xmit_size_goal / mss_now,
				      sk->sk_gso_max_segs);
			xmit_size_goal = tp->xmit_size_goal_segs * mss_now;
		}
	}
	return max(xmit_size_goal, mss_now);
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109

static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
			   int push_one, gfp_t gfp)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;
	unsigned int tso_segs, sent_pkts;
	int cwnd_quota;
	int result;

	sent_pkts = 0;

	if (!push_one) {
		/* Do MTU probing. */
		result = tcp_mtu_probe(sk);
		if (!result) {
			return false;
		} else if (result > 0) {
			sent_pkts = 1;
		}
	}
	/*遍历发送队列*/
	while ((skb = tcp_send_head(sk))) {
		unsigned int limit;

		tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
		BUG_ON(!tso_segs);

		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE)
			goto repair; /* Skip network transmission */

		cwnd_quota = tcp_cwnd_test(tp, skb);
		if (!cwnd_quota) {
			if (push_one == 2)
				/* Force out a loss probe pkt. */
				cwnd_quota = 1;
			else
				break;
		}

		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
			break;
		/*tso_segs=1表示无需tso分段*/
		if (tso_segs == 1 || !sk->sk_gso_max_segs) {
          	/* 根据nagle算法，计算是否需要推迟发送数据 */
			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
						     (tcp_skb_is_last(sk, skb) ?
						      nonagle : TCP_NAGLE_PUSH))))
				break;
		} else {
          	/*有多个tso分段*/
          	/*push所有skb*/
          	/*如果发送窗口剩余不多，并且预计下一个ack将很快到来(意味着可用窗口会增加)，则推迟发送*/
			if (!push_one && tcp_tso_should_defer(sk, skb))
				break;
		}

		limit = max_t(unsigned int, sysctl_tcp_limit_output_bytes,
			      sk->sk_pacing_rate >> 10);

		if (atomic_read(&sk->sk_wmem_alloc) > limit) {
			set_bit(TSQ_THROTTLED, &tp->tsq_flags);
			smp_mb__after_clear_bit();
			if (atomic_read(&sk->sk_wmem_alloc) > limit)
				break;
		}

      	/*下面的逻辑是：不用推迟发送，马上发送的情况*/
		limit = mss_now;
      	/*由于tso_segs被设置为skb->len/mss_now，所以开启gso时一定大于1*/
      	/*tso分段大于1且非urg模式*/
		if (tso_segs > 1 && sk->sk_gso_max_segs && !tcp_urg_mode(tp))
          	/*返回当前skb中可以发送的数据大小，通过mss和cwnd*/
			limit = tcp_mss_split_point(sk, skb, mss_now,
						    min_t(unsigned int,
							  cwnd_quota,
							  sk->sk_gso_max_segs));
		/* 当skb的长度大于限制时，需要调用tso_fragment分片,如果分段失败则暂不发送 */
		/*按limit切割成多个skb*/
		if (skb->len > limit &&
		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
			break;

		TCP_SKB_CB(skb)->when = tcp_time_stamp;
		/*发送，如果包被qdisc丢了，则退出循环，不继续发送了*/
		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
			break;

repair:
		/*更新sk_send_head和packets_out*/
		tcp_event_new_data_sent(sk, skb);
		tcp_minshall_update(tp, mss_now, skb);
		sent_pkts += tcp_skb_pcount(skb);

		if (push_one)
			break;
	}

	if (likely(sent_pkts)) {
		if (tcp_in_cwnd_reduction(sk))
			tp->prr_out += sent_pkts;

		/* Send one loss probe per tail loss episode. */
		if (push_one != 2)
			tcp_schedule_loss_probe(sk);
		tcp_cwnd_validate(sk);
		return false;
	}
	return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk));
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

/* Initialize TSO state of a skb.
 * This must be invoked the first time we consider transmitting
 * SKB onto the wire.
 */
static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb,
			     unsigned int mss_now)
{
	int tso_segs = tcp_skb_pcount(skb);

	if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
		tcp_set_skb_tso_segs(sk, skb, mss_now);
		tso_segs = tcp_skb_pcount(skb);
	}
	return tso_segs;
}

1
2
3
4
5
6
7

/* Initialize the device structure. */
static void vxlan_setup(struct net_device *dev)
{
	/* ... */
	dev->features   |= NETIF_F_GSO_SOFTWARE;
	/* ... */
}