主页 / 问题 / 1035297
Accepted
Andrey M
Asked:2020-10-16 20:00:31 +0000 UTC2020-10-16 20:00:31 +0000 UTC

使用以太网帧时在 Linux 下设置 10Gb 网卡

  • 772

我不确定这个问题是否符合规则，但仍然。

有一张 10Gb 的卡（品牌不重要，因为结果在另外两个与此不同的卡上重复）并Debian Buster通过光学直接连接到数据源机器。大约 9Gb 的数据流被组织起来，由大约 9000 字节的以太网帧组成，其中除了发送者的 MAC、接收者的 MAC、自定义 EtherType 和数据包计数器之外什么都没有。

实用程序wireshark, tshark,的测量结果tcpdump（尽管它们都通过同一个库工作）表明大约每隔一个包就会被跳过。自写的曲线代码显示大致相同。同时，在Windows 7同一个wireshark（分配了很大的缓冲区）下，它会捕获所有数据包。

使用 TCP 流量上的实用程序检查网络带宽iperf3可得到稳定的 9.8Gb。

# uname -a

Linux debian 4.19.0-6-amd64 #1 SMP Debian 4.19.67-2 (2019-08-28) x86_64 GNU/Linux


# ifconfig enp1s0

enp1s0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 12000
        inet 10.0.0.1  netmask 255.255.255.0  broadcast 10.0.255.255
        inet6 fe80::b2c5:54ff:feff:f37f  prefixlen 64  scopeid 0x20<link>
        ether b0:c5:54:ff:f3:7f  txqueuelen 3000  (Ethernet)
        RX packets 1501482032  bytes 13216245041216 (12.0 TiB)
        RX errors 0  dropped 1501472982  overruns 0  frame 0
        TX packets 75  bytes 7809 (7.6 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
        device interrupt 34  memory 0x90400000-90410000  



# ethtool -g enp1s0

Ring parameters for enp1s0:
Pre-set maximums:
RX:     1365
RX Mini:    0
RX Jumbo:   0
TX:     2048
Current hardware settings:
RX:     1365
RX Mini:    0
RX Jumbo:   0
TX:     2048

# ethtool -k enp1s0

Features for enp1s0:
rx-checksumming: on
tx-checksumming: on
    tx-checksum-ipv4: on
    tx-checksum-ip-generic: off [fixed]
    tx-checksum-ipv6: off [fixed]
    tx-checksum-fcoe-crc: off [fixed]
    tx-checksum-sctp: off [fixed]
scatter-gather: on
    tx-scatter-gather: on
    tx-scatter-gather-fraglist: on
tcp-segmentation-offload: on
    tx-tcp-segmentation: on
    tx-tcp-ecn-segmentation: off [fixed]
    tx-tcp-mangleid-segmentation: off
    tx-tcp6-segmentation: off [fixed]
udp-fragmentation-offload: off
generic-segmentation-offload: on
generic-receive-offload: on
large-receive-offload: off [fixed]
rx-vlan-offload: on
tx-vlan-offload: on
ntuple-filters: off [fixed]
receive-hashing: on
highdma: on
rx-vlan-filter: on
vlan-challenged: off [fixed]
tx-lockless: off [fixed]
netns-local: off [fixed]
tx-gso-robust: off [fixed]
tx-fcoe-segmentation: off [fixed]
tx-gre-segmentation: off [fixed]
tx-gre-csum-segmentation: off [fixed]
tx-ipxip4-segmentation: off [fixed]
tx-ipxip6-segmentation: off [fixed]
tx-udp_tnl-segmentation: off [fixed]
tx-udp_tnl-csum-segmentation: off [fixed]
tx-gso-partial: off [fixed]
tx-sctp-segmentation: off [fixed]
tx-esp-segmentation: off [fixed]
tx-udp-segmentation: off [fixed]
fcoe-mtu: off [fixed]
tx-nocache-copy: off
loopback: off [fixed]
rx-fcs: off [fixed]
rx-all: off [fixed]
tx-vlan-stag-hw-insert: off [fixed]
rx-vlan-stag-hw-parse: off [fixed]
rx-vlan-stag-filter: off [fixed]
l2-fwd-offload: off [fixed]
hw-tc-offload: off [fixed]
esp-hw-offload: off [fixed]
esp-tx-csum-hw-offload: off [fixed]
rx-udp_tunnel-port-offload: off [fixed]
tls-hw-tx-offload: off [fixed]
tls-hw-rx-offload: off [fixed]
rx-gro-hw: off [fixed]
tls-hw-record: off [fixed]

我不太了解网络堆栈，坦率地说，在设置网络时“浮动”，所以我请求社区的帮助，或者至少是一个可以挖掘的向量。特别感兴趣的是这条线RX errors 0 dropped 1501472982 overruns 0 frame 0

linux

2 个回答

  1. 在这个阶段，事情已经结束了：我太相信实用程序（wireshark, tshark, tcpdump）以及dropped 1501472982from的行ifconfig。这些实际上是什么dropped（也出现在上述程序的输出中）需要在每种情况下单独处理，但这些不是不可挽回的丢失数据包。
    就我而言，增加这些内核参数就足够了：

    net.core.rmem_max
net.core.wmem_max
net.core.rmem_default
net.core.wmem_default
net.core.optmem_max

    在现有数据流上，自行编写的代码（此https://github.com/hundeboll/rawsend的略微修改版本）捕获每个数据包，尝试测量最大吞吐量约为 9.9Gb。

    • 3
  2. Best Answer

    在 Linux 上的大多数情况下，您需要调整驱动程序、内核设置，并可能在内核之间重新分配中断。以下是 10g debian 机器的运行参数示例。

    # cat /etc/modprobe.d/ixgbe.conf 
options ixgbe MQ=1,1 
options ixgbe RSS=8,8 
options ixgbe InterruptThrottleRate=1,1 
options ixgbe DCA=2,2 
options ixgbe FCoE=0,0 
options ixgbe LRO=0,0


# cat /etc/sysctl.conf

# Kernel sysctl configuration file for Red Hat Linux
#
# For binary values, 0 is disabled, 1 is enabled.  See sysctl(8) and
# sysctl.conf(5) for more details.

# Controls IP packet forwarding
net.ipv4.ip_forward = 1

# Controls source route verification
net.ipv4.conf.default.rp_filter = 0

# Do not accept source routing
net.ipv4.conf.default.accept_source_route = 0

# Controls the System Request debugging functionality of the kernel
kernel.sysrq = 0

# Controls whether core dumps will append the PID to the core filename.
# Useful for debugging multi-threaded applications.
kernel.core_uses_pid = 1

# Controls the use of TCP syncookies
net.ipv4.tcp_syncookies = 1

# Controls the default maxmimum size of a mesage queue
kernel.msgmnb = 65536

# Controls the maximum size of a message, in bytes
kernel.msgmax = 65536

# Controls the maximum shared segment size, in bytes
kernel.shmmax = 68719476736

# Controls the maximum number of shared memory segments, in pages
kernel.shmall = 4294967296

# No swappiness
vm.swappiness = 0 
net.core.rmem_default = 8388608
net.core.wmem_default = 8388608
net.core.rmem_max = 8388608
net.core.wmem_max = 8388608

net.netfilter.nf_conntrack_max = 30000000
net.netfilter.nf_conntrack_tcp_timeout_established = 1800

net.ipv4.conf.all.arp_announce = 1 
net.ipv4.conf.lo.arp_announce = 1 

net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_fin_timeout = 20
net.core.netdev_max_backlog = 8192
net.ipv4.tcp_tw_recycle = 1 
net.ipv4.tcp_timestamps = 1 
net.core.somaxconn = 16384
net.ipv4.tcp_window_scaling = 1 
net.ipv4.tcp_max_orphans = 65536
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_tw_reuse = 1 
net.ipv4.conf.all.rp_filter = 0 
net.ipv4.tcp_orphan_retries = 0 
net.ipv4.udp_mem = 51200   76800   102400
net.ipv4.tcp_mem = 51200   76800   102400
net.ipv4.tcp_keepalive_probes = 5 
net.ipv4.conf.all.accept_source_route = 0 
net.ipv4.tcp_congestion_control = htcp
net.ipv4.tcp_keepalive_time = 1800
net.ipv4.tcp_synack_retries = 2 
net.ipv6.conf.all.disable_ipv6 = 1

vm.zone_reclaim_mode = 1
vm.min_free_kbytes = 512000
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 1024
net.ipv4.neigh.default.gc_thresh3 = 2048

kernel.panic = 60

    可以使用 intel 的脚本重新分配中断：https ://github.com/majek/ixgbe/blob/master/scripts/set_irq_affinity

    • 2

相关问题