Here’s a couple of scripts I want to keep around in case I need them again. I used these to test/benchmark a large NUMA machine’s network performance.

setup_fast_network_benchmark.sh

ethtool --features enp94s0 ntuple on

# node 0
ethtool -N enp94s0 flow-type tcp4 dst-port 5102 action 1
ethtool -N enp94s0 flow-type tcp4 dst-port 5104 action 2
ethtool -N enp94s0 flow-type tcp4 dst-port 5106 action 3
ethtool -N enp94s0 flow-type tcp4 dst-port 5108 action 4
ethtool -N enp94s0 flow-type tcp4 dst-port 5110 action 5
ethtool -N enp94s0 flow-type tcp4 dst-port 5112 action 6

# node 1
ethtool -N enp94s0 flow-type tcp4 dst-port 5101 action 12
ethtool -N enp94s0 flow-type tcp4 dst-port 5103 action 13
ethtool -N enp94s0 flow-type tcp4 dst-port 5105 action 14
ethtool -N enp94s0 flow-type tcp4 dst-port 5107 action 15


# node 0
irq=$(grep mlx5_comp1@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 04 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp2@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 10 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp3@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 40 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp4@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 100 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp5@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 400 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp6@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 1000 > /proc/irq/$irq/smp_affinity


# node 1
irq=$(grep mlx5_comp12@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 02 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp13@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 08 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp14@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 20 > /proc/irq/$irq/smp_affinity
irq=$(grep mlx5_comp15@ /proc/interrupts | cut -f1 -d: | egrep -o "[0-9]+")
echo 80 > /proc/irq/$irq/smp_affinity


#modprobe cpufreq_userspace
#cpupower frequency-set -g userspace
#cpupower -c all frequency-set -f 3.7GHz
cpupower frequency-set -g performance
cpupower set -b 0

setpci -s 5e:00.0 68.w=593e
ifup enp94s0
#tc qdisc add dev enp94s0 root fq

# let's not do this, instead map irq to cpu above
#cd /usr/local/src/mlnx-tools/ofed_scripts/
#./set_irq_affinity_cpulist.sh 0 enp94s0

do_iperf_clients.sh

iperf3 -c 192.168.56.174 -T cpu2 -p 5102 -A 2 -w 1M  &
iperf3 -c 192.168.56.174 -T cpu4 -p 5104 -A 4 -w 1M  -R &
iperf3 -c 192.168.56.174 -T cpu6 -p 5106 -A 6 -w 1M  &
iperf3 -c 192.168.56.174 -T cpu8 -p 5108 -A 8 -w 1M  -R &

#iperf3 -c 192.168.56.174 -T cpu1 -p 5101 -A 1 -w 1M  &
#iperf3 -c 192.168.56.174 -T cpu3 -p 5103 -A 3 -w 1M  -R &
#iperf3 -c 192.168.56.174 -T cpu5 -p 5105 -A 5 -w 1M  &
#iperf3 -c 192.168.56.174 -T cpu7 -p 5107 -A 7 -w 1M  -R &
#
#iperf3 -c 192.168.56.174 -T cpu10 -p 5110 -A 10 -w 1M  &
#iperf3 -c 192.168.56.174 -T cpu12 -p 5112 -A 12 -w 1M -R  &