vyos throughput optimizations

vyos-optimisations

Server 2 sockets,6 cores each, 2.4ghz

# Set ixgbe options
# Limit RSS queues to the number of physical cores per cpu
# Disable offload
# When you change this, you need to run the command and reboot for it to take.
echo "options ixgbe LRO=0,0 MQ=1,1 RSS=6,6 VMDQ=0,0 vxlan_rx=0,0" > /etc/modprobe.d/ixgbe.conf

# Shut down HT cores
for i in $(seq 1 2 23); do
  echo 0 > /sys/devices/system/cpu/cpu${i}/online
done

# Tune 10g interfaces.  Disable Offload Enable rx hashing
ethtool -K eth1 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on
ethtool -K eth3 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on

# Pin Interrupts for eth1 queues to physical cores on socket 1
let CPU=0
INTS=$(ls /sys/class/net/eth1/device/msi_irqs/ | sort -n)
for IRQ in $INTS; do
  echo $CPU > /proc/irq/$IRQ/smp_affinity_list
  let CPU+=2
done

# Pin Interrupts for eth3 queues to physical cores on socket 2
let CPU=12
INTS=$(ls /sys/class/net/eth3/device/msi_irqs/ | sort -n)
for IRQ in $INTS; do
  echo $CPU > /proc/irq/$IRQ/smp_affinity_list
  let CPU+=2
done

#  Configure MAX RPS flow counts for queues
echo 32768 > /proc/sys/net/core/rps_sock_flow_entries
for card in eth1 eth3; do
  for queue in $(seq 0 5); do
    echo 5461 > /sys/class/net/$card/queues/rx-$queue/rps_flow_cnt
  done
done

# Tune TCP Socket Buffers
echo 1 > /proc/sys/net/ipv4/tcp_window_scaling
echo 65536 1048576 4194304 > /proc/sys/net/ipv4/tcp_wmem
echo 65536 1048576 4194304 > /proc/sys/net/ipv4/tcp_rmem
echo 67108864 > /proc/sys/net/core/rmem_max
echo 67108864 > /proc/sys/net/core/wmem_max
echo 6711296 > /proc/sys/net/core/rmem_default  
echo 6711296 > /proc/sys/net/core/wmem_default
echo 40960 > /proc/sys/net/core/optmem_max

# Tune NetDev
echo 5000 > /proc/sys/net/core/netdev_max_backlog
echo 4000 > /proc/sys/net/core/netdev_budget

# Source from all unpriv ports
echo 1024 65000 > /proc/sys/net/ipv4/ip_local_port_range

# Increase conntrack hashsize to be 1/4 of nf_conntrack_max
echo 65536 > /sys/module/nf_conntrack/parameters/hashsize
echo 512 > /proc/sys/net/ipv4/neigh/default/gc_thresh1
echo 1024 > /proc/sys/net/ipv4/neigh/default/gc_thresh2
echo 2048 > /proc/sys/net/ipv4/neigh/default/gc_thresh3