Skip to content

Instantly share code, notes, and snippets.

@rkhozinov
Forked from RafPe/vyos-optimisations
Created May 16, 2017 08:03
Show Gist options
  • Save rkhozinov/2469ca6b6359d9e862c5ab99fd0bd5c3 to your computer and use it in GitHub Desktop.
Save rkhozinov/2469ca6b6359d9e862c5ab99fd0bd5c3 to your computer and use it in GitHub Desktop.
vyos throughput optimizations
Server 2 sockets,6 cores each, 2.4ghz
# Set ixgbe options
# Limit RSS queues to the number of physical cores per cpu
# Disable offload
# When you change this, you need to run the command and reboot for it to take.
echo "options ixgbe LRO=0,0 MQ=1,1 RSS=6,6 VMDQ=0,0 vxlan_rx=0,0" > /etc/modprobe.d/ixgbe.conf
# Shut down HT cores
for i in $(seq 1 2 23); do
echo 0 > /sys/devices/system/cpu/cpu${i}/online
done
# Tune 10g interfaces. Disable Offload Enable rx hashing
ethtool -K eth1 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on
ethtool -K eth3 rx on tx on sg on tso off gso off gro off lro off ntuple on rxhash on
# Pin Interrupts for eth1 queues to physical cores on socket 1
let CPU=0
INTS=$(ls /sys/class/net/eth1/device/msi_irqs/ | sort -n)
for IRQ in $INTS; do
echo $CPU > /proc/irq/$IRQ/smp_affinity_list
let CPU+=2
done
# Pin Interrupts for eth3 queues to physical cores on socket 2
let CPU=12
INTS=$(ls /sys/class/net/eth3/device/msi_irqs/ | sort -n)
for IRQ in $INTS; do
echo $CPU > /proc/irq/$IRQ/smp_affinity_list
let CPU+=2
done
# Configure MAX RPS flow counts for queues
echo 32768 > /proc/sys/net/core/rps_sock_flow_entries
for card in eth1 eth3; do
for queue in $(seq 0 5); do
echo 5461 > /sys/class/net/$card/queues/rx-$queue/rps_flow_cnt
done
done
# Tune TCP Socket Buffers
echo 1 > /proc/sys/net/ipv4/tcp_window_scaling
echo 65536 1048576 4194304 > /proc/sys/net/ipv4/tcp_wmem
echo 65536 1048576 4194304 > /proc/sys/net/ipv4/tcp_rmem
echo 67108864 > /proc/sys/net/core/rmem_max
echo 67108864 > /proc/sys/net/core/wmem_max
echo 6711296 > /proc/sys/net/core/rmem_default
echo 6711296 > /proc/sys/net/core/wmem_default
echo 40960 > /proc/sys/net/core/optmem_max
# Tune NetDev
echo 5000 > /proc/sys/net/core/netdev_max_backlog
echo 4000 > /proc/sys/net/core/netdev_budget
# Source from all unpriv ports
echo 1024 65000 > /proc/sys/net/ipv4/ip_local_port_range
# Increase conntrack hashsize to be 1/4 of nf_conntrack_max
echo 65536 > /sys/module/nf_conntrack/parameters/hashsize
echo 512 > /proc/sys/net/ipv4/neigh/default/gc_thresh1
echo 1024 > /proc/sys/net/ipv4/neigh/default/gc_thresh2
echo 2048 > /proc/sys/net/ipv4/neigh/default/gc_thresh3
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment