Skip to content

Instantly share code, notes, and snippets.

@eqhmcow
Last active December 16, 2020 03:15
Show Gist options
  • Save eqhmcow/c378c46a41aa5716767a0da811087dd4 to your computer and use it in GitHub Desktop.
Save eqhmcow/c378c46a41aa5716767a0da811087dd4 to your computer and use it in GitHub Desktop.
let me eat cake
### section 1
# disable CPU mitigations, disable CPU sleep (C states), disable CPU throttling (frequency scaling)
# edit /boot/grub/grub.cfg
# mitigations=off intel_idle.max_cstate=0 processor.max_cstate=1
echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor
echo performance > /sys/devices/system/cpu/cpufreq/policy1/scaling_governor
# you can verify the above were correctly disabled with "i7z" https://packages.ubuntu.com/i7z
### section 2
# disable / correctly configure NIC interrupt throttling / packet coalescing at the driver level (if necessary for your driver)
#rmmod r8169 ; rmmod r8168
#insmod r8168 use_dac=1 rx_copybreak=256 eee_enable=0 aspm=0 timer_count=0
#eqhmcow@OpenWrt:~$ cat /etc/modules.d/e1000e
#e1000e InterruptThrottleRate=0
# or after booting:
rmmod e1000e ; insmod e1000e InterruptThrottleRate=0
### section 3
# disable NIC offloading, configure NIC hardware buffers, etc
# also consider ethtool -C to disable coalescing if necessary for your driver
# high RX buffer; low TX buffer
# theory behind a high RX buffer: this gets packets into cake on our side quickly, to soak up
# and help mitigate other buffers that might be bloated
# also high RX reduces the chance we drop packets that we didn't want to drop
for i in eth0 eth1 ; do
for j in rx tx sg tso ufo gso gro lro ntuple rxhash ; do
/usr/sbin/ethtool -K $i $j off
done
/usr/sbin/ethtool -A $i autoneg off rx off tx off
/usr/sbin/ethtool -G $i rx 4094 tx 66
/usr/sbin/ethtool -G $i rx 4095 tx 65
/usr/sbin/ethtool -G $i rx 4096 tx 64
done
sleep 5
service network restart
sleep 5
### section 4
# default: 0
echo 1 > /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/net/eth0/napi_threaded
echo 1 > /sys/devices/pci0000:00/0000:00:1c.0/0000:02:00.0/net/eth1/napi_threaded
sleep 5
# IFF you use cake's "bandwidth" option, then "byte queue limits" don't matter much
# otherwise set the BQL max very low (3000 or less, down from the default of 1879048192)
echo 0 > /sys/devices/pci0000:00/0000:00:1c.0/0000:02:00.0/net/eth1/queues/tx-0/byte_queue_limits/limit_min
echo 0 > /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/net/eth0/queues/tx-0/byte_queue_limits/limit_min
# default 1879048192
#echo 3000 > /sys/devices/pci0000:00/0000:00:1c.0/0000:02:00.0/net/eth1/queues/tx-0/byte_queue_limits/limit_max
#echo 3000 > /sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/net/eth0/queues/tx-0/byte_queue_limits/limit_max
# 64 default
echo 4 > /proc/sys/net/core/dev_weight
# 20000 default
echo 0 > /proc/sys/net/core/netdev_budget_usecs
# 300 default
echo 0 > /proc/sys/net/core/netdev_budget
# 1000 default
echo 1000000000 > /proc/sys/net/core/netdev_max_backlog
# set up cake
WAN=eth1
LAN=eth0
/usr/sbin/tc qdisc del dev $WAN root 2> /dev/null
/usr/sbin/tc qdisc del dev $LAN root 2> /dev/null
# in my testing, around 40mbit is the best that wifi can do w/o latency increasing
BANDWIDTH=46mbit
UPBANDWIDTH=20mbit
/usr/sbin/tc qdisc add dev $WAN handle 1: root cake besteffort bandwidth $UPBANDWIDTH internet nat egress ack-filter dual-srchost ethernet memlimit 134217728
/usr/sbin/tc qdisc add dev $LAN handle 1: root cake besteffort bandwidth $BANDWIDTH internet ingress dual-dsthost ethernet memlimit 134217728
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment