Kernel tuning for dedicated linux server. /etc/sysctl.d/60-sysctl.conf

60-sysctl.conf

# Sysctl Configuration for High-Performance such as HTTP/2 Web Server
# Kawin Viriyaprasopsook <kawin.v@kkumail.com>
# place file in /etc/sysctl.d/60-sysctl.conf
# and run `sysctl --system`
# --------------------------------------------------------------------
# System options
# --------------------------------------------------------------------

# stop low-level messages on console
kernel.printk = 3 4 1 3

# tells the kernel to be slightly aggressive in swapping out memory pages
vm.swappiness = 10

# Increase number of incoming connections
# Default: 128
net.core.somaxconn = 32768

# Increase system file descriptor limit
fs.file-max = 2097152

# Increase the maximum number of allowed processes
kernel.pid_max = 65536

# Enable TCP window scaling for better performance
net.ipv4.tcp_window_scaling = 1

# Increase the maximum number of TCP connections
net.ipv4.tcp_max_syn_backlog = 8192

# Increase the maximum number of half-open connections
net.ipv4.tcp_max_tw_buckets = 1440000

# Enable TCP Fast Open to reduce connection setup time
net.ipv4.tcp_fastopen = 3

# Enable TCP timestamps for better RTT estimation
net.ipv4.tcp_timestamps = 1

# Enable TCP SACK (Selective Acknowledgment)
net.ipv4.tcp_sack = 1

# Use BBR TCP congestion control and set tcp_notsent_lowat to 16384 to ensure HTTP/2 prioritization works optimally
# Do a 'modprobe tcp_bbr' first (kernel > 4.9)
# Fall-back to htcp if bbr is unavailable (older kernels)
# Default: tcp_congestion_control=cubic, tcp_notsent_lowat=-1
net.ipv4.tcp_congestion_control = htcp
net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_notsent_lowat = 16384

# For servers with tcp-heavy workloads, enable 'fq' queue management scheduler (kernel > 3.12)
# Default: pfifo_fast
net.core.default_qdisc = fq

# Increase the maximum receive buffer size
net.core.rmem_max = 16777216

# Increase the maximum send buffer size
net.core.wmem_max = 16777216

# Enable SYN cookies to mitigate SYN flood attacks
net.ipv4.tcp_syncookies = 1

# Set the maximum number of SYN cookies
net.ipv4.tcp_max_syn_backlog = 4096

# Enable reverse path filtering (RPF) to prevent IP spoofing
net.ipv4.conf.all.rp_filter = 1

# Disable ICMP redirects
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0

# Disable source routing
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# Enable IP forwarding (if needed)
net.ipv4.ip_forward = 0

# Enable IP spoofing protection
net.ipv4.conf.all.log_martians = 1

# Enable SYN flood protection
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2

# Set the maximum number of connections per port
net.ipv4.ip_local_port_range = 1024 65535

# Optimize network buffers for 10G NIC
net.core.netdev_max_backlog = 30000
net.core.rmem_default = 31457280
net.core.wmem_default = 31457280
net.core.rmem_max = 67108864
net.core.wmem_max = 67108864

# Enable IP fragmentation defense
net.ipv4.ipfrag_high_thresh = 262144
net.ipv4.ipfrag_low_thresh = 196608
net.ipv4.ipfrag_time = 30

# Enable SYN proxy to protect against DDoS attacks
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_syncookies = 1
# try to reuse time-wait connections, but don't recycle them (recycle can break clients behind NAT)
# Default: tcp_tw_recycle=0, tcp_tw_reuse=0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
# Disable ICMP broadcasts
net.ipv4.icmp_echo_ignore_broadcasts = 1

# Disable ICMP timestamp requests
net.ipv4.icmp_timestamp_ignore_all = 1
# This will enusre that immediatly subsequent connections use the new values
net.ipv4.route.flush = 1
net.ipv6.route.flush = 1

65-k8s.conf

# Recommended settings for Kubernetes with IPVS and BBR on a 10Gbps network
# You may need to adjust some values depending on your specific hardware and use case.

# Increase maximum number of open files allowed
fs.file-max = 2097152

# Allow for more PIDs (to avoid "fork: resource temporarily unavailable" errors)
kernel.pid_max = 4194303

# Increase system IP port range
net.ipv4.ip_local_port_range = 1024 65535

# Enable TCP BBR congestion control algorithm
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

# Enable IP forwarding for Kubernetes routing
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1

# Enable IPVS-based connection tracking
net.ipv4.vs.conn_reuse_mode = 1
net.ipv4.vs.conntrack = 1

# Tune IPVS timeouts for Kubernetes
net.ipv4.vs.expire_nodest_conn = 1
net.ipv4.vs.expire_quiescent_template = 1

# Increase the maximum memory used for RPF (Reverse Path Forwarding) filtering
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2

# Increase TCP buffer sizes
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Increase the amount of memory allocated to network stack
net.core.optmem_max = 65535

# Increase the maximum amount of memory buffers used by sockets
net.ipv4.udp_mem = 65536 131072 262144
net.ipv4.tcp_mem = 65536 131072 262144

# Increase the maximum number of memory buffer space used by each socket
net.ipv4.udp_rmem_min = 16384
net.ipv4.udp_wmem_min = 16384
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Increase the maximum number of connections
net.ipv4.tcp_max_syn_backlog = 8192
net.core.somaxconn = 8192

# Enable fast recycling of TIME_WAIT sockets
net.ipv4.tcp_tw_reuse = 1

# Decrease the time default value for tcp_fin_timeout connection
net.ipv4.tcp_fin_timeout = 15

# Increase the maximum socket listen backlog
net.core.netdev_max_backlog = 65536

# Enable BBR congestion control algorithm for IPv6
net.ipv6.conf.all.accept_ra = 2
net.ipv6.conf.default.accept_ra = 2

# Disable IPv6 autoconfiguration
net.ipv6.conf.all.autoconf = 0
net.ipv6.conf.default.autoconf = 0
net.ipv6.conf.all.accept_ra = 0
net.ipv6.conf.default.accept_ra = 0

80-pve.conf

# Recommended settings for PVE and BBR on a 10Gbps network
# You may need to adjust some values depending on your specific hardware and use case.

# LXC: Failed to allocate directory watch: Too many open files
fs.file-max = 2097152
fs.aio-max-nr = 2097152
fs.inotify.max_user_instances = 512
fs.inotify.max_user_watches = 524288

# Allow for more PIDs (to avoid "fork: resource temporarily unavailable" errors)
kernel.pid_max = 4194303

# Increase system IP port range
net.ipv4.ip_local_port_range = 1024 65535

# Enable TCP BBR congestion control algorithm
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

# Increase the maximum memory used for RPF (Reverse Path Forwarding) filtering
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2

# Increase TCP buffer sizes
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Increase the amount of memory allocated to network stack
net.core.optmem_max = 65535

# Increase the maximum amount of memory buffers used by sockets
net.ipv4.udp_mem = 65536 131072 262144
net.ipv4.tcp_mem = 65536 131072 262144

# Increase the maximum number of memory buffer space used by each socket
net.ipv4.udp_rmem_min = 16384
net.ipv4.udp_wmem_min = 16384
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Increase the maximum number of connections
net.ipv4.tcp_max_syn_backlog = 8192
net.core.somaxconn = 8192

# Enable fast recycling of TIME_WAIT sockets
net.ipv4.tcp_tw_reuse = 1

# Decrease the time default value for tcp_fin_timeout connection
net.ipv4.tcp_fin_timeout = 15

# Increase the maximum socket listen backlog
net.core.netdev_max_backlog = 65536

# Disable IPv6 if not used
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

90-k8s.conf

# containerd: Failed to allocate directory watch: Too many open files
fs.file-max = 2097152
fs.aio-max-nr = 2097152
fs.inotify.max_user_instances = 512
fs.inotify.max_user_watches = 524288
# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#install-and-configure-prerequisites
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1