k3s

90-k8s.conf

# /etc/sysctl.d/90-k8s.conf

### FILE SYSTEM SETTINGS ###
fs.file-max = 2097152

### VIRTUAL MEMORY SETTINGS ###
# Decrease SWAP
vm.dirty_ratio = 15
vm.dirty_background_ratio = 8

# Panic and OOM Behavior
vm.panic_on_oom = 0
vm.overcommit_memory = 1

### CPU SCHEDULING SETTINGS ###
# Sets the time before the kernel considers migrating a process to another core
kernel.sched_migration_cost_ns = 2000000

# Group tasks by TTY
# kernel.sched_autogroup_enabled = 0

### NETWORKING SETTINGS ###

# General Networking
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.core.optmem_max = 25165824
net.core.netdev_max_backlog = 65536
net.core.somaxconn = 65535

# TCP Settings
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_rfc1337 = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_tw_buckets = 1440000
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 786432 1048576 26777216
net.ipv4.tcp_rmem = 8192 87380 33554432
net.ipv4.tcp_wmem = 8192 65536 33554432

# UDP Settings
net.ipv4.udp_mem = 65536 131072 262144
net.ipv4.udp_rmem_min = 16384
net.ipv4.udp_wmem_min = 16384

# # Local Port Range
# net.ipv4.ip_local_port_range = 2000 65535

# IPv4 Buffer Sizes
net.core.rmem_default = 31457280
net.core.rmem_max = 33554432
net.core.wmem_default = 31457280
net.core.wmem_max = 33554432

# Disable IPv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1

# Adjusting Maximum Backlog and SYN Backlog
net.core.netdev_max_backlog = 5000
net.ipv4.tcp_max_syn_backlog = 2048

### KERNEL HARDENING ###
kernel.panic = 10
kernel.panic_on_oops = 1
kernel.keys.root_maxbytes = 25000000

/etc/rancher/k3s/kubelet-config.yaml

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

featureGates:
NodeSwap: true

serializeImagePulls: true

memorySwap:
swapBehavior: "LimitedSwap"

kubeReserved:
cpu: "400m"
memory: "3Gi"
ephemeral-storage: "4Gi"

systemReserved:
cpu: "100m"
memory: "512Mi"
ephemeral-storage: "8Gi"

evictionHard:
memory.available: "128Mi"
nodefs.available: "10%"
nodefs.inodesFree: "5%"

imageMinimumGCAge: "2h"
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMaximumGCAge: "48h"

/etc/rancher/k3s/config.yaml - control-plane

write-kubeconfig-mode: "0644"

server: "https://192.168.0.102:6443"
token: "xxx"

tls-san:

• "k.vst.name"
• "api.k.vst.name"
• "external_ip"
• "192.168.0.101"
• "192.168.0.102"
• "192.168.0.103"
• "192.168.0.104"
• "192.168.0.105"
• "192.168.0.106"
• "192.168.0.107"
• "192.168.0.108"
• "192.168.0.109"

node-ip: "192.168.0.101"

etcd-s3: true
etcd-s3-timeout: "20m"
etcd-snapshot-schedule-cron: "0 6 * * 2,4"
etcd-snapshot-retention: 14
snapshot-retention: 14
etcd-s3-endpoint: storage.yandexcloud.net
etcd-s3-access-key: "xxx"
etcd-s3-secret-key: "xxx"
etcd-s3-bucket: bucket_name
etcd-s3-region: central1

disable-helm-controller: true
disable:

• "traefik"
• "servicelb"
• "metrics-server"
• "coredns"

embedded-registry: true

etcd-expose-metrics: true

kubelet-arg:

• "config=/etc/rancher/k3s/kubelet-config.yaml"
  kube-proxy-arg:
• "proxy-mode=ipvs"
• "ipvs-scheduler=rr"
• "ipvs-strict-arp"

/etc/rancher/k3s/kubelet-config.yaml - control-plane

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

featureGates:
NodeSwap: true

serializeImagePulls: true

memorySwap:
swapBehavior: "LimitedSwap"

kubeReserved:
cpu: "400m"
memory: "3Gi"
ephemeral-storage: "4Gi"

systemReserved:
cpu: "100m"
memory: "512Mi"
ephemeral-storage: "8Gi"

evictionHard:
memory.available: "128Mi"
nodefs.available: "10%"
nodefs.inodesFree: "5%"

imageMinimumGCAge: "2h"
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMaximumGCAge: "48h"

/etc/rancher/k3s/config.yaml - agent

server: "https://192.168.0.101:6443"
token: "xxx"

node-ip: "192.168.0.111"
embedded-registry: true

kubelet-arg:

• "config=/etc/rancher/k3s/kubelet-config.yaml"
  kube-proxy-arg:
• "proxy-mode=ipvs"
• "ipvs-scheduler=rr"
• "ipvs-strict-arp"

/etc/rancher/k3s/kubelet-config.yaml - agent

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

featureGates:
NodeSwap: true

serializeImagePulls: true

memorySwap:
swapBehavior: "LimitedSwap"

kubeReserved:
cpu: "200m"
memory: "384Mi"
ephemeral-storage: "1Gi"

systemReserved:
cpu: "100m"
memory: "512Mi"
ephemeral-storage: "4Gi"

evictionHard:
memory.available: "128Mi"
nodefs.available: "10%"
nodefs.inodesFree: "5%"

imageMinimumGCAge: "2h"
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMaximumGCAge: "48h"