Dockerfile

FROM ubuntu:20.04 AS opts

ENV KUBE_VERSION 1.18.5
ENV CRIO_VERSION 1.18.1
ENV COREDNS_VERSION 1.7.0
ENV KERNEL_VERSION 5.4.0-37-generic
ENV IMAGE_VERSION 1.0.0
ENV DEBIAN_FRONTEND noninteractive
ENV TZ UTC

FROM opts AS core

RUN rm -f /run/systemd/container

RUN apt-get update && apt-get upgrade -y
RUN yes | unminimize

FROM core AS downloads

RUN apt-get install -y wget
RUN wget https://github.com/coredns/coredns/releases/download/v${COREDNS_VERSION}/coredns_${COREDNS_VERSION}_linux_amd64.tgz
RUN wget https://github.com/cri-o/cri-o/releases/download/v${CRIO_VERSION}/crio-v${CRIO_VERSION}.tar.gz
RUN wget https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-node-linux-amd64.tar.gz

FROM core AS base

RUN apt-mark hold grub-pc
RUN apt-get install -y linux-image-${KERNEL_VERSION} live-boot systemd wget netplan.io ntp gnupg2 make squashfs-tools openssh-server iputils-ping htop vim pciutils lshw less iptables ntpdate ipmitool lvm2 curl

COPY --from=downloads coredns_${COREDNS_VERSION}_linux_amd64.tgz .
RUN tar -xpf coredns_${COREDNS_VERSION}_linux_amd64.tgz && \
  mv coredns /usr/bin/ && \
  chmod +x /usr/bin/coredns && \
  rm -rf coredns_${COREDNS_VERSION}_linux_amd64.tgz

COPY os/initramfs.conf /etc/initramfs-tools/initramfs.conf
COPY os/fstab /etc/fstab
COPY os/hosts /etc/hosts
COPY os/ntp.conf /etc/ntp.conf
COPY os/coredns.conf /etc/coredns.conf
COPY os/coredns.service /usr/lib/systemd/system/
COPY os/resolv.conf /etc/resolv.conf
COPY os/netplan.yaml /etc/netplan/config.yaml
COPY os/initialize-disks.sh /usr/local/sbin/initialize-disks.sh
COPY os/initialize-hostname.sh /usr/local/sbin/initialize-hostname.sh
RUN chmod +x /usr/local/sbin/initialize-disks.sh /usr/local/sbin/initialize-hostname.sh
COPY os/initialize-disks.service /usr/lib/systemd/system
COPY os/initialize-hostname.service /usr/lib/systemd/system
COPY os/systemd.preset /usr/lib/systemd/system-preset/00-tblflp.preset
COPY os/sshd_config /etc/ssh/sshd_config
COPY os/authorized_keys /root/.ssh/authorized_keys
COPY os/ntpd.service /usr/lib/systemd/system
COPY os/ntpdate.service /usr/lib/systemd/system
RUN chmod 400 /root/.ssh/authorized_keys && chown root /root/.ssh/authorized_keys
COPY secrets/shadow /etc/shadow
RUN mkdir -p /var/log/ntpstats && chown 101:101 /var/log/ntpstats

RUN update-initramfs -u
RUN systemctl enable ntpd ntpdate coredns initialize-disks initialize-hostname ssh dm-event
RUN systemctl disable ntp

FROM base AS node

RUN mkdir /mnt/ceph /mnt/slow1 /mnt/slow2 /mnt/fast /mnt/local /mnt/local/crio /mnt/local/containers /mnt/local/log /mnt/local/log/pods

# CRI-O

COPY --from=downloads crio-v${CRIO_VERSION}.tar.gz .
RUN tar -xpf crio-v${CRIO_VERSION}.tar.gz && \
  ( cd crio-v${CRIO_VERSION} && \
    make install ) && \
  rm -rf crio-v${CRIO_VERSION}/ crio-v${CRIO_VERSION}.tar.gz
COPY worker/99-cri.conf /etc/sysctl.d/99-cri.conf
COPY worker/crio.conf /etc/crio/crio.conf
COPY worker/storage.conf /etc/containers/storage.conf
RUN rm -f /usr/local/lib/systemd/system/crio.service
COPY worker/crio.service /usr/lib/systemd/system/crio.service
RUN systemctl enable crio

# KUBERNETES

COPY --from=downloads kubernetes-node-linux-amd64.tar.gz .
RUN tar -xpf kubernetes-node-linux-amd64.tar.gz && \
  mv kubernetes/node/bin/kubelet /usr/bin/ && \
  rm -rf kubernetes kubernetes-node-linux-amd64.tar.gz

COPY worker/kubelet.yaml /etc/kubelet.yaml
COPY worker/kubelet.service /usr/lib/systemd/system/
RUN mkdir /etc/kubelet /etc/kubelet/pki /etc/kubelet/static
COPY secrets/bootstrap.yaml /etc/kubelet/bootstrap.yaml
COPY secrets/pki/ca.crt /etc/pki/k8sca.crt
RUN systemctl enable kubelet

FROM node AS leader

COPY secrets/tokens /etc/kubernetes/tokens
COPY secrets/pki /etc/kubernetes/pki
COPY secrets/controller.yaml /etc/kubernetes/controller.kubeconfig
COPY secrets/scheduler.yaml /etc/kubernetes/scheduler.kubeconfig

FROM leader AS k8s-01

COPY leaders/k8s-01/kube-apiserver.yaml /etc/kubelet/static/
COPY leaders/k8s-01/kube-controller-manager.yaml /etc/kubelet/static/
COPY leaders/k8s-01/kube-scheduler.yaml /etc/kubelet/static/
COPY leaders/k8s-01/etcd.yaml /etc/kubelet/static/

FROM leader AS k8s-02

COPY leaders/k8s-02/kube-apiserver.yaml /etc/kubelet/static/
COPY leaders/k8s-02/kube-controller-manager.yaml /etc/kubelet/static/
COPY leaders/k8s-02/kube-scheduler.yaml /etc/kubelet/static/
COPY leaders/k8s-02/etcd.yaml /etc/kubelet/static/

FROM leader AS k8s-03

COPY leaders/k8s-03/kube-apiserver.yaml /etc/kubelet/static/
COPY leaders/k8s-03/kube-controller-manager.yaml /etc/kubelet/static/
COPY leaders/k8s-03/kube-scheduler.yaml /etc/kubelet/static/
COPY leaders/k8s-03/etcd.yaml /etc/kubelet/static/

FROM core AS ipxe

RUN apt-get install -y git gcc binutils make perl mtools liblzma-dev mkisofs syslinux
RUN git clone git://git.ipxe.org/ipxe.git
COPY pxe/boot.ipxe ipxe/src/boot.ipxe
RUN sed -ri "s/IMAGE_VERSION/${KUBE_VERSION}-${IMAGE_VERSION}/g" ipxe/src/boot.ipxe
RUN cd ipxe/src && make -j16 bin/undionly.kpxe EMBED=boot.ipxe

FROM core AS builder

RUN apt-get install -y squashfs-tools

RUN mkdir /tftp /roots /images

COPY --from=node / /roots/node
COPY --from=k8s-01 / /roots/k8s-01
COPY --from=k8s-02 / /roots/k8s-02
COPY --from=k8s-03 / /roots/k8s-03

RUN cp /roots/node/boot/initrd.img-${KERNEL_VERSION} /tftp/initrd.img-${KUBE_VERSION}-${IMAGE_VERSION}
RUN cp /roots/node/boot/vmlinuz-${KERNEL_VERSION} /tftp/vmlinuz-${KUBE_VERSION}-${IMAGE_VERSION}
RUN cp /roots/node/boot/config-${KERNEL_VERSION} /tftp/config-${KUBE_VERSION}-${IMAGE_VERSION}
RUN cp /roots/node/boot/System.map-${KERNEL_VERSION} /tftp/System.map-${KUBE_VERSION}-${IMAGE_VERSION}
RUN rm -rf /roots/*/boot

RUN mksquashfs /roots/node /images/node-${KUBE_VERSION}-${IMAGE_VERSION}.squashfs -comp lzo
RUN mksquashfs /roots/k8s-01 /images/k8s-01-${KUBE_VERSION}-${IMAGE_VERSION}.squashfs -comp lzo
RUN mksquashfs /roots/k8s-02 /images/k8s-02-${KUBE_VERSION}-${IMAGE_VERSION}.squashfs -comp lzo
RUN mksquashfs /roots/k8s-03 /images/k8s-03-${KUBE_VERSION}-${IMAGE_VERSION}.squashfs -comp lzo

COPY --from=ipxe ipxe/src/bin/undionly.kpxe /tftp/boot-${KUBE_VERSION}-${IMAGE_VERSION}.kpxe
RUN tar -zcpf /images/tftp-${KUBE_VERSION}-${IMAGE_VERSION}.tgz -C /tftp .

RUN rm -rf /roots /tftp

CMD cp -v /images/* /volume/

os-coredns.conf

. {
  bind 127.0.0.1 ::1
  forward . 1.0.0.1 1.1.1.1 4.2.2.1 4.2.2.2 4.2.2.3 4.2.2.4 8.8.8.8 8.8.4.4
  cache
}

os-coredns.service

[Unit]
Description=CoreDNS
Documentation=https://coredns.io/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/coredns -dns.port=53 -conf=/etc/coredns.conf
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

os-fstab

LABEL=local /mnt/local ext4 defaults 0 2

os-initialize-disks.service

[Unit]
Description=Initialize Disks
Documentation=http://tblflp.zone
After=dev-sda.device dev-sdb.device dev-sdc.device dev-sdd.device
Requires=dev-sda.device dev-sdb.device dev-sdc.device dev-sdd.device

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/initialize-disks.sh
RemainAfterExit=yes

[Install]
WantedBy=local-fs-pre.target

os-initialize-disks.sh

#!/bin/bash
initialize_disk() {
  local disk="${1}"
  local to_label="${2}"
  local label="$(e2label "${disk}")"
  if [[ "$label" != "$to_label" ]]
  then
    wipefs -af "${disk}"
    mkfs.ext4 -L "${to_label}" "${disk}"
  fi
}

slow1="$(ls -1 /dev/disk/by-id/ata-WDC_*|head -n 1)"
slow2="$(ls -1 /dev/disk/by-id/ata-WDC_*|tail -n 1)"
fast="$(ls -1 /dev/disk/by-id/ata-CT*|head -n 1)"
locl="$(ls -1 /dev/disk/by-id/ata-CT*|tail -n 1)"

if [[ "${fast}" == "${locl}" ]]
then
  echo "No separate fast and locl"
  exit 1
fi

initialize_disk "$locl" "local"

ln -s "${slow1}" /dev/disk/slow1
ln -s "${slow2}" /dev/disk/slow2
ln -s "${fast}" /dev/disk/fast

os-initialize-hostname.service

[Unit]
Description=Initialize Hostname
Documentation=http://tblflp.zone
After=network-online.target
Requires=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/initialize-hostname.sh
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target

os-initialize-hostname.sh

#!/bin/bash
for e in $(tr "\000" "\n" < /proc/1/environ); do
  eval "export $e"
done

nodenum=$(echo $pxeip | awk -F. '{print $4}')

host=$(printf "k8s-%02d" $nodenum)

node=$(printf "k8s-%02d.as53546.tblflp.zone" $nodenum)

hostnamectl set-hostname $node
cat <<EOF >/etc/nodeinfo.env
HOST=${host}
FQDN=${node}
ADMIN_IP=10.199.11.${nodenum}
ADMIN_HOST=${host}-admin.as53546.tblflp.zone
PUBLIC_IP=144.86.176.${nodenum}
PUBLIC_HOST=${host}-public.as53546.tblflp.zone
PRIVATE_IP=10.199.14.${nodenum}
PRIVATE_HOST=${host}-private.as53546.tblflp.zone
PXE_IP=${pxeip}
BOOT_SERVER=10.199.14.201
LB_IP=10.199.15.${nodenum}
LB_HOST=${host}-lb.as53546.tblflp.zone
STORAGE_IP=10.199.16.${nodenum}
STORAGE_HOST=${host}-storage.as53546.tblflp.zone
CLUSTER_IP=10.199.17.${nodenum}
CLUSTER_HOST=${host}-cluster.as53546.tblflp.zone
EOF

os-initramfs.conf

#
# initramfs.conf
# Configuration file for mkinitramfs(8). See initramfs.conf(5).
#
# Note that configuration options from this file can be overridden
# by config files in the /etc/initramfs-tools/conf.d directory.

#
# MODULES: [ most | netboot | dep | list ]
#
# most - Add most filesystem and all harddrive drivers.
#
# dep - Try and guess which modules to load.
#
# netboot - Add the base modules, network modules, but skip block devices.
#
# list - Only include modules from the 'additional modules' list
#

MODULES=netboot

#
# BUSYBOX: [ y | n | auto ]
#
# Use busybox shell and utilities.  If set to n, klibc utilities will be used.
# If set to auto (or unset), busybox will be used if installed and klibc will
# be used otherwise.
#

BUSYBOX=auto

#
# COMPCACHE_SIZE: [ "x K" | "x M" | "x G" | "x %" ]
#
# Amount of RAM to use for RAM-based compressed swap space.
#
# An empty value - compcache isn't used, or added to the initramfs at all.
# An integer and K (e.g. 65536 K) - use a number of kilobytes.
# An integer and M (e.g. 256 M) - use a number of megabytes.
# An integer and G (e.g. 1 G) - use a number of gigabytes.
# An integer and % (e.g. 50 %) - use a percentage of the amount of RAM.
#
# You can optionally install the compcache package to configure this setting
# via debconf and have userspace scripts to load and unload compcache.
#

COMPCACHE_SIZE=""

#
# COMPRESS: [ gzip | bzip2 | lz4 | lzma | lzop | xz ]
#

COMPRESS=lz4

#
# NFS Section of the config.
#

#
# DEVICE: ...
#
# Specify a specific network interface, like eth0
# Overridden by optional ip= or BOOTIF= bootarg
#

DEVICE=

#
# NFSROOT: [ auto | HOST:MOUNT ]
#

NFSROOT=auto

#
# RUNSIZE: ...
#
# The size of the /run tmpfs mount point, like 256M or 10%
# Overridden by optional initramfs.runsize= bootarg
#

RUNSIZE=10%

os-netplan.yaml

network:
  version: 2
  renderer: networkd
  ethernets:
    eth0:
      dhcp4: true
    eth1:
      dhcp4: true
    eth2:
      dhcp4: true
    eth3:
      dhcp4: true
    eth4:
      dhcp4: true
    eth5:
      dhcp4: true

os-ntp.conf

# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help

driftfile /var/lib/ntp/ntp.drift

# Enable this if you want statistics to be logged.
statsdir /var/log/ntpstats/

statistics loopstats peerstats clockstats rawstats sysstats
filegen loopstats file loopstats type day enable
filegen peerstats file peerstats type day enable
filegen clockstats file clockstats type day enable
filegen rawstats file rawstats type day enable
filegen sysstats file sysstats type day enable

# Specify one or more NTP servers.

peer k8s-01-private.as53546.tblflp.zone burst
peer k8s-02-private.as53546.tblflp.zone burst
peer k8s-03-private.as53546.tblflp.zone burst
peer k8s-04-private.as53546.tblflp.zone burst
peer k8s-05-private.as53546.tblflp.zone burst
peer k8s-06-private.as53546.tblflp.zone burst
peer k8s-07-private.as53546.tblflp.zone burst
peer k8s-08-private.as53546.tblflp.zone burst
peer k8s-09-private.as53546.tblflp.zone burst
peer k8s-10-private.as53546.tblflp.zone burst
peer k8s-11-private.as53546.tblflp.zone burst
peer k8s-12-private.as53546.tblflp.zone burst
peer k8s-13-private.as53546.tblflp.zone burst
peer k8s-14-private.as53546.tblflp.zone burst
peer k8s-15-private.as53546.tblflp.zone burst
peer k8s-16-private.as53546.tblflp.zone burst
peer k8s-17-private.as53546.tblflp.zone burst
peer k8s-18-private.as53546.tblflp.zone burst
peer k8s-19-private.as53546.tblflp.zone burst
peer k8s-20-private.as53546.tblflp.zone burst

# NIST
server time-a.nist.gov iburst
server time-b.nist.gov iburst
server time-c.nist.gov iburst
server wwv.nist.gov iburst
server time-a.timefreq.bldrdoc.gov iburst
server time-b.timefreq.bldrdoc.gov iburst
server time-c.timefreq.bldrdoc.gov iburst
server utcnist.colorado.edu iburst
server utcnist2.colorado.edu iburst
server nist-time-server.eoni.com iburst

# ISC
server clock.isc.org iburst

# MIT
server kerberos.mit.edu iburst

# UIUC
server ntp-0.cso.uiuc.edu iburst
server ntp-1.cso.uiuc.edu iburst
server ntp-2.cso.uiuc.edu iburst

# UNC
server level1e.cs.unc.edu iburst
server level1f.cs.unc.edu iburst
server level1g.cs.unc.edu iburst
server level1h.cs.unc.edu iburst

# HE
server clock.nyc.he.net iburst
server clock.sjc.he.net iburst
server clock.fmt.he.net iburst

# Deep Space Imaging
server t2.timegps.net iburst

# UMICH
server time.keneli.org iburst

# By default, exchange time with everybody, but don't allow configuration.
restrict -4 default kod notrap nomodify nopeer noquery limited
restrict -6 default kod notrap nomodify nopeer noquery limited

# Our peers and management.
restrict -4 k8s-01-private.as53546.tblflp.zone
restrict -4 k8s-02-private.as53546.tblflp.zone
restrict -4 k8s-03-private.as53546.tblflp.zone
restrict -4 k8s-04-private.as53546.tblflp.zone
restrict -4 k8s-05-private.as53546.tblflp.zone
restrict -4 k8s-06-private.as53546.tblflp.zone
restrict -4 k8s-07-private.as53546.tblflp.zone
restrict -4 k8s-08-private.as53546.tblflp.zone
restrict -4 k8s-09-private.as53546.tblflp.zone
restrict -4 k8s-10-private.as53546.tblflp.zone
restrict -4 k8s-11-private.as53546.tblflp.zone
restrict -4 k8s-12-private.as53546.tblflp.zone
restrict -4 k8s-13-private.as53546.tblflp.zone
restrict -4 k8s-14-private.as53546.tblflp.zone
restrict -4 k8s-15-private.as53546.tblflp.zone
restrict -4 k8s-16-private.as53546.tblflp.zone
restrict -4 k8s-17-private.as53546.tblflp.zone
restrict -4 k8s-18-private.as53546.tblflp.zone
restrict -4 k8s-19-private.as53546.tblflp.zone
restrict -4 k8s-20-private.as53546.tblflp.zone

# Local users may interrogate the ntp server more closely.
restrict 127.0.0.1
restrict ::1

os-ntpd.service

[Unit]
Description=NTP
Documentation=http://tblflp.zone
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/sbin/ntpd -g -u 101:101 -n
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

os-resolv.conf

search as53546.tblflp.zone tblflp.zone tblflp.net
nameserver 127.0.0.1

os-sshd_config

Include /etc/ssh/sshd_config.d/*.conf
AllowUsers *@10.199.0.0/16
ChallengeResponseAuthentication no
UsePAM yes
X11Forwarding yes
PrintMotd no
AcceptEnv LANG LC_*
Subsystem sftp /usr/lib/openssh/sftp-server

pxe-boot.ipxe

#!ipxe

set imgvers IMAGE_VERSION

prompt --key 0x02 --timeout 2000 Press Ctrl-B for the iPXE shell... || goto retry
shell
exit

:retry
dhcp || goto retry

set imgpfx node

iseq ${net0/ip} 10.199.14.1 && set imgpfx k8s-01 ||
iseq ${net0/ip} 10.199.14.2 && set imgpfx k8s-02 ||
iseq ${net0/ip} 10.199.14.3 && set imgpfx k8s-03 ||

initrd tftp://${net0/next-server}/pxe/initrd.img-${imgvers}
boot tftp://${net0/next-server}/pxe/vmlinuz-${imgvers} initrd=initrd.img-${imgvers} boot=live fetch=ftp://${net0/next-server}/${imgpfx}-${imgvers}.squashfs toram BOOTIF=${net0/mac} init=/lib/systemd/systemd net.ifnames=0 pxeip=${net0/ip}

worker-99-cri.conf

net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1

worker-crio.conf

[crio]
root = "/mnt/local/crio"
runroot = "/mnt/local/crio"
storage_driver = "overlay"
log_dir = "/mnt/local/log/pods"
version_file = "/var/run/crio/version"
version_file_persist = "/mnt/local/crio/version"

[crio.api]
listen = "/var/run/crio/crio.sock"
stream_address = "127.0.0.1"
stream_port = "0"
stream_enable_tls = false
stream_tls_cert = ""
stream_tls_key = ""
stream_tls_ca = ""
grpc_max_send_msg_size = 16777216
grpc_max_recv_msg_size = 16777216

[crio.runtime]
default_runtime = "runc"
no_pivot = false
decryption_keys_path = "/etc/crio/keys/"
conmon = ""
conmon_cgroup = "system.slice"
conmon_env = ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]
default_env = []
selinux = false
seccomp_profile = ""
apparmor_profile = "crio-default"
cgroup_manager = "systemd"
default_capabilities = [
        "CHOWN",
        "DAC_OVERRIDE",
        "FSETID",
        "FOWNER",
        "SETGID",
        "SETUID",
        "SETPCAP",
        "NET_BIND_SERVICE",
        "KILL",
]
default_sysctls = []
additional_devices = []
hooks_dir = ["/usr/share/containers/oci/hooks.d"]
default_mounts = []
pids_limit = 1024
log_size_max = -1
log_to_journald = false
container_exits_dir = "/var/run/crio/exits"
container_attach_socket_dir = "/var/run/crio"
bind_mount_prefix = ""
read_only = false
log_level = "info"
log_filter = ""
uid_mappings = ""
gid_mappings = ""
ctr_stop_timeout = 30
manage_ns_lifecycle = false
namespaces_dir = "/var/run"
pinns_path = ""

[crio.runtime.runtimes.runc]
runtime_path = ""
runtime_type = "oci"
runtime_root = "/run/runc"

[crio.image]
default_transport = "docker://"
global_auth_file = ""
pause_image = "k8s.gcr.io/pause:3.2"
pause_image_auth_file = ""
pause_command = "/pause"
signature_policy = ""
image_volumes = "mkdir"
registries = ["docker.io"]

[crio.network]
network_dir = "/etc/cni/net.d/"
plugin_dirs = ["/opt/cni/bin/"]

[crio.metrics]
enable_metrics = false
metrics_port = 9090

worker-crio.service

[Unit]
Description=Container Runtime Interface for OCI (CRI-O)
Documentation=https://github.com/cri-o/cri-o
Wants=network-online.target local-fs.target
After=network-online.target initialize-disks.service initialize-hostname.service crio.service

[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/crio
Environment=GOTRACEBACK=crash
ExecStartPre=-/bin/rm -rf /mnt/local/crio
ExecStart=/usr/local/bin/crio \
          $CRIO_CONFIG_OPTIONS \
          $CRIO_RUNTIME_OPTIONS \
          $CRIO_STORAGE_OPTIONS \
          $CRIO_NETWORK_OPTIONS \
          $CRIO_METRICS_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
TasksMax=infinity
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
OOMScoreAdjust=-999
TimeoutStartSec=0
Restart=on-abnormal

[Install]
WantedBy=multi-user.target

worker-kubelet.service

[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/home/
Wants=network-online.target
After=network-online.target initialize-hostname.service crio.service

[Service]
EnvironmentFile=/etc/nodeinfo.env
ExecStart=/usr/bin/kubelet --config /etc/kubelet.yaml --kubeconfig /etc/kubelet/kubeconfig.yaml --bootstrap-kubeconfig /etc/kubelet/bootstrap.yaml --cert-dir /etc/kubelet/pki --container-runtime remote --container-runtime-endpoint unix:///var/run/crio/crio.sock --cgroup-driver systemd --node-ip ${CLUSTER_IP}
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

worker-kubelet.yaml

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
staticPodPath: /etc/kubelet/static/
address: 0.0.0.0
readOnlyPort: 0
rotateCertificates: true
serverTLSBootstrap: true
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true
    cacheTTL: "2m"
  x509:
    clientCAFile: /etc/pki/k8sca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: "5m"
    cacheUnauthorizedTTL: "30s"
clusterDomain: cluster.local
clusterDNS:
- 10.195.1.5
hairpinMode: hairpin-veth
maxPods: 200
evictionHard:
  memory.available:  "200Mi"
  nodefs.available:  "10%"
  nodefs.inodesFree: "5%"
  imagefs.available: "15%"
enableControllerAttachDetach: true
featureGates:
  EphemeralContainers: true
  RotateKubeletClientCertificate: true
  RotateKubeletServerCertificate: true
volumePluginDir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/

worker-storage.conf

[storage]
driver = "overlay"
runroot = "/mnt/local/containers"
graphroot = "/mnt/local/containers"

[storage.options.overlay]
mountopt = "nodev"