Ubuntu 使用 kube-vip 部署 Kubernetes

README.md

环境准备

切换到root用户

运行初始化脚本 init-os.sh

安装kubeadm、kubelet和kubectl

运行安装脚本 install-k8s.sh

安装kube-vip

运行安装脚本 install-kube-vip

部署k8s集群

生成配置文件

kubeadm config print init-defaults --component-configs KubeletConfiguration > kubeadm-config.yaml

修改配置文件

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock # 使用 containerd 的 socket 地址
  taints: null

---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd  # 配置 cgroup driver

---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
controlPlaneEndpoint: 192.168.2.10:6443  # 设置控制平面 Endpoint 地址

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs # 配置kube-proxy ipvs模式

初始化master节点

kubeadm config images pull --config kubeadm-config.yaml
kubeadm init --upload-certs --config kubeadm-config.yaml

配置kubectl

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

添加控制平面节点

注意： 控制节点同样需要安装containerd、kubelet、kubeadm、kubectl、kube-vip

  kubeadm join 192.168.2.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:4a3134f5f3eeaf00cb05a0555d0f4532b818d0899ba242b4e6ba0ae13da64a77 \
        --control-plane --certificate-key 36aa8b9c9e48256546df021d6940cad5655a7fce912b13b093fc9c9d6caa86ca

添加worker节点

kubeadm join 192.168.2.10:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:4a3134f5f3eeaf00cb05a0555d0f4532b818d0899ba242b4e6ba0ae13da64a77

验证

kubectl get nodes

如果忘记了上面的 join 命令可以使用命令 kubeadm token create --print-join-command 重新获取。 如果是添加control-plane可以使用命令 kubeadm init phase upload-certs --upload-certs 获取加入证书， 在加入命令后添加 --control-plane --certificate-key xxxx 参数

安装CNI插件

calico

kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

cilium

CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
cilium install

init-os.sh

#!/usr/bin/env bash

# 禁用防火墙
sudo systemctl stop firewalld 1>/dev/null 2>/dev/null
sudo systemctl disable firewalld 1>/dev/null 2>/dev/null
sudo systemctl stop ufw 1>/dev/null 2>/dev/null
sudo systemctl disable ufw 1>/dev/null 2>/dev/null

# 关闭swap
sudo swapoff -a
sudo sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab

# 关闭selinux
# See https://github.com/kubernetes/website/issues/14457
if [ -f /etc/selinux/config ]; then
  sudo sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
fi
# for ubuntu: sudo apt install selinux-utils
# for centos: yum install selinux-policy
if command -v setenforce &> /dev/null
then
  setenforce 0
  getenforce
fi

# 修改内核参数
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 1
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
EOF

#See https://imroc.io/posts/kubernetes/troubleshooting-with-kubernetes-network/
sudo sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf

sudo sysctl --system

# 添加内核模块
sudo modinfo br_netfilter > /dev/null 2>&1
if [ $? -eq 0 ]; then
   sudo modprobe br_netfilter
   sudo mkdir -p /etc/modules-load.d
   echo 'br_netfilter' | sudo tee /etc/modules-load.d/kubekey-br_netfilter.conf
fi

sudo modinfo overlay > /dev/null 2>&1
if [ $? -eq 0 ]; then
   sudo modprobe overlay
   echo 'overlay' | sudo tee -a /etc/modules-load.d/kubekey-br_netfilter.conf
fi

sudo modprobe ip_vs
sudo modprobe ip_vs_rr
sudo modprobe ip_vs_wrr
sudo modprobe ip_vs_sh

cat << EOF | sudo tee /etc/modules-load.d/kube_proxy-ipvs.conf 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
EOF

sudo modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
   echo 'nf_conntrack_ipv4' | sudo tee /etc/modules-load.d/kube_proxy-ipvs.conf
else
   modprobe nf_conntrack
   echo 'nf_conntrack' | sudo tee /etc/modules-load.d/kube_proxy-ipvs.conf
fi

sudo systemctl restart systemd-modules-load.service

# Make sure the iptables utility doesn't use the nftables backend.
sudo update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true
sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true
sudo update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true
sudo update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true

# 安装依赖
sudo apt update
sudo apt install -y socat ipset ipvsadm conntrack chrony

install-k8s.sh

#!/usr/bin/env bash

# 安装 containerd
sudo apt install -y containerd

# 配置containerd
sudo mkdir /etc/containerd
containerd config default | sed 's/SystemdCgroup = false/SystemdCgroup = true/' | sudo tee /etc/containerd/config.toml
echo 'runtime-endpoint: unix:///run/containerd/containerd.sock' | sudo tee /etc/crictl.yaml
sudo systemctl restart containerd.service

# 安装使用 Kubernetes apt 仓库所需要的包：
sudo apt install -y apt-transport-https ca-certificates curl

# 下载并添加 Kubernetes 的官方 GPG 密钥：
sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg

# 添加 Kubernetes apt 仓库：
echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo  tee /etc/apt/sources.list.d/kubernetes.list

# 更新 apt 包索引，安装 kubelet、kubeadm 和 kubectl：
sudo apt update
sudo apt install -y kubelet kubeadm kubectl

# 防止kubelet被自动升级
sudo apt-mark hold kubelet kubeadm kubectl

install-kube-vip.sh

#!/usr/bin/env bash

sudo mkdir -p /etc/kubernetes/manifests/

# 配置vip地址
export VIP=192.168.2.10

# 设置网卡名称
export INTERFACE=eth1

# 安装kube-vip
KVVERSION=$(curl -s https://api.github.com/repos/kube-vip/kube-vip/releases/latest | grep tag_name | cut -d '"' -f 4)

sudo ctr image pull ghcr.io/kube-vip/kube-vip:$KVVERSION

sudo ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip \
/kube-vip manifest pod \
--interface $INTERFACE \
--vip $VIP \
--controlplane \
--services \
--arp \
--leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml

初始化命令

curl -sfL https://gist.githubusercontent.com/hydrz/ecdf89a15efeb8f15da3f21af83f46c7/raw/init-os.sh | sh -
curl -sfL https://gist.githubusercontent.com/hydrz/ecdf89a15efeb8f15da3f21af83f46c7/raw/install-k8s.sh | sh -