Skip to content

Instantly share code, notes, and snippets.

@detiber

detiber/README.md

Last active September 6, 2018 20:20
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save detiber/8d4344191035e98ff7e5f1df6e35d755 to your computer and use it in GitHub Desktop.
Save detiber/8d4344191035e98ff7e5f1df6e35d755 to your computer and use it in GitHub Desktop.
Download ZIP
kubeadm Stacked HA Upgrade
Raw
README.md

kubeadm Stacked HA Upgrade v1.11 Minor upgrades

Warning: hardcoded values below will need to be modified for testing against different hostnames and/or IPs

Common Steps

Install Docker

apt-get update
apt-get install -y apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
add-apt-repository "deb https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable"
apt-get update && apt-get install -y docker-ce=$(apt-cache madison docker-ce | grep 17.03 | head -1 | awk '{print $3}')

Install k8s Components

apt-get update && apt-get install -y apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
apt-get update
apt-get install -y kubelet=1.11.0-00 kubeadm=1.11.0-00 kubectl=1.11.0-00
apt-mark hold kubelet kubeadm kubectl

cp1 install

cp1 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.35.19:2379"
      advertise-client-urls: "https://172.31.35.19:2379"
      listen-peer-urls: "https://172.31.35.19:2380"
      initial-advertise-peer-urls: "https://172.31.35.19:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380"
    serverCertSANs:
      - ip-172-31-35-19
      - 172.31.35.19
    peerCertSANs:
      - ip-172-31-35-19
      - 172.31.35.19
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp1 Run kubeadm init

kubeadm init --config kubeadm-config.yaml

cp1 Copy Secrets to other hosts

USER=ubuntu # customizable
CONTROL_PLANE_IPS="172.31.32.249 172.31.41.139"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:etcd-ca.crt
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:etcd-ca.key
    scp /etc/kubernetes/admin.conf "${USER}"@$host:
done

cp2 install

cp2 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.32.249:2379"
      advertise-client-urls: "https://172.31.32.249:2379"
      listen-peer-urls: "https://172.31.32.249:2380"
      initial-advertise-peer-urls: "https://172.31.32.249:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380,ip-172-31-32-249=https://172.31.32.249:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-32-249
      - 172.31.32.249
    peerCertSANs:
      - ip-172-31-32-249
      - 172.31.32.249
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp2 Copy secrets in place

USER=ubuntu
mkdir -p /etc/kubernetes/pki/etcd
mv /home/${USER}/ca.crt /etc/kubernetes/pki/
mv /home/${USER}/ca.key /etc/kubernetes/pki/
mv /home/${USER}/sa.pub /etc/kubernetes/pki/
mv /home/${USER}/sa.key /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /home/${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
mv /home/${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key
mv /home/${USER}/admin.conf /etc/kubernetes/admin.conf

cp2 Run bootstrapping commands

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

export CP0_IP=172.31.35.19
export CP0_HOSTNAME=ip-172-31-35-19
export CP1_IP=172.31.32.249
export CP1_HOSTNAME=ip-172-31-32-249

export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

cp3 install

cp3 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.41.139:2379"
      advertise-client-urls: "https://172.31.41.139:2379"
      listen-peer-urls: "https://172.31.41.139:2380"
      initial-advertise-peer-urls: "https://172.31.41.139:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380,ip-172-31-32-249=https://172.31.32.249:2380,ip-172-31-41-139=https://172.31.41.139:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-41-139
      - 172.31.41.139
    peerCertSANs:
      - ip-172-31-41-139
      - 172.31.41.139
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp3 Copy secrets in place

USER=ubuntu
mkdir -p /etc/kubernetes/pki/etcd
mv /home/${USER}/ca.crt /etc/kubernetes/pki/
mv /home/${USER}/ca.key /etc/kubernetes/pki/
mv /home/${USER}/sa.pub /etc/kubernetes/pki/
mv /home/${USER}/sa.key /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /home/${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
mv /home/${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key
mv /home/${USER}/admin.conf /etc/kubernetes/admin.conf

cp3 Run bootstrapping commands

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

export CP0_IP=172.31.35.19
export CP0_HOSTNAME=ip-172-31-35-19
export CP1_IP=172.31.41.139
export CP1_HOSTNAME=ip-172-31-41-139

export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

Install CNI

cp1 install Calico CNI

kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

Install Node(s)

Node install

  kubeadm join kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443 --token izmm49.o46rya0kbj25g7uf --discovery-token-ca-cert-hash sha256:b3b0e0c4aa178112ffa0a12d7b235a135a2143d069d190c283c6bcb0eb2d3c1a

cp1 upgrade

cp1 upgrade kubeadm-config.yaml

Note: kubernetesVersion and etcd.local.extraArgs differ from install config

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.2
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.35.19:2379"
      advertise-client-urls: "https://172.31.35.19:2379"
      listen-peer-urls: "https://172.31.35.19:2380"
      initial-advertise-peer-urls: "https://172.31.35.19:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380,ip-172-31-32-249=https://172.31.32.249:2380,ip-172-31-41-139=https://172.31.41.139:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-35-19
      - 172.31.35.19
    peerCertSANs:
      - ip-172-31-35-19
      - 172.31.35.19
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp1 perform upgrade

apt-mark unhold kubeadm
apt-get install kubeadm=1.11.2-00
apt-mark hold kubeadm

kubeadm upgrade apply --config kubeadm-config.yaml v1.11.2

cp2 upgrade

cp2 upgrade kubeadm-config.yaml

Note: kubernetesVersion and etcd.local.extraArgs differ from install config

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.2
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.32.249:2379"
      advertise-client-urls: "https://172.31.32.249:2379"
      listen-peer-urls: "https://172.31.32.249:2380"
      initial-advertise-peer-urls: "https://172.31.32.249:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380,ip-172-31-32-249=https://172.31.32.249:2380,ip-172-31-41-139=https://172.31.41.139:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-32-249
      - 172.31.32.249
    peerCertSANs:
      - ip-172-31-32-249
      - 172.31.32.249
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp2 perform upgrade

apt-mark unhold kubeadm
apt-get install kubeadm=1.11.2-00
apt-mark hold kubeadm

kubeadm upgrade apply --config kubeadm-config.yaml v1.11.2

cp3 upgrade

cp3 upgrade kubeadm-config.yaml

Note: kubernetesVersion differs from install config

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.2
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.41.139:2379"
      advertise-client-urls: "https://172.31.41.139:2379"
      listen-peer-urls: "https://172.31.41.139:2380"
      initial-advertise-peer-urls: "https://172.31.41.139:2380"
      initial-cluster: "ip-172-31-35-19=https://172.31.35.19:2380,ip-172-31-32-249=https://172.31.32.249:2380,ip-172-31-41-139=https://172.31.41.139:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-41-139
      - 172.31.41.139
    peerCertSANs:
      - ip-172-31-41-139
      - 172.31.41.139
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp3 perform upgrade

apt-mark unhold kubeadm
apt-get install kubeadm=1.11.2-00
apt-mark hold kubeadm

kubeadm upgrade apply --config kubeadm-config.yaml v1.11.2

Common upgrade steps

apt-mark unhold kubelet kubectl
apt-get install kubelet=1.11.2-00 kubectl=1.11.2-00
apt-mark hold kubelet kubectl

Node(s) upgrade

apt-mark unhold kubeadm kubelet kubectl
apt-get install kubeadm=1.11.2-00 kubelet=1.11.2-00 kubectl=1.11.2-00
apt-mark hold kubeadm kubelet kubectl
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment