Skip to content

Instantly share code, notes, and snippets.

@detiber

detiber/README.md

Last active September 19, 2018 16:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save detiber/98742a1db1151aa9a8aaaa3c219e061d to your computer and use it in GitHub Desktop.
Save detiber/98742a1db1151aa9a8aaaa3c219e061d to your computer and use it in GitHub Desktop.
Download ZIP
kubeadm Stacked HA Upgrade v1.11.x to v1.12.x
Raw
README.md

kubeadm Stacked HA Upgrade v1.11.x to v1.12.x

Warning: hardcoded values below will need to be modified for testing against different hostnames and/or IPs

Common Steps

Install Docker

apt-get update && \
apt-get install -y apt-transport-https ca-certificates curl software-properties-common && \
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
add-apt-repository "deb https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable" && \
apt-get update && apt-get install -y docker-ce=$(apt-cache madison docker-ce | grep 17.03 | head -1 | awk '{print $3}')

Install k8s Components

apt-get update && apt-get install -y apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
apt-get update && \
apt-get install -y kubelet=1.11.3-00 kubeadm=1.11.3-00 kubectl=1.11.3-00 && \
apt-mark hold kubelet kubeadm kubectl

cp1 install

cp1 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.3
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.92.42:2379"
      advertise-client-urls: "https://172.31.92.42:2379"
      listen-peer-urls: "https://172.31.92.42:2380"
      initial-advertise-peer-urls: "https://172.31.92.42:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380"
    serverCertSANs:
      - ip-172-31-92-42
      - 172.31.92.42
    peerCertSANs:
      - ip-172-31-92-42
      - 172.31.92.42
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp1 Run kubeadm init

kubeadm init --config kubeadm-config.yaml

cp1 Copy Secrets to other hosts

USER=ubuntu # customizable
CONTROL_PLANE_IPS="172.31.89.186 172.31.90.42"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:etcd-ca.crt
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:etcd-ca.key
    scp /etc/kubernetes/admin.conf "${USER}"@$host:
done

cp2 install

cp2 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.3
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.89.186:2379"
      advertise-client-urls: "https://172.31.89.186:2379"
      listen-peer-urls: "https://172.31.89.186:2380"
      initial-advertise-peer-urls: "https://172.31.89.186:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380,ip-172-31-89-186=https://172.31.89.186:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-89-186
      - 172.31.89.186
    peerCertSANs:
      - ip-172-31-89-186
      - 172.31.89.186
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp2 Copy secrets in place

USER=ubuntu
mkdir -p /etc/kubernetes/pki/etcd
mv /home/${USER}/ca.crt /etc/kubernetes/pki/
mv /home/${USER}/ca.key /etc/kubernetes/pki/
mv /home/${USER}/sa.pub /etc/kubernetes/pki/
mv /home/${USER}/sa.key /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /home/${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
mv /home/${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key
mv /home/${USER}/admin.conf /etc/kubernetes/admin.conf

cp2 Run bootstrapping commands

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

export CP0_IP=172.31.92.42
export CP0_HOSTNAME=ip-172-31-92-42
export CP1_IP=172.31.89.186
export CP1_HOSTNAME=ip-172-31-89-186

export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

cp3 install

cp3 kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.3
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
api:
  controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.90.42:2379"
      advertise-client-urls: "https://172.31.90.42:2379"
      listen-peer-urls: "https://172.31.90.42:2380"
      initial-advertise-peer-urls: "https://172.31.90.42:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380,ip-172-31-89-186=https://172.31.89.186:2380,ip-172-31-90-42=https://172.31.90.42:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-90-42
      - 172.31.90.42
    peerCertSANs:
      - ip-172-31-90-42
      - 172.31.90.42
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp3 Copy secrets in place

USER=ubuntu
mkdir -p /etc/kubernetes/pki/etcd
mv /home/${USER}/ca.crt /etc/kubernetes/pki/
mv /home/${USER}/ca.key /etc/kubernetes/pki/
mv /home/${USER}/sa.pub /etc/kubernetes/pki/
mv /home/${USER}/sa.key /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /home/${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /home/${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
mv /home/${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key
mv /home/${USER}/admin.conf /etc/kubernetes/admin.conf

cp3 Run bootstrapping commands

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

export CP0_IP=172.31.92.42
export CP0_HOSTNAME=ip-172-31-92-42
export CP1_IP=172.31.90.42
export CP1_HOSTNAME=ip-172-31-90-42

export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

Install CNI

cp1 install Calico CNI

kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

Install Node(s)

Node install

kubeadm join kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443 --token m2ydfc.f10tfw32krqbp6kr --discovery-token-ca-cert-hash sha256:3fbac28428fb4eaf0e05ed7cbe1950bce806d4f33ba561574dfb8e7841766c8f

Copy pre-release v1.12 debs to hosts (from local dev machine)

for host in ec2-34-238-244-186.compute-1.amazonaws.com ec2-52-91-211-172.compute-1.amazonaws.com ec2-54-210-172-4.compute-1.amazonaws.com ec2-54-164-133-82.compute-1.amazonaws.com; do scp bazel-bin/build/debs/{cri-tools,kubeadm,kubectl,kubelet}.deb ubuntu@${host}:; done

Upgrade ordering todo

Following this process led to seeing transient pod crashes as different control plane nodes were in the process of being upgraded, some ordering and process twiddling is most likely needed to ensure stability/availability during the upgrade

cp1 upgrade

cp1 upgrade kubeadm-upgrade.yaml

Note: the apiVersion and other fields have changed from install config

apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v1.12.0-rc.1
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.92.42:2379"
      advertise-client-urls: "https://172.31.92.42:2379"
      listen-peer-urls: "https://172.31.92.42:2380"
      initial-advertise-peer-urls: "https://172.31.92.42:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380,ip-172-31-89-186=https://172.31.89.186:2380,ip-172-31-90-42=https://172.31.90.42:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-92-42
      - 172.31.92.42
    peerCertSANs:
      - ip-172-31-92-42
      - 172.31.92.42
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp1 perform upgrade

apt-mark unhold kubeadm && \
apt-get install /home/ubuntu/kubeadm.deb && \
apt-mark hold kubeadm && \
kubeadm upgrade apply --allow-experimental-upgrades --config kubeadm-upgrade.yaml v1.12.0-rc.1

cp2 upgrade

cp2 upgrade kubeadm-upgrade.yaml

Note: the apiVersion and other fields have changed from install config

apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v1.12.0-rc.1
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.89.186:2379"
      advertise-client-urls: "https://172.31.89.186:2379"
      listen-peer-urls: "https://172.31.89.186:2380"
      initial-advertise-peer-urls: "https://172.31.89.186:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380,ip-172-31-89-186=https://172.31.89.186:2380,ip-172-31-90-42=https://172.31.90.42:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-89-186
      - 172.31.89.186
    peerCertSANs:
      - ip-172-31-89-186
      - 172.31.89.186
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp2 perform upgrade

apt-mark unhold kubeadm && \
apt-get install /home/ubuntu/kubeadm.deb && \
apt-mark hold kubeadm && \
kubeadm upgrade apply --allow-experimental-upgrades --config kubeadm-upgrade.yaml v1.12.0-rc.1

cp3 upgrade

cp3 upgrade kubeadm-upgrade.yaml

Note: the apiVersion and other fields have changed from install config

apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v1.12.0-rc.1
apiServerCertSANs:
  - "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com"
controlPlaneEndpoint: "kubeadm-test-4c1724309e6e6aad.elb.us-east-1.amazonaws.com:443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://172.31.90.42:2379"
      advertise-client-urls: "https://172.31.90.42:2379"
      listen-peer-urls: "https://172.31.90.42:2380"
      initial-advertise-peer-urls: "https://172.31.90.42:2380"
      initial-cluster: "ip-172-31-92-42=https://172.31.92.42:2380,ip-172-31-89-186=https://172.31.89.186:2380,ip-172-31-90-42=https://172.31.90.42:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - ip-172-31-90-42
      - 172.31.90.42
    peerCertSANs:
      - ip-172-31-90-42
      - 172.31.90.42
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: "192.168.0.0/16"

cp3 perform upgrade

apt-mark unhold kubeadm && \
apt-get install /home/ubuntu/kubeadm.deb && \
apt-mark hold kubeadm && \
kubeadm upgrade apply --allow-experimental-upgrades --config kubeadm-upgrade.yaml v1.12.0-rc.1

Common upgrade steps

apt-mark unhold kubelet kubectl && \
apt-get install /home/ubuntu/kubelet.deb /home/ubuntu/kubectl.deb && \
apt-mark hold kubelet kubectl && \
systemctl restart kubelet

Node(s) upgrade

apt-mark unhold kubeadm kubelet kubectl && \
apt-get install /home/ubuntu/{kubelet,kubectl,kubeadm}.deb && \
apt-mark hold kubeadm kubelet kubectl && \
systemctl restart kubelet
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment