script to configure k8s storage on a pre-deployed CDK model

k8s-vsphere-storage.sh

#!/bin/bash

set -eu
###############################################################################
###############################################################################
# NOTE: the recommended way to use K8s/vSphere integration is with the
# vsphere-integrator charm. See details at:
# https://medium.com/@kwmonroe/the-road-to-k8s-vsphere-integration-eef8e1de64f2
###############################################################################
###############################################################################


###############################################################################
# Prerequisites
###############################################################################
# 0. vSphere cloud added and bootstrapped
#    - https://docs.jujucharms.com/2.4/en/help-vmware
# 1. CDK deployed (min k8s version 1.12)
#    - juju deploy canonical-kubernetes
# 2. Disk UUID enabled on VMs
#    - This step is necessary so that the VMDK always presents a consistent UUID
#      to the VM, thus allowing the disk to be mounted properly.
#    - govc: https://vmware.github.io/vsphere-storage-for-kubernetes/documentation/existing.html#step-3-enable-disk-uuid-on-node-virtual-machines
#    - vcenter: https://sort.veritas.com/public/documents/sfha/6.1/vmwareesx/productguides/html/sfhas_virtualization/ch09s05s01.htm
#    - sunek's blog: https://sunekjaergaard.blogspot.dk/2018/02/making-canonical-distribution-of.html
#
# These will be handled automatically when Juju supports disk uuid model config:
#    - https://bugs.launchpad.net/juju/+bug/1751858


###############################################################################
# Modify to suite your environment
###############################################################################
# IP/Port of your vsphere server
JUJU_VSPHERE_ENDPOINT="1.2.3.4"
JUJU_VSPHERE_PORT="443"

# Array of vsphere datacenter names, available in vCenter from:
#  vCenter Inventory Lists > Resources > Datacenters
JUJU_VSPHERE_REGIONS=(dc0)

# Login info for your vsphere server (same used when adding juju credentials)
JUJU_VSPHERE_USER="admin"
JUJU_VSPHERE_PASSWORD="password"

# Config used when bootstrapping (override to prevent discovery):
#  https://jujucharms.com/docs/2.3/help-vmware#bootstrapping
JUJU_VSPHERE_DATASTORE=$(juju model-config datastore 2>/dev/null || echo "")
JUJU_VSPHERE_EXTERNAL_NET=$(juju model-config external-network 2>/dev/null || echo "")

# VM folder created in your vsphere datacenter
JUJU_VSPHERE_FOLDER="kubernetes"

# Number of kubernetes master units in your deployment
NUMBER_OF_K8S_MASTERS=1


###############################################################################
# vSphere config template
###############################################################################
# From official vSphere docs:
#  https://vmware.github.io/vsphere-storage-for-kubernetes/documentation/existing.html
# As well as HOWTO from sunek:
#  https://sunekjaergaard.blogspot.dk/2018/02/making-canonical-distribution-of.html
VSPHERE_CONF=$(cat <<EOF
[Global]
        # properties in this section will be used for all specified vCenters unless overriden in VirtualCenter section.
        datacenters = "${JUJU_VSPHERE_REGIONS[*]}"
        insecure-flag = "1" #set to 1 if the vCenter uses a self-signed cert
        port = "$JUJU_VSPHERE_PORT" #Optional
        vm-uuid="VMware-VM_UUID" # we will set this value on each VM

[VirtualCenter "$JUJU_VSPHERE_ENDPOINT"]
        user = "$JUJU_VSPHERE_USER"
        password = "$JUJU_VSPHERE_PASSWORD"

[Workspace]
        # Specify properties which will be used for various vSphere Cloud Provider functionality.
        # e.g. Dynamic provisioing, Storage Profile Based Volume provisioning etc.
        server = "$JUJU_VSPHERE_ENDPOINT"
        datacenter = "${JUJU_VSPHERE_REGIONS[0]}"
        default-datastore = "$JUJU_VSPHERE_DATASTORE" #Datastore to use for provisioning volumes using storage classes/dynamic provisioning
        folder = "$JUJU_VSPHERE_FOLDER"
        resourcepool-path = "" # Used for dummy VM creation. Optional
[Disk]
        scsicontrollertype = pvscsi
[Network]
        public-network = "$JUJU_VSPHERE_EXTERNAL_NET"
EOF
)


###############################################################################
# Configure applications
###############################################################################
VSPHERE_LOCAL_CONF_FILE=$(mktemp /tmp/vsphere.conf.XXXX)
echo "Creating $VSPHERE_LOCAL_CONF_FILE"
echo "${VSPHERE_CONF}" > $VSPHERE_LOCAL_CONF_FILE

echo "Updating k8s masters"
for i in $(seq 0 $((NUMBER_OF_K8S_MASTERS-1))); do
  MASTER="kubernetes-master/${i}"
  echo "Transfering config to $MASTER"
  juju scp $VSPHERE_LOCAL_CONF_FILE ${MASTER}:vsphere.conf

  echo "Setting UUID in the $MASTER config"
  juju ssh ${MASTER} 'uuid=$(sudo cat /sys/class/dmi/id/product_uuid); sed -i -e "s/VM_UUID/$uuid/" vsphere.conf'

  echo "Moving $MASTER config to /root/cdk"
  juju ssh ${MASTER} "sudo chown root:root /home/ubuntu/vsphere.conf; sudo mv /home/ubuntu/vsphere.conf /root/cdk/"
done

echo "Configuring k8s-master to use the vsphere provider"
juju config kubernetes-master controller-manager-extra-args="cloud-provider=vsphere cloud-config=/root/cdk/vsphere.conf" \
  api-extra-args="cloud-provider=vsphere cloud-config=/root/cdk/vsphere.conf"

echo "Configuring k8s-workers to use the vsphere provider"
juju config kubernetes-worker kubelet-extra-args="cloud-provider=vsphere"

cat <<EOM

NOTE: manual configuration is still required. You must ssh to each
kubernetes-worker and update the node ProviderID. For example:

juju ssh kubernetes-worker/0
uuid=\$(sudo cat /sys/class/dmi/id/product_uuid); /snap/bin/kubectl patch node \$(hostname -s) -p '{ "spec": { "providerID": "vsphere://\$uuid)" } }'

Once node config is complete, you may define a K8s storage class and
persistent volume claim to dynamically provision vSphere storage.
For example:

kubectl create -f - <<EOY
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: mystorage
provisioner: kubernetes.io/vsphere-volume
parameters:
  diskformat: zeroedthick
EOY

kubectl create -f - <<EOY
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: testclaim
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 100Mi
  storageClassName: mystorage
EOY

EOM