run.sh

#!/usr/bin/env bash
help () {
    local "${@}"
    echo
    if [ "${NO_CREDENTIALS_SPECIFIED}" ]; then
      echo
      echo "cat <path-to-credentials-file>/<file-name>"
      echo "[default]"
      echo "aws_access_key_id = yourIdHash"
      echo "aws_secret_access_key = yourSecretHash"
      echo
    else
      echo "Parameters are passed as key=value pairs, except -r"
      echo "Usage: run -d=packet -c=mgmt-capi-cluster-1"
      echo "optional: -r  ** restore previous backup of mgmt cluster**"
      echo "optional: -s  ** path to a velero credentials file in S3 format**"
      echo "optional: -s  ** path to a velero credentials file in S3 format detailed below**"
    fi
  }

declare CLUSTERNAME='mgmt-capi-cluster-1'

for i in "$@"
do
  case $i in
      -d=*|--datacentre=*)
      DATACENTRE="${i#*=}"
      shift # past argument=value
      ;;
      -c=*|--clustername=*)
      CLUSTERNAME="${i#*=}"
      shift # past argument with no value
      ;;
      -r|--restore)
        RESTORE=1
        shift
        ;;
      -s=*|--velero-credentials=*)
        VELERO_CREDENTIALS="${i#*=}"
        shift
        ;;
      -h)
        help
        exit 0
        shift
        ;;
      *)
        echo "unknown option $@"
        help
        exit 1
      ;;
  esac
done


get_credentials_head () {
  cat < "$VELERO_CREDENTIALS" | head -1 | grep '\[default\]' > /dev/null 2>&1
  declare -r status=${PIPESTATUS[2]}
  echo $status
  return $status
}

get_aws_access_key_id () {
  cat < "$VELERO_CREDENTIALS" | grep 'aws_access_key_id' | grep '^aws_access_key_id = .*' > /dev/null 2>&1
  declare -r status=${PIPESTATUS[2]}
  echo $status
  return $status
}

get_aws_secret_access_key () {
  cat < "$VELERO_CREDENTIALS" | grep 'aws_secret_access_key' | grep '^aws_access_key_id = .*' > /dev/null 2>&1
  declare -r status=${PIPESTATUS[2]}
  echo $status
  return $status
}

if [ -z "$DATACENTRE" ]; then
  echo "Please specify a datacentre, exiting without running!"
  exit 1
fi

# Probably a cleaner way to handle this in bash, but this will do for now.
if [ "$RESTORE" ]; then
  if [ -f "$VELERO_CREDENTIALS" ];then
    declare get_credentials_head_value=$(get_credentials_head)
    if [ ! "${get_credentials_head_value}" ];then
      echo "please set your aws_access_key_id"
      exit 1
    fi

    declare get_aws_access_key_id_value=$(get_aws_access_key_id)
    if [ ! "${get_aws_access_key_id_value}" ];then
      echo "please set your aws_access_key_id"
      exit 1
    fi

    declare get_aws_secret_access_key_value=$(get_aws_secret_access_key)
    if [ ! "${get_aws_secret_access_key_value}" ];then
      echo "please set your aws_access_key_id"
      exit 1
    fi
  else
    echo "Restore option specified but file: $VELERO_CREDENTIALS does not exist"
    exit 1
  fi
fi
#func_result=$(get_credentials_head)
#echo "where is func output: $func_result"
#if [ ! "${func_result}" == '0' ];then
#  echo true
#fi

#echo "DATACENTRE    = ${DATACENTRE}"
#echo "RESTORE       = ${RESTORE}"
#echo "CLUSTERNAME   = ${CLUSTERNAME}"


set -x

#echo "declared cluster is: ${MGMT_CLUSTER_NAME}"
#############################################################################
### Parameters are passed as key=value pairs
### Usage: deployment_rollout_status \
###          ROLLOUT_NAME="${YOUR_DEPLOYMENT_NAME}" \
###          ROLLOUT_NAMESPACE="${YOUR_DEPLOYMENT_NAMESPACE}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
deployment_rollout_status () {
    local "${@}"
      ROLLOUT_STATUS_CMD="kubectl rollout status deployment/${ROLLOUT_NAME} --namespace ${ROLLOUT_NAMESPACE}"
    ATTEMPTS=0
    until $ROLLOUT_STATUS_CMD || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
}

#############################################################################
### Parameters are passed as key=value pairs
### Usage: deployment_rollout_status \
###          ROLLOUT_NAME="${YOUR_DEPLOYMENT_NAME}" \
###          ROLLOUT_NAMESPACE="${YOUR_DEPLOYMENT_NAMESPACE}"
###          ROLLOUT_CLUSTER="${YOUR_DEPLOYMENT_CUSTER}" # OPTIONAL: only for target clusters
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
target_deployment_rollout_status () {
    local "${@}"
    ROLLOUT_STATUS_CMD="kubectl --kubeconfig ${HOME}/.kube/${ROLLOUT_CLUSTER}.kubeconfig rollout status deployment/${ROLLOUT_NAME} --namespace ${ROLLOUT_NAMESPACE}"
    ATTEMPTS=0
    until $ROLLOUT_STATUS_CMD || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
}

#############################################################################
### Parameters are passed as key=value pairs
### Usage: namespace_rollout_status ROLLOUT_NAME="${YOUR_NAMESPACE_NAME}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
namespace_rollout_status () {
    local "${@}"
    ROLLOUT_STATUS_CMD="kubectl get namespace ${ROLLOUT_NAME}"
    ATTEMPTS=0
    until $ROLLOUT_STATUS_CMD || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
}

#############################################################################
### Parameters are passed as key=value pairs
### Usage: kubeconfig_rollout_status DATACENTRE="${YOUR_DATACENTRE}" CLUSTER_NAME="${YOUR_TARGET_CLUSTER}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
kubeconfig_rollout_status () {
    local "${@}"
    # the pipe gives a false success
    KUBECONFIG_ROLLOUT_STATUS_CMD() {
      kubectl -n "${1}" get secret "${2}"-kubeconfig -o=jsonpath='{.data.value}' &> /dev/null
    }
    ATTEMPTS=0
    until KUBECONFIG_ROLLOUT_STATUS_CMD "${DATACENTRE}" "${CLUSTER_NAME}" || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
    # on successfully finding the kubeconfig, exit loop and output it as expected.
    kubectl -n "${DATACENTRE}" get secret "${CLUSTER_NAME}-kubeconfig" -o=jsonpath='{.data.value}' | { base64 -d 2>/dev/null ; } > "${HOME}/.kube/${CLUSTER_NAME}.kubeconfig"
}

#############################################################################
### Parameters are passed as key=value pairs
### Usage: control_plane_ready_rollout_status DATACENTRE="${YOUR_DATACENTRE}"  CLUSTER_NAME="${YOUR_TARGET_CLUSTER}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
control_plane_ready_rollout_status () {
    local "${@}"
    ROLLOUT_STATUS_CMD="kubectl --kubeconfig "${HOME}/.kube/${CLUSTER_NAME}.kubeconfig" -n ${DATACENTRE} get clusters ${CLUSTER_NAME} -o jsonpath='{.status.controlPlaneReady}"
    ATTEMPTS=0
    until [[ "${ROLLOUT_STATUS_CMD[*]}" == 'true' ]] || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
}

#############################################################################
### Parameters are passed as key=value pairs
### Waits till the secret exists
### Usage: secret_ready_status CLUSTER_NAME=${YOUR_TARGET_CLUSTER} DATACENTRE=${YOUR_DATACENTRE}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
secret_ready_status () {
    local "${@}"
    ROLLOUT_STATUS_CMD="kubectl get secret ${CLUSTER_NAME}-kubeconfig --namespace ${DATACENTRE}"
    ATTEMPTS=0
    until [[ "${ROLLOUT_STATUS_CMD[*]}" ]] || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        sleep 10
    done
}

#############################################################################
### Parameters are passed as key=value pairs
### Waits till the secret exists
### Usage: velero_restore_status CLUSTER_NAME=${YOUR_TARGET_CLUSTER}"
### Optionally you can pass ROLLOUT_ATTEMPTS=${A_NUMBER} default is 60
#############################################################################
velero_restore_status () {
    local "${@}"
    # the pipe gives a false success and so must be used in a nested function
    VELERO_RESTORE_STATUS_CMD() {
      velero --kubeconfig ${HOME}/.kube/${CLUSTER_NAME}.kubeconfig get restores | egrep -v 'STATUS' | awk '{print $3}' | egrep -v 'Completed' | wc -l
    }
    ATTEMPTS=0
    until VELERO_RESTORE_STATUS_CMD "${CLUSTER_NAME}" eq "0" || [[ "$ATTEMPTS" -eq ${ROLLOUT_ATTEMPTS:=60} ]]; do
        ATTEMPTS=$ATTEMPTS+1
        echo "Waiting for Velero restores to complete"
        sleep 10
    done
}

mkdir -p ${HOME}/tmp
mkdir -p ${HOME}/.kube

if [[ ${DATACENTRE} == 'bremen' ]]; then
  export NO_PROXY=10.0.0.0/8,172.17.0.0/16,127.0.0.1,.tmo,.telekom.de,.svc,localhost,*.cert-manager.io,*.capi-webhook-system.svc,*.cluster.x-k8s.io,*.machinedeployment.cluster.x-k8s.io,*.svc,10.0.0.0/8,192.168.0.0/16,172.17.0.0/16,*.sce-dcn.net,*.tmo,*.internal.t-online.de,*.detemobil.de
  export http_proxy=http://10.208.220.80:3128
  export https_proxy=http://10.208.220.80:3128
fi

declare -r MGMT_CLUSTER_NAME="${CLUSTERNAME}"
kind delete cluster
# Sleep required to enable iptables changes and reload to complete,
# otherwise cluster creation results in an exit code 125
sleep 15
#
# exit on failure of the kind cluster creation. Do not remove or modify this set section, it is critical to have
# a functioning kind cluster and it is better to exit than proceed with possible issues.
#
set -e
kind create cluster
set +e

kubectl cluster-info
git pull
clusterctl init --config "${HOME}/.cluster-api/clusterctl-${DATACENTRE}.yaml" -v 6
clusterctl init --config "${HOME}/.cluster-api/clusterctl-${DATACENTRE}.yaml" --infrastructure vsphere --watching-namespace "${DATACENTRE}" --target-namespace "capv-system-${DATACENTRE}" -v 6

sleep 5
kubectl apply -f kind-bootstrap/${DATACENTRE}Namespaces.yaml

helm upgrade -i helm-operator fluxcd/helm-operator \
  --set helm.versions=v3 \
  --version 1.0.1 \
  --namespace default

deployment_rollout_status ROLLOUT_NAME="helm-operator" ROLLOUT_NAMESPACE="default"

helm upgrade -i flux fluxcd/flux \
  --version 1.3.0 \
  --namespace default -f "kind-bootstrap/flux_${DATACENTRE}.yaml"

secret_ready_status DATACENTRE="${DATACENTRE}" CLUSTER_NAME="${MGMT_CLUSTER_NAME}"

kubeconfig_rollout_status DATACENTRE="${DATACENTRE}" CLUSTER_NAME="${MGMT_CLUSTER_NAME}"

target_deployment_rollout_status ROLLOUT_NAME="calico-kube-controllers" ROLLOUT_NAMESPACE="kube-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"

kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" create namespace "${DATACENTRE}"

if [[ ${DATACENTRE} != 'packet' ]]; then
  kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" create namespace refsaone
fi

clusterctl init --config "${HOME}/.cluster-api/clusterctl-${DATACENTRE}.yaml" \
                --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" -v 6

clusterctl init --config "${HOME}/.cluster-api/clusterctl-${DATACENTRE}.yaml" \
                --infrastructure vsphere \
                --watching-namespace "${DATACENTRE}" \
                --target-namespace "capv-system-${DATACENTRE}" \
                --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" -v 6

if [[ ${DATACENTRE} == 'bremen' ]]; then

  clusterctl init --config "${HOME}/.cluster-api/clusterctl-refsa.yaml" \
                  --infrastructure vsphere \
                  --watching-namespace refsaone \
                  --target-namespace capv-system-refsaone \
                  --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" -v 6
fi

target_deployment_rollout_status ROLLOUT_NAME="cert-manager" ROLLOUT_NAMESPACE="cert-manager" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
target_deployment_rollout_status ROLLOUT_NAME="capi-kubeadm-control-plane-controller-manager" ROLLOUT_NAMESPACE="capi-kubeadm-control-plane-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
target_deployment_rollout_status ROLLOUT_NAME="capi-controller-manager" ROLLOUT_NAMESPACE="capi-webhook-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"

echo "Uninstalling flux from Kind cluster"
helm uninstall flux
echo "Uninstalling helm-operator from Kind cluster"
helm uninstall helm-operator
sleep 60

# Missing capv creds in the target cluster.
kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
        apply -f "masterClusterComponents/${DATACENTRE}/${MGMT_CLUSTER_NAME}/capv-manager-boot-credentials.yaml" --namespace "kube-system"
kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
        apply -f "masterClusterComponents/${DATACENTRE}/${MGMT_CLUSTER_NAME}/capv-manager-boot-credentials-${DATACENTRE}.yaml" --namespace "capv-system-${DATACENTRE}"
sleep 120

if [ "$RESTORE" ]; then
  #
  # Restore valero install & CRDs
  #
  ~/bin/velero --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
      install \
      --provider aws \
      --plugins velero/velero-plugin-for-aws:v1.0.0 \
      --bucket velero \
      --prefix "${MGMT_CLUSTER_NAME}" \
      --secret-file ./velero-credentials \
      --use-volume-snapshots=false \
      --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=https://minio.das-schiff.io

  target_deployment_rollout_status ROLLOUT_NAME="velero" ROLLOUT_NAMESPACE="velero" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"

  #
  # Restore backup configurations for restore to process against.
  #

  for backup in $(~/bin/mc ls minio/velero/${MGMT_CLUSTER_NAME}/backups | awk '{print $5}' | sed "s/\/$//g");do
    ~/bin/mc cat \
    "minio/velero/${MGMT_CLUSTER_NAME}/backups/${backup}/velero-backup.json" | kubectl \
      --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
      apply -f -
    sleep 3
  done

  for backup in $(~/bin/mc ls minio/velero/${MGMT_CLUSTER_NAME}/backups | egrep -v 'schiff-system' | awk '{print $5}' | sed "s/\/$//g");do
    echo "Restoring backup of namespace: ${backup}"
    ~/bin/velero --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
          restore create \
          --namespace-mappings "default:${backup}" \
          --from-backup "${backup}"
  done

  velero_restore_status CLUSTER_NAME="${MGMT_CLUSTER_NAME}"
  sleep 60
else
  clusterctl move --to-kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" -v 6 -n "${DATACENTRE}"
  sleep 60
fi

kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
        apply -f "masterClusterComponents/${DATACENTRE}/${MGMT_CLUSTER_NAME}/00-namespaces.yaml"

helm --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
  upgrade -i helm-operator fluxcd/helm-operator \
  --set helm.versions=v3 \
  --version 1.0.1 \
  --namespace schiff-system

if [ "$RESTORE" ]; then
  # Wait for some critical components to restore before resuming the remaining schiff-system restore
  target_deployment_rollout_status ROLLOUT_NAME="helm-operator" ROLLOUT_NAMESPACE="schiff-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
  target_deployment_rollout_status ROLLOUT_NAME="thanos-prometheus-operator-operator" ROLLOUT_NAMESPACE="monitoring-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
  target_deployment_rollout_status ROLLOUT_NAME="thanos-prometheus-operator-grafana" ROLLOUT_NAMESPACE="monitoring-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
  target_deployment_rollout_status ROLLOUT_NAME="metallb-controller" ROLLOUT_NAMESPACE="metallb-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
else
  sleep 15
fi
kubectl --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
        apply -f "masterClusterComponents/${DATACENTRE}/${MGMT_CLUSTER_NAME}/flux-${DATACENTRE}ConfigMap.yaml" \
        --namespace schiff-system

if [ "$RESTORE" ]; then
  for backup in $(~/bin/mc ls minio/velero/${MGMT_CLUSTER_NAME}/backups | egrep 'schiff-system' | awk '{print $5}' | sed "s/\/$//g");do
    echo "Restoring backup of namespace: ${backup}"
    ~/bin/velero --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
          restore create \
          --namespace-mappings "default:${backup}" \
          --from-backup "${backup}"
  done
  target_deployment_rollout_status ROLLOUT_NAME="flux" ROLLOUT_NAMESPACE="schiff-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
  target_deployment_rollout_status ROLLOUT_NAME="das-schiff-flux-clusters-${DATACENTRE}" ROLLOUT_NAMESPACE="schiff-system" ROLLOUT_CLUSTER="${MGMT_CLUSTER_NAME}"
fi
sleep 30

helm --kubeconfig "${HOME}/.kube/${MGMT_CLUSTER_NAME}.kubeconfig" \
        upgrade -i flux fluxcd/flux \
        --version 1.3.0 \
        --set git.url="https://x-access-token:tDozsovYdJBFzzEP5Rqu@gitlab.dol.telekom.de/schiff/platform/infrastructure/alpha3-environments.git" \
        --set git.branch="packet_sync" \
        --set git.path="masterClusterComponents/${DATACENTRE}/${MGMT_CLUSTER_NAME}" \
        --set git.git-timeout=20s  \
        --set git.sync-interval=1m \
        --set git.git-ci-skip=false \
        --set git.automation-interval=1m \
        --namespace schiff-system