Testing process to expand k8s PVCs without losing data

expand_k8s_pvc.sh

# Testing process to expand k8s PVCs without losing data

#region Install
# Check the current cluster
kubectl config current-context

# Add bitnami repo
helm repo list
helm repo add bitnami https://charts.bitnami.com/bitnami

# List all current releases
helm list -A
helm status rabbitmq

# Install test application (helm)
kubectl create namespace rabbitmq

# helm upgrade rabbitmq bitnami/rabbitmq --set=installCRDs=true --install --atomic --namespace rabbitmq --debug --dry-run
# helm install rabbitmq bitnami/rabbitmq --namespace rabbitmq
# used for testing Method 1
helm upgrade rabbitmq bitnami/rabbitmq --install --atomic --namespace rabbitmq --set=replicaCount=2 --set=persistence.size=1Gi --debug

# used for testing Method 2
helm upgrade rabbitmq bitnami/rabbitmq --install --atomic --namespace rabbitmq --set=replicaCount=3 --set=persistence.size=1Gi --debug

# Follow helm notes for login info
echo "Username      : user"
echo "Password      : $(kubectl get secret --namespace rabbitmq rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 --decode)"
echo "ErLang Cookie : $(kubectl get secret --namespace rabbitmq rabbitmq -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode)"

# Expose the RabbitMQ Management interface
# open http://127.0.0.1:15672/#/
kubectl port-forward --namespace rabbitmq svc/rabbitmq 15672:15672
#endregion Install



#region pre-checks
# Show statefulset config for your application, eg: rabbitmq
# Note the volumeClaimTemplates storageClassName, eg: default
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml

# Ensure storage class allowVolumeExpansion is "true"
kubectl get storageclass default --output yaml

# Note down the current disk space used within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h

    Filesystem      Size  Used Avail Use% Mounted on
    /dev/sdd        976M  315M  646M  33% /bitnami/rabbitmq/mnesia

# List all PVCs
kubectl --namespace rabbitmq get pvc
#endregion pre-checks



#region Method 1: REQUIRES DOWNTIME - Scale down replicas (from 2 to 0), so that the AKS disk state is: "Unattached"
# Start monitoring in separate terminals
kubectl --namespace rabbitmq get pod --watch
kubectl --namespace rabbitmq get pvc --watch

# ? IMPORTANT: Backup the statefulset YAML - needed to recreate afterwards
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml > rabbitmq-statefulset.yaml

# Update the exported rabbitmq-statefulset.yaml with the new volumeClaimTemplates.spec.resources.requests.storage value (eg: from 1Gi to 2Gi)

# Scale down statefulset to 0 replicas
# ? IMPORTANT: Wait until all AKS disk states show: "Unattached"
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=0

# Delete the StatefulSet but leave its pod(s)
# this is required as you cannot update/patch "volumeClaimTemplates.spec.resources.requests.storage"
kubectl --namespace rabbitmq delete statefulsets rabbitmq --cascade=false

# Patch every PVC (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-0 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
kubectl --namespace rabbitmq patch pvc data-rabbitmq-1 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'

# Recreate using the exported/amended YAML from earlier
# ? IMPORTANT: Ensure the exported rabbitmq-statefulset.yaml has the new volumeClaimTemplates.spec.resources.requests.storage value (eg: 2Gi)
kubectl --namespace rabbitmq apply -f rabbitmq-statefulset.yaml

# * All pods should now be back online, with the attached PVCs showing the new disk capacity
# Validate the new disk size (2Gi) within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h

    Filesystem      Size  Used Avail Use% Mounted on
    /dev/sdd        2.0G  497M  1.5G  26% /bitnami/rabbitmq/mnesia

# Scale up +1 to test PVC storage size for a fresh pod
# should show the new disk capacity (eg: 2Gi)
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=3

# Scale back to original replica amount
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=2

# Delete orphaned PVC for the previous replica, as statefulset PVCs won't auto-delete / clean-up
kubectl --namespace rabbitmq delete pvc data-rabbitmq-2
#endregion Method 1: REQUIRES DOWNTIME - Scale down replicas (from 2 to 0), so that the AKS disk state is: "Unattached"



#region Method 2: NO DOWNTIME REQUIRED (at least 1 pod running at all times)j
# Start monitoring in separate terminals
kubectl --namespace rabbitmq get pod --watch
kubectl --namespace rabbitmq get pvc --watch

# ? IMPORTANT: Backup the statefulset YAML - needed to recreate afterwards
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml > rabbitmq-statefulset.yaml
# Update the exported rabbitmq-statefulset.yaml with the new volumeClaimTemplates.spec.resources.requests.storage value (eg: from 1Gi to 2Gi)

# Delete the StatefulSet but leave its pod(s)
# this is required as you cannot update/patch "volumeClaimTemplates.spec.resources.requests.storage"
kubectl --namespace rabbitmq delete statefulsets rabbitmq --cascade=false

# Delete only first pod (second and third pods are still running)
# ? IMPORTANT: Wait until the first pod AKS disk state is: "Unattached"
kubectl --namespace rabbitmq delete pod rabbitmq-0

# Patch first pod PVC (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-0 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'

# Recreate using the exported YAML from earlier
# ? IMPORTANT: Ensure the exported rabbitmq-statefulset.yaml has the new volumeClaimTemplates.spec.resources.requests.storage value (eg: 2Gi)
kubectl --namespace rabbitmq apply -f rabbitmq-statefulset.yaml

# Scale down statefulset to 1 replica, so the second and third pod is terminated
# ? IMPORTANT: Wait until replica AKS disk states show: "Unattached"
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=1

# Patch second and third PVCs (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-1 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
kubectl --namespace rabbitmq patch pvc data-rabbitmq-2 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'

# Scale back to original replica amount, so cluster can rebalance
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=3

# * All pods should now be back online, with the attached PVCs showing the new disk capacity
# Validate the new disk space used within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h

    Filesystem      Size  Used Avail Use% Mounted on
    /dev/sdd        2.0G  507M  1.5G  26% /bitnami/rabbitmq/mnesia
#endregion Method 2: NO DOWNTIME REQUIRED (at least 1 pod running at all times)



#region Validation
# Check PVCs have been resized
kubectl --namespace rabbitmq describe pvc data-rabbitmq-0
kubectl --namespace rabbitmq describe pvc data-rabbitmq-1
kubectl --namespace rabbitmq describe pvc data-rabbitmq-2
#endregion Validation



# Cleanup
helm uninstall rabbitmq --namespace rabbitmq
kubectl delete namespace rabbitmq