Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Testing process to expand k8s PVCs without losing data
# Testing process to expand k8s PVCs without losing data
#region Install
# Check the current cluster
kubectl config current-context
# Add bitnami repo
helm repo list
helm repo add bitnami https://charts.bitnami.com/bitnami
# List all current releases
helm list -A
helm status rabbitmq
# Install test application (helm)
kubectl create namespace rabbitmq
# helm upgrade rabbitmq bitnami/rabbitmq --set=installCRDs=true --install --atomic --namespace rabbitmq --debug --dry-run
# helm install rabbitmq bitnami/rabbitmq --namespace rabbitmq
# used for testing Method 1
helm upgrade rabbitmq bitnami/rabbitmq --install --atomic --namespace rabbitmq --set=replicaCount=2 --set=persistence.size=1Gi --debug
# used for testing Method 2
helm upgrade rabbitmq bitnami/rabbitmq --install --atomic --namespace rabbitmq --set=replicaCount=3 --set=persistence.size=1Gi --debug
# Follow helm notes for login info
echo "Username : user"
echo "Password : $(kubectl get secret --namespace rabbitmq rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 --decode)"
echo "ErLang Cookie : $(kubectl get secret --namespace rabbitmq rabbitmq -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode)"
# Expose the RabbitMQ Management interface
# open http://127.0.0.1:15672/#/
kubectl port-forward --namespace rabbitmq svc/rabbitmq 15672:15672
#endregion Install
#region pre-checks
# Show statefulset config for your application, eg: rabbitmq
# Note the volumeClaimTemplates storageClassName, eg: default
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml
# Ensure storage class allowVolumeExpansion is "true"
kubectl get storageclass default --output yaml
# Note down the current disk space used within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sdd 976M 315M 646M 33% /bitnami/rabbitmq/mnesia
# List all PVCs
kubectl --namespace rabbitmq get pvc
#endregion pre-checks
#region Method 1: REQUIRES DOWNTIME - Scale down replicas (from 2 to 0), so that the AKS disk state is: "Unattached"
# Start monitoring in separate terminals
kubectl --namespace rabbitmq get pod --watch
kubectl --namespace rabbitmq get pvc --watch
# ? IMPORTANT: Backup the statefulset YAML - needed to recreate afterwards
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml > rabbitmq-statefulset.yaml
# Update the exported rabbitmq-statefulset.yaml with the new volumeClaimTemplates.spec.resources.requests.storage value (eg: from 1Gi to 2Gi)
# Scale down statefulset to 0 replicas
# ? IMPORTANT: Wait until all AKS disk states show: "Unattached"
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=0
# Delete the StatefulSet but leave its pod(s)
# this is required as you cannot update/patch "volumeClaimTemplates.spec.resources.requests.storage"
kubectl --namespace rabbitmq delete statefulsets rabbitmq --cascade=false
# Patch every PVC (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-0 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
kubectl --namespace rabbitmq patch pvc data-rabbitmq-1 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
# Recreate using the exported/amended YAML from earlier
# ? IMPORTANT: Ensure the exported rabbitmq-statefulset.yaml has the new volumeClaimTemplates.spec.resources.requests.storage value (eg: 2Gi)
kubectl --namespace rabbitmq apply -f rabbitmq-statefulset.yaml
# * All pods should now be back online, with the attached PVCs showing the new disk capacity
# Validate the new disk size (2Gi) within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sdd 2.0G 497M 1.5G 26% /bitnami/rabbitmq/mnesia
# Scale up +1 to test PVC storage size for a fresh pod
# should show the new disk capacity (eg: 2Gi)
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=3
# Scale back to original replica amount
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=2
# Delete orphaned PVC for the previous replica, as statefulset PVCs won't auto-delete / clean-up
kubectl --namespace rabbitmq delete pvc data-rabbitmq-2
#endregion Method 1: REQUIRES DOWNTIME - Scale down replicas (from 2 to 0), so that the AKS disk state is: "Unattached"
#region Method 2: NO DOWNTIME REQUIRED (at least 1 pod running at all times)j
# Start monitoring in separate terminals
kubectl --namespace rabbitmq get pod --watch
kubectl --namespace rabbitmq get pvc --watch
# ? IMPORTANT: Backup the statefulset YAML - needed to recreate afterwards
kubectl --namespace rabbitmq get statefulset rabbitmq --output yaml > rabbitmq-statefulset.yaml
# Update the exported rabbitmq-statefulset.yaml with the new volumeClaimTemplates.spec.resources.requests.storage value (eg: from 1Gi to 2Gi)
# Delete the StatefulSet but leave its pod(s)
# this is required as you cannot update/patch "volumeClaimTemplates.spec.resources.requests.storage"
kubectl --namespace rabbitmq delete statefulsets rabbitmq --cascade=false
# Delete only first pod (second and third pods are still running)
# ? IMPORTANT: Wait until the first pod AKS disk state is: "Unattached"
kubectl --namespace rabbitmq delete pod rabbitmq-0
# Patch first pod PVC (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-0 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
# Recreate using the exported YAML from earlier
# ? IMPORTANT: Ensure the exported rabbitmq-statefulset.yaml has the new volumeClaimTemplates.spec.resources.requests.storage value (eg: 2Gi)
kubectl --namespace rabbitmq apply -f rabbitmq-statefulset.yaml
# Scale down statefulset to 1 replica, so the second and third pod is terminated
# ? IMPORTANT: Wait until replica AKS disk states show: "Unattached"
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=1
# Patch second and third PVCs (spec.resources.requests.storage) in the StatefulSet, to increase its capacity (eg: from 1Gi to 2Gi)
kubectl --namespace rabbitmq patch pvc data-rabbitmq-1 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
kubectl --namespace rabbitmq patch pvc data-rabbitmq-2 --patch '{\"spec\": {\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}}}'
# Scale back to original replica amount, so cluster can rebalance
kubectl --namespace rabbitmq scale statefulset rabbitmq --replicas=3
# * All pods should now be back online, with the attached PVCs showing the new disk capacity
# Validate the new disk space used within application container
kubectl --namespace rabbitmq exec -it rabbitmq-0 -- df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sdd 2.0G 507M 1.5G 26% /bitnami/rabbitmq/mnesia
#endregion Method 2: NO DOWNTIME REQUIRED (at least 1 pod running at all times)
#region Validation
# Check PVCs have been resized
kubectl --namespace rabbitmq describe pvc data-rabbitmq-0
kubectl --namespace rabbitmq describe pvc data-rabbitmq-1
kubectl --namespace rabbitmq describe pvc data-rabbitmq-2
#endregion Validation
# Cleanup
helm uninstall rabbitmq --namespace rabbitmq
kubectl delete namespace rabbitmq
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment