Last active
April 17, 2024 20:13
-
-
Save caruccio/be825aa39d53535217494369cc793dbd to your computer and use it in GitHub Desktop.
Migrate EBS Volume based PVs across AWS availability zones
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
if [ $# -lt 6 ]; then | |
echo "Usage: $0 [source-namespace] [source-pvc-name] [target-namespace] [target-pvc-name] [target-aws-zone] [target-pv-name] [kind/workload=None]" | |
echo "Clone EBS, PV and PVC from source to target. Will stop kind/workload if defined." | |
exit | |
fi | |
set -eu | |
SOURCE_NAMESPACE=$1 | |
SOURCE_PVCNAME=$2 | |
TARGET_NAMESPACE=$3 | |
TARGET_PVCNAME=$4 | |
TARGET_ZONE=$5 | |
TARGET_PVNAME=$6 | |
if [ $# -gt 6 ]; then | |
DEPLOYMENTOBJ=$7 | |
REPLICAS=$(kubectl -n $SOURCE_NAMESPACE get $DEPLOYMENTOBJ --template={{.spec.replicas}}) | |
else | |
DEPLOYMENTOBJ="" | |
REPLICAS=0 | |
fi | |
## Nao precisa mexer a partir daqui | |
SOURCE_PVNAME=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.volumeName}}) | |
SOURCE_VOLUMEID=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{or .spec.awsElasticBlockStore.volumeID ""}}{{or .spec.csi.volumeHandle ""}}' | awk -F/ '{print $NF}') | |
SOURCE_STORAGE=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.resources.requests.storage}}) | |
SOURCE_VOLUMEMODE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template={{.spec.volumeMode}}) | |
if [ -z "$SOURCE_VOLUMEMODE" ]; then | |
SOURCE_VOLUMEMODE=Filesystem | |
fi | |
TARGET_STORAGE=$SOURCE_STORAGE | |
TARGET_VOLUMEMODE=$SOURCE_VOLUMEMODE | |
cat <<EOF | |
Summary: | |
${SOURCE_NAMESPACE@A} | |
${SOURCE_PVCNAME@A} | |
${SOURCE_PVNAME@A} | |
${SOURCE_VOLUMEID@A} | |
${SOURCE_STORAGE@A} | |
${SOURCE_VOLUMEMODE@A} | |
${TARGET_NAMESPACE@A} | |
${TARGET_PVCNAME@A} | |
${TARGET_PVNAME@A} | |
${TARGET_ZONE@A} | |
${TARGET_STORAGE@A} | |
${TARGET_VOLUMEMODE@A} | |
${DEPLOYMENTOBJ@A} | |
${REPLICAS@A} | |
EOF | |
read -p 'Press ENTER to continue' | |
echo | |
if [ -v DEPLOYMENTOBJ ] && [ $REPLICAS -gt 0 ]; then | |
echo "Scaling down $DEPLOYMENTOBJ: $REPLICAS -> 0" | |
kubectl -n $SOURCE_NAMESPACE scale --replicas=0 $DEPLOYMENTOBJ | |
while true; do | |
r="$(kubectl -n $SOURCE_NAMESPACE get $DEPLOYMENTOBJ --template={{.status.replicas}})" | |
[ "$r" == "0" ] && break || true | |
[ "$r" == "<no value>" ] && break || true | |
echo waiting pods to die... | |
sleep 1 | |
done | |
while sleep 0.1; do | |
kubectl -n $SOURCE_NAMESPACE get pod --no-headers | |
read -p "Press ENTER to continue" -t 3 || { echo; continue; } | |
break | |
done | |
fi | |
DESCRIPTION="cloned from ns=${SOURCE_NAMESPACE}, pvc=${SOURCE_PVCNAME}, pv=$SOURCE_PVNAME, volumeId=$SOURCE_VOLUMEID" | |
echo "Waiting volume $SOURCE_VOLUMEID to become available..." | |
echo "Use \`aws ec2 detach-volume --volume-id $SOURCE_VOLUMEID\` to force detach" | |
aws ec2 wait volume-available --volume-id $SOURCE_VOLUMEID | |
echo "Creating snapshot from $SOURCE_VOLUMEID... " | |
SNAPSHOTID=$(aws ec2 create-snapshot --volume-id $SOURCE_VOLUMEID --description "$DESCRIPTION" --output text --query SnapshotId) | |
aws ec2 wait snapshot-completed --filter Name=snapshot-id,Values=$SNAPSHOTID | |
echo ${SNAPSHOTID@A} | |
echo "Creating volume from snapshot $SNAPSHOTID... " | |
TAGSPEC="ResourceType=volume,Tags=[{Key=Name,Value=$TARGET_NAMESPACE-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]" | |
TARGET_VOLUMEID=$(aws ec2 create-volume \ | |
--availability-zone $TARGET_ZONE \ | |
--snapshot-id $SNAPSHOTID \ | |
--volume-type gp2 \ | |
--output text \ | |
--query VolumeId \ | |
--tag-specifications "$TAGSPEC") | |
echo ${TARGET_VOLUMEID@A} | |
echo Creating new PV/PVC... | |
kubectl apply -f - <<EOF | |
--- | |
apiVersion: v1 | |
kind: PersistentVolumeClaim | |
metadata: | |
name: $TARGET_PVCNAME | |
namespace: $TARGET_NAMESPACE | |
spec: | |
accessModes: | |
- ReadWriteOnce | |
resources: | |
requests: | |
storage: ${TARGET_STORAGE} | |
volumeMode: ${TARGET_VOLUMEMODE} | |
volumeName: ${TARGET_PVNAME} | |
--- | |
apiVersion: v1 | |
kind: PersistentVolume | |
metadata: | |
labels: | |
failure-domain.beta.kubernetes.io/region: ${TARGET_ZONE:0:-1} | |
failure-domain.beta.kubernetes.io/zone: $TARGET_ZONE | |
name: ${TARGET_PVNAME} | |
spec: | |
accessModes: | |
- ReadWriteOnce | |
awsElasticBlockStore: | |
fsType: ext4 | |
volumeID: aws://$TARGET_ZONE/$TARGET_VOLUMEID | |
capacity: | |
storage: ${TARGET_STORAGE} | |
claimRef: | |
apiVersion: v1 | |
kind: PersistentVolumeClaim | |
name: $TARGET_PVCNAME | |
namespace: $TARGET_NAMESPACE | |
persistentVolumeReclaimPolicy: Retain | |
volumeMode: Filesystem | |
EOF | |
if [ -v DEPLOYMENTOBJ ]; then | |
echo "Now \`kubeclt edit $DEPLOYMENTOBJ\` to use new pvc/$TARGET_PVCNAME and then \`kubectl scale --replicas=1 $DEPLOYMENTOBJ\`" | |
fi | |
#if [ -v DEPLOYMENTOBJ ] && [ $REPLICAS -gt 0 ]; then | |
# echo "Scaling back $DEPLOYMENTOBJ: 0 -> $REPLICAS" | |
# kubectl -n $SOURCE_NAMESPACE scale --replicas=$REPLICAS $DEPLOYMENTOBJ | |
#fi |
Thank you so much for this. I have made some update to fit my use:
- Add source zone name info
- Add cluster name var. Use into some volume tags
- Update AWS volume tag to current volume
- Add waiter snapshot progress to skip max attempt error
- Switch to gp3
- Update source pv to retain policy
my diff:
--- migrate-pv-to-zone.sh.orig 2024-03-29 11:08:26.755378764 +0100
+++ migrate-pv-to-zone.sh 2024-03-29 11:12:55.651384780 +0100
@@ -1,7 +1,7 @@
#!/bin/bash
-if [ $# -lt 6 ]; then
- echo "Usage: $0 [source-namespace] [source-pvc-name] [target-namespace] [target-pvc-name] [target-aws-zone] [target-pv-name] [kind/workload=None]"
+if [ $# -lt 5 ]; then
+ echo "Usage: $0 <source-namespace> <source-pvc-name> <target-namespace> <target-pvc-name> <target-aws-zone> [<target-pv-name>] [<kind/workload=None>]"
echo "Clone EBS, PV and PVC from source to target. Will stop kind/workload if defined."
exit
fi
@@ -13,7 +13,9 @@
TARGET_NAMESPACE=$3
TARGET_PVCNAME=$4
TARGET_ZONE=$5
-TARGET_PVNAME=$6
+#TARGET_PVNAME=$6
+# Make target pvc name compliant with csi drivers if not defined
+TARGET_PVNAME=${6:-"pvc-$(cat /proc/sys/kernel/random/uuid)"}
if [ $# -gt 6 ]; then
DEPLOYMENTOBJ=$7
@@ -25,9 +27,11 @@
## Nao precisa mexer a partir daqui
+CLUSTER_NAME=`kubectl config current-context`
SOURCE_PVNAME=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.volumeName}})
SOURCE_VOLUMEID=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{or .spec.awsElasticBlockStore.volumeID ""}}{{or .spec.csi.volumeHandle ""}}' | awk -F/ '{print $NF}')
SOURCE_STORAGE=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.resources.requests.storage}})
+SOURCE_ZONE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{ (index (index (index .spec.nodeAffinity.required.nodeSelectorTerms 0).matchExpressions 0).values 0)}}')
SOURCE_VOLUMEMODE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template={{.spec.volumeMode}})
if [ -z "$SOURCE_VOLUMEMODE" ]; then
@@ -43,6 +47,7 @@
${SOURCE_PVCNAME@A}
${SOURCE_PVNAME@A}
${SOURCE_VOLUMEID@A}
+ ${SOURCE_ZONE@A}
${SOURCE_STORAGE@A}
${SOURCE_VOLUMEMODE@A}
@@ -87,21 +92,30 @@
echo "Creating snapshot from $SOURCE_VOLUMEID... "
SNAPSHOTID=$(aws ec2 create-snapshot --volume-id $SOURCE_VOLUMEID --description "$DESCRIPTION" --output text --query SnapshotId)
+SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
+while [ $SNAPSHOTPROGRESS != "100%" ]
+do
+ sleep 15
+ echo "Snapshot ID: $SNAPSHOTID $SNAPSHOTPROGRESS"
+ SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
+done
aws ec2 wait snapshot-completed --filter Name=snapshot-id,Values=$SNAPSHOTID
echo ${SNAPSHOTID@A}
echo "Creating volume from snapshot $SNAPSHOTID... "
-TAGSPEC="ResourceType=volume,Tags=[{Key=Name,Value=$TARGET_NAMESPACE-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]"
+TAGSPEC="ResourceType=volume,Tags=[{Key=ebs.csi.aws.com/cluster,Value=true},{Key=kubernetes.io/cluster/$CLUSTER_NAME,Value=owned},{Key=CSIVolumeName,Value=$TARGET_PVNAME},{Key=kubernetesCluster,Value=$CLUSTER_NAME},{Key=Name,Value=$CLUSTER_NAME-dynamic-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]"
TARGET_VOLUMEID=$(aws ec2 create-volume \
--availability-zone $TARGET_ZONE \
--snapshot-id $SNAPSHOTID \
- --volume-type gp2 \
+ --volume-type gp3 \
--output text \
--query VolumeId \
--tag-specifications "$TAGSPEC")
echo ${TARGET_VOLUMEID@A}
echo Creating new PV/PVC...
+kubectl patch pv $SOURCE_PVNAME -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
+kubectl delete -n $TARGET_NAMESPACE pvc $TARGET_PVCNAME ||:
kubectl apply -f - <<EOF
---
apiVersion: v1
A good alternative is this tool: https://github.com/utkuozdemir/pv-migrate
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hello @caruccio ,
great, it works for me (TM) :-) .... I made two further changes.
1.) Auto-generate the target-pv-name in the style the CSI driver does it by itself
2.) Auto-Delete the target-pvc name . So, if you set source-pvc and target-pvc to the same value it will delete the pvc and re-create it with the same name. (In my case the pods are already in pending state, so the pvc can be deleted and re-created and the pod start immediately (cool!).
I use it like this:
./migrate-pv-to-zone.sh kubecost kubecost-cost-analyzer kubecost kubecost-cost-analyzer eu-central-1b
Here is my diff proposal: