Migrate EBS Volume based PVs across AWS availability zones

migrate-pv-to-zone.sh

#!/bin/bash

if [ $# -lt 6 ]; then
    echo "Usage: $0 [source-namespace] [source-pvc-name] [target-namespace] [target-pvc-name] [target-aws-zone] [target-pv-name] [kind/workload=None]"
    echo "Clone EBS, PV and PVC from source to target. Will stop kind/workload if defined."
    exit
fi

set -eu

SOURCE_NAMESPACE=$1
SOURCE_PVCNAME=$2
TARGET_NAMESPACE=$3
TARGET_PVCNAME=$4
TARGET_ZONE=$5
TARGET_PVNAME=$6

if [ $# -gt 6 ]; then
    DEPLOYMENTOBJ=$7
    REPLICAS=$(kubectl -n $SOURCE_NAMESPACE get $DEPLOYMENTOBJ --template={{.spec.replicas}})
else
    DEPLOYMENTOBJ=""
    REPLICAS=0
fi

## Nao precisa mexer a partir daqui

SOURCE_PVNAME=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.volumeName}})
SOURCE_VOLUMEID=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{or .spec.awsElasticBlockStore.volumeID ""}}{{or .spec.csi.volumeHandle ""}}' | awk -F/ '{print $NF}')
SOURCE_STORAGE=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.resources.requests.storage}})
SOURCE_VOLUMEMODE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template={{.spec.volumeMode}})

if [ -z "$SOURCE_VOLUMEMODE" ]; then
    SOURCE_VOLUMEMODE=Filesystem
fi

TARGET_STORAGE=$SOURCE_STORAGE
TARGET_VOLUMEMODE=$SOURCE_VOLUMEMODE

cat <<EOF
Summary:
    ${SOURCE_NAMESPACE@A}
    ${SOURCE_PVCNAME@A}
    ${SOURCE_PVNAME@A}
    ${SOURCE_VOLUMEID@A}
    ${SOURCE_STORAGE@A}
    ${SOURCE_VOLUMEMODE@A}

    ${TARGET_NAMESPACE@A}
    ${TARGET_PVCNAME@A}
    ${TARGET_PVNAME@A}
    ${TARGET_ZONE@A}
    ${TARGET_STORAGE@A}
    ${TARGET_VOLUMEMODE@A}

    ${DEPLOYMENTOBJ@A}
    ${REPLICAS@A}
EOF

read -p 'Press ENTER to continue'
echo

if [ -v DEPLOYMENTOBJ ] && [ $REPLICAS -gt 0 ]; then
    echo "Scaling down $DEPLOYMENTOBJ: $REPLICAS -> 0"
    kubectl -n $SOURCE_NAMESPACE scale --replicas=0 $DEPLOYMENTOBJ
    while true; do
        r="$(kubectl -n $SOURCE_NAMESPACE get $DEPLOYMENTOBJ --template={{.status.replicas}})"
        [ "$r" == "0" ] && break || true
        [ "$r" == "<no value>" ] && break || true
        echo waiting pods to die...
        sleep 1
    done

    while sleep 0.1; do
        kubectl -n $SOURCE_NAMESPACE get pod --no-headers
        read -p "Press ENTER to continue" -t 3 || { echo; continue; }
        break
    done

fi

DESCRIPTION="cloned from ns=${SOURCE_NAMESPACE}, pvc=${SOURCE_PVCNAME}, pv=$SOURCE_PVNAME, volumeId=$SOURCE_VOLUMEID"

echo "Waiting volume $SOURCE_VOLUMEID to become available..."
echo "Use \`aws ec2 detach-volume --volume-id $SOURCE_VOLUMEID\` to force detach"
aws ec2 wait volume-available --volume-id $SOURCE_VOLUMEID

echo "Creating snapshot from $SOURCE_VOLUMEID... "
SNAPSHOTID=$(aws ec2 create-snapshot --volume-id $SOURCE_VOLUMEID --description "$DESCRIPTION" --output text --query SnapshotId)
aws ec2 wait snapshot-completed --filter Name=snapshot-id,Values=$SNAPSHOTID
echo ${SNAPSHOTID@A}

echo "Creating volume from snapshot $SNAPSHOTID... "
TAGSPEC="ResourceType=volume,Tags=[{Key=Name,Value=$TARGET_NAMESPACE-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]"
TARGET_VOLUMEID=$(aws ec2 create-volume \
    --availability-zone $TARGET_ZONE \
    --snapshot-id $SNAPSHOTID \
    --volume-type gp2 \
    --output text \
    --query VolumeId \
    --tag-specifications "$TAGSPEC")
echo ${TARGET_VOLUMEID@A}

echo Creating new PV/PVC...
kubectl apply -f - <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: $TARGET_PVCNAME
  namespace: $TARGET_NAMESPACE
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: ${TARGET_STORAGE}
  volumeMode: ${TARGET_VOLUMEMODE}
  volumeName: ${TARGET_PVNAME}
---
apiVersion: v1
kind: PersistentVolume
metadata:
  labels:
    failure-domain.beta.kubernetes.io/region: ${TARGET_ZONE:0:-1}
    failure-domain.beta.kubernetes.io/zone: $TARGET_ZONE
  name: ${TARGET_PVNAME}
spec:
  accessModes:
  - ReadWriteOnce
  awsElasticBlockStore:
    fsType: ext4
    volumeID: aws://$TARGET_ZONE/$TARGET_VOLUMEID
  capacity:
    storage: ${TARGET_STORAGE}
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: $TARGET_PVCNAME
    namespace: $TARGET_NAMESPACE
  persistentVolumeReclaimPolicy: Retain
  volumeMode: Filesystem
EOF

if [ -v DEPLOYMENTOBJ ]; then
    echo "Now \`kubeclt edit $DEPLOYMENTOBJ\` to use new pvc/$TARGET_PVCNAME and then \`kubectl scale --replicas=1 $DEPLOYMENTOBJ\`"
fi

#if [ -v DEPLOYMENTOBJ ] && [ $REPLICAS -gt 0 ]; then
#    echo "Scaling back $DEPLOYMENTOBJ: 0 -> $REPLICAS"
#    kubectl -n $SOURCE_NAMESPACE scale --replicas=$REPLICAS $DEPLOYMENTOBJ
#fi

Thx, that saved the time and headaches.
In my case it is EKS + CSI Plugin. In this case the volumeId path a different path in the object:

SOURCE_VOLUMEID=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME '--template={{.spec.csi.volumeHandle}}')

Nice! Will add this to script.

@derjohn please validade this works for you.
Thanks

Hello @caruccio ,
great, it works for me (TM) :-) .... I made two further changes.
1.) Auto-generate the target-pv-name in the style the CSI driver does it by itself
2.) Auto-Delete the target-pvc name . So, if you set source-pvc and target-pvc to the same value it will delete the pvc and re-create it with the same name. (In my case the pods are already in pending state, so the pvc can be deleted and re-created and the pod start immediately (cool!).
I use it like this:
./migrate-pv-to-zone.sh kubecost kubecost-cost-analyzer kubecost kubecost-cost-analyzer eu-central-1b

Here is my diff proposal:

--- migrate-pv-to-zone.sh.orig  2024-01-13 12:03:59.522382481 +0100
+++ migrate-pv-to-zone.sh       2024-01-13 12:03:51.123353050 +0100
@@ -2,4 +2,4 @@
 
-if [ $# -lt 6 ]; then
-    echo "Usage: $0 [source-namespace] [source-pvc-name] [target-namespace] [target-pvc-name] [target-aws-zone] [target-pv-name] [kind/workload=None]"
+if [ $# -lt 5 ]; then
+    echo "Usage: $0 <source-namespace> <source-pvc-name> <target-namespace> <target-pvc-name> <target-aws-zone> [<target-pv-name>] [<kind/workload=None>]"
     echo "Clone EBS, PV and PVC from source to target. Will stop kind/workload if defined."
@@ -15,3 +15,3 @@
 TARGET_ZONE=$5
-TARGET_PVNAME=$6
+TARGET_PVNAME=${6:-"pvc-$(cat /proc/sys/kernel/random/uuid)"}
 
@@ -104,2 +104,3 @@
 echo Creating new PV/PVC...
+kubectl delete -n $TARGET_NAMESPACE pvc $TARGET_PVCNAME ||:
 kubectl apply -f - <<EOF

Thank you so much for this. I have made some update to fit my use:

• Add source zone name info
• Add cluster name var. Use into some volume tags
• Update AWS volume tag to current volume
• Add waiter snapshot progress to skip max attempt error
• Switch to gp3
• Update source pv to retain policy

my diff:

--- migrate-pv-to-zone.sh.orig  2024-03-29 11:08:26.755378764 +0100
+++ migrate-pv-to-zone.sh       2024-03-29 11:12:55.651384780 +0100
@@ -1,7 +1,7 @@
 #!/bin/bash

-if [ $# -lt 6 ]; then
-    echo "Usage: $0 [source-namespace] [source-pvc-name] [target-namespace] [target-pvc-name] [target-aws-zone] [target-pv-name] [kind/workload=None]"
+if [ $# -lt 5 ]; then
+    echo "Usage: $0 <source-namespace> <source-pvc-name> <target-namespace> <target-pvc-name> <target-aws-zone> [<target-pv-name>] [<kind/workload=None>]"
     echo "Clone EBS, PV and PVC from source to target. Will stop kind/workload if defined."
     exit
 fi
@@ -13,7 +13,9 @@
 TARGET_NAMESPACE=$3
 TARGET_PVCNAME=$4
 TARGET_ZONE=$5
-TARGET_PVNAME=$6
+#TARGET_PVNAME=$6
+# Make target pvc name compliant with csi drivers if not defined
+TARGET_PVNAME=${6:-"pvc-$(cat /proc/sys/kernel/random/uuid)"}

 if [ $# -gt 6 ]; then
     DEPLOYMENTOBJ=$7
@@ -25,9 +27,11 @@

 ## Nao precisa mexer a partir daqui

+CLUSTER_NAME=`kubectl config current-context`
 SOURCE_PVNAME=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.volumeName}})
 SOURCE_VOLUMEID=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{or .spec.awsElasticBlockStore.volumeID ""}}{{or .spec.csi.volumeHandle ""}}' | awk -F/ '{print $NF}')
 SOURCE_STORAGE=$(kubectl -n $SOURCE_NAMESPACE get pvc $SOURCE_PVCNAME --template={{.spec.resources.requests.storage}})
+SOURCE_ZONE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template='{{ (index (index (index .spec.nodeAffinity.required.nodeSelectorTerms 0).matchExpressions 0).values 0)}}')
 SOURCE_VOLUMEMODE=$(kubectl -n $SOURCE_NAMESPACE get pv $SOURCE_PVNAME --template={{.spec.volumeMode}})

 if [ -z "$SOURCE_VOLUMEMODE" ]; then
@@ -43,6 +47,7 @@
     ${SOURCE_PVCNAME@A}
     ${SOURCE_PVNAME@A}
     ${SOURCE_VOLUMEID@A}
+    ${SOURCE_ZONE@A}
     ${SOURCE_STORAGE@A}
     ${SOURCE_VOLUMEMODE@A}

@@ -87,21 +92,30 @@

 echo "Creating snapshot from $SOURCE_VOLUMEID... "
 SNAPSHOTID=$(aws ec2 create-snapshot --volume-id $SOURCE_VOLUMEID --description "$DESCRIPTION" --output text --query SnapshotId)
+SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
+while [ $SNAPSHOTPROGRESS != "100%"  ]
+do
+    sleep 15
+    echo "Snapshot ID: $SNAPSHOTID $SNAPSHOTPROGRESS"
+    SNAPSHOTPROGRESS=$(aws ec2 describe-snapshots --snapshot-ids $SNAPSHOTID --query "Snapshots[*].Progress" --output text)
+done
 aws ec2 wait snapshot-completed --filter Name=snapshot-id,Values=$SNAPSHOTID
 echo ${SNAPSHOTID@A}

 echo "Creating volume from snapshot $SNAPSHOTID... "
-TAGSPEC="ResourceType=volume,Tags=[{Key=Name,Value=$TARGET_NAMESPACE-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]"
+TAGSPEC="ResourceType=volume,Tags=[{Key=ebs.csi.aws.com/cluster,Value=true},{Key=kubernetes.io/cluster/$CLUSTER_NAME,Value=owned},{Key=CSIVolumeName,Value=$TARGET_PVNAME},{Key=kubernetesCluster,Value=$CLUSTER_NAME},{Key=Name,Value=$CLUSTER_NAME-dynamic-$TARGET_PVNAME},{Key=kubernetes.io/created-for/pv/name,Value=$TARGET_PVNAME},{Key=kubernetes.io/created-for/pvc/name,Value=$TARGET_PVCNAME},{Key=kubernetes.io/created-for/pvc/namespace,Value=$TARGET_NAMESPACE}]"
 TARGET_VOLUMEID=$(aws ec2 create-volume \
     --availability-zone $TARGET_ZONE \
     --snapshot-id $SNAPSHOTID \
-    --volume-type gp2 \
+    --volume-type gp3 \
     --output text \
     --query VolumeId \
     --tag-specifications "$TAGSPEC")
 echo ${TARGET_VOLUMEID@A}

 echo Creating new PV/PVC...
+kubectl patch pv $SOURCE_PVNAME -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
+kubectl delete -n $TARGET_NAMESPACE pvc $TARGET_PVCNAME ||:
 kubectl apply -f - <<EOF
 ---
 apiVersion: v1
 

A good alternative is this tool: https://github.com/utkuozdemir/pv-migrate