Re-create AWS EC2 volumes created in an EKS cluster from their snapshots and add PersistentVolume and PersistentVolumeClaim back to cluster

restore-snapshots.sh

#!/bin/bash

## Command to get list of snapshots I need
# aws-vault exec wildfire -- aws ec2 describe-snapshots --owner self --output json \
#   --filters Name=tag:KubernetesCluster,Values=main-prod-wf  "Name=tag:kubernetes.io/created-for/pvc/name,Values=claim-*" \
#   --query "Snapshots[?(StartTime>='2024-01-17T19:30:00')]" \
#   | jq '.[] | .SnapshotId' \
#   | sed 's/"//g' \
#   > snap_list.txt

while IFS= read -r snap_id
do
  echo "$snap_id";
  snapshot=$(aws-vault exec wildfire -- aws ec2 describe-snapshots --owner self --snapshot-ids "$snap_id")

  # Pull out the tags on the snapshot and save in create-volume file format for tags
  tags=$(echo "$snapshot" | jq -r '.Snapshots[] | [{"ResourceType": "volume", Tags: .Tags}]' )

  # Get the claim_name from the tags
  claim_name=$(echo "$tags" | jq -r '.[].Tags[] | select( .Key == "kubernetes.io/created-for/pvc/name" ) | .Value')
  # De-dupe the snap-list
  echo "$snap_id" > "auto-$claim_name.txt"
done < snap_list.txt

head -n1 -q auto-claim-*.txt > snap_list_dedupe.txt

while IFS= read -r snap_id
do
  echo "Snap ID from file: $snap_id"

  snapshot=$(aws-vault exec wildfire -- aws ec2 describe-snapshots --owner self --snapshot-ids "$snap_id")

  # Pull out the tags on the snapshot and save in create-volume file format for tags
  tags=$(echo "$snapshot" | jq -r '.Snapshots[] | [{"ResourceType": "volume", Tags: .Tags}]' )

  # Get the claim_name from the tags
  claim_name=$(echo "$tags" | jq -r '.[].Tags[] | select( .Key == "kubernetes.io/created-for/pvc/name" ) | .Value')

  username="${claim_name#*claim-}"
  echo "Claim name: $claim_name :: username: $username"
  pv_name=$(echo "$tags" | jq -r '.[].Tags[] | select( .Key == "kubernetes.io/created-for/pv/name" ) | .Value')


  # Output tags
  echo "$tags" > "$claim_name-tags.json";

  # Create the new volume
  volume_id=$(aws-vault exec wildfire -- aws ec2 create-volume --volume-type gp2 --snapshot-id "$snap_id" --availability-zone "us-east-2b" --tag-specifications "file://$claim_name-tags.json" | jq -r '.VolumeId');
  echo "New volume_id: $volume_id";

  # Create k8s pv / pvc manifest and apply
  cat > "$claim_name-pv-new.yaml" <<EOF;
---
apiVersion: v1
kind: PersistentVolume
metadata:
  annotations:
    pv.kubernetes.io/migrated-to: ebs.csi.aws.com
    pv.kubernetes.io/provisioned-by: kubernetes.io/aws-ebs
    volume.kubernetes.io/provisioner-deletion-secret-name: ""
    volume.kubernetes.io/provisioner-deletion-secret-namespace: ""
  finalizers:
  - kubernetes.io/pv-protection
  - external-attacher/ebs-csi-aws-com
  labels:
    topology.kubernetes.io/region: us-east-2
    topology.kubernetes.io/zone: us-east-2b
  name: $pv_name
spec:
  accessModes:
  - ReadWriteOnce
  awsElasticBlockStore:
    fsType: ext4
    volumeID: $volume_id
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: topology.kubernetes.io/zone
              operator: In
              values:
                - us-east-2b
            - key: topology.kubernetes.io/region
              operator: In
              values:
                - us-east-2
  capacity:
    storage: 250Gi
  persistentVolumeReclaimPolicy: Delete
  storageClassName: gp2
  volumeMode: Filesystem
EOF

  cat > "$claim_name-pvc-new.yaml" <<EOF;
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    hub.jupyter.org/username: $username
    pv.kubernetes.io/bind-completed: "yes"
    pv.kubernetes.io/bound-by-controller: "yes"
    volume.beta.kubernetes.io/storage-provisioner: ebs.csi.aws.com
    volume.kubernetes.io/selected-node: ip-10-10-52-56.us-east-2.compute.internal
    volume.kubernetes.io/storage-provisioner: ebs.csi.aws.com
  finalizers:
  - kubernetes.io/pvc-protection
  labels:
    app: jupyterhub
    chart: jupyterhub-2.0.0
    component: singleuser-storage
    heritage: jupyterhub
    hub.jupyter.org/username: $username
    release: jupyterhub-jupyterhub
  name: $claim_name
  namespace: jupyterhub
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 250Gi
  storageClassName: gp2
  volumeMode: Filesystem
  volumeName: $pv_name
EOF

  # Using k8s role, add to cluster
  aws-vault exec main-prod-wf-platform-admin -- kubectl apply -f "$claim_name-pv-new.yaml"
  aws-vault exec main-prod-wf-platform-admin -- kubectl apply -f "$claim_name-pvc-new.yaml"

done < snap_list_dedupe.txt