Backup Script for EC2 EBS Volumes - The script takes snapshots of volumes if marked with certain tags and deletes the old ones once the are older than the specified days. Perfect for Jenkins.

volume-ec2-backup.sh

#!/bin/bash

# The script will generate new snapshots
# every execution. It does not look at the amount
# of snapshots, only at their age.

# ---------- CONSTANTS ----------
retention_period_days=30
last_date_to_back=$(date --date "$retention_period_days days ago")
last_date_to_back_seconds=$(date +%s --date "$retention_period_days days ago")

volume_tag_backup="BackupRequired"
volume_tag_backup_indicator="True"

sleep_seconds=3600 # 1 hour

# Boolean var to throw an error in case any of the snapshots has finished with errors
success_backup=true

# -------------------------------


echo "--- Starting Amazon EC2 Volumes Backup Script ---"
echo
echo "All the volumes marked with the following tag will be backed up:"
echo "TAG=$volume_tag_backup VALUE=$volume_tag_backup_indicator"
echo
echo "The RETENTION PERIOD is $retention_period_days days."
echo "All the snapshots taken before $last_date_to_back will be deleted."
echo
echo "The script will wait $sleep_seconds seconds to check the status of the snapshots."
echo

# Array to store the ids of the created snpashots
# The array will append a new id every time a new snashot is created
# by the script. This array will be used to validate that the status
# of the snapshots is "complete".
taken_snapshot_ids_array=()


# Receives an undetermined number of strings
# Shows the strings with a timestamp
log() {
        echo "[$(date +"%Y-%m-%d"+"%T")]: $*"
}

# Sleeps the program
# Receives the seconds to sleep the program
wait_snapshots() {
        echo "Waiting $sleep_seconds seconds."
        sleep $sleep_seconds
}

# Receives a string representing a volumeID
# Received string structure: <quotation>VolumeID<quotation><coma>
# The coma does not always appear
# Removes the comas and quotation characters
clean_id_field() {
        clean_name=$(echo $1 | cut -d "\"" -f 2)
        echo $clean_name
}

# To identify the volumes uses the tag defined in the 
# constants.
# returns the VolumeIDs for the volumes required to back up
get_list_volumes_id_to_backup() {
        volume_list_raw=$(aws ec2 describe-volumes --region us-west-2 --no-paginate --filters Name=tag-key,Values=$volume_tag_backup Name=tag-value,Values=$volume_tag_backup_indicator --query Volumes[*].VolumeId | grep vol)
        for raw_volume_id in $volume_list_raw; do
                volume_id=$(clean_id_field $raw_volume_id)
                echo $volume_id
        done
}


# Receive a string representing a volume ID
# Returns the list of ids of the snapshots assigned to the volume
get_list_snapshots_id() {
        snapshot_list_raw=$(aws ec2 describe-snapshots --region us-west-2 --filters Name=volume-id,Values="$1" --query Snapshots[*].SnapshotId | grep snap)
        for raw_snapshot_id in $snapshot_list_raw; do
                snapshot_id=$(clean_id_field $raw_snapshot_id)
                echo $snapshot_id
        done
}

# Receives a volumeId and take and snapshot of it
# Could throw an error related with the number of consecutive
# calls to the command create-snapshot. Acceptable error
# Error expected: SnapshotCreationPerVolumeRateExceeded
take_snapshot() {
        echo
        echo "Taking snapshot from volume $1."
        taken_snapshot_id=$(aws ec2 create-snapshot --volume-id $1 --description "Automated Back Up Snapshot" | grep SnapshotId | cut -d ":" -f 2 | cut -d "\"" -f 2)
        echo "Created new snapshot. Snapshot ID: $taken_snapshot_id"
        # Add the id of the created snapshot into the global array
        # of snapshots.
        taken_snapshot_ids_array+=($taken_snapshot_id)
}

# Receives a snapshot ID and deletes it
# If the snapshot is in use this function will fail
# this error is acceptable
# The expected error is: InvalidSnapshot.InUse
delete_snapshot() {
        echo "Deleting snapshot $1."
        aws ec2 delete-snapshot --snapshot-id $1
}

# Uses the array with the snapshots to
# validate that the statuses.
# If the status is "pending" it will log a warning
# If the status is "erro" it will throw an error
check_status_snapshots() {
        echo
        echo "Checking status taken snapshots."
        # Iterate over the already triggered snapshots ids
        for snapshot_taken_id in "${taken_snapshot_ids_array[@]}"; do
                snapshot_status=$(aws ec2 describe-snapshots --region us-west-2 --filters Name=snapshot-id,Values=$snapshot_taken_id --query "Snapshots[*].State" | grep -Fv '[' | grep -Fv ']' | cut -d "\"" -f 2)
                if [ "$snapshot_status" == "completed" ]; then
                        echo "SpanshotID: $snapshot_taken_id completed."
                elif [ "$snapshot_status" == "pending" ]; then
                        echo "WARNING: SnapshotID: $snapshot_taken_id still PENDING after $sleep_seconds."
                else
                        echo "ERROR: SnapshotID: $snapshot_taken_id final status is ERROR."
                        success_backup=false
                fi

        done
}

# Checks the global variable success_backup
# If some snapshot has ended in an error status
# shows an error message through stderr
# and ends the script with errors
check_success_backup_process(){
        if $success_backup; then
                echo
                echo "Backup process finished succesfully."
                exit 0
        else
                echo
                echo "Error in the backup process." 1>&2
                exit 1
        fi
}



# MAIN LOOP
volumes_to_back=$(get_list_volumes_id_to_backup)
for volume_id in $volumes_to_back; do
        list_snapshots_id=$(get_list_snapshots_id $volume_id)
        snapshot_counter=$(echo $list_snapshots_id | wc -l)
        if (($snapshot_counter < 1)) || [ -z "$list_snapshots_id" ]; then
                echo "No previous snapshots for the volume: $volume_id"
        else
                for snapshot_id in $list_snapshots_id; do
                        echo "Analyzing - Volume: $volume_id | Snapshot: $snapshot_id"
                        start_time=$(aws ec2 describe-snapshots --region us-west-2 --filters Name=snapshot-id,Values=$snapshot_id --query "Snapshots[*].StartTime" | grep -Fv '[' | grep -Fv ']' | cut -d "\"" -f 2)
                        start_time_seconds=$(date --date=$start_time +%s)

                        if (($start_time_seconds < $last_date_to_back_seconds)); then
                                echo "The snapshot $snapshot_id created the $start_time will be deleted."
                                delete_snapshot $snapshot_id
                        fi
                done
        fi
        # Take a snapshot every time the script is called.
        take_snapshot $volume_id
done


# Sleep the program and wait
# a predefined amount of time
# allowing the snapshots to complete
wait_snapshots

# Validate the snapshot status
# This could vary between pending | completed | error
check_status_snapshots

# Check if the bakcup process has finished succesfully
check_success_backup_process