ebs-backup.sh

#!/bin/bash
set -e -o nounset -o pipefail

# Credentials for user EBSbackup
# with inline policy that allows:
#   ec2:CreateSnapshot
#   ec2:CreateTags
#   ec2:DeleteSnapshot
#   ec2:DescribeSnapshots
#   ec2:DescribeVolumes
#
# fixme: remove from script into CI server and export via ENV
export AWS_ACCESS_KEY_ID="XXXXXXXXXXXXXXX"
export AWS_SECRET_ACCESS_KEY="xxxxxxxxxxx"

main() {
  # only backing up the single Jenkins volume right now but could easily
  # be configured for more regions/volumes
  local region=us-west-2
  local volume_id=vol-43834cfd
  local retention=604800

  ebs_backup $volume_id $region $retention
}

# creates a EBS volume snapshot and removes old snapshots
# takes a EBS volume-id, AWS region and backup retention in seconds as args
ebs_backup() {
  local volume_id=$1
  local region=$2
  local retention=$3
  local snapshot_desc="$volume_id-backup-$(date +%Y-%m-%d)"

  echo -n "creating snapshot $snapshot_desc"
  local snapshot_id=$(aws ec2 create-snapshot --region $region --output=text --description $snapshot_desc --volume-id $volume_id --query SnapshotId)
  echo -n " with id $snapshot_id"

  # tag the created snapshot with 'author: backup' so we
  # can filter for those snapshots during our cleanup
  aws ec2 create-tags --region $region --resource $snapshot_id --tags Key=author,Value=backup
  echo " and tagged it"

  echo "getting list of snapshots"
  # get an array of all the snapshot ids
  local snapshot_list=($(aws ec2 describe-snapshots --region $region --output=text --filters "Name=volume-id,Values=$volume_id" "Name=tag:author,Values=backup" --query Snapshots[].SnapshotId))
  local snapshot_id
  local jobs
  local max_jobs=10 # AWS EBS API requests are terribly slow (>500ms/req) so process in parallel

  while [ ${#snapshot_list[@]} -gt 0 ]          # as long as there are snapshot-ids in the list
  do
    jobs=($(jobs -pr))                          # get list of currently running jobs
    if [ ${#jobs[@]} -lt $max_jobs ]; then      # if list is shorted than max_jobs
      snapshot_id=${snapshot_list[0]}           # get first snapshot-id in snapshot_list
      unset snapshot_list[0]                    # shift
      if [ ${#snapshot_list[@]} -gt 0 ]; then
        snapshot_list=( "${snapshot_list[@]}" ) # pack
      fi

      echo "found existing snapshot $snapshot_id"
      snapshot_cleanup $snapshot_id $region $retention & # check if snapshot needs cleanup
    else          # if job queue is full
      sleep 0.1   # idle for a while
    fi
  done

  wait            # wait for all snapshot_cleanup jobs to complete
}

# removes old snapshots
# takes a EBS snapshot-id, AWS region and backup retention in seconds as args
snapshot_cleanup() {
  local snapshot_id=$1
  local region=$2
  local retention=$3
  local delete_after=$(($(date +%s)-$retention))

  local snapshot_date=$(aws ec2 describe-snapshots --region $region --output=text --snapshot-ids $snapshot_id --query Snapshots[].StartTime)
  echo "$snapshot_id was created on $snapshot_date"

  if [ $(uname) == 'Linux' ]; then    # on Linux assume GNU date
    local snapshot_date_utime=$(date -d "$snapshot_date" +%s)
  else                                # else assume BSD date
    local snapshot_date_utime=$(date -j -f '%Y-%m-%dT%H:%M:%S' "${snapshot_date%.*}" +%s)
  fi

  if (( $snapshot_date_utime <= $delete_after ))
  then
    aws ec2 delete-snapshot --region $region --snapshot-id $snapshot_id
    echo "removed expired snapshot $snapshot_id"
  fi
}

main