AWS SSM Document for performing Consistent Snapshots

cleanup.py

"""
Deletes old EC2 Snapshots created from the ConsistentSnapshot AWS RunCommand.
"""
import re
from datetime import datetime
from collections import defaultdict
from operator import itemgetter

import boto3

OWNERS = ['<owner_id_here>']
MINIMUM = 3  # Minimum number of snapshots to keep
SNAP_REGEX = re.compile(r"ConsistentSnapshot\(i-\w+\) for ([\w-]+)")


def cleanup():
    """
    Calls to EC2 to get all snapshots created by ConsistentSnapshot, then
    makes sure to keep the MINIMUM number of snapshots for all SERVER groups, deleting the rest.
    """
    ec2 = boto3.client('ec2')
    server_snaps = defaultdict(list)
    snapshots_response = ec2.describe_snapshots(OwnerIds=OWNERS)

    for snap in snapshots_response['Snapshots']:
        matches = SNAP_REGEX.findall(snap['Description'])
        if matches:
            server_snaps[matches[0]].append(snap)

    for server, snaps in server_snaps.items():
        len_snaps = len(snaps)
        print("Filtering {}'s {} snapshots to keep the {} most recent".format(
            server, len_snaps, MINIMUM))
        if len_snaps > MINIMUM:
            ordered_snaps = sorted(snaps, key=itemgetter('StartTime'))
            oldest_snaps = [s for s in ordered_snaps][:len_snaps - MINIMUM]
            for old_snap in oldest_snaps:
                ec2.delete_snapshot(SnapshotId=old_snap['SnapshotId'])
                print("Deleted '{}'".format(old_snap['SnapshotId']))


def handle(event, _):
    """
    Deletes old EC2 Snapshots created from the ConsistentSnapshot AWS RunCommand.

    Args:
        event (dict): CloudWatch Event dictionary. Contains the event time.
        _ (object): Ignored AWS Lambda context object.
    Returns:
        str: The event time in a string format.
    """
    start = datetime.now()
    print('Beginning clean_old_snapshots at {}'.format(str(start)))
    try:
        cleanup()
    except:
        print('Cleanup failed!')
        raise
    else:
        print('Cleanup was successful!')
        return event['time']
    finally:
        end = datetime.now()
        print('Finished in {:d} second(s).'.format((end - start).seconds))
        print('clean_old_snapshots complete at {}'.format(str(end)))

cw_events.tf

# Variables coming from apex
variable "apex_function_clean_old_snapshots" {}

resource "aws_cloudwatch_event_rule" "weekly_sunday_night" {
    name = "weekly-sunday-night"
    description = "Fires every week on Sunday night"
    schedule_expression = "cron(0 8 ? * SUN *)"
}

resource "aws_cloudwatch_event_target" "clean_old_snapshots_every_week" {
    rule = "${aws_cloudwatch_event_rule.weekly_sunday_night.name}"
    target_id = "clean_old_snapshots"
    arn = "${var.apex_function_clean_old_snapshots}"
}

resource "aws_lambda_permission" "allow_cloudwatch_to_call_clean_old_snapshots" {
    statement_id = "AllowExecutionFromCloudWatch"
    action = "lambda:InvokeFunction"
    function_name = "${var.apex_function_clean_old_snapshots}"
    principal = "events.amazonaws.com"
    source_arn = "${aws_cloudwatch_event_rule.weekly_sunday_night.arn}"
}

ssm-doc.json

{
   "schemaVersion": "2.2",
   "description": "Consistent EC2 Snapshot",
   "mainSteps":[
      {
         "action":"aws:runShellScript",
         "name":"ConsistentSnapshotLinux",
         "precondition":{
            "StringEquals":[
               "platformType",
               "Linux"
            ]
         },
         "inputs":{
            "runCommand":[
              "sync",
              "for target in $(findmnt -nlo TARGET -t ext4); do fsfreeze -f $target; done",
              "instance=`curl -s http://169.254.169.254/latest/meta-data/instance-id`",
              "region=`curl -s 169.254.169.254/latest/meta-data/placement/availability-zone`",
              "region=${region::-1}",
              "name=`aws ec2 describe-instances --instance-ids $instance --output text --query \"Reservations[*].Instances[*].[Tags[?Key=='Name'].Value]\" --region $region`",
              "volumes=`aws ec2 describe-instance-attribute --instance-id $instance --attribute blockDeviceMapping --output text --query BlockDeviceMappings[*].Ebs.VolumeId --region $region`",
              "for volume in $(echo $volumes | tr \" \" \"\\n\")",
              "do aws ec2 create-snapshot --volume-id $volume --description \"Created by ConsistentSnapshot($instance) for $name from $volume\" --region $region > /dev/null 2>&1",
              "done",
              "for target in $(findmnt -nlo TARGET -t ext4); do fsfreeze -u $target; done"
            ]
         }
      }
   ]
}

Built from this tutorial. Made a tweak in the EC2 snapshot description to include the name of the instance. Also pulled the MySQL pieces as I didn't need them.

Also added a cleanup.py & cw_events.tf to show a setup to clean up older ConsistentSnapshot images on a weekly basis.