Create a gist now

Instantly share code, notes, and snippets.

@tclancy /AWS Backup
Last active Dec 19, 2015

What would you like to do?
Simple, unsubtle hack to back up files matching two patterns to AWS. Designed for a Django project right now, but no real need for the coupling.
#!/usr/bin/env python
import datetime
import os
import time
from zipfile import ZipFile, ZIP_DEFLATED
from django.conf import settings
from django.core.mail import mail_admins
from django.core.management.base import BaseCommand
from boto.s3.connection import S3Connection
from boto.s3.key import Key
DAYS_TO_KEEP_FILES = 5
DEBUGGING = False
def display_progress(bytes_sent, total_bytes):
print "%.1f%% complete" % (100.0 * (bytes_sent / total_bytes))
class Command(BaseCommand):
"""
Back up database to S3 Storage. Boto docs at:
http://docs.pythonboto.org/en/latest/s3_tut.html
Also delete files older than 5 days if they aren't the most
recent backup
"""
args = "<dry_run>"
help = """Move most recent database files to Amazon and delete any older files
Dry run = just show files"""
def handle(self, *args, **options):
errors = []
dry_run = len(args) > 0 and args[0] == "yes"
# create our connection
conn = S3Connection(settings.AWS_ID, settings.AWS_KEY)
bucket = conn.get_bucket(settings.AWS_BUCKET)
threshold = time.mktime(
(datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_KEEP_FILES)
).timetuple())
directory = settings.DATABASE_BACKUP_PATH
files = [os.path.join(directory, f) for f in os.listdir(directory) if not os.path.isdir(f)]
to_backup = {}
if files:
# find most recent files matching
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
try:
to_backup[name] = max([f for f in files if f.find(pattern) > -1], key=lambda x: os.stat(x).st_mtime)
except ValueError:
errors.append("Could not find a %s file (pattern = '%s')" % (name, pattern))
to_delete = [f for f in files if f not in to_backup and os.path.getmtime(f) < threshold]
else:
errors.append("NO FILES FOUND!")
if dry_run:
if errors:
print "ERRORS ENCOUNTERED - would not have completed/ deleted any files"
for e in errors:
print e
print "Would have backed up the following:"
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
if name in to_backup:
f = to_backup[name]
print "%s: %s (%s)" % (name, f, time.ctime(os.stat(f).st_mtime))
print "TO DELETE:"
for d in to_delete:
print d
return
for name, filepath in to_backup.iteritems():
if not self._backup_file(bucket, filepath, name):
errors.append("Did not backup %s" % filepath)
if errors:
body = "The following errors were encountered during today's backup:" + os.linesep
for e in errors:
body += e + os.linesep
body += "No files were deleted because of the errors"
print body
if not DEBUGGING:
mail_admins("Amazon Database Backup Problems", body, fail_silently=True)
return
if not DEBUGGING:
for d in to_delete:
try:
os.unlink(d)
except Exception:
pass
def _backup_file(self, bucket, filepath, key):
if not filepath:
return False
print "Starting backup of %s" % filepath
k = Key(bucket)
k.key = key
zip_name = filepath + ".zip"
try:
with ZipFile(zip_name, "w") as myzip:
myzip.write(filepath, os.path.basename(filepath), ZIP_DEFLATED)
k.set_contents_from_filename(zip_name, cb=display_progress)
try:
os.unlink(zip_name)
except Exception:
pass
return True
except Exception:
pass
Owner

tclancy commented Jul 2, 2013

The callback function is either useless or I'm getting round off problems:

0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
100.0% complete

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment