Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Django management command to tar files and send to S3
import datetime
import os
import tarfile
import time
from django.conf import settings
from django.core.mail import mail_admins
from django.core.management.base import BaseCommand
from boto.s3.connection import S3Connection
from boto.s3.key import Key
DAYS_TO_KEEP_FILES = 2
DEBUGGING = False
def display_progress(bytes_sent, total_bytes):
print "%.1f%% complete" % (100.0 * (bytes_sent / total_bytes))
class Command(BaseCommand):
"""
Back up database to S3 Storage. Boto docs at:
http://docs.pythonboto.org/en/latest/s3_tut.html
Also delete files older than DAYS_TO_KEEP_FILES days if they aren't the most recent backup
"""
args = "<dry_run>"
help = """Move most recent database files to Amazon and delete any older files
Dry run = just show files"""
def handle(self, *args, **options):
errors = []
dry_run = len(args) > 0 and args[0] == "yes"
# create our connection
conn = S3Connection(settings.AWS_ID, settings.AWS_KEY)
bucket = conn.get_bucket(settings.AWS_BUCKET)
threshold = time.mktime(
(datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_KEEP_FILES)
).timetuple())
directory = settings.DATABASE_BACKUP_PATH
files = [os.path.join(directory, f) for f in os.listdir(directory) if not os.path.isdir(f)]
to_backup = {}
to_delete = []
if files:
# find most recent files matching
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
try:
to_backup[name] = max([
f for f in files if f.find(pattern) > -1 and f.find(".tar.") == -1
], key=lambda x: os.stat(x).st_mtime)
except ValueError:
errors.append("Could not find a %s file (pattern = '%s')" % (name, pattern))
to_delete = [f for f in files if f not in to_backup and os.path.getmtime(f) < threshold]
else:
errors.append("NO FILES FOUND!")
if dry_run:
if errors:
print "ERRORS ENCOUNTERED - would not have completed/ deleted any files"
for e in errors:
print e
print "Would have backed up the following:"
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
if name in to_backup:
f = to_backup[name]
print "%s: %s (%s)" % (name, f, time.ctime(os.stat(f).st_mtime))
print "TO DELETE:"
for d in to_delete:
print d
return
for name, filepath in to_backup.iteritems():
if not self._backup_file(bucket, filepath, name):
errors.append("Did not backup %s" % filepath)
if errors:
body = "The following errors were encountered during today's backup:" + os.linesep
for e in errors:
body += e + os.linesep
body += "No files were deleted because of the errors"
print body
if not DEBUGGING:
mail_admins("Amazon Database Backup Problems", body, fail_silently=True)
return
if not DEBUGGING:
for d in to_delete:
try:
os.unlink(d)
except Exception:
pass
def _backup_file(self, bucket, filepath, key):
"""
Back SQL files to Amazon after tar/gzip
"""
if not filepath:
return False
print "Starting backup of %s" % filepath
k = Key(bucket)
k.key = key
tar_name = filepath + ".tar.bz2"
tar = tarfile.open(tar_name, "w:bz2")
tar.add(filepath)
k.set_contents_from_filename(tar_name, cb=display_progress)
return True
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment