Skip to content

Instantly share code, notes, and snippets.

Created May 14, 2017 17:15
Show Gist options
  • Save tclancy/b84af39f2cd527fa63a2e663e5abc44a to your computer and use it in GitHub Desktop.
Save tclancy/b84af39f2cd527fa63a2e663e5abc44a to your computer and use it in GitHub Desktop.
Django management command to tar files and send to S3
import datetime
import os
import tarfile
import time
from django.conf import settings
from django.core.mail import mail_admins
from import BaseCommand
from boto.s3.connection import S3Connection
from boto.s3.key import Key
def display_progress(bytes_sent, total_bytes):
print "%.1f%% complete" % (100.0 * (bytes_sent / total_bytes))
class Command(BaseCommand):
Back up database to S3 Storage. Boto docs at:
Also delete files older than DAYS_TO_KEEP_FILES days if they aren't the most recent backup
args = "<dry_run>"
help = """Move most recent database files to Amazon and delete any older files
Dry run = just show files"""
def handle(self, *args, **options):
errors = []
dry_run = len(args) > 0 and args[0] == "yes"
# create our connection
conn = S3Connection(settings.AWS_ID, settings.AWS_KEY)
bucket = conn.get_bucket(settings.AWS_BUCKET)
threshold = time.mktime(
( - datetime.timedelta(days=DAYS_TO_KEEP_FILES)
directory = settings.DATABASE_BACKUP_PATH
files = [os.path.join(directory, f) for f in os.listdir(directory) if not os.path.isdir(f)]
to_backup = {}
to_delete = []
if files:
# find most recent files matching
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
to_backup[name] = max([
f for f in files if f.find(pattern) > -1 and f.find(".tar.") == -1
], key=lambda x: os.stat(x).st_mtime)
except ValueError:
errors.append("Could not find a %s file (pattern = '%s')" % (name, pattern))
to_delete = [f for f in files if f not in to_backup and os.path.getmtime(f) < threshold]
errors.append("NO FILES FOUND!")
if dry_run:
if errors:
print "ERRORS ENCOUNTERED - would not have completed/ deleted any files"
for e in errors:
print e
print "Would have backed up the following:"
for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
if name in to_backup:
f = to_backup[name]
print "%s: %s (%s)" % (name, f, time.ctime(os.stat(f).st_mtime))
print "TO DELETE:"
for d in to_delete:
print d
for name, filepath in to_backup.iteritems():
if not self._backup_file(bucket, filepath, name):
errors.append("Did not backup %s" % filepath)
if errors:
body = "The following errors were encountered during today's backup:" + os.linesep
for e in errors:
body += e + os.linesep
body += "No files were deleted because of the errors"
print body
mail_admins("Amazon Database Backup Problems", body, fail_silently=True)
for d in to_delete:
except Exception:
def _backup_file(self, bucket, filepath, key):
Back SQL files to Amazon after tar/gzip
if not filepath:
return False
print "Starting backup of %s" % filepath
k = Key(bucket)
k.key = key
tar_name = filepath + ".tar.bz2"
tar =, "w:bz2")
k.set_contents_from_filename(tar_name, cb=display_progress)
return True
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment