s3_backup.py

#!/usr/bin/env python

import datetime
import fnmatch
from functools import partial
import gzip
import json
import os
import shutil
import socket
import sys
import tarfile
import tempfile

import boto
from boto.s3.key import Key

BUCKET = os.environ.get('BACKUP_BUCKET', 'your_bucket')
# Contains files named {name}.conf
# Each file is JSON with the keys:
#  path: absolute path to backup
#  exclude: list of patterns to exclude (globbing works via fnmatch)
CONFIG_DIR = '/etc/backup.d'


def backup(backup_type):
    print("Starting backup")
    bucket = boto.connect_s3().get_bucket(BUCKET)
    backup_configs = read_configs(CONFIG_DIR)
    tmp_dir = tempfile.mkdtemp('backups')
    for config in backup_configs:
        config['tmp_dir'] = tmp_dir
        backup_file = backup_for_config(config)
        upload(backup_file, bucket, backup_type)
    shutil.rmtree(tmp_dir)


def read_configs(config_dir):
    master_config = []
    for filepath in os.listdir(config_dir):
        if not filepath.endswith('.conf'):
            continue
        with open(os.path.join(config_dir, filepath), 'r') as f:
            print('Reading {}...'.format(filepath))
            config = json.loads(f.read())
            # name is the filename without the path and without ".conf"
            config['name'] = os.path.split(filepath)[-1][:-5]
            master_config.append(config)
    if not master_config:
        print("No configurations found in {}!".format(config_dir))
    return master_config


def backup_for_config(config):
    if os.path.isdir(config['path']):
        return create_tarball(config)
    return create_gzip(config)


def _exclude_filter(exclude_list, full_path, tarinfo):
    for exclusion in exclude_list:
        if not exclusion.startswith('*'):
            # convert relative paths to absolute (excluding the preceeding "/")
            exclusion = os.path.join(full_path, exclusion)[1:]
        if fnmatch.fnmatch(tarinfo.name, exclusion):
            return None
    print('\t+ {}'.format(tarinfo.name))
    return tarinfo


def create_tarball(config):
    print('Creating tarball of {}...'.format(config['path']))
    tarball_path = os.path.join(config['tmp_dir'],
                                config['name'] + '.tar.gz')
    tarball = tarfile.open(tarball_path, 'w:gz')
    tarball.add(config['path'],
                filter=partial(_exclude_filter,
                               config.get('exclude', []),
                               config['path']))
    tarball.close()
    return tarball_path


def create_gzip(config):
    print('Creating gzip of {}...'.format(config['path']))
    to_backup = open('file.txt', 'rb')
    gzip_path = os.path.join(config['tmp_dir'], config['name'] + '.gz')
    gzipped = gzip.open(gzip_path, 'wb')
    gzipped.writelines(to_backup)
    gzipped.close()
    to_backup.close()


def upload(file_path, bucket, key_prefix):
    print('Uploading {}...'.format(file_path))
    hostname = socket.getfqdn()
    filename = '_'.join([datetime.date.today().isoformat(),
                         os.path.split(file_path)[-1]])
    key = Key(bucket)
    key.key = os.path.join(key_prefix, hostname, filename)
    key.set_contents_from_filename(file_path)
    print('Archived as s3://{}/{}'.format(BUCKET, key.key))


if __name__ == '__main__':
    backup_type = sys.argv[1]
    backup(backup_type)