Example of using Luigi for backup to S3

luigi_backup.py

import os
import boto3
import luigi
import shutil
import datetime
from luigi.s3 import S3Target

# Globals
today = datetime.datetime.today().strftime(format='%Y_%m_%d')
folder_to_archive = ''
archive_name = '' + today
archive_type = ''
s3_bucket_name = ''


class CompressDocumentsFolder(luigi.ExternalTask):
    """
    Archives the  user's specified directory
    """

    def run(self):
        shutil.make_archive(archive_name, archive_type, folder_to_archive)

    def output(self):
        return luigi.LocalTarget(archive_name + "." + archive_type)


class UploadArchive(luigi.Task):
    """
    Uploads the archive to S3
    """

    def requires(self):
        return CompressDocumentsFolder()

    def run(self):
        s3 = boto3.resource('s3')
        bucket = s3.Bucket(s3_bucket_name)
        bucket.upload_file(archive_name + "." + archive_type,  archive_name)
        os.remove(archive_name + "." + archive_type)
        return

    def output(self):
        full_s3_path = 's3://' + s3_bucket_name + '/' + archive_name
        return S3Target(full_s3_path)


if __name__ == "__main__":
    luigi.run(main_task_cls=UploadArchive)