Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Example of using Luigi for backup to S3
import os
import boto3
import luigi
import shutil
import datetime
from luigi.s3 import S3Target
# Globals
today = datetime.datetime.today().strftime(format='%Y_%m_%d')
folder_to_archive = ''
archive_name = '' + today
archive_type = ''
s3_bucket_name = ''
class CompressDocumentsFolder(luigi.ExternalTask):
"""
Archives the user's specified directory
"""
def run(self):
shutil.make_archive(archive_name, archive_type, folder_to_archive)
def output(self):
return luigi.LocalTarget(archive_name + "." + archive_type)
class UploadArchive(luigi.Task):
"""
Uploads the archive to S3
"""
def requires(self):
return CompressDocumentsFolder()
def run(self):
s3 = boto3.resource('s3')
bucket = s3.Bucket(s3_bucket_name)
bucket.upload_file(archive_name + "." + archive_type, archive_name)
os.remove(archive_name + "." + archive_type)
return
def output(self):
full_s3_path = 's3://' + s3_bucket_name + '/' + archive_name
return S3Target(full_s3_path)
if __name__ == "__main__":
luigi.run(main_task_cls=UploadArchive)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.