Skip to content

Instantly share code, notes, and snippets.

@tommyready
Last active November 21, 2019 00:38
Show Gist options
  • Save tommyready/8e8718e4c73bf466da19ba592724b127 to your computer and use it in GitHub Desktop.
Save tommyready/8e8718e4c73bf466da19ba592724b127 to your computer and use it in GitHub Desktop.
Using Python to Zip a Directory then Upload to AWS S3
import os, sys
import math
import boto
import string
import shutil
from os import path
from os import stat
from array import *
from time import gmtime, strftime
from boto.s3.key import Key
import boto.s3.connection
class backupper():
root_dir = '/Users/me/Documents/test/'
backup_path = '/Users/me/Documents/backup/'
backup_file = strftime('%Y-%m-%d-%H-%M-%S', gmtime())
aws_host = 'us-east-2'
aws_key = '<aws-key>'
aws_secret_key = '<aws-secret-key>'
def uploadBackup(self):
s3 = boto.s3.connect_to_region(self.aws_host,
aws_access_key_id=self.aws_key,
aws_secret_access_key=self.aws_secret_key,
is_secure=True,
calling_format=boto.s3.connection.OrdinaryCallingFormat())
b = s3.get_bucket('backups')
file_path = self.backup_path + self.backup_file + '.zip'
filename = os.path.basename(file_path)
k = b.new_key(filename)
mp = b.initiate_multipart_upload(filename)
source_size = os.stat(file_path).st_size
bytes_per_chunk = 5000*1024*1024
chunks_count = int(math.ceil(source_size / float(bytes_per_chunk)))
for i in range(chunks_count):
offset = i * bytes_per_chunk
remaining_bytes = source_size - offset
bytes = min([bytes_per_chunk, remaining_bytes])
part_num = i + 1
print "uploading part " + str(part_num) + " of " + str(chunks_count)
with open(file_path, 'r') as fp:
fp.seek(offset)
mp.upload_part_from_file(fp=fp, part_num=part_num, size=bytes)
if len(mp.get_all_parts()) == chunks_count:
mp.complete_upload()
print "upload to S3 complete"
else:
mp.cancel_upload()
print "upload to S3 failed"
def doBackup(self):
print "Backing up to: " + self.backup_path
shutil.make_archive(self.backup_path + self.backup_file, 'zip', self.root_dir)
self.uploadBackup();
if __name__ == '__main__':
b = backupper()
b.doBackup()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment