Last active
November 16, 2019 12:13
-
-
Save valarpirai/d9e8976ca4a19f986fdf5a20c0fc8af2 to your computer and use it in GitHub Desktop.
Upload Directory to AWS S3 using boto3 python SDK. Files will be uploaded if last modified time newer than existing file. Also sets file MIME type.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import os | |
import sys | |
import boto3 | |
from datetime import datetime, timezone | |
from urllib.request import pathname2url | |
import mimetypes | |
# upload_dir_s3.py /path/to/local/folder thebucketname /path/to/s3/folder | |
# ./upload_dir_s3.py test/ my-bucket static-resources/dev | |
# ./upload_dir_s3.py /home/resources/ my-bucket static-resources/dev | |
if len(sys.argv) < 4: | |
print('Missing required parameters') | |
sys.exit() | |
local_directory, bucket, destination = sys.argv[1:4] | |
client = boto3.client('s3') | |
def upload_file(local_path, bucket, s3_path): | |
print('****** Uploading %s...' % s3_path) | |
url = pathname2url(local_path) | |
file_mimetype = mimetypes.guess_type(url)[0] | |
client.upload_file(local_path, bucket, s3_path) | |
# Set Content-Type | |
if file_mimetype is not None: | |
client.copy_object(Bucket=bucket, Key=s3_path, ContentType=file_mimetype, MetadataDirective="REPLACE", CopySource=bucket + "/" + s3_path) | |
# Set Public read ACL | |
client.put_object_acl(ACL='public-read', Bucket=bucket, Key=s3_path) | |
def delete_angularjs_files(): | |
# Delete angular DIST every time | |
print('Deleting AngularJs files') | |
angularprefix = destination + '/angulardist/' | |
result = client.list_objects(Bucket=bucket, Prefix=angularprefix, Delimiter='/') | |
if 'Contents' in result: | |
auglarfiles = result['Contents'] | |
else: | |
auglarfiles = [] | |
for o in auglarfiles: | |
client.delete_object(Bucket=bucket, Key=o.get('Key')) | |
print('***** Deleted AngularJs files') | |
# delete_angularjs_files() | |
# enumerate local files recursively | |
for root, dirs, files in os.walk(local_directory): | |
for filename in files: | |
# construct the full local path | |
local_path = os.path.join(root, filename) | |
# print(local_path) | |
# construct the full path | |
relative_path = os.path.relpath(local_path, local_directory) | |
s3_path = os.path.join(destination, relative_path) | |
# relative_path = os.path.relpath(os.path.join(root, filename)) | |
print('Searching "%s" in "%s"' % (s3_path, bucket)) | |
try: | |
head_res = client.head_object(Bucket=bucket, Key=s3_path) | |
# print('Path found on S3! Skipping %s...' % s3_path) | |
# print(head_res) | |
remote_last_modified = head_res['LastModified'] | |
local_last_modified = datetime.fromtimestamp(os.path.getmtime(local_path), tz=timezone.utc) | |
# print(remote_last_modified) | |
# print(local_last_modified) | |
# Check last modified time | |
if remote_last_modified <= local_last_modified: | |
# print('Local file is modified') | |
try: | |
print('Deleting on S3 : ' + s3_path) | |
client.delete_object(Bucket=bucket, Key=s3_path) | |
upload_file(local_path, bucket, s3_path) | |
except: | |
print('Unable to delete %s...' % s3_path) | |
# else: | |
# print('Local file not modified') | |
except Exception as e: | |
# print(e) | |
upload_file(local_path, bucket, s3_path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
boto3==1.5.23 | |
botocore==1.8.37 | |
docutils==0.14 | |
jmespath==0.9.3 | |
python-dateutil==2.6.1 | |
s3transfer==0.1.12 | |
six==1.11.0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Usage Example
Create virtual environment and install the requirements.txt
See boto3 document for setting up AWS S3 credentials
upload_dir_s3.py /path/to/local/folder thebucketname /path/to/s3/folder
./upload_dir_s3.py test/ my-bucket static-resources/dev
./upload_dir_s3.py /home/resources/ my-bucket static-resources/dev