Upload Directory to AWS S3 using boto3 python SDK. Files will be uploaded if last modified time newer than existing file. Also sets file MIME type.

_upload_dir_s3.py

#!/usr/bin/env python

import os
import sys
import boto3
from datetime import datetime, timezone
from urllib.request import pathname2url
import mimetypes

# upload_dir_s3.py /path/to/local/folder thebucketname /path/to/s3/folder
# ./upload_dir_s3.py test/ my-bucket static-resources/dev
# ./upload_dir_s3.py /home/resources/ my-bucket static-resources/dev

if len(sys.argv) < 4:
    print('Missing required parameters')
    sys.exit()

local_directory, bucket, destination = sys.argv[1:4]

client = boto3.client('s3')

def upload_file(local_path, bucket, s3_path):
    print('****** Uploading %s...' % s3_path)
    url = pathname2url(local_path)
    file_mimetype = mimetypes.guess_type(url)[0]
    client.upload_file(local_path, bucket, s3_path)

    # Set Content-Type
    if file_mimetype is not None:
        client.copy_object(Bucket=bucket, Key=s3_path, ContentType=file_mimetype, MetadataDirective="REPLACE", CopySource=bucket + "/" + s3_path)
    # Set Public read ACL
    client.put_object_acl(ACL='public-read', Bucket=bucket, Key=s3_path)

def delete_angularjs_files():
    # Delete angular DIST every time
    print('Deleting AngularJs files')
    angularprefix = destination + '/angulardist/'
    result = client.list_objects(Bucket=bucket, Prefix=angularprefix, Delimiter='/')
    if 'Contents' in result:
        auglarfiles = result['Contents']
    else:
        auglarfiles = []
    for o in auglarfiles:
        client.delete_object(Bucket=bucket, Key=o.get('Key'))
    print('***** Deleted AngularJs files')
    

# delete_angularjs_files()

# enumerate local files recursively
for root, dirs, files in os.walk(local_directory):

    for filename in files:

        # construct the full local path
        local_path = os.path.join(root, filename)
        # print(local_path)

        # construct the full path
        relative_path = os.path.relpath(local_path, local_directory)
        s3_path = os.path.join(destination, relative_path)

        # relative_path = os.path.relpath(os.path.join(root, filename))

        print('Searching "%s" in "%s"' % (s3_path, bucket))
        try:
            head_res = client.head_object(Bucket=bucket, Key=s3_path)
            # print('Path found on S3! Skipping %s...' % s3_path)
            # print(head_res)

            remote_last_modified = head_res['LastModified']
            local_last_modified = datetime.fromtimestamp(os.path.getmtime(local_path), tz=timezone.utc)

            # print(remote_last_modified)
            # print(local_last_modified)

            # Check last modified time
            if remote_last_modified <= local_last_modified:
                # print('Local file is modified')
                try:
                    print('Deleting on S3 : ' + s3_path)
                    client.delete_object(Bucket=bucket, Key=s3_path)
                    upload_file(local_path, bucket, s3_path)
                except:
                    print('Unable to delete %s...' % s3_path)
            # else:
            #     print('Local file not modified')
        except Exception as e:
            # print(e)
            upload_file(local_path, bucket, s3_path)

requirements.txt

boto3==1.5.23
botocore==1.8.37
docutils==0.14
jmespath==0.9.3
python-dateutil==2.6.1
s3transfer==0.1.12
six==1.11.0

Usage Example

Create virtual environment and install the requirements.txt

See boto3 document for setting up AWS S3 credentials

upload_dir_s3.py /path/to/local/folder thebucketname /path/to/s3/folder
./upload_dir_s3.py test/ my-bucket static-resources/dev
./upload_dir_s3.py /home/resources/ my-bucket static-resources/dev