boto3 aws s3 sync

s3_folder_download_upload.py

import boto3

def pull_s3_prefix(dst_dir, bucket, prefix):
    client = boto3.client('s3')
    resource = boto3.resource('s3')
    download_dir(client, resource, prefix, prefix, dst_dir, bucket)

def download_dir(client, resource, prefix, start_prefix, local, bucket ):
    paginator = client.get_paginator('list_objects')
    for result in paginator.paginate(Bucket=bucket, Delimiter='/', Prefix=prefix):
        if result.get('CommonPrefixes') is not None:
            for subdir in result.get('CommonPrefixes'):
                download_dir(client, resource, subdir.get('Prefix'), start_prefix, local, bucket)
        if result.get('Contents') is not None:
            for file in result.get('Contents'):
                key_relative = file.get('Key').replace(start_prefix, '')
                if not os.path.exists(os.path.dirname(local + os.sep + key_relative)):
                  os.makedirs(os.path.dirname(local + os.sep + key_relative))
                local_path = local + os.sep + key_relative
                s3_path = file.get('Key')
                print(f"Downloading {s3_path} -> {local_path}")
                resource.meta.client.download_file(bucket, s3_path, local_path)


def upload_dir_s3(source_dir, dst_bucket, dst_prefix=''):
    client = boto3.client('s3')

    # enumerate local files recursively
    for root, dirs, files in os.walk(source_dir):

      for filename in files:

        # construct the full local path
        local_path = os.path.join(root, filename)

        # construct the full Dropbox path
        relative_path = os.path.relpath(local_path, source_dir)
        s3_path = os.path.join(dst_prefix, relative_path)
        # s3_path = "{dst_prefix}{s3_path}"
        # relative_path = os.path.relpath(os.path.join(root, filename))

        try:
            print("Uploading %s..." % s3_path)
            client.upload_file(local_path, dst_bucket, s3_path)

            # try:
                # client.delete_object(Bucket=bucket, Key=s3_path)
            # except:
                # print "Unable to delete %s..." % s3_path
        except Exception as e:
            logger.exception(f"Failed to upload {local_path} to {s3_path}")