Python S3 bucket download

__init__.py

import boto3, errno, os

def mkdir_p(path):
    # mkdir -p functionality from https://stackoverflow.com/a/600612/2448314
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get_s3_path_filename(key):
    key = str(key)
    return key.replace(key.split('/')[-1],""),  key.split('/')[-1]

def download_s3_bucket(bucket_name, local_folder, aws_user_with_s3_access):
    session = boto3.Session(profile_name=aws_user_with_s3_access)
    s3_client = session.resource('s3')
    s3_bucket = s3_client.Bucket(bucket_name)
    for obj in s3_bucket.objects.all():
        s3_path, s3_filename = get_s3_path_filename(obj.key)
        local_folder_path = os.path.join(*[os.curdir,local_folder, s3_path])
        local_fullpath = os.path.join(*[local_folder_path, s3_filename])
        mkdir_p(local_folder_path)
        s3_bucket.download_file(obj.key, local_fullpath)

README.md

Usage

1. Install boto3
2. Create IAM user with a similar policy

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads",
                "s3:ListMultipartUploadParts",
                "s3:GetObject",
                "s3:GetBucketLocation",
            ],
            "Resource": [
                "arn:aws:s3:::your_bucket_name"
            ]
        }
    ]
}

3. Create a profile in ~/.aws/credentials with access details of this IAM user as explained in boto documentation

download_s3_bucket(bucket_name = your_bucket_name, local_folder = "/tmp/s3_bucket", aws_user_with_s3_access = profile_name)