Boto3 S3 helper

S3Storage.md

Following example class shows how to use boto3 to upload files to s3 using a programmable configuration

from uuid import uuid1

import boto3
from botocore.client import Config
from boto3.session import Session

class S3Storage(object):
    def __init__(self,
                 s3_host=AWS_S3_HOST,
                 s3_bucket=AWS_S3_BUCKET,
                 aws_access_key_id=AWS_ACCESS_KEY_ID,
                 aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                 aws_region=AWS_REGION):
        self.host = s3_host
        self.bucket_name = s3_bucket
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.aws_region = aws_region
        self._s3_resource = None
        self._s3_client = None

    @property
    def s3_resource(self):
        """
        Provide v4 Session for S3
        """
        self._s3_resource = self._s3_resource or Session(aws_access_key_id=self.aws_access_key_id,
                                                         aws_secret_access_key=self.aws_secret_access_key,
                                                         region_name=self.region).resource('s3', config=Config(
            signature_version='s3v4'))
        return self._s3_resource

    @property
    def s3_client(self):
        """
        Provides access to v4 Client for s3
        """
        self._s3_client = self._s3_client or boto3.client('s3',
                                                          config=Config(signature_version='s3v4'))
        return self._s3_client

    @property
    def bucket(self):
        """
        Provides access to the s3 Bucket object
        """
        return self.s3_resource.Bucket(self.bucket_name)

    def write(self, df, filename):
        """
        Uploads a pandas dataframe as a csv file to S3. Overrides existing. Makes the file public. Returns link to the file.
        """
        bucket = self.bucket
        tmpfile = "/tmp/{}.tmp".format(uuid1().hex)
        df.to_csv(tmpfile, encoding='utf-8', index=False)
        obj = bucket.Object(filename)
        obj.upload_file(tmpfile)
        acl = self.s3_resource.ObjectAcl(self.bucket_name, filename)
        acl.load()
        acl.put(ACL='public-read')
        return self.get_link(feedtype)

    def get_link(self, filename):
        """
        Returns presigned url for the file in the bucket.
        """
        return self.s3_client.generate_presigned_url(
            ClientMethod='get_object',
            Params={
                'Bucket': self.bucket_name,
                'Key': filename
            }
        )