Buffered, file-like class to wrap amazon S3 / boto multipart upload

s3_multipart.py

from StringIO import StringIO
import boto.s3
from django.conf import settings

class MultiPartUploader:
    counter = 0

    def __init__(self, bucket_name, key_name, buf_size=32768):
        self.connection = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
        self.bucket = self.connection.get_bucket(bucket_name)
        self.mp = self.bucket.initiate_multipart_upload(key_name)
        self.buf_size = buf_size
        self.buffer = StringIO()


    def write(self,s):
        self.buffer.write(s)
        if self.buffer.len > self.buf_size:
            self.flush()

    def flush(self):
        if self.buffer.len:
            self.counter+=1
            self.mp.upload_part_from_file(self.buffer, self.counter)
            self.buffer.close()
            self.buffer = StringIO()

    def close(self):
        self.mp.flush()
        self.mp.complete_upload()

Example usage is:

from my_app.models import MyModel
from django.core import serializers
fd = MultiPartUploader('my_bucket','my_models.xml')
serializers.get_serializer('xml')().serialize(MyModel.objects.all(), stream=fd)
fd.close()

fd.close() is critical and has to happen, or the upload will need to manually be set to 'complete'