mediasync-style storage backend using django-storages

mediasync.py

import base64
import hashlib
import mimetypes
import os
import threading

from boto.s3.key import Key
from django.conf import settings
from django.core.files.storage import get_storage_class
from storages.backends.s3boto import S3BotoStorage

STATIC_LOCAL = getattr(settings, 'STATIC_LOCAL', settings.DEBUG)

_threadlocals = threading.local()

def checksum(data):
    pos = data.tell()
    data.seek(0)
    checksum = hashlib.md5(data.read())
    hexdigest = checksum.hexdigest()
    b64digest = base64.b64encode(checksum.digest())
    data.seek(pos)
    return (hexdigest, b64digest)

class MediaSyncMiddleware(object):

    def process_request(self, request):
        enc = request.META.get('HTTP_ACCEPT_ENCODING', '')
        _threadlocals.client_supports_gzip = 'gzip' in enc

class MediaSyncStorage(S3BotoStorage):

    def _get_key(self, name):
        k = self.bucket.get_key(self._encode_name(name))
        if not k:
            k = self.bucket.new_key(self._encode_name(name))
        return k

    def _save(self, name, content):
    
        cleaned_name = self._clean_name(name)
        name = self._normalize_name(cleaned_name)
        content.name = cleaned_name

        content_type = getattr(content, 'content_type', mimetypes.guess_type(name)[0] or Key.DefaultContentType)

        (hexdigest, b64digest) = checksum(content)

        headers = self.headers.copy()
        headers['Content-Type'] = content_type

        # sync normal file

        k = self._get_key(name)

        s3digest = (k.get_metadata('mediasync-checksum') or '').replace(' ', '+')

        print s3digest, ':',  b64digest, ':', s3digest == b64digest

        if s3digest != b64digest:

            k.set_metadata('mediasync-checksum', b64digest)
            k.set_contents_from_file(content, headers=headers, policy=self.acl,
                reduced_redundancy=self.reduced_redundancy,
                md5=(hexdigest, b64digest))

            # sync gzipped file

            if self.gzip and content_type in self.gzip_content_types:

                name = '%s.gz' % name

                k = self._get_key(name)

                content = self._compress_content(content)
                headers['Content-Encoding'] = 'gzip'
                headers["Content-Disposition"] = 'inline; filename="%sgz"' % name.split('/')[-1]

                k.set_contents_from_file(content, headers=headers, policy=self.acl,
                    reduced_redundancy=self.reduced_redundancy)

        return cleaned_name

    def url(self, name):

        if STATIC_LOCAL:
            return os.path.join(settings.STATIC_URL, name)

        name = self._normalize_name(self._clean_name(name))
        content_type = mimetypes.guess_type(name)[0] or Key.DefaultContentType

        if self.gzip and content_type in self.gzip_content_types:
            if _threadlocals.client_supports_gzip:
                name = '%s.gz' % name

        if self.custom_domain:
            return "%s://%s/%s" % ('https' if self.secure_urls else 'http', self.custom_domain, name)
        else:
            return self.connection.generate_url(self.querystring_expire, method='GET', \
                bucket=self.bucket.name, key=self._encode_name(name), query_auth=self.querystring_auth, \
                force_http=not self.secure_urls)

class CachedMediaSyncStorage(MediaSyncStorage):
    """
    S3 storage backend that saves the files locally, too.
    """
    def __init__(self, *args, **kwargs):
        super(CachedMediaSyncStorage, self).__init__(*args, **kwargs)
        self.local_storage = get_storage_class(
            "compressor.storage.CompressorFileStorage")()

    def save(self, name, content):
        self.local_storage._save(name, content)
        name = super(CachedMediaSyncStorage, self).save(name, content)
        return name

Hmmm....

Hmmmm?

Hmmm!

Am I correct in assuming this is in reference to the mixed whitespace and/or use of thread-local?

No, looks useful.

Whew!