Skip to content

Instantly share code, notes, and snippets.

@nicolov
Last active October 14, 2015 02:05
Show Gist options
  • Save nicolov/7d993cc12203e7b81e08 to your computer and use it in GitHub Desktop.
Save nicolov/7d993cc12203e7b81e08 to your computer and use it in GitHub Desktop.
Domain sharding on Django for media files
# put storage.py in one off your apps or create a new one
DEFAULT_FILE_STORAGE = 'yourapp.storage.DomainShardingStorage'
# for easy_thumbnails
THUMBNAIL_DEFAULT_STORAGE = DEFAULT_FILE_STORAGE
SHARDED_MEDIA_DOMAINS_NUMBER = 2
SHARDED_MEDIA_URL_TEMPLATE = '/media/' if DEBUG else 'http://s{0}.afancycdn.com/media/'
# Domain sharding on Django for media files
# credits to: https://github.com/coagulant/django-webperf
import hashlib, os
from django.utils.six.moves.urllib.parse import urljoin
from django.core.files.storage import FileSystemStorage
from django.utils.encoding import filepath_to_uri
from django.utils._os import abspathu
from django.conf import settings
class DomainShardingStorage(FileSystemStorage):
"""
Standard filesystem storage with domain sharding for urls
"""
def __init__(self, base_url=None, **kwargs):
FileSystemStorage.__init__(self, base_url, **kwargs)
if base_url is None:
base_url = settings.SHARDED_MEDIA_URL_TEMPLATE
elif not base_url.endswith('/'):
base_url += '/'
self.base_url = base_url
def get_domain_number_for_uri(self, uri):
md5 = hashlib.md5()
md5.update(uri)
integer_hash = int(md5.hexdigest(), 16)
domain_number = integer_hash % settings.SHARDED_MEDIA_DOMAINS_NUMBER + 1
return domain_number
def url(self, name):
if self.base_url is None:
raise ValueError("This file is not accessible via a URL.")
filepath = filepath_to_uri(name)
domain_number = self.get_domain_number_for_uri(filepath)
sharded_base_url = self.base_url.format(domain_number)
return urljoin(sharded_base_url, filepath)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment