Skip to content

Instantly share code, notes, and snippets.

@taylorhughes
Last active December 14, 2015 14:59
Show Gist options
  • Save taylorhughes/5104743 to your computer and use it in GitHub Desktop.
Save taylorhughes/5104743 to your computer and use it in GitHub Desktop.
import os.path
import json
import logging
from django.conf import settings
from django.contrib.staticfiles.storage import CachedFilesMixin
from storages.backends.s3boto import S3BotoStorage
INDEX_FILENAME = os.path.join(os.path.dirname(__file__), 'static-index.json')
class S3HashedFilesStorage(CachedFilesMixin, S3BotoStorage):
"""
A bit of a hack to prevent S3BotoStorage + cached files from actually
requesting files from S3 when a new process starts up -- which can take
several seconds.
Instead, during `collectstatic` (in post_process) we build a JSON file
to use as a map from static asset name -> final serving URL (on S3).
This file is saved and deployed with the app. When the process starts,
it reads that file once and keeps it in memory.
(Note that memcache is not a good solution for this case -- we want
each worker to serve the copy of JS/CSS/etc. that goes with the
HTML it is serving, rather than serving what is freshest in memcache.
Also making a memcache hit for every static resource seems silly.)
"""
_index_dict = None
_building_index = False
def url_from_index(self, name):
if self._index_dict is None:
try:
with open(INDEX_FILENAME, 'r') as index_file:
self._index_dict = json.load(index_file)
logging.info('Loaded static index file (%d items)', len(self._index_dict))
except IOError:
self._index_dict = {}
if not settings.DEBUG:
logging.warn('Failed to load index file.')
url = self._index_dict.get(name)
if not url and not settings.DEBUG:
logging.warn('Could not find static file: %s', name)
return url
def url(self, name, force=False):
url = None
if not self._building_index:
url = self.url_from_index(name)
if not url or force:
url = super(S3HashedFilesStorage, self).url(name, force=force)
return url
def post_process(self, paths, dry_run=False, **options):
result = super(S3HashedFilesStorage, self).post_process(paths, dry_run=dry_run, **options)
# During a dry run, don't actually build the JSON index. This makes a
# lot of sense, but one extra reason is that files aren't actually copied
# to S3 during a dry run so S3BotoStorage won't include them yet and then
# the below (self.url(path)) explodes.
if not dry_run:
# Prevent url() from trying to read the existing json file, which we
# are right in the middle of building.
self._building_index = True
paths_to_url = {}
for path in paths.keys():
paths_to_url[path] = self.url(path)
with open(INDEX_FILENAME, 'w+') as index_file:
json.dump(paths_to_url, index_file)
self._building_index = False
return result
@taylorhughes
Copy link
Author

Updated to fix an issue with new files + dry run.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment