Whitenoise extension to speed application startup

base.py

from whitenoise.middleware import WhiteNoiseMiddleware


class WhiteNoiseMiddlewareFileLoader(WhiteNoiseMiddleware):
    """
    Implements static file discovery method that parallels logic from parent class.
    Duplicated here to:
    * Enable Storage class to generate a pickle file to cache Whitenoise metadata
    * Avoid circular imports
    """

    def load_files_from_dirs(self, ignore_autorefresh=False):
        """
        Rebuilld self.files from a clean slate.
        Intended to match corresponding behavior of parent class __init__ method.
        """
        self.files = {}
        if self.static_root:
            self.add_files(self.static_root, prefix=self.static_prefix)
        if self.root:
            self.add_files(self.root)
        if self.use_finders and (ignore_autorefresh or not self.autorefresh):
            self.add_files_from_finders()

middleware.py

import warnings

from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage

from whitenoise.base import WhiteNoise

from .base import WhiteNoiseMiddlewareFileLoader


class PickledWhiteNoiseMiddleware(WhiteNoiseMiddlewareFileLoader):
    """
    Allows stock Whitenoise to load static files data from a pickle file generated
    during collectstatic to save time during production initialization.
    """

    def __init__(self, get_response=None, settings=settings):
        """
        Only intended difference with base class is pickle handling when
        staticfiles_storage implements a `load_whitenoise_files_from_pickle()` method.
        """
        self.get_response = get_response
        self.configure_from_settings(settings)
        # Pass None for `application`
        WhiteNoise.__init__(self, None)
        if not self.autorefresh:
            try:
                self.files = staticfiles_storage.load_whitenoise_files_from_pickle()
            except AttributeError:
                warnings.warn("Use pickled Whitenoise storage class for faster load time")
        if not self.files:
            self.load_files_from_dirs()

storage.py

import pickle
import time

from django.contrib.staticfiles.storage import ManifestFilesMixin, StaticFilesStorage
from django.core.files.base import ContentFile

from whitenoise.storage import CompressedManifestStaticFilesStorage

from .base import WhiteNoiseMiddlewareFileLoader


class PickledWhitenoiseFilesMixin(ManifestFilesMixin):
    """
    Storage class mixin intended for use with Whitenoise that:
    * Generates a pickle file during `collectstatic` to cache Whitenoise metadata
    * Enables Whitenoise middleware to load metadata from the pickle for faster startup

    Pickle is used because the Whitenoise metadata includes complex data types that
    can't be directly serialized by standard JSON.
    """

    pickle_path = "staticfiles_whitenoise.pkl"
    pickle_protocol = None  # use pickle default
    pickle_version = "1.0"  # pickle format standard

    def post_process(self, *args, **kwargs):
        yield from super().post_process(*args, **kwargs)
        if not kwargs.get("dry_run"):
            self.generate_pickle()

    def generate_pickle(self):
        """
        Write out a full current files snapshot to the pickle file.
        """
        start_time = time.time()
        self.whitenoise_pickler = WhiteNoiseMiddlewareFileLoader()
        self.whitenoise_pickler.load_files_from_dirs(ignore_autorefresh=True)
        self.dump_pickle()
        end_time = time.time()
        print(
            "Generated Whitenoise pickle file with "
            f"{len(self.whitenoise_pickler.files):,} entries "
            f"in {end_time - start_time:.1f} seconds"
        )

    def dump_pickle(self):
        payload = {
            "files": self.whitenoise_pickler.files,
            "version": self.pickle_version,
        }
        pickle_data = pickle.dumps(payload, protocol=self.pickle_protocol)
        if self.exists(self.pickle_path):
            self.delete(self.pickle_path)
        self._save(self.pickle_path, ContentFile(pickle_data))

    def read_pickle(self):
        # Paraphrased from stock ManifestFilesMixin.read_manifest()
        try:
            with self.open(self.pickle_path) as pickle_file:
                return pickle_file.read()
        except IOError:
            return None

    def load_whitenoise_files_from_pickle(self):
        """
        A hook for middleware to load the Whitenoise file data from the pickle.
        """
        # Paraphrased from ManifestFilesMixin.load_manifest()
        pickle_data = self.read_pickle()
        if pickle_data is None:
            return {}
        try:
            stored = pickle.loads(pickle_data)
            version = stored.get("version")
            if version == "1.0":
                return stored.get("files", {})
        except pickle.UnpicklingError:
            pass
        raise ValueError(
            "Couldn't load pickle '%s' (version %s)"
            % (self.pickle_path, self.pickle_version)
        )


class PickledManifestStaticFilesStorage(
    PickledWhitenoiseFilesMixin, StaticFilesStorage
):
    """
    A static file system storage backend which also saves:
    * hashed copies of the files it saves
    * a manifest to correlate base URLs with current hashed versions
    * a pickle file with Whitenoise metadata for faster application start-up
    """

    pass


class PickledCompressedManifestStaticFilesStorage(
    PickledWhitenoiseFilesMixin, CompressedManifestStaticFilesStorage
):
    """
    A static file system storage backend which also saves:
    * compressed and hashed copies of the files it saves
    * a manifest to correlate base URLs with current hashed versions
    * a pickle file with Whitenoise metadata for faster application start-up

    Optionally, deletes the non-hashed files (i.e. those without the hash in their name)
    """

    pass