django_prefetch_limit.py

#!/usr/bin/env python
# -*- coding:utf-8 -*-

# Stolen from: https://mlvin.xyz/django-single-file-project.html

import datetime
import inspect
import os
import sys

from types import ModuleType

import django

from django.conf import settings

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

# The current name of the file, which will be the name of our app
APP_LABEL, _ = os.path.splitext(os.path.basename(os.path.abspath(__file__)))

# Migrations folder need to be created, and django needs to be told where it is
APP_MIGRATION_MODULE = '%s_migrations' % APP_LABEL
APP_MIGRATION_PATH = os.path.join(BASE_DIR, APP_MIGRATION_MODULE)

# Create the folder and a __init__.py if they don't exist
if not os.path.exists(APP_MIGRATION_PATH):
    os.makedirs(APP_MIGRATION_PATH)
    open(os.path.join(APP_MIGRATION_PATH, '__init__.py'), 'w').close()

# Hack to trick Django into thinking this file is actually a package
sys.modules[APP_LABEL] = sys.modules[__name__]
sys.modules[APP_LABEL].__path__ = [os.path.abspath(__file__)]

settings.configure(
    DEBUG=True,
    ROOT_URLCONF='%s.urls' % APP_LABEL,
    MIDDLEWARE=(),
    INSTALLED_APPS=[APP_LABEL],
    MIGRATION_MODULES={APP_LABEL: APP_MIGRATION_MODULE},
    SITE_ID=1,
    DATABASES={
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
        }
    },
    LOGGING={
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'simple': {
                'format': "%(levelname)s %(message)s",
            },
        },
        'handlers': {
            'console': {
                'level': 'DEBUG',
                'class': 'logging.StreamHandler',
                'formatter': 'simple',
            }
        },
        'loggers': {
            'django.db.backends': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': False},
            'django.db.backends.schema': {'level': 'ERROR'},  # Causes sql logs to duplicate -- really annoying
        }
    },
    STATIC_URL='/static/'
)

django.setup()

from django.apps import apps  # noqa: E402 isort:skip


# Setup the AppConfig so we don't have to add the app_label to all our models
def get_containing_app_config(module):
    if module == '__main__':
        return apps.get_app_config(APP_LABEL)
    return apps._get_containing_app_config(module)


apps._get_containing_app_config = apps.get_containing_app_config
apps.get_containing_app_config = get_containing_app_config


# Your code below this line
# ##############################################################################

from django.db import models  # noqa: E402 isort:skip
from django.test import TestCase  # noqa: E402 isort:skip


class Worker(models.Model):
    name = models.CharField(max_length=128)


class Job(models.Model):
    worker = models.ForeignKey(Worker, on_delete=models.PROTECT)
    type = models.CharField(max_length=64)
    position = models.CharField(max_length=64)
    location = models.CharField(max_length=128)
    start_date = models.DateField()


urlpatterns = []


class SimpleTestCase(TestCase):
    """
    Simple test that shows your example running.
    """

    SOLUTION_EXPECTED = [
        ('A', 'east', datetime.date(2019, 5, 11)),
        ('B', 'west', datetime.date(2019, 1, 1))
    ]

    MY_SOLUTION_NUM_QUERIES = 2
    HAKEN_NUM_QUERIES = 3

    def setUp(self):
        worker_a = Worker.objects.create(name='A')
        worker_b = Worker.objects.create(name='B')

        Job.objects.create(worker=worker_a, location='east', start_date=datetime.date(2018, 1, 21))
        Job.objects.create(worker=worker_a, location='south', start_date=datetime.date(2018, 9, 13))
        Job.objects.create(worker=worker_a, location='east', start_date=datetime.date(2019, 5, 11))
        Job.objects.create(worker=worker_b, location='west', start_date=datetime.date(2019, 1, 1))

    def test_my_solution(self):
        from django.db import models  # needs to be here becuase of the hack that is having a single django file  # noqa

        actual = []
        expected = self.SOLUTION_EXPECTED

        with self.assertNumQueries(self.MY_SOLUTION_NUM_QUERIES):
            subqry = models.Subquery(Job.objects.filter(worker_id=models.OuterRef('worker_id'))
                                                .order_by('-start_date').values('id')[:1])
            workers = Worker.objects.prefetch_related(models.Prefetch('job_set',
                                                                      queryset=Job.objects.filter(id__in=subqry)))

            for worker in workers:
                latest_job = list(worker.job_set.all())[0]
                actual.append((worker.name, latest_job.location, latest_job.start_date))

        self.assertEqual(actual, expected)

    def test_haken_lid(self):  # Håken Lid
        from django.db import models  # needs to be here becuase of the hack that is having a single django file  # noqa

        actual = []
        expected = self.SOLUTION_EXPECTED

        with self.assertNumQueries(self.HAKEN_NUM_QUERIES):
            for worker in Worker.objects.all():
                latest_job = worker.job_set.latest('start_date')
                actual.append((worker.name, latest_job.location, latest_job.start_date))

        self.assertEqual(actual, expected)


class ComplexTestCase(SimpleTestCase):
    """
    Shows that if you have more and more workers the other solution will continue to grow, my solution stays the same
    with the same results
    """

    SOLUTION_EXPECTED = [
        ('A', 'south', datetime.date(2018, 2, 2)),
        ('B', 'east', datetime.date(2018, 4, 4)),
        ('C', 'north', datetime.date(2018, 6, 6)),
        ('D', 'west', datetime.date(2018, 8, 8)),
    ]

    MY_SOLUTION_NUM_QUERIES = 2
    HAKEN_NUM_QUERIES = 5

    def setUp(self):
        worker_a = Worker.objects.create(name='A')
        worker_b = Worker.objects.create(name='B')
        worker_c = Worker.objects.create(name='C')
        worker_d = Worker.objects.create(name='D')

        Job.objects.create(worker=worker_a, location='east', start_date=datetime.date(2018, 1, 1))
        Job.objects.create(worker=worker_a, location='south', start_date=datetime.date(2018, 2, 2))

        Job.objects.create(worker=worker_b, location='north', start_date=datetime.date(2018, 3, 3))
        Job.objects.create(worker=worker_b, location='east', start_date=datetime.date(2018, 4, 4))

        Job.objects.create(worker=worker_c, location='west', start_date=datetime.date(2018, 5, 5))
        Job.objects.create(worker=worker_c, location='north', start_date=datetime.date(2018, 6, 6))

        Job.objects.create(worker=worker_d, location='east', start_date=datetime.date(2018, 7, 7))
        Job.objects.create(worker=worker_d, location='west', start_date=datetime.date(2018, 8, 8))


class O_N_TestCase(SimpleTestCase):
    """
    Shows that if you have more and more workers the other solution will continue to grow, my solution stays the same
    with the same results
    """

    SOLUTION_EXPECTED = [
        ('A', 'north', datetime.date(2018, 1, 1)),
        ('B', 'north', datetime.date(2018, 1, 1)),
        ('C', 'north', datetime.date(2018, 1, 1)),
        ('D', 'north', datetime.date(2018, 1, 1)),
        ('E', 'north', datetime.date(2018, 1, 1)),
        ('F', 'north', datetime.date(2018, 1, 1)),
        ('G', 'north', datetime.date(2018, 1, 1)),
        ('H', 'north', datetime.date(2018, 1, 1)),
        ('I', 'north', datetime.date(2018, 1, 1)),
        ('J', 'north', datetime.date(2018, 1, 1)),
        ('K', 'north', datetime.date(2018, 1, 1)),
        ('L', 'north', datetime.date(2018, 1, 1)),
        ('M', 'north', datetime.date(2018, 1, 1)),
        ('N', 'north', datetime.date(2018, 1, 1)),
        ('O', 'north', datetime.date(2018, 1, 1)),
        ('P', 'north', datetime.date(2018, 1, 1)),
        ('Q', 'north', datetime.date(2018, 1, 1)),
        ('R', 'north', datetime.date(2018, 1, 1)),
        ('S', 'north', datetime.date(2018, 1, 1)),
        ('T', 'north', datetime.date(2018, 1, 1)),
        ('U', 'north', datetime.date(2018, 1, 1)),
        ('V', 'north', datetime.date(2018, 1, 1)),
        ('W', 'north', datetime.date(2018, 1, 1)),
        ('X', 'north', datetime.date(2018, 1, 1)),
        ('Y', 'north', datetime.date(2018, 1, 1)),
        ('Z', 'north', datetime.date(2018, 1, 1)),
    ]

    MY_SOLUTION_NUM_QUERIES = 2
    HAKEN_NUM_QUERIES = 27

    def setUp(self):
        import string
        self.maxDiff = None

        for letter in string.ascii_uppercase:
            worker = Worker.objects.create(name=letter)
            Job.objects.create(worker=worker, location='north', start_date=datetime.date(2018, 1, 1))


# Your code above this line
# ##############################################################################


# Used so you can do 'from <name of file>.models import *'
models_module = ModuleType('%s.models' % (APP_LABEL))
tests_module = ModuleType('%s.tests' % (APP_LABEL))
urls_module = ModuleType('%s.urls' % (APP_LABEL))
urls_module.urlpatterns = urlpatterns
for variable_name, value in list(locals().items()):
    # We are only interested in models
    if inspect.isclass(value) and issubclass(value, models.Model):
        setattr(models_module, variable_name, value)
    # We are only interested in tests
    if inspect.isclass(value) and issubclass(value, TestCase):
        setattr(tests_module, variable_name, value)

# Setup the fake modules
sys.modules[models_module.__name__] = models_module
sys.modules[tests_module.__name__] = tests_module
sys.modules[urls_module.__name__] = urls_module
sys.modules[APP_LABEL].models = models_module
sys.modules[APP_LABEL].tests = tests_module
sys.modules[APP_LABEL].urls = urls_module


if __name__ == "__main__":
    # Hack to fix tests
    argv = [arg for arg in sys.argv if not arg.startswith('-')]
    if len(argv) == 2 and argv[1] == 'test':
        sys.argv.append(APP_LABEL)

    from django.core.management import execute_from_command_line
    execute_from_command_line(sys.argv)
else:
    from django.core.wsgi import get_wsgi_application
    get_wsgi_application()

In order to run the code above run the following:

curl -L https://gist.githubusercontent.com/kingbuzzman/ac2ada9c27196fc90c1b75f2d01a6271/raw/django_prefetch_limit.py > django_prefetch_limit.py
pip install django==2.1 
python django_prefetch_limit.py makemigrations
python django_prefetch_limit.py test

If you want the REAL copy and paste version:

docker run -it --rm python:3.7 bash -c '
curl -L https://gist.githubusercontent.com/kingbuzzman/ac2ada9c27196fc90c1b75f2d01a6271/raw/django_prefetch_limit.py > django_prefetch_limit.py
pip install django==2.1 
python django_prefetch_limit.py makemigrations
python django_prefetch_limit.py test
'

Answers stackoverflow question: https://stackoverflow.com/questions/57169445/django-reverse-foreign-key/57170596