-
-
Save noviluni/d86adfa24843c7b8ed10c183a9df2afe to your computer and use it in GitHub Desktop.
Large Table Paginator for Django: Scale Django admin pages and avoid timeouts.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from django.contrib.admin import ModelAdmin | |
from .paginator import LargeTablePaginator | |
class MyTableAdmin(ModelAdmin): | |
... | |
paginator = LargeTablePaginator | |
show_full_result_count = False # Recommended to avoid another count() | |
... | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from django.core.paginator import Paginator | |
from django.db import connection, transaction, OperationalError | |
from django.utils.functional import cached_property | |
class LargeTablePaginator(Paginator): | |
""" | |
Combination of ideas from: | |
- https://gist.github.com/safar/3bbf96678f3e479b6cb683083d35cb4d | |
- https://medium.com/@hakibenita/optimizing-django-admin-paginator-53c4eb6bfca3 | |
Overrides the count method of QuerySet objects to avoid timeouts. | |
- Try to get the real count limiting the queryset execution time to 150 ms. | |
- If count takes longer than 150 ms the database kills the query and raises OperationError. In that case, | |
get an estimate instead of actual count when not filtered (this estimate can be stale and hence not fit for | |
situations where the count of objects actually matter). | |
- If any other exception occured fall back to default behaviour. | |
""" | |
@cached_property | |
def count(self): | |
""" | |
Returns an estimated number of objects, across all pages. | |
""" | |
try: | |
with transaction.atomic(), connection.cursor() as cursor: | |
# Limit to 150 ms | |
cursor.execute('SET LOCAL statement_timeout TO 5;') | |
return super().count | |
except OperationalError: | |
pass | |
if not self.object_list.query.where: | |
try: | |
with transaction.atomic(), connection.cursor() as cursor: | |
# Obtain estimated values (only valid with PostgreSQL) | |
cursor.execute( | |
"SELECT reltuples FROM pg_class WHERE relname = %s", | |
[self.object_list.query.model._meta.db_table] | |
) | |
estimate = int(cursor.fetchone()[0]) | |
return estimate | |
except Exception: | |
# If any other exception occurred fall back to default behaviour | |
pass | |
return super().count |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment