Skip to content

Instantly share code, notes, and snippets.

@noviluni
Forked from safar/admin.py
Last active March 7, 2024 16:18
Show Gist options
  • Save noviluni/d86adfa24843c7b8ed10c183a9df2afe to your computer and use it in GitHub Desktop.
Save noviluni/d86adfa24843c7b8ed10c183a9df2afe to your computer and use it in GitHub Desktop.
Large Table Paginator for Django: Scale Django admin pages and avoid timeouts.
from django.contrib.admin import ModelAdmin
from .paginator import LargeTablePaginator
class MyTableAdmin(ModelAdmin):
...
paginator = LargeTablePaginator
show_full_result_count = False # Recommended to avoid another count()
...
from django.core.paginator import Paginator
from django.db import connection, transaction, OperationalError
from django.utils.functional import cached_property
class LargeTablePaginator(Paginator):
"""
Combination of ideas from:
- https://gist.github.com/safar/3bbf96678f3e479b6cb683083d35cb4d
- https://medium.com/@hakibenita/optimizing-django-admin-paginator-53c4eb6bfca3
Overrides the count method of QuerySet objects to avoid timeouts.
- Try to get the real count limiting the queryset execution time to 150 ms.
- If count takes longer than 150 ms the database kills the query and raises OperationError. In that case,
get an estimate instead of actual count when not filtered (this estimate can be stale and hence not fit for
situations where the count of objects actually matter).
- If any other exception occured fall back to default behaviour.
"""
@cached_property
def count(self):
"""
Returns an estimated number of objects, across all pages.
"""
try:
with transaction.atomic(), connection.cursor() as cursor:
# Limit to 150 ms
cursor.execute('SET LOCAL statement_timeout TO 5;')
return super().count
except OperationalError:
pass
if not self.object_list.query.where:
try:
with transaction.atomic(), connection.cursor() as cursor:
# Obtain estimated values (only valid with PostgreSQL)
cursor.execute(
"SELECT reltuples FROM pg_class WHERE relname = %s",
[self.object_list.query.model._meta.db_table]
)
estimate = int(cursor.fetchone()[0])
return estimate
except Exception:
# If any other exception occurred fall back to default behaviour
pass
return super().count
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment