Django management command to delete old celery tasks from database using batches of adaptive size
import time | |
from django.core.management.base import BaseCommand | |
from django.db.models import Max, Min | |
from django.db.utils import ProgrammingError | |
from djcelery.models import TaskMeta | |
class Command(BaseCommand): | |
help = 'Delete old celery tasks' | |
def handle(self, *args, **options): | |
result = TaskMeta.objects.aggregate(Max('id'), Min('id')) | |
id_min, id_max = result['id__min'], result['id__max'] | |
self.stdout.write( | |
"Total: %r, ranging from %r to %r" | |
% (TaskMeta.objects.count(), id_min, id_max) | |
) | |
if id_min is None or id_max is None: | |
self.stdout.write("Nothing to do, exiting") | |
return | |
id_max -= 100 | |
self.stdout.write("Decreased finish down to %d" % id_max) | |
step = 100 | |
start = id_min | |
while start < id_max: | |
finish = start + step | |
try: | |
deleted, _ = TaskMeta.objects.filter(id__gte=start, id__lt=finish).delete() | |
self.stdout.write( | |
"For range %d-%d deleted %d objects" | |
% (start, finish, deleted) | |
) | |
step = int(step * 1.05) | |
start = finish | |
except ProgrammingError as exc: | |
self.stderr.write(str(exc)) | |
time.sleep(60) | |
step = max(10, step / 2) |
This comment has been minimized.
This comment has been minimized.
I made a short write-up about this management command at https://peterdemin.github.io/cleanup_celery.html |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
invoke with