List duplicate celery tasks in redis queues

gistfile1.py

""" List duplicate celery tasks in redis queues """
import datetime
import redis
import json
import base64
import pickle
from collections import Counter
import argparse
import sys


parser = argparse.ArgumentParser(description='List duplicate celery tasks in redis queues')
parser.add_argument('queue', metavar='queue', type=str, help='celery queue to inspect')
parser.add_argument('--host', dest='host', type=str, default='localhost', help='redis host')
parser.add_argument('--port', dest='port', type=int, default=6379, help='redis port')
parser.add_argument('--limit', dest='limit', type=int, default=None, help='limit to top n most duplicated')

queue, host, limit, port = tuple(vars(parser.parse_args()).values())

redis = redis.StrictRedis(host=host, port=port)
tasks = []
queue_len = redis.llen(queue)

for x in redis.lrange(queue, 0, -1):
    j = json.loads(x)
    decoded = base64.b64decode(j['body'])
    depickled = pickle.loads(decoded)
    joined = depickled['task'] + ' ' + str(depickled['args']) + str(depickled['kwargs'])
    tasks.append(joined)

tots = Counter(tasks)
num_dupes = sum([x for x in tots.values() if x > 1])

for task, count in [x for x in tots.most_common(limit) if x[1] > 1]:
    print '%d\t%s' % (count, task)
print
percent = '{0:.2%}'.format(float(num_dupes) / float(queue_len)) if num_dupes > 0 else '0%'
print '\'%s\' queue contains %d tasks and %d of them (%s) have duplicates' % (queue, queue_len, num_dupes, percent)