Script for monitoring celery worker usage, for https://github.com/gem/oq-engine

cluster_status.py

#!/usr/bin/env python
import sys
from celery.task.control import inspect

def ping_celery():
    ins = inspect()

    # Count the number of total worker processes
    total_workers = 0
    # ... and active tasks
    num_active_tasks = 0

    all_stats = ins.stats()
    if all_stats is None:
       print "No active workers"
       sys.exit(0)

    hostnames = []

    for hostname, stats in all_stats.items():
        num_procs = len(stats['pool']['processes'])
        total_workers += num_procs
        hostnames.append(hostname)



    # Now ping the workers
    # and show individual stats:
    ping = ins.ping()
    active = ins.active()

    for host in hostnames:
        print '=========='
        print 'Host: %s' % host
        if ping[host] == 'pong':
            print 'Status: Online'
        else:
            print 'Status: Not Responding'
        print 'Worker processes: %s' % len(
            all_stats[host]['pool']['processes'])

        worker_activity = active.get(host)
        if worker_activity is not None:
            print 'Active tasks: %s' % len(worker_activity)
            num_active_tasks += len(worker_activity)


    print '=========='
    print
    print 'Total workers:       %s' % total_workers
    print 'Active tasks:        %s' % num_active_tasks
    print 'Cluster utilization: %.2f%%' % (
        (float(num_active_tasks) / total_workers) * 100
    )


if __name__ == "__main__":
    ping_celery()