Benchmark the Pulp Tasking System

benchmark_tasking.py

import uuid
import os
import time

from datetime import datetime


os.environ["DJANGO_SETTINGS_MODULE"] = "pulpcore.app.settings"
import django
django.setup()


from pulpcore.app.tasks.benchmark import do_nothing
from pulpcore.plugin.tasking import enqueue_with_reservation
from pulpcore.plugin.models import Task

def dispatch(num_tasks):
    async_results = []
    for i in range(num_tasks):
        unique_lock = str(uuid.uuid4())
        async_results.append(enqueue_with_reservation(do_nothing, [unique_lock]))
    return async_results


def wait(async_result_ids):
    while Task.objects.filter(pk__in=async_result_ids).exclude(state='completed').count():
        time.sleep(0.5)


def compute_throughput(async_result_ids):
    tasks = Task.objects.filter(pk__in=async_result_ids)
    first_task_created_time = tasks.order_by('pulp_created').first().pulp_created
    last_task_completed_time = tasks.order_by('-finished_at').first().finished_at
    return (last_task_completed_time - first_task_created_time).total_seconds()


def main():
    num_tasks = 500
    dispatch_start = datetime.now()
    async_results = dispatch(num_tasks)
    sec_duration = (datetime.now() - dispatch_start).total_seconds()

    print(f'dispatched\t{num_tasks} in {sec_duration:.3f} seconds\t{(num_tasks / sec_duration):.3f} task/sec')

    async_result_ids = [result.id for result in async_results]
    wait(async_result_ids)

    total_seconds = compute_throughput(async_result_ids)
    print(f'completed\t{num_tasks} in {total_seconds:.3f} seconds\t{(num_tasks / total_seconds):.3f} task/sec')


if __name__ == "__main__":
    main()