Part of the blog post - Celery - Groups & Loops which explore a few questions I had with Celery and parallelism on it using groups and implementing the same feature using blocking for loop and non-blocking for loop - https://adikrishnan.in/2018/05/30/celery-groups-loops-parallelism/

run.py

# Refer https://adikrishnan.in/2018/05/30/celery-groups-loops-parallelism/ for understanding the concept.
from tasks import run_ping, hello, loop_service, group_service

hello.delay()
loop_service.delay()
group_service.delay()

tasks.py

# Refer https://adikrishnan.in/2018/05/30/celery-groups-loops-parallelism/ for understanding the concept.
import time
import datetime
import subprocess
from celery import Celery, group

conf = {
    "broker_url": "redis://localhost:6379/7",
    "result_backend": "redis://localhost:6379/7",
    "result_expires": "600"
}

celery_app = Celery()
celery_app.config_from_object(conf)

NUM_OF_ITEMS = 5


@celery_app.task
def hello():
    print("Bello Merld at {}".format(datetime.datetime.now().isoformat()))


@celery_app.task
def run_ping(item_num, task_type):
    start = time.time()
    cmd = "ping adikrishnan.in -c 4"
    subprocess.check_output(cmd.split(), universal_newlines=True)
    end = time.time()
    print("Item Number: {}, Type: {}, Total time: {:.2f} sec".format(item_num, task_type, end - start))


@celery_app.task
def loop_service():
    start = time.time()
    for item in range(NUM_OF_ITEMS):
        run_ping.delay(item, "Loop")
        # run_ping("item, "Loop")
    end = time.time()
    print("{} ended, Total time: {:.2f} sec".format(loop_service.__name__, end - start))


@celery_app.task
def group_service():
    start = time.time()
    group(run_ping.s(item, "group") for item in range(NUM_OF_ITEMS))()
    end = time.time()
    print("{} ended, Total time: {:.2f} sec".format(group_service.__name__, end - start))