Part of the blog post - Celery - Groups & Loops which explore a few questions I had with Celery and parallelism on it using groups and implementing the same feature using blocking for loop and non-blocking for loop - https://adikrishnan.in/2018/05/30/celery-groups-loops-parallelism/
# Refer https://adikrishnan.in/2018/05/30/celery-groups-loops-parallelism/ for understanding the concept. | |
import time | |
import datetime | |
import subprocess | |
from celery import Celery, group | |
conf = { | |
"broker_url": "redis://localhost:6379/7", | |
"result_backend": "redis://localhost:6379/7", | |
"result_expires": "600" | |
} | |
celery_app = Celery() | |
celery_app.config_from_object(conf) | |
NUM_OF_ITEMS = 5 | |
@celery_app.task | |
def hello(): | |
print("Bello Merld at {}".format(datetime.datetime.now().isoformat())) | |
@celery_app.task | |
def run_ping(item_num, task_type): | |
start = time.time() | |
cmd = "ping adikrishnan.in -c 4" | |
subprocess.check_output(cmd.split(), universal_newlines=True) | |
end = time.time() | |
print("Item Number: {}, Type: {}, Total time: {:.2f} sec".format(item_num, task_type, end - start)) | |
@celery_app.task | |
def loop_service(): | |
start = time.time() | |
for item in range(NUM_OF_ITEMS): | |
run_ping.delay(item, "Loop") | |
# run_ping("item, "Loop") | |
end = time.time() | |
print("{} ended, Total time: {:.2f} sec".format(loop_service.__name__, end - start)) | |
@celery_app.task | |
def group_service(): | |
start = time.time() | |
group(run_ping.s(item, "group") for item in range(NUM_OF_ITEMS))() | |
end = time.time() | |
print("{} ended, Total time: {:.2f} sec".format(group_service.__name__, end - start)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment