Setting up Celery & Redis to a Django Project.

__init__.py

"""
This file will be added in the main Django Project Folder which already contains settings.py
"""

from .celery import app as celery_app


__all__ = ("celery_app",) 

README.md

Adding Redis and Celery to a Django Project

Terminal 1

Run standalone docker container for Redis. 
cd to <path-to-repo>/redis
Build - sudo docker build -t redis .
Run - sudo docker run -p 6379:6379 --log-opt max-size=10m redis

Terminal 2

Run the celery command

celery --app=djangoAppName worker --loglevel=info &

celery-exit.sh

echo "
Execute the below command if this fails to start
> chmod +x ./celery-exit.sh
"

echo "Exiting Flower"
lsof -ti tcp:5555 | xargs kill -9

echo "Exiting Celery"
pkill -f celery

celery-start.sh

echo "
Execute the below command if this fails to start
> chmod +x ./celery-start.sh
"
echo "Redis must be running before running this script, command is below.
> sudo docker run -p 6379:6379 --log-opt max-size=10m redis
"

echo "Starting Celery with beat scheduler"
celery --app=INSERTAPPNAME worker --beat --scheduler django --loglevel=info &
# Refer https://django-celery-beat.readthedocs.io/en/latest/ to understand params.

read -p "Pause time 5 seconds" -t 5

echo "Starting Flower to monitor celery tasks"
celery --app=INSERTAPPNAME flower --loglevel=info &

celery.py

"""
This file will be added in the main Django Project Folder which already contains settings.py
"""

import os

from celery import Celery


os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoAppName.settings")
app = Celery("djangoAppName")
app.config_from_object("django.conf:settings", namespace="CELERY")
app.autodiscover_tasks()

dockerfile

RUN apt-get install -y redis-server

dockerfile-entrypoint.sh

echo "Starting Redis"
/usr/bin/redis-server &

echo "Starting Celery Worker"
celery --app=djangoAppName worker --loglevel=info &

redis-dockerfile

### Dockerfile for Redis. See setup.md to understand how to run Redis + Celery on local.

FROM        ubuntu:20.04
RUN         apt-get update && apt-get install -y redis-server
EXPOSE      6379

# Since docker container is like a new different server, protected-mode no is required for the localhost to
# connect to this container. For local dev, securities can be turned off.
ENTRYPOINT  ["/usr/bin/redis-server", "--protected-mode no"]

settings.py

# Celery Related
CELERY_BROKER_URL = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'

tasks.py

import time, logging

from celery import shared_task


@shared_task
def create_task(task_type):
    """
    Example Celery Task
    """
    logging.info("New task received")
    time.sleep(int(task_type) * 20)
    return True

views.py

from . import tasks


class CeleryExampleView(APIView):
    def post(self, request):
        task_type = request.data.get('task_type')
        task = tasks.create_task.delay(int(task_type))
        return JsonResponse({"task_id": str(task)}, status=202)