Django Celery Redis Setup

credis_requirements.txt

Django==1.3.1
django-celery==2.3.3
redis==2.4.9
celerymon

ctest.tasks.py

import time

from celery.task import task, periodic_task
from celery.schedules import crontab

from django.core.mail import send_mail

@task
def addnums (x, y):
  for i in range(0, 10):
    time.sleep(1)
    
  ret = x + y
  print "TEST"
  print ret
  
  return ret
  
@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*"))
def annoy_me ():
  print "Sending Mail"
  send_mail(
            'Custom Report', 'Here is the message.', 'paul.m.bailey@gmail.com',
            ['paul.m.bailey@gmail.com'], fail_silently=False
           )
  

screen dump

paul@saturn:~$ cd ENVs
paul@saturn:~/ENVs$ virtualenv CREDIS
New python executable in CREDIS/bin/python
Installing setuptools............done.
Installing pip...............done.
paul@saturn:~/ENVs$ source CREDIS/bin/activate
(CREDIS)paul@saturn:~/ENVs$ cd
(CREDIS)paul@saturn:~$ cd git
(CREDIS)paul@saturn:~/git$ cd djmeetup/
(CREDIS)paul@saturn:~/git/djmeetup$ cd djmeetu
(CREDIS)paul@saturn:~/git/djmeetup$ pip install -r credis_requirements.txt 
Successfully installed Django django-celery redis celerymon django-picklefield celery importlib anyjson kombu python-dateutil ordereddict amqplib
(CREDIS)paul@saturn:~/git/djmeetup$ django-admin startproject credis
(CREDIS)paul@saturn:~/git/djmeetup$ cd credis
(CREDIS)paul@saturn:~/git/djmeetup/credis$ ls
(CREDIS)paul@saturn:~/git/djmeetup/credis$ nano settings.py 
(CREDIS)paul@saturn:~/git/djmeetup/credis$ nano urls.py 
(CREDIS)paul@saturn:~/git/djmeetup/credis$ ./manage.py syncdb
Creating tables ...
Creating table auth_permission
Creating table auth_group_permissions
Creating table auth_group
Creating table auth_user_user_permissions
Creating table auth_user_groups
Creating table auth_user
Creating table auth_message
Creating table django_content_type
Creating table django_session
Creating table django_site
Creating table django_admin_log
Creating table celery_taskmeta
Creating table celery_tasksetmeta
Creating table djcelery_intervalschedule
Creating table djcelery_crontabschedule
Creating table djcelery_periodictasks
Creating table djcelery_periodictask
Creating table djcelery_workerstate
Creating table djcelery_taskstate

You just installed Django's auth system, which means you don't have any superusers defined.
Would you like to create one now? (yes/no): yes
Username (Leave blank to use 'paul'): 
E-mail address: paul.m.bailey@gmail.com
Password: 
Password (again): 
Superuser created successfully.
Installing custom SQL ...
Installing indexes ...
No fixtures found.
(CREDIS)paul@saturn:~/git/djmeetup/credis$ ./manage.py celeryd -l info -E
/home/paul/ENVs/CREDIS/lib/python2.6/site-packages/djcelery/loaders.py:84: UserWarning: Using settings.DEBUG leads to a memory leak, never use this setting in production environments!
  warnings.warn("Using settings.DEBUG leads to a memory leak, never "
[2012-01-02 19:56:55,278: WARNING/MainProcess]  

 -------------- celery@saturn v2.4.6
---- **** -----
--- * ***  * -- [Configuration]
-- * - **** ---   . broker:      redis://localhost:6379/0
- ** ----------   . loader:      djcelery.loaders.DjangoLoader
- ** ----------   . logfile:     [stderr]@INFO
- ** ----------   . concurrency: 4
- ** ----------   . events:      ON
- *** --- * ---   . beat:        OFF
-- ******* ----
--- ***** ----- [Queues]
 --------------   . celery:      exchange:celery (direct) binding:celery
                  

[Tasks]
  . ctest.tasks.addnums
  . ctest.tasks.annoy_me

[2012-01-02 19:56:55,292: INFO/PoolWorker-1] child process calling self.run()
[2012-01-02 19:56:55,296: INFO/PoolWorker-2] child process calling self.run()
[2012-01-02 19:56:55,300: INFO/PoolWorker-3] child process calling self.run()
[2012-01-02 19:56:55,306: INFO/PoolWorker-4] child process calling self.run()
[2012-01-02 19:56:55,308: WARNING/MainProcess] celery@saturn has started.
[2012-01-02 19:59:44,220: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[71d84f08-1318-4c37-a8f8-fc1391028df2]
[2012-01-02 19:59:54,236: WARNING/PoolWorker-3] TEST
[2012-01-02 19:59:54,236: WARNING/PoolWorker-3] 3
[2012-01-02 19:59:54,251: INFO/MainProcess] Task ctest.tasks.addnums[71d84f08-1318-4c37-a8f8-fc1391028df2] succeeded in 10.0264480114s: 3
^C[2012-01-02 20:00:12,243: WARNING/MainProcess] celeryd: Hitting Ctrl+C again will terminate all running tasks!
[2012-01-02 20:00:12,243: WARNING/MainProcess] celeryd: Warm shutdown (MainProcess)
[2012-01-02 20:00:13,922: INFO/PoolWorker-4] process shutting down
[2012-01-02 20:00:13,923: INFO/PoolWorker-4] process exiting with exitcode 0
[2012-01-02 20:00:13,925: INFO/PoolWorker-1] process shutting down
[2012-01-02 20:00:13,925: INFO/PoolWorker-2] process shutting down
[2012-01-02 20:00:13,925: INFO/PoolWorker-2] process exiting with exitcode 0
[2012-01-02 20:00:13,925: INFO/PoolWorker-1] process exiting with exitcode 0
[2012-01-02 20:00:13,925: INFO/PoolWorker-3] process shutting down
[2012-01-02 20:00:13,926: INFO/PoolWorker-3] process exiting with exitcode 0
[2012-01-02 20:00:15,661: INFO/MainProcess] process shutting down
(CREDIS)paul@saturn:~/git/djmeetup/credis$ ./manage.py celeryd -l info -E -B
/home/paul/ENVs/CREDIS/lib/python2.6/site-packages/djcelery/loaders.py:84: UserWarning: Using settings.DEBUG leads to a memory leak, never use this setting in production environments!
  warnings.warn("Using settings.DEBUG leads to a memory leak, never "
[2012-01-02 20:00:22,989: WARNING/MainProcess]  

 -------------- celery@saturn v2.4.6
---- **** -----
--- * ***  * -- [Configuration]
-- * - **** ---   . broker:      redis://localhost:6379/0
- ** ----------   . loader:      djcelery.loaders.DjangoLoader
- ** ----------   . logfile:     [stderr]@INFO
- ** ----------   . concurrency: 4
- ** ----------   . events:      ON
- *** --- * ---   . beat:        ON
-- ******* ----
--- ***** ----- [Queues]
 --------------   . celery:      exchange:celery (direct) binding:celery
                  

[Tasks]
  . ctest.tasks.addnums
  . ctest.tasks.annoy_me

[2012-01-02 20:00:23,003: INFO/PoolWorker-2] child process calling self.run()
[2012-01-02 20:00:23,007: INFO/PoolWorker-3] child process calling self.run()
[2012-01-02 20:00:23,010: INFO/PoolWorker-4] child process calling self.run()
[2012-01-02 20:00:23,015: INFO/PoolWorker-5] child process calling self.run()
[2012-01-02 20:00:23,021: WARNING/MainProcess] celery@saturn has started.
[2012-01-02 20:00:23,021: INFO/Beat] child process calling self.run()
[2012-01-02 20:00:23,022: INFO/Beat] Celerybeat: Starting...
[2012-01-02 20:00:50,870: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[b1bc1960-ff5f-417f-bb4f-6974f86a9560]
[2012-01-02 20:01:00,040: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[6009460d-7d57-428f-9b91-e2c00b9bb02c]
[2012-01-02 20:01:00,058: WARNING/PoolWorker-2] Sending Mail
[2012-01-02 20:01:00,774: INFO/MainProcess] Task ctest.tasks.annoy_me[6009460d-7d57-428f-9b91-e2c00b9bb02c] succeeded in 0.71470284462s: None
[2012-01-02 20:01:00,902: WARNING/PoolWorker-5] TEST
[2012-01-02 20:01:00,903: WARNING/PoolWorker-5] 3
[2012-01-02 20:01:01,341: INFO/MainProcess] Task ctest.tasks.addnums[b1bc1960-ff5f-417f-bb4f-6974f86a9560] succeeded in 10.4493460655s: 3
[2012-01-02 20:02:00,133: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[931208fa-dbc4-4559-bd9e-841c1aacc3a2]
[2012-01-02 20:02:00,141: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 20:02:00,152: INFO/MainProcess] Task ctest.tasks.annoy_me[931208fa-dbc4-4559-bd9e-841c1aacc3a2] succeeded in 0.00985503196716s: None
[2012-01-02 20:03:00,146: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[2ceee291-7ac9-47f8-8a70-122f80e51b0f]
[2012-01-02 20:03:00,161: WARNING/PoolWorker-4] Sending Mail
[2012-01-02 20:03:00,173: INFO/MainProcess] Task ctest.tasks.annoy_me[2ceee291-7ac9-47f8-8a70-122f80e51b0f] succeeded in 0.0122110843658s: None
[2012-01-02 20:04:00,134: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[d03e863c-7d4d-4ae4-9b07-ad0dddf56c2c]
[2012-01-02 20:04:00,146: WARNING/PoolWorker-3] Sending Mail
[2012-01-02 20:04:00,156: INFO/MainProcess] Task ctest.tasks.annoy_me[d03e863c-7d4d-4ae4-9b07-ad0dddf56c2c] succeeded in 0.0104629993439s: None
[2012-01-02 20:05:00,130: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[e680b02c-9615-413d-be5a-907348e67c3c]
[2012-01-02 20:05:00,165: WARNING/PoolWorker-2] Sending Mail
[2012-01-02 20:05:00,172: INFO/MainProcess] Task ctest.tasks.annoy_me[e680b02c-9615-413d-be5a-907348e67c3c] succeeded in 0.00630903244019s: None
[2012-01-02 20:06:00,010: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[a27447e6-e988-42cf-a9a7-a2570abaabed]
[2012-01-02 20:06:00,050: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 20:06:00,056: INFO/MainProcess] Task ctest.tasks.annoy_me[a27447e6-e988-42cf-a9a7-a2570abaabed] succeeded in 0.00538992881775s: None
[2012-01-02 20:07:00,130: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[a987be4d-dd4c-459e-aee8-dc8ba784ac77]
[2012-01-02 20:07:00,134: WARNING/PoolWorker-3] Sending Mail
[2012-01-02 20:07:00,141: INFO/MainProcess] Task ctest.tasks.annoy_me[a987be4d-dd4c-459e-aee8-dc8ba784ac77] succeeded in 0.00667595863342s: None
[2012-01-02 20:07:26,170: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[e2d0d516-9cfc-4efa-93ab-d0b457b25525]
[2012-01-02 20:07:36,177: WARNING/PoolWorker-4] TEST
[2012-01-02 20:07:36,178: WARNING/PoolWorker-4] 1001000
[2012-01-02 20:07:36,191: INFO/MainProcess] Task ctest.tasks.addnums[e2d0d516-9cfc-4efa-93ab-d0b457b25525] succeeded in 10.0168790817s: 1001000
[2012-01-02 20:08:00,127: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[e3ff6826-e160-4dce-9517-49214eea0e3e]
[2012-01-02 20:08:00,154: WARNING/PoolWorker-2] Sending Mail
[2012-01-02 20:08:00,160: INFO/MainProcess] Task ctest.tasks.annoy_me[e3ff6826-e160-4dce-9517-49214eea0e3e] succeeded in 0.00493717193604s: None
[2012-01-02 20:09:00,123: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[8eea288f-0f13-458f-a1aa-dd13e993a775]
[2012-01-02 20:09:00,141: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 20:09:00,148: INFO/MainProcess] Task ctest.tasks.annoy_me[8eea288f-0f13-458f-a1aa-dd13e993a775] succeeded in 0.00571012496948s: None
[2012-01-02 20:10:00,114: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[3b62fe3a-9086-49eb-81fd-8a6c1a8db795]
[2012-01-02 20:10:00,127: WARNING/PoolWorker-4] Sending Mail
[2012-01-02 20:10:00,133: INFO/MainProcess] Task ctest.tasks.annoy_me[3b62fe3a-9086-49eb-81fd-8a6c1a8db795] succeeded in 0.00604200363159s: None
[2012-01-02 20:11:00,003: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[8a6cef1b-87fc-4a81-9ad2-ac61295e21f1]
[2012-01-02 20:11:00,013: WARNING/PoolWorker-3] Sending Mail
[2012-01-02 20:11:00,021: INFO/MainProcess] Task ctest.tasks.annoy_me[8a6cef1b-87fc-4a81-9ad2-ac61295e21f1] succeeded in 0.00662398338318s: None
^C[2012-01-02 20:11:57,462: WARNING/MainProcess] celeryd: Hitting Ctrl+C again will terminate all running tasks!
[2012-01-02 20:11:57,463: WARNING/MainProcess] celeryd: Warm shutdown (MainProcess)
[2012-01-02 20:11:57,463: INFO/MainProcess] Celerybeat: Shutting down...
[2012-01-02 20:11:59,774: INFO/PoolWorker-2] process shutting down
[2012-01-02 20:11:59,774: INFO/PoolWorker-3] process shutting down
[2012-01-02 20:11:59,774: INFO/PoolWorker-5] process shutting down
[2012-01-02 20:11:59,774: INFO/PoolWorker-2] process exiting with exitcode 0
[2012-01-02 20:11:59,774: INFO/PoolWorker-3] process exiting with exitcode 0
[2012-01-02 20:11:59,774: INFO/PoolWorker-5] process exiting with exitcode 0
[2012-01-02 20:11:59,776: INFO/PoolWorker-4] process shutting down
[2012-01-02 20:11:59,777: INFO/PoolWorker-4] process exiting with exitcode 0
[2012-01-02 20:12:01,520: INFO/MainProcess] process shutting down
(CREDIS)paul@saturn:~/git/djmeetup/credis$ exit
exit

settings.py

# Django settings for credis project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '/home/paul/credis.db',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'f%3f^zy50x#pqoi&8f_&4gu0u42n*=@^mhvrg039*qfqh!6+fh'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'credis.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'djcelery',
    'ctest'
)


import djcelery
djcelery.setup_loader()

BROKER_TRANSPORT = 'redis'
BROKER_HOST = 'localhost'
BROKER_PORT = 6379
BROKER_VHOST = '0'