Django Celery Database Setup

ceasy_requirements.txt

Django==1.3.1
django-celery==2.3.3
django-kombu==0.9.4

ctest.tasks.py

import time

from celery.task import task, periodic_task
from celery.schedules import crontab

from django.core.mail import send_mail

@task
def addnums (x, y):
  for i in range(0, 10):
    time.sleep(1)
    
  ret = x + y
  print "TEST"
  print ret
  
  return ret
  
@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*"))
def annoy_me ():
  print "Sending Mail"
  send_mail(
            'Custom Report', 'Here is the message.', 'paul.m.bailey@gmail.com',
            ['paul.m.bailey@gmail.com'], fail_silently=False
           )
  

screen dump

paul@saturn:~/ENVs$ cd ENVs
paul@saturn:~/ENVs$ ls
CM  events  ide  neutron_dev  newsletter  restaurant
paul@saturn:~/ENVs$ virtualenvs  CEASY
New python executable in CEASY/bin/python
Installing setuptools............done.
Installing pip...............done.
paul@saturn:~/ENVs$ source CEASY/bin/activate
(CEASY)paul@saturn:~/ENVs$ cd
(CEASY)paul@saturn:~$ cd git
(CEASY)paul@saturn:~/git$ cd djmeetup/
(CEASY)paul@saturn:~/git/djmeetup$ ls
ceasy_requirements.txt  credis_requirements.txt  tasks.py
(CEASY)paul@saturn:~/git/djmeetup$ pip install -r ceasy_requirements.txt 
Successfully installed Django django-celery django-kombu django-picklefield celery importlib anyjson kombu python-dateutil ordereddict amqplib
Cleaning up...
(CEASY)paul@saturn:~/git/djmeetup$ django-admin startproject cesa  asy
(CEASY)paul@saturn:~/git/djmeetup$ cd ceasy
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ ./manage.py startapp ctest
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ cp ../tasks.py ctest/
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ nano settings.py
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ ./manage.py syncdb
Creating tables ...
Creating table auth_permission
Creating table auth_group_permissions
Creating table auth_group
Creating table auth_user_user_permissions
Creating table auth_user_groups
Creating table auth_user
Creating table auth_message
Creating table django_content_type
Creating table django_session
Creating table django_site
Creating table celery_taskmeta
Creating table celery_tasksetmeta
Creating table djcelery_intervalschedule
Creating table djcelery_crontabschedule
Creating table djcelery_periodictasks
Creating table djcelery_periodictask
Creating table djcelery_workerstate
Creating table djcelery_taskstate
Creating table djkombu_queue
Creating table djkombu_message
You just installed Django's auth system, which means you don't have any superusers defined.
Would you like to create one now? (yes/no): yes
Username (Leave blank to use 'paul'): 
E-mail address: paul.m.baileygmail.com
Password: 
Password (again): 
Superuser created successfully.
Installing custom SQL ...
Installing indexes ...
No fixtures found.
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ ./manage.py help
Type 'manage.py help <subcommand>' for help on a specific subcommand.

Available subcommands:
  camqadm
  celerybeat
  celerycam
  celeryctl
  celeryd
  celeryd_detach
  celeryd_multi
  celeryev
  celerymon
  changepassword
  clean_kombu_messages
  cleanup
  compilemessages
  createcachetable
  createsuperuser
  dbshell
  diffsettings
  djcelerymon
  dumpdata
  flush
  inspectdb
  loaddata
  makemessages
  reset
  runfcgi
  runserver
  shell
  sql
  sqlall
  sqlclear
  sqlcustom
  sqlflush
  sqlindexes
  sqlinitialdata
  sqlreset
  sqlsequencereset
  startapp
  syncdb
  test
  testserver
  validate
Usage: manage.py subcommand [options] [args]

Options:
  -v VERBOSITY, --verbosity=VERBOSITY
                        Verbosity level; 0=minimal output, 1=normal output,
                        2=all output
  --settings=SETTINGS   The Python path to a settings module, e.g.
                        "myproject.settings.main". If this isn't provided, the
                        DJANGO_SETTINGS_MODULE environment variable will be
                        used.
  --pythonpath=PYTHONPATH
                        A directory to add to the Python path, e.g.
                        "/home/djangoprojects/myproject".
  --traceback           Print traceback on exception
  --version             show program's version number and exit
  -h, --help            show this help message and exit

(CEASY)paul@saturn:~/git/djmeetup/ceasy$ ./manage.py celeryd -l info
/home/paul/ENVs/CEASY/lib/python2.6/site-packages/djcelery/loaders.py:84: UserWarning: Using settings.DEBUG leads to a memory leak, never use this setting in production environments!
  warnings.warn("Using settings.DEBUG leads to a memory leak, never "
[2012-01-02 19:25:55,708: WARNING/MainProcess]  

 -------------- celery@saturn v2.4.6
---- **** -----
--- * ***  * -- [Configuration]
-- * - **** ---   . broker:      djkombu.transport.DatabaseTransport://localhost//
- ** ----------   . loader:      djcelery.loaders.DjangoLoader
- ** ----------   . logfile:     [stderr]@INFO
- ** ----------   . concurrency: 4
- ** ----------   . events:      OFF
- *** --- * ---   . beat:        OFF
-- ******* ----
--- ***** ----- [Queues]
 --------------   . celery:      exchange:celery (direct) binding:celery
                  

[Tasks]
  . ctest.tasks.addnums
  . ctest.tasks.annoy_me

[2012-01-02 19:25:55,758: INFO/PoolWorker-1] child process calling self.run()
[2012-01-02 19:25:55,763: INFO/PoolWorker-2] child process calling self.run()
[2012-01-02 19:25:55,767: INFO/PoolWorker-3] child process calling self.run()
[2012-01-02 19:25:55,770: INFO/PoolWorker-4] child process calling self.run()
[2012-01-02 19:25:55,773: WARNING/MainProcess] celery@saturn has started.
[2012-01-02 19:28:36,188: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[7e4872a5-3cde-4b71-9ad0-7adea24ec38b]
[2012-01-02 19:28:46,249: WARNING/PoolWorker-3] TEST
[2012-01-02 19:28:46,249: WARNING/PoolWorker-3] 3
[2012-01-02 19:28:46,856: INFO/MainProcess] Task ctest.tasks.addnums[7e4872a5-3cde-4b71-9ad0-7adea24ec38b] succeeded in 10.618170023s: 3
[2012-01-02 19:29:26,324: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[dc695af2-4f28-4951-b180-5c15654a64ec]
[2012-01-02 19:29:36,381: WARNING/PoolWorker-4] TEST
[2012-01-02 19:29:36,381: WARNING/PoolWorker-4] 3
[2012-01-02 19:29:36,394: INFO/MainProcess] Task ctest.tasks.addnums[dc695af2-4f28-4951-b180-5c15654a64ec] succeeded in 10.0243198872s: 3
[2012-01-02 19:31:21,614: INFO/MainProcess] Got task from broker: ctest.tasks.addnums[3b7d5f4a-7ba7-4383-8877-7455bcd9f57d] eta:[2012-01-02 19:31:21.587442]
[2012-01-02 19:31:31,689: WARNING/PoolWorker-2] TEST
[2012-01-02 19:31:31,690: WARNING/PoolWorker-2] 4
[2012-01-02 19:31:31,701: INFO/MainProcess] Task ctest.tasks.addnums[3b7d5f4a-7ba7-4383-8877-7455bcd9f57d] succeeded in 10.0232799053s: 4
^C[2012-01-02 19:34:11,022: WARNING/MainProcess] celeryd: Hitting Ctrl+C again will terminate all running tasks!
[2012-01-02 19:34:11,023: WARNING/MainProcess] celeryd: Warm shutdown (MainProcess)
[2012-01-02 19:34:12,326: INFO/PoolWorker-2] process shutting down
[2012-01-02 19:34:12,326: INFO/PoolWorker-1] process shutting down
[2012-01-02 19:34:12,326: INFO/PoolWorker-2] process exiting with exitcode 0
[2012-01-02 19:34:12,326: INFO/PoolWorker-1] process exiting with exitcode 0
[2012-01-02 19:34:12,326: INFO/PoolWorker-4] process shutting down
[2012-01-02 19:34:12,327: INFO/PoolWorker-4] process exiting with exitcode 0
[2012-01-02 19:34:12,329: INFO/PoolWorker-3] process shutting down
[2012-01-02 19:34:12,329: INFO/PoolWorker-3] process exiting with exitcode 0
[2012-01-02 19:34:13,048: INFO/MainProcess] process shutting down
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ ./manage.py celeryd -l info -B
/home/paul/ENVs/CEASY/lib/python2.6/site-packages/djcelery/loaders.py:84: UserWarning: Using settings.DEBUG leads to a memory leak, never use this setting in production environments!
  warnings.warn("Using settings.DEBUG leads to a memory leak, never "
[2012-01-02 19:37:14,342: WARNING/MainProcess]  

 -------------- celery@saturn v2.4.6
---- **** -----
--- * ***  * -- [Configuration]
-- * - **** ---   . broker:      djkombu.transport.DatabaseTransport://localhost//
- ** ----------   . loader:      djcelery.loaders.DjangoLoader
- ** ----------   . logfile:     [stderr]@INFO
- ** ----------   . concurrency: 4
- ** ----------   . events:      OFF
- *** --- * ---   . beat:        ON
-- ******* ----
--- ***** ----- [Queues]
 --------------   . celery:      exchange:celery (direct) binding:celery
                  

[Tasks]
  . ctest.tasks.addnums
  . ctest.tasks.annoy_me

[2012-01-02 19:37:14,357: INFO/PoolWorker-2] child process calling self.run()
[2012-01-02 19:37:14,363: INFO/PoolWorker-3] child process calling self.run()
[2012-01-02 19:37:14,368: INFO/PoolWorker-4] child process calling self.run()
[2012-01-02 19:37:14,374: INFO/PoolWorker-5] child process calling self.run()
[2012-01-02 19:37:14,383: WARNING/MainProcess] celery@saturn has started.
[2012-01-02 19:37:14,384: INFO/Beat] child process calling self.run()
[2012-01-02 19:37:14,385: INFO/Beat] Celerybeat: Starting...
[2012-01-02 19:38:04,550: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[9705231a-2f71-4d2e-9ab4-36ec6c1f5a02]
[2012-01-02 19:38:04,558: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 19:38:06,028: INFO/MainProcess] Task ctest.tasks.annoy_me[9705231a-2f71-4d2e-9ab4-36ec6c1f5a02] succeeded in 1.47033309937s: None
[2012-01-02 19:39:04,703: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[a9d9d02c-c3dd-4236-99ea-15a554e7944d]
[2012-01-02 19:39:04,743: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 19:39:04,779: INFO/MainProcess] Task ctest.tasks.annoy_me[a9d9d02c-c3dd-4236-99ea-15a554e7944d] succeeded in 0.0358719825745s: None
[2012-01-02 19:40:04,856: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[4bc3e04a-a8a8-4133-92df-720343a776e7]
[2012-01-02 19:40:04,879: WARNING/PoolWorker-2] Sending Mail
[2012-01-02 19:40:04,889: INFO/MainProcess] Task ctest.tasks.annoy_me[4bc3e04a-a8a8-4133-92df-720343a776e7] succeeded in 0.0109131336212s: None
[2012-01-02 19:41:05,012: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[2002ecda-3691-46b6-b4cb-abe6167b4ce2]
[2012-01-02 19:41:05,063: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 19:41:05,068: INFO/MainProcess] Task ctest.tasks.annoy_me[2002ecda-3691-46b6-b4cb-abe6167b4ce2] succeeded in 0.00570011138916s: None
[2012-01-02 19:42:00,154: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[5f09fc28-4221-49a2-9f5f-4179c513e7dc]
[2012-01-02 19:42:00,197: WARNING/PoolWorker-2] Sending Mail
[2012-01-02 19:42:00,389: INFO/MainProcess] Task ctest.tasks.annoy_me[5f09fc28-4221-49a2-9f5f-4179c513e7dc] succeeded in 0.192721128464s: None
[2012-01-02 19:43:00,303: INFO/MainProcess] Got task from broker: ctest.tasks.annoy_me[50e972e6-b2da-442c-afb0-bd8828d28f29]
[2012-01-02 19:43:00,330: WARNING/PoolWorker-5] Sending Mail
[2012-01-02 19:43:00,335: INFO/MainProcess] Task ctest.tasks.annoy_me[50e972e6-b2da-442c-afb0-bd8828d28f29] succeeded in 0.00494885444641s: None
^C[2012-01-02 19:43:16,553: WARNING/MainProcess] celeryd: Hitting Ctrl+C again will terminate all running tasks!
[2012-01-02 19:43:16,554: WARNING/MainProcess] celeryd: Warm shutdown (MainProcess)
[2012-01-02 19:43:16,554: INFO/MainProcess] Celerybeat: Shutting down...
[2012-01-02 19:43:18,780: INFO/PoolWorker-4] process shutting down
[2012-01-02 19:43:18,780: INFO/PoolWorker-3] process shutting down
[2012-01-02 19:43:18,780: INFO/PoolWorker-2] process shutting down
[2012-01-02 19:43:18,780: INFO/PoolWorker-3] process exiting with exitcode 0
[2012-01-02 19:43:18,780: INFO/PoolWorker-4] process exiting with exitcode 0
[2012-01-02 19:43:18,781: INFO/PoolWorker-2] process exiting with exitcode 0
[2012-01-02 19:43:18,783: INFO/PoolWorker-5] process shutting down
[2012-01-02 19:43:18,784: INFO/PoolWorker-5] process exiting with exitcode 0
[2012-01-02 19:43:19,642: INFO/MainProcess] process shutting down
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ 
(CEASY)paul@saturn:~/git/djmeetup/ceasy$ exit

settings.py

# Django settings for ceasy project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '/home/paul/ceasy.db',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'ta@5i!-rrw9=^n5c)0s6q9rtxm-dh2yd2+9ore9-r%0h-6z=(j'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'ceasy.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'djcelery',
    'djkombu',
    'ctest',
)


import djcelery
djcelery.setup_loader()

BROKER_BACKEND = 'djkombu.transport.DatabaseTransport'
CELERY_RESULT_BACKEND = 'database'