Skip to content

Instantly share code, notes, and snippets.

View ask's full-sized avatar
🎯
Focusing

Ask Solem ask

🎯
Focusing
View GitHub Profile
@ask
ask / get_results.py
Created June 22, 2011 17:37 — forked from lrvick/get_results.py
Get async celery results from subtasks
from tasks import task1
from celery.task import TaskSet
from celery.result import AsyncResult, ResultSet
def get_results(queries):
query_procs = task1.delay(queries).get().join()
results = []
@ask
ask / gist:866335
Created March 11, 2011 18:34
Ideas for an advanced Celery tutorial
* Routing tasks, and how to set up broadcast exchanges
* Kombu
* How to publish/consume messages
* Handling errors, and connection pools
* How virtual transports works and creating new transports.
* Monitoring and events
* celeryev and djcelerymon
* Create custom event consumer
* Logging (sentry, syslog, error e-mails)
* Debugging tasks
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer, Publisher
@task()
def send_emails(n=10):
conn = send_emails.establish_connection()
pub = Publisher(conn, exchange="sendemail",
routing_key="sendemail_emails")
try:
from datetime import datetime, timedelta
from dateutil.parser import parse as iso8601
from celery.task.base import Task
from celery.utils import timedelta_seconds
class ExpireTask(Task):
"""Task with an expiration time.
"libMemcached cache backend"
from django.core.cache.backends.base import (BaseCache,
InvalidCacheBackendError)
from django.utils.encoding import smart_unicode, smart_str
try:
import pylibmc as memcache
is_pylibmc = True
except ImportError:
# Convert Sphinx formatted files to regular RST (e.g. for PyPI README's)
from __future__ import with_statement
import re
import sys
RE_CODE_BLOCK = re.compile(r'.. code-block:: (.+?)\s*$')
RE_REFERENCE = re.compile(r':(.+?):`(.+?)`')
def replace_code_block(lines, pos):
@ask
ask / celery_resque_worker.py
Created November 6, 2009 15:39
Use Celery as a Resque worker
from carrot import serialization
from ghettoq.taproot import Redis
class Resque(Redis):
def message_to_python(self, raw_message):
payload = serialization.decode(raw_message,
content_type="application/json",
content_encoding="utf-8")
task_class = payload["class"]
class TokenBucketQueue(object):
"""An implementation of the token bucket algorithm.
See http://en.wikipedia.org/wiki/Token_Bucket
Most of this code was stolen from an entry in the ASPN Python Cookbook:
http://code.activestate.com/recipes/511490/
:param fill_rate: see :attr:`fill_rate`.
:keyword capacity: see :attr:`capacity`.
class RefreshAllFeeds(PeriodicTask):
"""Periodic Task to refresh all the feeds.
Splits the feeds into slices, depending how many feeds there are in
total and how many iterations you want it to run in.
:keyword iterations: The number of iterations you want the
work to complete in (default: 4).
"""
Index: Lib/multiprocessing/pool.py
===================================================================
--- Lib/multiprocessing/pool.py (revision 74797)
+++ Lib/multiprocessing/pool.py (working copy)
@@ -12,11 +12,14 @@
# Imports
#
+import os
+import errno